New upstream version 8.1.0

fsck/source/database/Buffer.h

@@ -0,0 +1,47 @@
+#ifndef BUFFER_H_
+#define BUFFER_H_
+
+#include <database/Set.h>
+
+template<typename Data>
+class Buffer {
+   private:
+      Set<Data>* set;
+      SetFragment<Data>* currentFragment;
+      size_t fragmentSize;
+
+      Buffer(const Buffer&);
+      Buffer& operator=(const Buffer&);
+
+   public:
+      Buffer(Set<Data>& set, size_t fragmentSize)
+         : set(&set), currentFragment(NULL), fragmentSize(fragmentSize)
+      {}
+
+      ~Buffer()
+      {
+         flush();
+      }
+
+      void flush()
+      {
+         if(currentFragment)
+         {
+            currentFragment->flush();
+            currentFragment = NULL;
+         }
+      }
+
+      void append(const Data& data)
+      {
+         if(!currentFragment)
+            currentFragment = set->newFragment();
+
+         currentFragment->append(data);
+
+         if(currentFragment->size() * sizeof(Data) >= fragmentSize)
+            flush();
+      }
+};
+
+#endif

fsck/source/database/Chunk.h

@@ -0,0 +1,55 @@
+#ifndef CHUNK_H_
+#define CHUNK_H_
+
+#include <common/fsck/FsckChunk.h>
+#include <database/EntryID.h>
+
+#include <boost/tuple/tuple.hpp>
+#include <boost/tuple/tuple_comparison.hpp>
+
+namespace db {
+
+struct Chunk {
+   static const unsigned SAVED_PATH_SIZE = 43;
+
+   EntryID id;             /* 12 */
+   uint32_t targetID;      /* 16 */
+   uint32_t buddyGroupID;  /* 20 */
+
+   /* savedPath layout is ... weird.
+    * we have two layouts:
+    *   - <u16 _b16>/<u16 _b16>/<id _string>
+    *   - u<origOwnerUID _b16>/<(ts >> 16) _b16>/<(ts >> 12) & 0xF _b16>/<id _string>
+    *
+    * where u16 are random 16 bit unsigneds each. that gives maximum path lengths of
+    *   - 2 + 1 + 2 + 26
+    *   - 9 + 1 + 4 + 1 + 1 + 1 + 26
+    * add a terminating NUL to each, the maximum path length becomes 44 bytes,
+    * which is also conveniently aligned for uint_64t followers */
+   char savedPath[SAVED_PATH_SIZE + 1]; /* 64 */
+
+   int64_t fileSize;       /* 72 */
+   int64_t usedBlocks;     /* 80 */
+
+   uint32_t uid;           /* 84 */
+   uint32_t gid;           /* 88 */
+
+   typedef boost::tuple<EntryID, uint32_t, uint32_t> KeyType;
+
+   KeyType pkey() const
+   {
+      return KeyType(id, targetID, buddyGroupID);
+   }
+
+   operator FsckChunk() const
+   {
+      Path path(savedPath);
+
+      return FsckChunk(id.str(), targetID, path, fileSize, usedBlocks, 0, 0, 0, uid,
+         gid, buddyGroupID);
+   }
+};
+
+}
+
+#endif

fsck/source/database/ContDir.h

@@ -0,0 +1,26 @@
+#ifndef CONTDIR_H_
+#define CONTDIR_H_
+
+#include <common/fsck/FsckContDir.h>
+#include <database/EntryID.h>
+
+namespace db {
+
+struct ContDir {
+   EntryID id;                 /* 12 */
+   uint32_t saveNodeID;        /* 16 */
+   uint32_t isBuddyMirrored:1; /* 20 */
+
+   typedef EntryID KeyType;
+
+   KeyType pkey() const { return id; }
+
+   operator FsckContDir() const
+   {
+      return FsckContDir(id.str(), NumNodeID(saveNodeID), isBuddyMirrored);
+   }
+};
+
+}
+
+#endif

fsck/source/database/Cursor.h

@@ -0,0 +1,67 @@
+#ifndef CURSOR_H_
+#define CURSOR_H_
+
+#include <boost/make_shared.hpp>
+#include <boost/shared_ptr.hpp>
+
+template<typename Obj>
+class Cursor
+{
+   public:
+      typedef Obj ElementType;
+
+   private:
+      class SourceBase
+      {
+         public:
+            virtual ~SourceBase() {}
+
+            virtual bool step() = 0;
+            virtual ElementType* get() = 0;
+      };
+
+      template<typename Inner>
+      class Source : public SourceBase
+      {
+         public:
+            Source(Inner inner)
+               : inner(inner)
+            {
+            }
+
+            bool step()
+            {
+               return inner.step();
+            }
+
+            ElementType* get()
+            {
+               return inner.get();
+            }
+
+         private:
+            Inner inner;
+      };
+
+   public:
+      template<typename Inner>
+      explicit Cursor(Inner inner)
+         : source(boost::make_shared<Source<Inner> >(inner) )
+      {
+      }
+
+      bool step()
+      {
+         return source->step();
+      }
+
+      ElementType* get()
+      {
+         return source->get();
+      }
+
+   private:
+      boost::shared_ptr<SourceBase> source;
+};
+
+#endif

fsck/source/database/DirEntry.h

@@ -0,0 +1,67 @@
+#ifndef DIRENTRY_H_
+#define DIRENTRY_H_
+
+#include <common/fsck/FsckDirEntry.h>
+#include <database/EntryID.h>
+
+namespace db {
+
+struct DirEntry
+{
+   EntryID id;                          /* 12 */
+   EntryID parentDirID;                 /* 24 */
+
+   uint32_t entryOwnerNodeID;           /* 28 */
+   uint32_t inodeOwnerNodeID;           /* 32 */
+
+   uint32_t saveNodeID;                 /* 36 */
+   int32_t saveDevice;                  /* 40 */
+   uint64_t saveInode;                  /* 48 */
+
+   union {
+      struct {
+         char text[16];               /* +16 */
+      } inlined;
+      struct {
+         /* take name from a specific offset in some other file,
+          * identified by a u64 for convenience */
+         uint64_t fileID;             /* +8 */
+         uint64_t fileOffset;         /* +16 */
+      } extended;
+   } name;                              /* 64 */
+
+   // the identifying key for dir entries is actually (parent, name). since storing the name as
+   // part of the key is very inefficient, and existence of duplicate names per parent can be
+   // easily excluded, we instead number all dir entries to create a surrogate key (seqNo), which
+   // will also work as (parent, seqNo).
+   //
+   // the actual primary key of the table is (id, parent, GFID, seqNo) though. this is because
+   // most querys only need the id, a few queries need the full fsid linking identifier
+   // (id, parent, GFID), and deletion always need (parent, seqNo). this also means that the
+   // deletion tracker set for dir entries will be rather large per entry, but it does maek queries
+   // more efficient.
+   uint64_t seqNo;                      /* 72 */
+
+   uint32_t entryType:16;
+   uint32_t fileNameInlined:1;
+   uint32_t hasInlinedInode:1;
+   uint32_t isBuddyMirrored:1;          /* 76 */
+
+   typedef boost::tuple<EntryID, EntryID, uint32_t, int32_t, uint32_t, uint64_t> KeyType;
+
+   KeyType pkey() const
+   {
+      return KeyType(id, parentDirID, saveNodeID, saveDevice, saveInode, seqNo);
+   }
+
+   operator FsckDirEntry() const
+   {
+      return FsckDirEntry(id.str(), "[<unresolved>]", parentDirID.str(),
+         NumNodeID(entryOwnerNodeID), NumNodeID(inodeOwnerNodeID), FsckDirEntryType(entryType),
+         hasInlinedInode, NumNodeID(saveNodeID), saveDevice, saveInode, isBuddyMirrored, seqNo);
+   }
+};
+
+}
+
+#endif

fsck/source/database/DirInode.h

@@ -0,0 +1,40 @@
+#ifndef DIRINODE_H_
+#define DIRINODE_H_
+
+#include <common/fsck/FsckDirInode.h>
+#include <database/EntryID.h>
+
+namespace db {
+
+struct DirInode {
+   EntryID id;                /* 12 */
+   EntryID parentDirID;       /* 24 */
+
+   uint32_t parentNodeID;       /* 28 */
+   uint32_t ownerNodeID;        /* 32 */
+   uint32_t saveNodeID;         /* 36 */
+
+   uint32_t stripePatternType:16;
+   uint32_t readable:1;
+   uint32_t isBuddyMirrored:1;
+   uint32_t isMismirrored:1;    /* 40 */
+
+   uint64_t size;               /* 48 */
+   uint64_t numHardlinks;       /* 56 */
+
+   typedef EntryID KeyType;
+
+   EntryID pkey() const { return id; }
+
+   operator FsckDirInode() const
+   {
+      return FsckDirInode(id.str(), parentDirID.str(), NumNodeID(parentNodeID),
+         NumNodeID(ownerNodeID), size, numHardlinks, UInt16Vector(),
+         FsckStripePatternType(stripePatternType), NumNodeID(saveNodeID), isBuddyMirrored,
+         readable, isMismirrored);
+   }
+};
+
+}
+
+#endif

fsck/source/database/DiskList.h

@@ -0,0 +1,149 @@
+#ifndef DISKLIST_H_
+#define DISKLIST_H_
+
+#include <common/threading/Mutex.h>
+#include <common/toolkit/serialization/Serialization.h>
+
+#include <cerrno>
+#include <stdexcept>
+#include <string>
+#include <mutex>
+#include <vector>
+
+#include <fcntl.h>
+#include <stdio.h>
+#include <unistd.h>
+
+struct DiskListDoesNotExist : public std::runtime_error
+{
+   DiskListDoesNotExist(const std::string& file)
+      : runtime_error(file)
+   {}
+};
+
+template<typename Data>
+class DiskList;
+
+template<typename Data>
+class DiskListCursor
+{
+   friend class DiskList<Data>;
+
+   public:
+      bool step()
+      {
+         uint64_t itemLen;
+
+         {
+            ssize_t readRes = ::pread(fd, &itemLen, sizeof(itemLen), offset);
+
+            if (readRes == 0)
+               return false;
+
+            if (readRes < (ssize_t) sizeof(itemLen))
+               throw std::runtime_error("could not read from disk list file: " + std::string(strerror(errno)));
+         }
+
+         itemLen = LE_TO_HOST_64(itemLen);
+
+         std::unique_ptr<char[]> itemBuf(new char[itemLen]);
+
+         ssize_t readRes = ::pread(fd, itemBuf.get(), itemLen, offset + sizeof(itemLen));
+
+         if (readRes < (ssize_t) itemLen)
+            throw std::runtime_error("could not read from disk list file: " + std::string(strerror(errno)));
+
+         Deserializer des(itemBuf.get(), itemLen);
+
+         des % item;
+         if (!des.good())
+            throw std::runtime_error("could not read from disk list file: " + std::string(strerror(errno)));
+
+         offset += sizeof(itemLen) + itemLen;
+         return true;
+      }
+
+      Data* get()
+      {
+         return &item;
+      }
+
+   private:
+      int fd;
+      size_t offset;
+      Data item;
+
+      DiskListCursor(int fd):
+         fd(fd), offset(0)
+      {
+      }
+};
+
+template<typename Data>
+class DiskList {
+   private:
+      std::string file;
+      int fd;
+      uint64_t itemCount;
+      Mutex mtx;
+
+      DiskList(const DiskList&);
+      DiskList& operator=(const DiskList&);
+
+   public:
+      DiskList(const std::string& file, bool allowCreate = true) :
+         file(file), itemCount(0)
+      {
+         fd = ::open(file.c_str(), O_RDWR | (allowCreate ? O_CREAT : 0), 0660);
+         if(fd < 0)
+         {
+            int eno = errno;
+            if(!allowCreate && errno == ENOENT)
+               throw DiskListDoesNotExist(file);
+            else
+               throw std::runtime_error("could not open disk list file " + file + ": " + strerror(eno));
+         }
+      }
+
+      ~DiskList()
+      {
+         if(fd >= 0)
+            close(fd);
+      }
+
+      const std::string filename() const { return file; }
+
+      void append(const Data& data)
+      {
+         std::lock_guard<Mutex> lock(mtx);
+
+         boost::scoped_array<char> buffer;
+
+         ssize_t size = serializeIntoNewBuffer(data, buffer);
+         if (size < 0)
+            throw std::runtime_error("error serializing disk list item");
+
+         uint64_t bufSize = HOST_TO_LE_64(size);
+         if (::write(fd, &bufSize, sizeof(bufSize)) < (ssize_t) sizeof(bufSize))
+            throw std::runtime_error("error writing disk list item: " + std::string(strerror(errno)));
+
+         if (::write(fd, buffer.get(), size) < size)
+            throw std::runtime_error("error writing disk list item: " + std::string(strerror(errno)));
+      }
+
+      DiskListCursor<Data> cursor()
+      {
+         return DiskListCursor<Data>(fd);
+      }
+
+      void clear()
+      {
+         if (fd >= 0)
+         {
+            if (ftruncate(fd, 0) < 0)
+               throw std::runtime_error("error clearing disk list: " + std::string(strerror(errno)));
+         }
+      }
+};
+
+#endif

fsck/source/database/Distinct.h

@@ -0,0 +1,88 @@
+#ifndef DISTINCT_H_
+#define DISTINCT_H_
+
+#include <boost/type_traits/decay.hpp>
+#include <boost/utility/result_of.hpp>
+
+template<typename Source, typename KeyExtract>
+class Distinct
+{
+   private:
+      typedef typename boost::decay<
+            typename boost::result_of<KeyExtract(typename Source::ElementType&)>::type
+         >::type KeyType;
+
+   public:
+      typedef typename Source::ElementType ElementType;
+      typedef typename Source::MarkerType MarkerType;
+
+   public:
+      Distinct(Source source, KeyExtract keyExtract)
+         : source(source), keyExtract(keyExtract), hasKey(false)
+      {}
+
+      bool step()
+      {
+         if(!this->source.step() )
+            return false;
+
+         while(this->hasKey && this->key == this->keyExtract(*this->source.get() ) )
+         {
+            if(!this->source.step() )
+               return false;
+         }
+
+         this->hasKey = true;
+         this->key = this->keyExtract(*this->source.get() );
+         return true;
+      }
+
+      ElementType* get()
+      {
+         return this->source.get();
+      }
+
+      MarkerType mark() const
+      {
+         return this->source.mark();
+      }
+
+      void restore(MarkerType mark)
+      {
+         this->source.restore(mark);
+         this->hasKey = true;
+         this->key = this->keyExtract(*this->source.get() );
+      }
+
+   private:
+      Source source;
+      KeyType key;
+      KeyExtract keyExtract;
+      bool hasKey;
+};
+
+
+namespace db {
+
+template<typename KeyExtract>
+struct DistinctOp
+{
+   KeyExtract keyExtract;
+
+   template<typename Source>
+   friend Distinct<Source, KeyExtract> operator|(Source source, const DistinctOp& op)
+   {
+      return Distinct<Source, KeyExtract>(source, op.keyExtract);
+   }
+};
+
+template<typename KeyExtract>
+inline DistinctOp<KeyExtract> distinctBy(KeyExtract keyExtract)
+{
+   DistinctOp<KeyExtract> result = {keyExtract};
+   return result;
+}
+
+}
+
+#endif

fsck/source/database/EntryID.h

@@ -0,0 +1,163 @@
+#ifndef ENTRYID_H_
+#define ENTRYID_H_
+
+#include <common/storage/Metadata.h>
+
+#include <algorithm>
+#include <cstdio>
+#include <exception>
+#include <string>
+
+#include <boost/tuple/tuple.hpp>
+#include <boost/tuple/tuple_comparison.hpp>
+
+namespace db {
+
+struct EntryID {
+   unsigned sequence;
+   unsigned timestamp;
+   unsigned nodeID;
+
+   // required for queries, since they use boost::tuple to avoid writing out huge comparison
+   // operators
+   EntryID()
+      : sequence(0), timestamp(0), nodeID(0)
+   {}
+
+   EntryID(unsigned sequence, unsigned timestamp, unsigned nodeID)
+      : sequence(sequence), timestamp(timestamp), nodeID(nodeID)
+   {
+      if(nodeID == 0 && (timestamp != 0 || sequence > 3) )
+         throw std::exception();
+   }
+
+   bool isSpecial() const { return nodeID == 0; }
+
+   bool operator<(const EntryID& other) const
+   {
+      return boost::make_tuple(sequence, timestamp, nodeID)
+         < boost::make_tuple(other.sequence, other.timestamp, other.nodeID);
+   }
+
+   bool operator==(const EntryID& other) const
+   {
+      return this->sequence == other.sequence
+         && this->timestamp == other.timestamp
+         && this->nodeID == other.nodeID;
+   }
+
+   bool operator!=(const EntryID& other) const
+   {
+      return !(*this == other);
+   }
+
+   bool isDisposalDir() const
+   {
+      return *this == disposal() || *this == mdisposal();
+   }
+
+   bool isRootDir() const
+   {
+      return *this == root();
+   }
+
+   std::string str() const
+   {
+      if(isSpecial() )
+      {
+         if(*this == anchor() )
+            return "";
+
+         if(*this == root() )
+            return META_ROOTDIR_ID_STR;
+
+         if(*this == disposal() )
+            return META_DISPOSALDIR_ID_STR;
+
+         if(*this == mdisposal())
+            return META_MIRRORDISPOSALDIR_ID_STR;
+
+         throw std::exception();
+      }
+
+      char buffer[3*8 + 2 + 1];
+      sprintf(buffer, "%X-%X-%X", sequence, timestamp, nodeID);
+      return buffer;
+   }
+
+   static EntryID anchor() { return EntryID(0, 0, 0); }
+   static EntryID root() { return EntryID(1, 0, 0); }
+   static EntryID disposal() { return EntryID(2, 0, 0); }
+   static EntryID mdisposal() { return EntryID(3, 0, 0); }
+
+   static EntryID fromStr(const std::string& str)
+   {
+      auto pair = tryFromStr(str);
+      if (!pair.first)
+         throw std::exception();
+
+      return pair.second;
+   }
+
+   static std::pair<bool, EntryID> tryFromStr(const std::string& str)
+   {
+      if (str.empty())
+         return {true, anchor()};
+
+      if (str == META_ROOTDIR_ID_STR)
+         return {true, root()};
+
+      if (str == META_DISPOSALDIR_ID_STR)
+         return {true, disposal()};
+
+      if (str == META_MIRRORDISPOSALDIR_ID_STR)
+         return {true, mdisposal()};
+
+      std::string::size_type sep1;
+      std::string::size_type sep2;
+
+      sep1 = str.find('-');
+      sep2 = str.find('-', sep1 + 1);
+
+      if (sep1 == str.npos || sep2 == str.npos)
+         return {false, {}};
+
+      std::string seqStr = str.substr(0, sep1);
+      std::string tsStr = str.substr(sep1 + 1, sep2 - (sep1 + 1));
+      std::string nodeStr = str.substr(sep2 + 1);
+
+      if (seqStr.empty() || tsStr.empty() || nodeStr.empty() ||
+            seqStr.size() > 8 || tsStr.size() > 8 || nodeStr.size() > 8)
+         return {false, {}};
+
+      struct
+      {
+         static bool notHex(char c)
+         {
+            return !std::isxdigit(c);
+         }
+
+         bool operator()(const std::string& str) const
+         {
+            return std::count_if(str.begin(), str.end(), notHex) == 0;
+         }
+      } onlyHex;
+
+      if (!onlyHex(seqStr) || !onlyHex(tsStr) || !onlyHex(nodeStr) )
+         return {false, {}};
+
+      unsigned seq;
+      unsigned ts;
+      unsigned node;
+
+      std::sscanf(seqStr.c_str(), "%x", &seq);
+      std::sscanf(tsStr.c_str(), "%x", &ts);
+      std::sscanf(nodeStr.c_str(), "%x", &node);
+
+      return {true, EntryID(seq, ts, node)};
+   }
+};
+
+}
+
+#endif

fsck/source/database/FileInode.h

@@ -0,0 +1,99 @@
+#ifndef FILEINODE_H_
+#define FILEINODE_H_
+
+#include <common/fsck/FsckFileInode.h>
+#include <database/EntryID.h>
+
+namespace db {
+
+struct FileInode {
+   EntryID id;                   /* 12 */
+   EntryID parentDirID;          /* 24 */
+   EntryID origParentEntryID;    /* 36 */
+
+   uint32_t parentNodeID;        /* 40 */
+   uint32_t saveNodeID;          /* 44 */
+   uint32_t origParentUID;       /* 48 */
+
+   uint32_t uid;                 /* 52 */
+   uint32_t gid;                 /* 56 */
+
+   uint64_t fileSize;            /* 64 */
+   uint64_t usedBlocks;          /* 72 */
+
+   uint64_t numHardlinks;        /* 80 */
+
+   uint64_t saveInode;           /* 88 */
+   int32_t saveDevice;           /* 92 */
+
+   uint32_t chunkSize;           /* 96 */
+
+   enum {
+      NTARGETS = 6
+   };
+   uint16_t targets[NTARGETS];   /* 108 */
+
+   uint32_t isInlined:1;
+   uint32_t pathInfoFlags:4;
+   uint32_t stripePatternType:4;
+   uint32_t stripePatternSize:20;
+   uint32_t readable:1;
+   uint32_t isBuddyMirrored:1;
+   uint32_t isMismirrored:1;     /* 112 */
+
+   typedef EntryID KeyType;
+
+   EntryID pkey() const { return id; }
+
+   FsckFileInode toInodeWithoutStripes() const
+   {
+      PathInfo info(origParentUID, origParentEntryID.str(), pathInfoFlags);
+
+      return FsckFileInode(id.str(), parentDirID.str(), NumNodeID(parentNodeID), info, uid, gid,
+            fileSize, numHardlinks, usedBlocks, {}, FsckStripePatternType(stripePatternType),
+            chunkSize, NumNodeID(saveNodeID), saveInode, saveDevice, isInlined, isBuddyMirrored,
+            readable, isMismirrored);
+   }
+
+   friend std::ostream& operator<<(std::ostream& os, FileInode const& obj)
+   {
+         os << "-------------" << "\n";
+         os << "FsckFileInode" << "\n";
+         os << "-------------" << "\n";
+         os << "EntryID: " << obj.id.str() << "\n";
+         os << "Parent dir's entryID: " << obj.parentDirID.str() << "\n";
+         os << "Orig parent dir's entryID: " << obj.origParentEntryID.str() << "\n";
+         os << "parent nodeID: " << obj.parentNodeID << "\n";
+         os << "save nodeID: " << obj.saveNodeID << "\n";
+         os << "origParentUID: " << obj.origParentUID << "\n";
+         os << "uid: " << obj.uid << "\n";
+         os << "gid: " << obj.gid << "\n";
+         os << "fileSize: " << obj.fileSize << "\n";
+         os << "usedBlocks: " << obj.usedBlocks << "\n";
+         os << "numHardlinks: " << obj.numHardlinks << "\n";
+         os << "saveInode: " << obj.saveInode << "\n";
+         os << "saveDevice: " << obj.saveDevice << "\n";
+         os << "chunkSize: " << obj.chunkSize << "\n";
+
+         os << "Stripe targets: [ ";
+
+         for (int i=0; i< NTARGETS; i++)
+         {
+            os << obj.targets[i] << " ";
+         }
+         os << "]\n";
+
+         os << "isInlined: " << obj.isInlined << "\n";
+         os << "pathInfoFlags: " << obj.pathInfoFlags << "\n";
+         os << "stripePatternType: " << obj.stripePatternType << "\n";
+         os << "stripePatternSize: " << obj.stripePatternSize << "\n";
+         os << "readable: " << obj.readable << "\n";
+         os << "isBuddyMirrored: " << obj.isBuddyMirrored << "\n";
+         os << "isMismirrored: " << obj.isMismirrored << "\n\n";
+         return os;
+   }
+};
+
+}
+
+#endif

fsck/source/database/Filter.h

@@ -0,0 +1,72 @@
+#ifndef FILTER_H_
+#define FILTER_H_
+
+template<typename Source, class Pred>
+class Filter
+{
+   public:
+      typedef typename Source::ElementType ElementType;
+      typedef typename Source::MarkerType MarkerType;
+
+   public:
+      Filter(Source source, Pred pred)
+         : source(source), pred(pred)
+      {
+      }
+
+      bool step()
+      {
+         while(this->source.step() )
+         {
+            if(this->pred(*get() ) )
+               return true;
+         }
+
+         return false;
+      }
+
+      ElementType* get()
+      {
+         return this->source.get();
+      }
+
+      MarkerType mark() const
+      {
+         return this->source.mark();
+      }
+
+      void restore(MarkerType mark)
+      {
+         this->source.restore(mark);
+      }
+
+   private:
+      Source source;
+      Pred pred;
+};
+
+
+namespace db {
+
+template<typename Pred>
+struct FilterOp
+{
+   Pred pred;
+
+   template<typename Source>
+   friend Filter<Source, Pred> operator|(Source source, const FilterOp& op)
+   {
+      return Filter<Source, Pred>(source, op.pred);
+   }
+};
+
+template<typename Pred>
+inline FilterOp<Pred> where(Pred pred)
+{
+   FilterOp<Pred> result = {pred};
+   return result;
+}
+
+}
+
+#endif

fsck/source/database/FsID.h

@@ -0,0 +1,38 @@
+#ifndef FSID_H_
+#define FSID_H_
+
+#include <common/fsck/FsckFsID.h>
+#include <database/EntryID.h>
+
+#include <boost/tuple/tuple.hpp>
+#include <boost/tuple/tuple_comparison.hpp>
+
+namespace db {
+
+struct FsID {
+   EntryID id;                  /* 12 */
+   EntryID parentDirID;         /* 24 */
+
+   uint32_t saveNodeID;         /* 28 */
+   int32_t saveDevice;          /* 32 */
+   uint64_t saveInode;          /* 40 */
+
+   uint64_t isBuddyMirrored:1;  /* 48 */
+
+   typedef boost::tuple<EntryID, EntryID, uint32_t, int32_t, uint32_t> KeyType;
+
+   KeyType pkey() const
+   {
+      return KeyType(id, parentDirID, saveNodeID, saveDevice, saveInode);
+   }
+
+   operator FsckFsID() const
+   {
+      return FsckFsID(id.str(), parentDirID.str(), NumNodeID(saveNodeID), saveDevice, saveInode,
+            isBuddyMirrored);
+   }
+};
+
+}
+
+#endif

fsck/source/database/FsckDB.cpp

@@ -0,0 +1,40 @@
+#include "FsckDB.h"
+
+#include <program/Program.h>
+#include <common/toolkit/StorageTk.h>
+#include <common/toolkit/StringTk.h>
+#include <database/FsckDBException.h>
+#include <database/FsckDBTable.h>
+#include <toolkit/FsckTkEx.h>
+
+#include <cstdio>
+
+FsckDB::FsckDB(const std::string& databasePath, size_t fragmentSize, size_t nameCacheLimit,
+      bool allowCreate)
+   : log("FsckDB"),
+     databasePath(databasePath),
+     dentryTable(new FsckDBDentryTable(databasePath, fragmentSize, nameCacheLimit, allowCreate) ),
+     fileInodesTable(new FsckDBFileInodesTable(databasePath, fragmentSize, allowCreate) ),
+     dirInodesTable(new FsckDBDirInodesTable(databasePath, fragmentSize, allowCreate) ),
+     chunksTable(new FsckDBChunksTable(databasePath, fragmentSize, allowCreate) ),
+     contDirsTable(new FsckDBContDirsTable(databasePath, fragmentSize, allowCreate) ),
+     fsIDsTable(new FsckDBFsIDsTable(databasePath, fragmentSize, allowCreate) ),
+     usedTargetIDsTable(new FsckDBUsedTargetIDsTable(databasePath, fragmentSize, allowCreate) ),
+     modificationEventsTable(new FsckDBModificationEventsTable(databasePath, fragmentSize,
+         allowCreate) ),
+     malformedChunks(databasePath + "/malformedChunks")
+{
+}
+
+void FsckDB::clear()
+{
+   this->dentryTable->clear();
+   this->fileInodesTable->clear();
+   this->dirInodesTable->clear();
+   this->chunksTable->clear();
+   this->contDirsTable->clear();
+   this->fsIDsTable->clear();
+   this->usedTargetIDsTable->clear();
+   this->modificationEventsTable->clear();
+   this->malformedChunks.clear();
+}

fsck/source/database/FsckDB.h

@@ -0,0 +1,178 @@
+/*
+ * This class is intended to be the interface to the sqlite DB. It can be created once in the
+ * applicationm, and can then be used in all threads, as sqlite claims to be thread-safe in
+ * "serialized" mode (which is the default, http://sqlite.org/threadsafe.html).
+ */
+
+#ifndef FSCKDB_H_
+#define FSCKDB_H_
+
+#include <common/app/log/LogContext.h>
+#include <common/Common.h>
+#include <common/fsck/FsckChunk.h>
+#include <common/fsck/FsckContDir.h>
+#include <common/fsck/FsckDirEntry.h>
+#include <common/fsck/FsckDirInode.h>
+#include <common/fsck/FsckFileInode.h>
+#include <common/fsck/FsckFsID.h>
+#include <common/fsck/FsckTargetID.h>
+#include <common/fsck/FsckDuplicateInodeInfo.h>
+#include <common/nodes/TargetMapper.h>
+#include <common/storage/striping/StripePattern.h>
+#include <common/storage/StorageDefinitions.h>
+#include <database/ContDir.h>
+#include <database/Cursor.h>
+#include <database/DirEntry.h>
+#include <database/DirInode.h>
+#include <database/DiskList.h>
+#include <database/EntryID.h>
+#include <database/FileInode.h>
+#include <database/Filter.h>
+#include <database/VectorSource.h>
+#include <toolkit/FsckDefinitions.h>
+
+#include <boost/scoped_ptr.hpp>
+
+class FsckDBDentryTable;
+class FsckDBFileInodesTable;
+class FsckDBDirInodesTable;
+class FsckDBChunksTable;
+class FsckDBContDirsTable;
+class FsckDBFsIDsTable;
+class FsckDBUsedTargetIDsTable;
+class FsckDBModificationEventsTable;
+class FsckDBDentryErrorTable;
+class FsckDBDirInodeErrorTable;
+class FsckDBFsIDErrorTable;
+class FsckDBFileInodeErrorTable;
+class FsckDBChunkErrorTable;
+class FsckDBContDirErrorTable;
+class FsckDBStripeTargetErrorTable;
+
+namespace checks {
+
+struct InodeAttribs
+{
+   uint64_t size;
+   uint64_t nlinks;
+};
+
+struct OptionalInodeAttribs
+{
+   boost::optional<uint64_t> size;
+   boost::optional<uint64_t> nlinks;
+
+   void reset() {
+       size.reset();
+       nlinks.reset();
+   }
+};
+
+typedef std::pair<db::EntryID, std::set<FsckDuplicateInodeInfo>> DuplicatedInode;
+}
+
+class FsckDB
+{
+   friend class TestDatabase;
+
+   public:
+      // FsckDB.cpp
+      FsckDB(const std::string& databasePath, size_t fragmentSize, size_t nameCacheLimit,
+         bool allowCreate);
+
+      void clear();
+
+      // FsckDBChecks.cpp
+      Cursor<checks::DuplicatedInode> findDuplicateInodeIDs();
+      Cursor<std::list<FsckChunk> > findDuplicateChunks();
+      Cursor<std::list<db::ContDir>> findDuplicateContDirs();
+
+      Cursor<db::DirEntry> findMismirroredDentries();
+      Cursor<db::DirInode> findMismirroredDirectories();
+      Cursor<db::FileInode> findMismirroredFiles();
+
+      Cursor<db::DirEntry> findDanglingDirEntries();
+      Cursor<db::DirEntry> findDirEntriesWithBrokenByIDFile();
+      Cursor<FsckFsID> findOrphanedFsIDFiles();
+      Cursor<FsckDirInode> findInodesWithWrongOwner();
+      Cursor<std::pair<db::DirEntry, NumNodeID> > findDirEntriesWithWrongOwner();
+      Cursor<FsckDirInode> findOrphanedDirInodes();
+      Cursor<FsckFileInode> findOrphanedFileInodes();
+      Cursor<FsckChunk> findOrphanedChunks();
+
+      Cursor<FsckDirInode> findInodesWithoutContDir();
+
+      Cursor<FsckContDir> findOrphanedContDirs();
+
+      Cursor<std::pair<FsckFileInode, checks::OptionalInodeAttribs> > findWrongInodeFileAttribs();
+
+      Cursor<std::pair<FsckDirInode, checks::InodeAttribs> > findWrongInodeDirAttribs();
+
+      Cursor<db::DirEntry> findFilesWithMissingStripeTargets(TargetMapper* targetMapper,
+         MirrorBuddyGroupMapper* buddyGroupMapper);
+      Cursor<std::pair<FsckChunk, FsckFileInode> > findChunksWithWrongPermissions();
+      Cursor<std::pair<FsckChunk, FsckFileInode> > findChunksInWrongPath();
+      Cursor<db::FileInode> findFilesWithMultipleHardlinks();
+
+   private:
+      LogContext log;
+      std::string databasePath;
+
+      boost::scoped_ptr<FsckDBDentryTable> dentryTable;
+      boost::scoped_ptr<FsckDBFileInodesTable> fileInodesTable;
+      boost::scoped_ptr<FsckDBDirInodesTable> dirInodesTable;
+      boost::scoped_ptr<FsckDBChunksTable> chunksTable;
+      boost::scoped_ptr<FsckDBContDirsTable> contDirsTable;
+      boost::scoped_ptr<FsckDBFsIDsTable> fsIDsTable;
+      boost::scoped_ptr<FsckDBUsedTargetIDsTable> usedTargetIDsTable;
+      boost::scoped_ptr<FsckDBModificationEventsTable> modificationEventsTable;
+      DiskList<FsckChunk> malformedChunks;
+
+   public:
+      FsckDBDentryTable* getDentryTable()
+      {
+         return this->dentryTable.get();
+      }
+
+      FsckDBFileInodesTable* getFileInodesTable()
+      {
+         return this->fileInodesTable.get();
+      }
+
+      FsckDBDirInodesTable* getDirInodesTable()
+      {
+         return this->dirInodesTable.get();
+      }
+
+      FsckDBChunksTable* getChunksTable()
+      {
+         return this->chunksTable.get();
+      }
+
+      FsckDBContDirsTable* getContDirsTable()
+      {
+         return this->contDirsTable.get();
+      }
+
+      FsckDBFsIDsTable* getFsIDsTable()
+      {
+         return this->fsIDsTable.get();
+      }
+
+      FsckDBUsedTargetIDsTable* getUsedTargetIDsTable()
+      {
+         return this->usedTargetIDsTable.get();
+      }
+
+      FsckDBModificationEventsTable* getModificationEventsTable()
+      {
+         return this->modificationEventsTable.get();
+      }
+
+      DiskList<FsckChunk>* getMalformedChunksList()
+      {
+         return &malformedChunks;
+      }
+};
+
+#endif /* FSCKDB_H_ */

fsck/source/database/FsckDBException.cpp

@@ -0,0 +1,2 @@
+#include "FsckDBException.h"
+

fsck/source/database/FsckDBException.h

@@ -0,0 +1,8 @@
+#ifndef FSCKDBEXCEPTION_H
+#define FSCKDBEXCEPTION_H
+
+#include <common/toolkit/NamedException.h>
+
+DECLARE_NAMEDEXCEPTION(FsckDBException, "FsckDBException")
+
+#endif /*FSCKDBEXCEPTION_H*/

fsck/source/database/FsckDBTable.cpp

@@ -0,0 +1,509 @@
+#include "FsckDBTable.h"
+
+#include <database/FsckDBException.h>
+#include <toolkit/FsckTkEx.h>
+
+#include <boost/make_shared.hpp>
+
+static db::DirEntry fsckDirEntryToDbDirEntry(const FsckDirEntry& dentry)
+{
+   db::DirEntry result = {
+      db::EntryID::fromStr(dentry.getID() ),
+      db::EntryID::fromStr(dentry.getParentDirID() ),
+
+      dentry.getEntryOwnerNodeID().val(),
+      dentry.getInodeOwnerNodeID().val(),
+
+      dentry.getSaveNodeID().val(), dentry.getSaveDevice(), dentry.getSaveInode(),
+
+      {},
+
+      dentry.getInternalID(),
+
+      dentry.getEntryType(),
+      dentry.getName().size() < sizeof(result.name.inlined.text),
+      dentry.getHasInlinedInode(),
+      dentry.getIsBuddyMirrored(),
+   };
+
+   if(result.fileNameInlined) {
+      ::memset(result.name.inlined.text, '\0', sizeof(result.name.inlined.text));
+      ::strncpy(result.name.inlined.text, dentry.getName().c_str(),
+            sizeof(result.name.inlined.text) - 1);
+   }
+
+   return result;
+}
+
+void FsckDBDentryTable::insert(FsckDirEntryList& dentries, const BulkHandle* handle)
+{
+   NameBuffer& names = handle ? *handle->nameBuffer : getNameBuffer(0);
+
+   for(FsckDirEntryListIter it = dentries.begin(), end = dentries.end(); it != end; ++it)
+   {
+      db::DirEntry dentry = fsckDirEntryToDbDirEntry(*it);
+      ByParent link = { dentry.parentDirID, dentry.id, dentry.entryType };
+
+      if(!dentry.fileNameInlined)
+      {
+         dentry.name.extended.fileID = names.id();
+         dentry.name.extended.fileOffset = names.put(it->getName() );
+      }
+
+      if(handle)
+      {
+         handle->dentries->append(dentry);
+         handle->byParent->append(link);
+      }
+      else
+      {
+         this->table.insert(dentry);
+         this->byParent.insert(link);
+      }
+   }
+}
+
+void FsckDBDentryTable::updateFieldsExceptParent(FsckDirEntryList& dentries)
+{
+   for(FsckDirEntryListIter it = dentries.begin(), end = dentries.end(); it != end; ++it)
+   {
+      db::DirEntry dentry = fsckDirEntryToDbDirEntry(*it);
+
+      this->table.remove(dentry.pkey() );
+      this->table.insert(dentry);
+   }
+}
+
+void FsckDBDentryTable::remove(FsckDirEntryList& dentries)
+{
+   for(FsckDirEntryListIter it = dentries.begin(), end = dentries.end(); it != end; ++it)
+      this->table.remove(fsckDirEntryToDbDirEntry(*it).pkey() );
+}
+
+Table<db::DirEntry>::QueryType FsckDBDentryTable::get()
+{
+   this->table.commitChanges();
+   return this->table.cursor();
+}
+
+Table<FsckDBDentryTable::ByParent>::QueryType FsckDBDentryTable::getByParent()
+{
+   this->byParent.commitChanges();
+   return this->byParent.cursor();
+}
+
+std::pair<bool, db::DirEntry> FsckDBDentryTable::getAnyFor(db::EntryID id)
+{
+   this->table.commitChanges();
+
+   struct ops
+   {
+      static db::EntryID onlyID(const db::DirEntry::KeyType& key) { return boost::get<0>(key); }
+   };
+
+   return this->table.getByKeyProjection(id, ops::onlyID);
+}
+
+std::string FsckDBDentryTable::getNameOf(const db::DirEntry& dentry)
+{
+   if(dentry.fileNameInlined)
+      return dentry.name.inlined.text;
+
+   return getNameBuffer(dentry.name.extended.fileID).get(dentry.name.extended.fileOffset);
+}
+
+std::string FsckDBDentryTable::getPathOf(const db::DirEntry& dentry)
+{
+   std::string result = getNameOf(dentry);
+
+   db::EntryID parent = dentry.parentDirID;
+   unsigned depth = 0;
+
+   while(!parent.isSpecial() )
+   {
+      depth++;
+      if(depth > 255)
+         return "[<unresolved>]/" + result;
+
+      const NameCacheEntry* cacheEntry = getFromCache(parent);
+      if(cacheEntry)
+      {
+         result = cacheEntry->name + "/" + result;
+         parent = cacheEntry->parent;
+         continue;
+      }
+
+      std::pair<bool, db::DirEntry> item = getAnyFor(parent);
+      if(!item.first)
+         return "[<unresolved>]/" + result;
+
+      std::string parentName = getNameOf(item.second);
+
+      result = parentName + "/" + result;
+      parent = item.second.parentDirID;
+
+      addToCache(item.second, parentName);
+   }
+
+   return getNameOf(parent) + "/" + result;
+}
+
+
+
+void FsckDBFileInodesTable::insert(FsckFileInodeList& fileInodes, const BulkHandle* handle)
+{
+   for(FsckFileInodeListIter it = fileInodes.begin(), end = fileInodes.end(); it != end; ++it)
+   {
+      const std::vector<uint16_t>& stripes = it->getStripeTargets();
+
+      db::FileInode inode = {
+         db::EntryID::fromStr(it->getID() ),
+         db::EntryID::fromStr(it->getParentDirID() ),
+         db::EntryID::fromStr(it->getPathInfo()->getOrigParentEntryID() ),
+
+         it->getParentNodeID().val(),
+         it->getSaveNodeID().val(),
+         it->getPathInfo()->getOrigUID(),
+
+         it->getUserID(), it->getGroupID(),
+
+         (uint64_t) it->getFileSize(), it->getUsedBlocks(),
+
+         it->getNumHardLinks(),
+
+         it->getSaveInode(),
+         it->getSaveDevice(),
+
+         it->getChunkSize(),
+
+         {},
+
+         it->getIsInlined(),
+         uint32_t(it->getPathInfo()->getFlags() ),
+         it->getStripePatternType(),
+         uint32_t(stripes.size() ),
+         it->getReadable(),
+         it->getIsBuddyMirrored(),
+         it->getIsMismirrored(),
+      };
+
+      db::StripeTargets extraTargets = { inode.id, {}, 0 };
+
+      for(size_t i = 0; i < stripes.size(); i++)
+      {
+         if(i < inode.NTARGETS)
+            inode.targets[i] = stripes[i];
+         else
+         {
+            size_t offsetInPattern = (i - inode.NTARGETS) % extraTargets.NTARGETS;
+
+            if(offsetInPattern == 0)
+               extraTargets.firstTargetIndex = i;
+
+            extraTargets.targets[offsetInPattern] = stripes[i];
+
+            if(offsetInPattern == extraTargets.NTARGETS - 1)
+            {
+               if(handle)
+                  std::get<1>(*handle)->append(extraTargets);
+               else
+                  this->targets.insert(extraTargets);
+
+               extraTargets.firstTargetIndex = 0;
+            }
+         }
+      }
+
+      if(handle)
+      {
+         std::get<0>(*handle)->append(inode);
+         if(extraTargets.firstTargetIndex != 0)
+            std::get<1>(*handle)->append(extraTargets);
+      }
+      else
+      {
+         this->inodes.insert(inode);
+         if(extraTargets.firstTargetIndex != 0)
+            this->targets.insert(extraTargets);
+      }
+   }
+}
+
+void FsckDBFileInodesTable::update(FsckFileInodeList& inodes)
+{
+   for(FsckFileInodeListIter it = inodes.begin(), end = inodes.end(); it != end; ++it)
+   {
+      FsckFileInodeList list(1, *it);
+
+      remove(list);
+      insert(list);
+   }
+}
+
+void FsckDBFileInodesTable::remove(FsckFileInodeList& fileInodes)
+{
+   for(FsckFileInodeListIter it = fileInodes.begin(), end = fileInodes.end(); it != end; ++it)
+   {
+      this->inodes.remove(db::EntryID::fromStr(it->getID() ) );
+      this->targets.remove(db::EntryID::fromStr(it->getID() ) );
+   }
+}
+
+FsckDBFileInodesTable::InodesQueryType FsckDBFileInodesTable::getInodes()
+{
+   this->inodes.commitChanges();
+   return
+      this->inodes.cursor()
+      | db::groupBy(UniqueInlinedInode() )
+      | db::select(SelectFirst() );
+}
+
+Table<db::StripeTargets, true>::QueryType FsckDBFileInodesTable::getTargets()
+{
+   this->targets.commitChanges();
+   return this->targets.cursor();
+}
+
+UInt16Vector FsckDBFileInodesTable::getStripeTargetsByKey(const db::EntryID& id)
+{
+   UInt16Vector allStripeTargets;
+   std::pair<bool, db::FileInode> result = this->get(id.str());
+
+   if (!result.first)
+      return allStripeTargets;
+
+   // firstly copy stripeTargets from FileInode object
+   auto inode = result.second;
+   uint32_t targetArrSize = inode.NTARGETS;
+   uint32_t numTargets = (inode.stripePatternSize > targetArrSize) ? targetArrSize
+      : inode.stripePatternSize;
+
+   std::copy(inode.targets, inode.targets + numTargets, std::back_inserter(allStripeTargets));
+
+   // if extraTargets are present then get them from targets table
+   uint32_t numExtraTargets = inode.stripePatternSize - numTargets;
+   if (numExtraTargets)
+   {
+      this->targets.commitChanges();
+      auto dbRes = this->targets.getAllByKey(id);
+
+      while (dbRes.step())
+      {
+         auto elem = dbRes.get();
+
+         for (int i=0; i<elem->NTARGETS; i++)
+         {
+            allStripeTargets.push_back(elem->targets[i]);
+            numExtraTargets -= 1;
+            if (numExtraTargets == 0) break;
+         }
+
+         if (numExtraTargets == 0) break;
+      }
+   }
+
+   return allStripeTargets;
+}
+
+std::pair<bool, db::FileInode> FsckDBFileInodesTable::get(std::string id)
+{
+   this->inodes.commitChanges();
+   return this->inodes.getByKey(db::EntryID::fromStr(id) );
+}
+
+
+
+void FsckDBDirInodesTable::insert(FsckDirInodeList& dirInodes, const BulkHandle* handle)
+{
+   for(FsckDirInodeListIter it = dirInodes.begin(), end = dirInodes.end(); it != end; ++it)
+   {
+      db::DirInode inode = {
+         db::EntryID::fromStr(it->getID() ),
+         db::EntryID::fromStr(it->getParentDirID() ),
+
+         it->getParentNodeID().val(),
+         it->getOwnerNodeID().val(),
+         it->getSaveNodeID().val(),
+
+         it->getStripePatternType(),
+         it->getReadable(),
+         it->getIsBuddyMirrored(),
+         it->getIsMismirrored(),
+
+         (uint64_t) it->getSize(),
+         it->getNumHardLinks(),
+      };
+
+      if(handle)
+         (*handle)->append(inode);
+      else
+         this->table.insert(inode);
+   }
+}
+
+void FsckDBDirInodesTable::update(FsckDirInodeList& inodes)
+{
+   for(FsckDirInodeListIter it = inodes.begin(), end = inodes.end(); it != end; ++it)
+   {
+      FsckDirInodeList list(1, *it);
+
+      remove(list);
+      insert(list);
+   }
+}
+
+void FsckDBDirInodesTable::remove(FsckDirInodeList& dirInodes)
+{
+   for(FsckDirInodeListIter it = dirInodes.begin(), end = dirInodes.end(); it != end; ++it)
+      this->table.remove(db::EntryID::fromStr(it->getID() ) );
+}
+
+Table<db::DirInode>::QueryType FsckDBDirInodesTable::get()
+{
+   this->table.commitChanges();
+   return this->table.cursor();
+}
+
+std::pair<bool, FsckDirInode> FsckDBDirInodesTable::get(std::string id)
+{
+   this->table.commitChanges();
+   return this->table.getByKey(db::EntryID::fromStr(id) );
+}
+
+
+
+static db::Chunk fsckChunkToDbChunk(FsckChunk& chunk)
+{
+   db::Chunk result = {
+      db::EntryID::fromStr(chunk.getID() ),
+      chunk.getTargetID(), chunk.getBuddyGroupID(),
+      {},
+      chunk.getFileSize(), chunk.getUsedBlocks(),
+      chunk.getUserID(), chunk.getGroupID(),
+   };
+
+   strncpy(result.savedPath, chunk.getSavedPath()->str().c_str(),
+      sizeof(result.savedPath) - 1);
+
+   return result;
+}
+
+void FsckDBChunksTable::insert(FsckChunkList& chunks, const BulkHandle* handle)
+{
+   for(FsckChunkListIter it = chunks.begin(), end = chunks.end(); it != end; ++it)
+   {
+      if(handle)
+         (*handle)->append(fsckChunkToDbChunk(*it) );
+      else
+         this->table.insert(fsckChunkToDbChunk(*it) );
+   }
+}
+
+void FsckDBChunksTable::update(FsckChunkList& chunks)
+{
+   for(FsckChunkListIter it = chunks.begin(), end = chunks.end(); it != end; ++it)
+   {
+      db::Chunk chunk = fsckChunkToDbChunk(*it);
+
+      this->table.remove(chunk.pkey() );
+      this->table.insert(chunk);
+   }
+}
+
+void FsckDBChunksTable::remove(const db::Chunk::KeyType& id)
+{
+   this->table.remove(id);
+}
+
+Table<db::Chunk>::QueryType FsckDBChunksTable::get()
+{
+   this->table.commitChanges();
+   return this->table.cursor();
+}
+
+
+
+static db::ContDir fsckContDirToDbContDir(FsckContDir& contDir)
+{
+   return {
+      db::EntryID::fromStr(contDir.getID()),
+      contDir.getSaveNodeID().val(),
+      contDir.getIsBuddyMirrored(),
+   };
+}
+
+void FsckDBContDirsTable::insert(FsckContDirList& contDirs, const BulkHandle* handle)
+{
+   for(FsckContDirListIter it = contDirs.begin(), end = contDirs.end(); it != end; ++it)
+   {
+      if(handle)
+         (*handle)->append(fsckContDirToDbContDir(*it) );
+      else
+         this->table.insert(fsckContDirToDbContDir(*it) );
+   }
+}
+
+Table<db::ContDir>::QueryType FsckDBContDirsTable::get()
+{
+   this->table.commitChanges();
+   return this->table.cursor();
+}
+
+
+
+static db::FsID fsckFsIDToDbFsID(FsckFsID& id)
+{
+   return {
+      db::EntryID::fromStr(id.getID() ),
+      db::EntryID::fromStr(id.getParentDirID() ),
+      id.getSaveNodeID().val(), id.getSaveDevice(), id.getSaveInode(),
+      id.getIsBuddyMirrored(),
+   };
+}
+
+void FsckDBFsIDsTable::insert(FsckFsIDList& fsIDs, const BulkHandle* handle)
+{
+   for(FsckFsIDListIter it = fsIDs.begin(), end = fsIDs.end(); it != end; ++it)
+   {
+      if(handle)
+         (*handle)->append(fsckFsIDToDbFsID(*it) );
+      else
+         this->table.insert(fsckFsIDToDbFsID(*it) );
+   }
+}
+
+void FsckDBFsIDsTable::remove(FsckFsIDList& fsIDs)
+{
+   for(FsckFsIDListIter it = fsIDs.begin(), end = fsIDs.end(); it != end; ++it)
+      this->table.remove(fsckFsIDToDbFsID(*it).pkey() );
+}
+
+Table<db::FsID>::QueryType FsckDBFsIDsTable::get()
+{
+   this->table.commitChanges();
+   return this->table.cursor();
+}
+
+
+
+void FsckDBUsedTargetIDsTable::insert(FsckTargetIDList& targetIDs, const BulkHandle& handle)
+{
+   for(FsckTargetIDListIter it = targetIDs.begin(), end = targetIDs.end(); it != end; ++it)
+   {
+      db::UsedTarget target = { it->getID(), it->getTargetIDType() };
+      handle->append(target);
+   }
+}
+
+
+
+void FsckDBModificationEventsTable::insert(const FsckModificationEventList& events,
+   const BulkHandle& handle)
+{
+   for(std::list<FsckModificationEvent>::const_iterator it = events.begin(), end = events.end();
+         it != end; ++it)
+   {
+      db::ModificationEvent event = { db::EntryID::fromStr(it->getEntryID() ) };
+      handle->append(event);
+   }
+}

fsck/source/database/FsckDBTable.h

@@ -0,0 +1,641 @@
+#ifndef FsckDBTable_h_drBbZVcUQetDB8UVnMd1Uj
+#define FsckDBTable_h_drBbZVcUQetDB8UVnMd1Uj
+
+#include <common/fsck/FsckModificationEvent.h>
+#include <database/Chunk.h>
+#include <database/ContDir.h>
+#include <database/DirEntry.h>
+#include <database/DirInode.h>
+#include <database/FileInode.h>
+#include <database/FsID.h>
+#include <database/Group.h>
+#include <database/Table.h>
+#include <database/ModificationEvent.h>
+#include <database/StripeTargets.h>
+#include <database/UsedTarget.h>
+
+class FsckDBDentryTable
+{
+   public:
+      struct ByParent
+      {
+         db::EntryID parent; /* 12 */
+         db::EntryID target; /* 24 */
+         uint32_t entryType; /* 28 */
+
+         typedef std::pair<db::EntryID, db::EntryID> KeyType;
+
+         KeyType pkey() const { return KeyType(parent, target); }
+      };
+
+      class NameBuffer
+      {
+         public:
+            NameBuffer(const std::string& path, unsigned id):
+               fileID(id)
+            {
+               fd = ::open(path.c_str(), O_RDWR | O_CREAT, 0660);
+               if(fd < 0)
+                  throw std::runtime_error("could not open name file");
+
+               streamBuf.reserve(4096 * 1024);
+
+               writePos = ::lseek(fd, 0, SEEK_END);
+               if (writePos < 0)
+               {
+                  close(fd);
+                  fd = -1;
+                  throw std::runtime_error("could not open name file");
+               }
+            }
+
+            NameBuffer(NameBuffer&&) = delete;
+            NameBuffer& operator=(NameBuffer&&) = delete;
+
+            ~NameBuffer()
+            {
+               if (fd >= 0)
+               {
+                  flush();
+                  close(fd);
+               }
+            }
+
+            uint64_t put(const std::string& name)
+            {
+               if (streamBuf.size() + name.size() + 1 > streamBuf.capacity())
+                  flush();
+
+               if (name.size() > streamBuf.capacity())
+                  streamBuf.reserve(name.size());
+
+               uint64_t result = writePos + streamBuf.size();
+               streamBuf.insert(streamBuf.end(), name.begin(), name.end());
+               streamBuf.push_back(0);
+
+               return result;
+            }
+
+            std::string get(uint64_t offset)
+            {
+               flush();
+
+               char buffer[255 + 1];
+               ssize_t readRes = ::pread(fd, buffer, sizeof(buffer), offset);
+               // if the read did not fill the full buffer, we can
+               //   a) have an error, or
+               //   b) have a short read.
+               // errors should be reported directly. short reads are likely caused be trying to
+               // read past the end of the file. since we cannot possible have a sane read that
+               // stretches beyond writePos here (as we have flushed the write buffer), we can
+               // reliably detect both.
+               if (readRes < 0 || ssize_t(offset) + readRes > writePos)
+                  throw std::runtime_error("could not read name file: " + std::string(strerror(errno)));
+
+               return buffer;
+            }
+
+            unsigned id() const { return fileID; }
+
+         private:
+            unsigned fileID;
+            std::vector<char> streamBuf;
+
+            int fd;
+            ssize_t writePos;
+
+            void flush()
+            {
+               ssize_t writeRes = ::pwrite(fd, &streamBuf[0], streamBuf.size(), writePos);
+               if (writeRes < 0 || size_t(writeRes) < streamBuf.size())
+                  throw std::runtime_error("error in flush: " + std::string(strerror(errno)));
+
+               writePos += writeRes;
+               streamBuf.resize(0);
+            }
+      };
+
+      struct BulkHandle
+      {
+         boost::shared_ptr<Buffer<db::DirEntry> > dentries;
+         boost::shared_ptr<Buffer<ByParent> > byParent;
+         NameBuffer* nameBuffer;
+      };
+
+   private:
+      struct NameCacheEntry
+      {
+         db::EntryID id;
+         db::EntryID parent;
+         std::string name;
+
+         std::list<NameCacheEntry*>::iterator lruLink;
+      };
+
+   public:
+      FsckDBDentryTable(const std::string& dbPath, size_t fragmentSize, size_t nameCacheLimit,
+            bool allowCreate)
+         : dbPath(dbPath),
+           table(dbPath + "/dentries", fragmentSize, allowCreate),
+           byParent(dbPath + "/dentriesbyparent", fragmentSize, allowCreate),
+           insertSeqNo(0),
+           nameCacheLimit(nameCacheLimit)
+      {
+      }
+
+      void insert(FsckDirEntryList& dentries, const BulkHandle* handle = NULL);
+      void updateFieldsExceptParent(FsckDirEntryList& dentries);
+      void remove(FsckDirEntryList& dentries);
+
+      Table<db::DirEntry>::QueryType get();
+      std::pair<bool, db::DirEntry> getAnyFor(db::EntryID id);
+
+      Table<ByParent>::QueryType getByParent();
+
+      std::string getNameOf(const db::DirEntry& dentry);
+      std::string getPathOf(const db::DirEntry& dentry);
+
+   private:
+      std::string dbPath;
+      Table<db::DirEntry> table;
+      Table<ByParent> byParent;
+
+      std::map<uint64_t, boost::shared_ptr<NameBuffer> > nameBuffers;
+      uint64_t insertSeqNo;
+
+      std::map<db::EntryID, NameCacheEntry> nameCache;
+      std::list<NameCacheEntry*> nameCacheLRU;
+      size_t nameCacheLimit;
+
+      NameBuffer& getNameBuffer(unsigned id)
+      {
+         if(this->nameBuffers.count(id) )
+            return *this->nameBuffers[id];
+
+         const std::string& nextNameFile = dbPath + "/dentrynames."
+            + StringTk::intToStr(id);
+
+         boost::shared_ptr<NameBuffer> buf = boost::make_shared<NameBuffer>(
+            nextNameFile, id);
+         this->nameBuffers[id] = buf;
+         return *buf;
+      }
+
+      void addToCache(const db::DirEntry& entry, const std::string& name)
+      {
+         if(nameCache.size() >= nameCacheLimit)
+         {
+            nameCache.erase(nameCacheLRU.front()->id);
+            nameCacheLRU.pop_front();
+         }
+
+         NameCacheEntry cacheEntry = { entry.id, entry.parentDirID, name, nameCacheLRU.end() };
+         std::pair<std::map<db::EntryID, NameCacheEntry>::iterator, bool> pos = nameCache.insert(
+            std::make_pair(entry.id, cacheEntry) );
+
+         if(!pos.second)
+            return;
+
+         pos.first->second.lruLink = nameCacheLRU.insert(nameCacheLRU.end(), &pos.first->second);
+      }
+
+      const NameCacheEntry* getFromCache(db::EntryID id)
+      {
+         std::map<db::EntryID, NameCacheEntry>::iterator pos = nameCache.find(id);
+         if(pos == nameCache.end() )
+            return NULL;
+
+         nameCacheLRU.splice(nameCacheLRU.end(), nameCacheLRU, pos->second.lruLink);
+         return &pos->second;
+      }
+
+   public:
+      void clear()
+      {
+         this->table.clear();
+         this->byParent.clear();
+         this->insertSeqNo = 0;
+         this->nameCache.clear();
+         this->nameCacheLRU.clear();
+      }
+
+      BulkHandle newBulkHandle()
+      {
+         BulkHandle result = {
+            this->table.bulkInsert(),
+            this->byParent.bulkInsert(),
+            &getNameBuffer(this->nameBuffers.size() ),
+         };
+         return result;
+      }
+
+      void flush(BulkHandle& handle)
+      {
+         handle = {{}, {}, nullptr};
+      }
+
+      void insert(FsckDirEntryList& dentries, const BulkHandle& handle)
+      {
+         insert(dentries, &handle);
+      }
+
+      std::string getNameOf(db::EntryID id)
+      {
+         if(id == db::EntryID::anchor() || id == db::EntryID::root() )
+            return "";
+         else
+         if(id == db::EntryID::disposal() )
+            return "[<disposal>]";
+         else
+         if(id == db::EntryID::mdisposal() )
+            return "[<mdisposal>]";
+
+         std::pair<bool, db::DirEntry> item = getAnyFor(id);
+         if(!item.first)
+            return "[<unresolved>]";
+         else
+            return getNameOf(item.second);
+      }
+
+      std::string getPathOf(db::EntryID id)
+      {
+         std::pair<bool, db::DirEntry> item = getAnyFor(id);
+         if(!item.first)
+            return "[<unresolved>]";
+         else
+            return getPathOf(item.second);
+      }
+
+      void commitChanges()
+      {
+         table.commitChanges();
+      }
+};
+
+class FsckDBFileInodesTable
+{
+   public:
+      typedef std::tuple<
+         boost::shared_ptr<Buffer<db::FileInode> >,
+         boost::shared_ptr<Buffer<db::StripeTargets> > > BulkHandle;
+
+      struct UniqueInlinedInode
+      {
+         typedef boost::tuple<db::EntryID, uint64_t, uint32_t, int32_t> KeyType;
+         typedef db::FileInode ProjType;
+         typedef int GroupType;
+
+         KeyType key(const db::FileInode& fi)
+         {
+            return boost::make_tuple(fi.id, fi.saveInode, fi.saveNodeID, fi.saveDevice);
+         }
+
+         db::FileInode project(const db::FileInode& fi)
+         {
+            return fi;
+         }
+
+         void step(const db::FileInode& fi) {}
+
+         GroupType finish()
+         {
+            return 0;
+         }
+      };
+
+      struct SelectFirst
+      {
+         typedef db::FileInode result_type;
+
+         db::FileInode operator()(std::pair<db::FileInode, int>& pair) const
+         {
+            return pair.first;
+         }
+      };
+
+      typedef
+         Select<
+            Group<
+               Table<db::FileInode>::QueryType,
+               UniqueInlinedInode>,
+            SelectFirst> InodesQueryType;
+
+   public:
+      FsckDBFileInodesTable(const std::string& dbPath, size_t fragmentSize, bool allowCreate)
+         : inodes(dbPath + "/fileinodes", fragmentSize, allowCreate),
+           targets(dbPath + "/filetargets", fragmentSize, allowCreate)
+      {
+      }
+
+      void insert(FsckFileInodeList& fileInodes, const BulkHandle* handle = NULL);
+      void update(FsckFileInodeList& inodes);
+      void remove(FsckFileInodeList& fileInodes);
+
+      InodesQueryType getInodes();
+      Table<db::StripeTargets, true>::QueryType getTargets();
+      UInt16Vector getStripeTargetsByKey(const db::EntryID& id);
+
+      std::pair<bool, db::FileInode> get(std::string id);
+
+   private:
+      Table<db::FileInode> inodes;
+      Table<db::StripeTargets, true> targets;
+
+   public:
+      void clear()
+      {
+         inodes.clear();
+         targets.clear();
+      }
+
+      void remove(db::EntryID id)
+      {
+         inodes.remove(id);
+         targets.remove(id);
+      }
+
+      BulkHandle newBulkHandle()
+      {
+         return BulkHandle(this->inodes.bulkInsert(), this->targets.bulkInsert() );
+      }
+
+      void flush(BulkHandle& handle)
+      {
+         handle = {};
+      }
+
+      void insert(FsckFileInodeList& fileInodes, const BulkHandle& handle)
+      {
+         insert(fileInodes, &handle);
+      }
+
+      void commitChanges()
+      {
+         inodes.commitChanges();
+         targets.commitChanges();
+      }
+};
+
+class FsckDBDirInodesTable
+{
+   public:
+      typedef boost::shared_ptr<Buffer<db::DirInode> > BulkHandle;
+
+   public:
+      FsckDBDirInodesTable(const std::string& dbPath, size_t fragmentSize, bool allowCreate)
+         : table(dbPath + "/dirinodes", fragmentSize, allowCreate)
+      {
+      }
+
+      void insert(FsckDirInodeList& dirInodes, const BulkHandle* handle = NULL);
+      void update(FsckDirInodeList& inodes);
+      void remove(FsckDirInodeList& dirInodes);
+
+      Table<db::DirInode>::QueryType get();
+      std::pair<bool, FsckDirInode> get(std::string id);
+
+   private:
+      Table<db::DirInode> table;
+
+   public:
+      void clear()
+      {
+         this->table.clear();
+      }
+
+      BulkHandle newBulkHandle()
+      {
+         return this->table.bulkInsert();
+      }
+
+      void flush(BulkHandle& handle)
+      {
+         handle = {};
+      }
+
+      void insert(FsckDirInodeList& fileInodes, const BulkHandle& handle)
+      {
+         insert(fileInodes, &handle);
+      }
+
+      void commitChanges()
+      {
+         table.commitChanges();
+      }
+};
+
+class FsckDBChunksTable
+{
+   public:
+      typedef boost::shared_ptr<Buffer<db::Chunk> > BulkHandle;
+
+   public:
+      FsckDBChunksTable(const std::string& dbPath, size_t fragmentSize, bool allowCreate)
+         : table(dbPath + "/chunks", fragmentSize, allowCreate)
+      {
+      }
+
+      void insert(FsckChunkList& chunks, const BulkHandle* handle = NULL);
+      void update(FsckChunkList& chunks);
+      void remove(const db::Chunk::KeyType& id);
+
+      Table<db::Chunk>::QueryType get();
+
+   private:
+      Table<db::Chunk> table;
+
+   public:
+      void clear()
+      {
+         table.clear();
+      }
+
+      BulkHandle newBulkHandle()
+      {
+         return this->table.bulkInsert();
+      }
+
+      void flush(BulkHandle& handle)
+      {
+         handle = {};
+      }
+
+      void insert(FsckChunkList& chunks, const BulkHandle& handle)
+      {
+         insert(chunks, &handle);
+      }
+
+      void commitChanges()
+      {
+         table.commitChanges();
+      }
+};
+
+class FsckDBContDirsTable
+{
+   public:
+      typedef boost::shared_ptr<Buffer<db::ContDir> > BulkHandle;
+
+   public:
+      FsckDBContDirsTable(const std::string& dbPath, size_t fragmentSize, bool allowCreate)
+         : table(dbPath + "/contdirs", fragmentSize, allowCreate)
+      {
+      }
+
+      void insert(FsckContDirList& contDirs, const BulkHandle* handle = NULL);
+
+      Table<db::ContDir>::QueryType get();
+
+   private:
+      Table<db::ContDir> table;
+
+   public:
+      void clear()
+      {
+         table.clear();
+      }
+
+      BulkHandle newBulkHandle()
+      {
+         return this->table.bulkInsert();
+      }
+
+      void flush(BulkHandle& handle)
+      {
+         handle = {};
+      }
+
+      void insert(FsckContDirList& contDirs, const BulkHandle& handle)
+      {
+         insert(contDirs, &handle);
+      }
+
+      void commitChanges()
+      {
+         table.commitChanges();
+      }
+};
+
+class FsckDBFsIDsTable
+{
+   public:
+      typedef boost::shared_ptr<Buffer<db::FsID> > BulkHandle;
+
+   public:
+      FsckDBFsIDsTable(const std::string& dbPath, size_t fragmentSize, bool allowCreate)
+         : table(dbPath + "/fsids", fragmentSize, allowCreate)
+      {
+      }
+
+      void insert(FsckFsIDList& fsIDs, const BulkHandle* handle = NULL);
+      void remove(FsckFsIDList& fsIDs);
+
+      Table<db::FsID>::QueryType get();
+
+   private:
+      Table<db::FsID> table;
+
+   public:
+      void clear()
+      {
+         this->table.clear();
+      }
+
+      BulkHandle newBulkHandle()
+      {
+         return this->table.bulkInsert();
+      }
+
+      void flush(BulkHandle& handle)
+      {
+         handle = {};
+      }
+
+      void insert(FsckFsIDList& fsIDs, const BulkHandle& handle)
+      {
+         insert(fsIDs, &handle);
+      }
+
+      void commitChanges()
+      {
+         table.commitChanges();
+      }
+};
+
+template<typename Data>
+class FsckUniqueTable
+{
+   public:
+      typedef boost::shared_ptr<Buffer<Data> > BulkHandle;
+
+   protected:
+      FsckUniqueTable(const std::string& name, size_t bufferSize, bool allowCreate)
+         : set(name, allowCreate), bufferSize(bufferSize)
+      {}
+
+   private:
+      Set<Data> set;
+      size_t bufferSize;
+      std::vector<boost::weak_ptr<Buffer<Data> > > bulkHandles;
+
+   public:
+      void clear()
+      {
+         set.clear();
+         bulkHandles.clear();
+      }
+
+      BulkHandle newBulkHandle()
+      {
+         BulkHandle result(new Buffer<Data>(set, bufferSize) );
+         bulkHandles.push_back(result);
+         return result;
+      }
+
+      void flush(BulkHandle& handle)
+      {
+         handle = {};
+      }
+
+      SetFragmentCursor<Data> get()
+      {
+         if(!bulkHandles.empty() )
+         {
+            for(size_t i = 0; i < bulkHandles.size(); i++)
+            {
+               if(!bulkHandles[i].expired() )
+                  throw std::runtime_error("still has open bulk handles");
+            }
+
+            bulkHandles.clear();
+            set.makeUnique();
+         }
+
+         return set.cursor();
+      }
+};
+
+class FsckDBUsedTargetIDsTable : public FsckUniqueTable<db::UsedTarget>
+{
+   public:
+      FsckDBUsedTargetIDsTable(const std::string& dbPath, size_t fragmentSize, bool allowCreate)
+         : FsckUniqueTable<db::UsedTarget>(dbPath + "/usedtargets", fragmentSize, allowCreate)
+      {
+      }
+
+      void insert(FsckTargetIDList& targetIDs, const BulkHandle& handle);
+};
+
+class FsckDBModificationEventsTable : public FsckUniqueTable<db::ModificationEvent>
+{
+   public:
+      FsckDBModificationEventsTable(const std::string& dbPath, size_t fragmentSize,
+         bool allowCreate)
+         : FsckUniqueTable<db::ModificationEvent>(dbPath + "/modevents", fragmentSize, allowCreate)
+      {
+      }
+
+      void insert(const FsckModificationEventList& events, const BulkHandle& handle);
+};
+
+#endif

fsck/source/database/Group.h

@@ -0,0 +1,131 @@
+#ifndef GROUP_H_
+#define GROUP_H_
+
+#include <boost/utility/result_of.hpp>
+
+#include <utility>
+
+/*
+ * requires Ops : struct {
+ *    typedef <key-type> KeyType;
+ *    typedef <proj-type> ProjType;
+ *    typedef <group-type> GroupType;
+ *
+ *    KeyType key(Source::ElementType&);
+ *    ProjType project(Source::ElementType&);
+ *    void step(Source::ElementType&);
+ *
+ *    GroupType finish();
+ * }
+ *
+ * yields sequence (Source::ElementType, <row-type>)
+ */
+template<typename Source, typename Ops>
+class Group
+{
+   private:
+      typedef typename Ops::KeyType KeyType;
+      typedef typename Ops::ProjType ProjType;
+      typedef typename Ops::GroupType GroupType;
+
+   public:
+      typedef std::pair<ProjType, GroupType> ElementType;
+      typedef typename Source::MarkerType MarkerType;
+
+   public:
+      Group(Source source, Ops ops)
+         : source(source), ops(ops), sourceIsActive(false)
+      {
+      }
+
+      bool step()
+      {
+         if(!this->sourceIsActive && !this->source.step() )
+            return false;
+
+         groupCurrentSet();
+         return true;
+      }
+
+      ElementType* get()
+      {
+         return &current;
+      }
+
+      MarkerType mark() const
+      {
+          return this->currentMark;
+      }
+
+      void restore(MarkerType mark)
+      {
+         this->source.restore(mark);
+         groupCurrentSet();
+      }
+
+   private:
+      void groupCurrentSet()
+      {
+         typename Source::ElementType* row = this->source.get();
+
+         this->currentMark = this->source.mark();
+         this->current.first = this->ops.project(*row);
+         this->ops.step(*row);
+
+         KeyType currentKey = this->ops.key(*row);
+
+         while(true)
+         {
+            if(!this->source.step() )
+            {
+               this->sourceIsActive = false;
+               break;
+            }
+
+            row = this->source.get();
+
+            if(!(currentKey == this->ops.key(*row) ) )
+            {
+               this->sourceIsActive = true;
+               break;
+            }
+
+            this->ops.step(*row);
+         }
+
+         this->current.second = this->ops.finish();
+      }
+
+   private:
+      Source source;
+      Ops ops;
+      ElementType current;
+      typename Source::MarkerType currentMark;
+      bool sourceIsActive;
+};
+
+
+namespace db {
+
+template<typename Ops>
+struct GroupOp
+{
+   Ops ops;
+
+   template<typename Source>
+   friend Group<Source, Ops> operator|(Source source, const GroupOp& fn)
+   {
+      return Group<Source, Ops>(source, fn.ops);
+   }
+};
+
+template<typename Ops>
+inline GroupOp<Ops> groupBy(Ops ops)
+{
+   GroupOp<Ops> result = {ops};
+   return result;
+}
+
+}
+
+#endif

fsck/source/database/LeftJoinEq.h

@@ -0,0 +1,189 @@
+#ifndef LEFTJOINEQ_H_
+#define LEFTJOINEQ_H_
+
+#include <common/Common.h>
+
+#include <cstddef>
+#include <utility>
+
+#include <boost/type_traits/decay.hpp>
+#include <boost/utility/result_of.hpp>
+
+template<typename Left, typename Right, typename KeyExtract>
+class LeftJoinEq
+{
+   private:
+      enum State {
+         s_next_left,
+         s_first_right,
+         s_next_right,
+         s_only_left,
+      };
+
+      typedef typename boost::decay<
+            typename boost::result_of<KeyExtract(typename Left::ElementType&)>::type
+         >::type LeftKey;
+      typedef typename boost::decay<
+            typename boost::result_of<KeyExtract(typename Right::ElementType&)>::type
+         >::type RightKey;
+
+   public:
+      typedef std::pair<typename Left::ElementType, typename Right::ElementType*> ElementType;
+
+      struct MarkerType
+      {
+         typename Left::MarkerType leftMark;
+         typename Right::MarkerType rightMark;
+         State state;
+
+         bool hasRightMark;
+         typename Right::MarkerType innerRightMark;
+      };
+
+   public:
+      LeftJoinEq(Left left, Right right, KeyExtract keyExtract)
+         : left(left), right(right), keyExtract(keyExtract), state(s_next_left),
+           hasRightMark(false)
+      {
+         if(!this->right.step() )
+            this->state = s_only_left;
+      }
+
+      bool step()
+      {
+         switch(this->state)
+         {
+            case s_only_left:
+               if(!this->left.step() )
+                  return false;
+
+               return makeCurrent(false);
+
+            case s_next_left:
+               if(!this->left.step() )
+               {
+                  this->state = s_only_left;
+                  return false;
+               }
+
+               if(this->hasRightMark &&
+                     this->keyExtract(*this->left.get() ) == this->rightKeyAtMark)
+                  this->right.restore(this->rightMark);
+               else
+                  this->hasRightMark = false;
+
+               BEEGFS_FALLTHROUGH;
+
+            case s_first_right: {
+               while(rightKey() < leftKey() )
+               {
+                  if(!this->right.step() )
+                  {
+                     this->state = s_only_left;
+                     return makeCurrent(false);
+                  }
+               }
+
+               if(leftKey() == rightKey() )
+               {
+                  this->state = s_next_right;
+                  this->hasRightMark = true;
+                  this->rightKeyAtMark = rightKey();
+                  this->rightMark = this->right.mark();
+                  return makeCurrent(true);
+               }
+
+               this->state = s_next_left;
+               return makeCurrent(false);
+            }
+
+            case s_next_right: {
+               if(!this->right.step() )
+               {
+                  this->state = s_next_left;
+                  return step();
+               }
+
+               if(leftKey() == rightKey() )
+               {
+                  this->state = s_next_right;
+                  return makeCurrent(true);
+               }
+
+               this->state = s_next_left;
+               return step();
+            }
+         }
+
+         return false;
+      }
+
+      ElementType* get()
+      {
+         return &current;
+      }
+
+      MarkerType mark() const
+      {
+         MarkerType result = { this->left.mark(), this->right.mark(), this->state,
+            this->hasRightMark, this->rightMark };
+         return result;
+      }
+
+      void restore(MarkerType mark)
+      {
+         this->hasRightMark = mark.hasRightMark;
+         if(mark.hasRightMark)
+         {
+            this->rightMark = mark.innerRightMark;
+            this->right.restore(mark.innerRightMark);
+            this->rightKeyAtMark = rightKey();
+         }
+
+         this->left.restore(mark.leftMark);
+         this->right.restore(mark.rightMark);
+         this->state = mark.state;
+
+         if(this->state == s_only_left)
+            makeCurrent(false);
+         else
+            makeCurrent(leftKey() == rightKey() );
+      }
+
+   private:
+      bool makeCurrent(bool hasRight)
+      {
+         if(hasRight)
+            this->current = ElementType(*this->left.get(), this->right.get() );
+         else
+            this->current = ElementType(*this->left.get(), nullptr);
+
+         return true;
+      }
+
+      LeftKey leftKey() { return this->keyExtract(*this->left.get() ); }
+      RightKey rightKey() { return this->keyExtract(*this->right.get() ); }
+
+   private:
+      Left left;
+      Right right;
+      KeyExtract keyExtract;
+      ElementType current;
+      State state;
+
+      bool hasRightMark;
+      RightKey rightKeyAtMark;
+      typename Right::MarkerType rightMark;
+};
+
+namespace db {
+
+template<typename KeyExtract, typename Left, typename Right>
+inline LeftJoinEq<Left, Right, KeyExtract> leftJoinBy(KeyExtract ex, Left left, Right right)
+{
+   return LeftJoinEq<Left, Right, KeyExtract>(left, right, ex);
+}
+
+}
+
+#endif

fsck/source/database/ModificationEvent.h

@@ -0,0 +1,18 @@
+#ifndef MODFICATIONEVENT_H_
+#define MODFICATIONEVENT_H_
+
+#include <database/EntryID.h>
+
+namespace db {
+
+struct ModificationEvent
+{
+   EntryID id;
+
+   typedef EntryID KeyType;
+   EntryID pkey() const { return id; }
+};
+
+}
+
+#endif

fsck/source/database/Select.h

@@ -0,0 +1,74 @@
+#ifndef SELECT_H_
+#define SELECT_H_
+
+#include <boost/utility/result_of.hpp>
+
+template<typename Source, typename Fn>
+class Select
+{
+   public:
+      typedef typename boost::result_of<Fn(typename Source::ElementType&)>::type ElementType;
+      typedef typename Source::MarkerType MarkerType;
+
+   public:
+      Select(Source source, Fn fn)
+         : source(source), fn(fn)
+      {
+      }
+
+      bool step()
+      {
+         if(!this->source.step() )
+            return false;
+
+         this->current = this->fn(*this->source.get() );
+         return true;
+      }
+
+      ElementType* get()
+      {
+         return &current;
+      }
+
+      MarkerType mark() const
+      {
+         return this->source.mark();
+      }
+
+      void restore(MarkerType mark)
+      {
+         this->source.restore(mark);
+         this->current = this->fn(*this->source.get() );
+      }
+
+   private:
+      Source source;
+      Fn fn;
+      ElementType current;
+};
+
+
+namespace db {
+
+template<typename Pred>
+struct SelectOp
+{
+   Pred pred;
+
+   template<typename Source>
+   friend Select<Source, Pred> operator|(Source source, const SelectOp& op)
+   {
+      return Select<Source, Pred>(source, op.pred);
+   }
+};
+
+template<typename Pred>
+inline SelectOp<Pred> select(Pred pred)
+{
+   SelectOp<Pred> result = {pred};
+   return result;
+}
+
+}
+
+#endif

fsck/source/database/Set.h

@@ -0,0 +1,380 @@
+#ifndef SET_H_
+#define SET_H_
+
+#include <common/threading/Mutex.h>
+#include <database/SetFragment.h>
+#include <database/SetFragmentCursor.h>
+#include <database/Union.h>
+
+#include <iomanip>
+#include <fstream>
+#include <limits>
+#include <map>
+#include <mutex>
+#include <sstream>
+
+#include <limits.h>
+
+#include <boost/format.hpp>
+
+template<typename Data>
+class Set {
+   public:
+      typedef SetFragment<Data> Fragment;
+      typedef SetFragmentCursor<Data> Cursor;
+
+   private:
+      static const unsigned VERSION = 1;
+
+      std::string basename;
+      unsigned nextID;
+      std::map<std::string, Fragment*> openFragments;
+      bool dropped;
+
+      Mutex mutex;
+
+      typedef typename std::map<std::string, Fragment*>::iterator FragmentIter;
+
+      Set(const Set&);
+      Set& operator=(const Set&);
+
+      std::string fragmentName(unsigned id)
+      {
+         std::stringstream ss;
+         ss << basename << '.' << std::setw(5) << std::setfill('0') << id;
+         return ss.str();
+      }
+
+      Fragment* getFragment(const std::string& file, bool allowCreate)
+      {
+         if(openFragments.count(file) )
+            return openFragments[file];
+
+         Fragment* fragment = new Fragment(file, allowCreate);
+         openFragments[file] = fragment;
+         return fragment;
+      }
+
+      void acceptFragment(Fragment* foreign)
+      {
+         const unsigned newID = nextID++;
+         const std::string& file = fragmentName(newID);
+
+         foreign->rename(file);
+
+         openFragments[file] = foreign;
+         saveConfig();
+      }
+
+      void removeFragment(const std::string& file)
+      {
+         Fragment* fragment = openFragments[file];
+         openFragments.erase(file);
+         fragment->drop();
+         delete fragment;
+      }
+
+      void loadConfig(bool allowCreate)
+      {
+         std::ifstream in( (basename + ".t").c_str() );
+
+         if(!in)
+         {
+            if(allowCreate)
+               return;
+
+            throw std::runtime_error("could not read set description file " + basename);
+         }
+
+         {
+            char code;
+            unsigned version;
+            in >> code >> version;
+            if (code != 'v' || version != VERSION) {
+               throw std::runtime_error(str(
+                        boost::format("cannot read set %1% with version %2%")
+                           % basename
+                           % version));
+            }
+         }
+
+         in >> nextID;
+
+         unsigned count = 0;
+         in >> count;
+
+         in.ignore(1, '\n');
+
+         while(count > 0)
+         {
+            std::string file;
+
+            std::getline(in, file);
+
+            if(!in)
+               break;
+
+            getFragment(basename + file, false);
+            count--;
+         }
+
+         if(!in || in.peek() != std::ifstream::traits_type::eof() )
+            throw std::runtime_error("bad set description for " + basename);
+      }
+
+      void saveConfig()
+      {
+         std::ofstream out((basename + ".t").c_str(), std::ostream::trunc);
+
+         out << 'v' << VERSION << '\n';
+         out << nextID << '\n';
+         out << openFragments.size() << '\n';
+
+         for (FragmentIter it = openFragments.begin(), end = openFragments.end(); it != end; ++it) {
+            if (it->second->filename().find(basename) != 0)
+               throw std::runtime_error("set corrupted: " + basename);
+
+            out << it->second->filename().substr(basename.size()) << '\n';
+         }
+      }
+
+      static std::string makeAbsolute(const std::string& path)
+      {
+         std::string::size_type spos = path.find_last_of('/');
+         if (spos == path.npos)
+            spos = 0;
+         else
+            spos += 1;
+
+         // can't/don't want to use ::basename because it may return static buffers
+         const std::string basename = path.substr(spos);
+
+         if (basename.empty() || basename == "." || basename == "..")
+            throw std::runtime_error("bad path");
+
+         if (path[0] == '/')
+            return path;
+
+         char cwd[PATH_MAX];
+
+         if (::getcwd(cwd, sizeof(cwd)) == NULL)
+            throw std::runtime_error("bad path");
+
+         return cwd + ('/' + path);
+      }
+
+   public:
+      Set(const std::string& basename, bool allowCreate = true)
+         : basename(makeAbsolute(basename) ), nextID(0), dropped(false)
+      {
+         loadConfig(allowCreate);
+      }
+
+      ~Set()
+      {
+         if(dropped)
+            return;
+
+         saveConfig();
+
+         for(FragmentIter it = openFragments.begin(), end = openFragments.end(); it != end; ++it)
+            delete it->second;
+      }
+
+      size_t size()
+      {
+         size_t result = 0;
+
+         for(FragmentIter it = openFragments.begin(), end = openFragments.end(); it != end; ++it)
+            result += it->second->size();
+
+         return result;
+      }
+
+      Fragment* newFragment()
+      {
+         const std::lock_guard<Mutex> lock(mutex);
+
+         Fragment* result = getFragment(fragmentName(nextID++), true);
+         saveConfig();
+
+         return result;
+      }
+
+      void sort()
+      {
+         if(openFragments.empty() )
+            newFragment();
+
+         std::multimap<size_t, Fragment*> sortedFragments;
+
+         for(FragmentIter it = openFragments.begin(), end = openFragments.end(); it != end; ++it)
+            it->second->flush();
+
+         for(FragmentIter it = openFragments.begin(), end = openFragments.end(); it != end; ++it)
+         {
+            it->second->sort();
+            sortedFragments.insert(std::make_pair(it->second->size(), it->second) );
+         }
+
+         static const unsigned MERGE_WIDTH = 4;
+
+         if(sortedFragments.size() == 1)
+            return;
+
+         saveConfig();
+
+         while(sortedFragments.size() > 1)
+         {
+            Fragment* inputs[MERGE_WIDTH] = {};
+            unsigned inputCount = 0;
+
+            for(inputCount = 0; inputCount < MERGE_WIDTH; inputCount++)
+            {
+               if(sortedFragments.empty() )
+                  break;
+
+               inputs[inputCount] = sortedFragments.begin()->second;
+               sortedFragments.erase(sortedFragments.begin());
+            }
+
+            SetFragment<Data>* merged = getFragment(fragmentName(nextID++), true);
+
+            struct op
+            {
+               static typename Data::KeyType key(const Data& d) { return d.pkey(); }
+            };
+            typedef typename Data::KeyType (*key_t)(const Data&);
+            typedef Union<Cursor, Cursor, key_t> L1Union;
+
+            switch(inputCount)
+            {
+            case 2: {
+               L1Union u(Cursor(*inputs[0]), (Cursor(*inputs[1]) ), op::key);
+               while(u.step() )
+                  merged->append(*u.get() );
+               break;
+            }
+
+            case 3: {
+               Union<L1Union, SetFragmentCursor<Data>, key_t> u(
+                  L1Union(Cursor(*inputs[0]), (Cursor(*inputs[1]) ), op::key),
+                  Cursor(*inputs[2]),
+                  op::key);
+               while(u.step() )
+                  merged->append(*u.get() );
+               break;
+            }
+
+            case 4: {
+               Union<L1Union, L1Union, key_t> u(
+                  L1Union(Cursor(*inputs[0]), (Cursor(*inputs[1]) ), op::key),
+                  L1Union(Cursor(*inputs[2]), (Cursor(*inputs[3]) ), op::key),
+                  op::key);
+               while(u.step() )
+                  merged->append(*u.get() );
+               break;
+            }
+
+            default:
+               throw std::runtime_error("");
+            }
+
+            merged->flush();
+
+            sortedFragments.insert(std::make_pair(merged->size(), merged) );
+
+            for (unsigned i = 0; i < inputCount; i++)
+               removeFragment(inputs[i]->filename() );
+
+            saveConfig();
+         }
+      }
+
+      Cursor cursor()
+      {
+         sort();
+
+         return Cursor(*openFragments.begin()->second);
+      }
+
+      template<typename Key, typename Fn>
+      std::pair<bool, Data> getByKeyProjection(const Key& key, Fn fn)
+      {
+         sort();
+
+         if(openFragments.empty() )
+            return std::make_pair(false, Data() );
+
+         return openFragments.begin()->second->getByKeyProjection(key, fn);
+      }
+
+      std::pair<bool, Data> getByKey(const typename Data::KeyType& key)
+      {
+         struct ops
+         {
+            static typename Data::KeyType key(const typename Data::KeyType& key) { return key; }
+         };
+
+         return getByKeyProjection(key, ops::key);
+      }
+
+      void clear()
+      {
+         while(!openFragments.empty() )
+            removeFragment(openFragments.begin()->first);
+
+         nextID = 0;
+         saveConfig();
+      }
+
+      void drop()
+      {
+         clear();
+         if(::unlink( (basename + ".t").c_str() ) )
+            throw std::runtime_error("could not unlink set file " + basename + ": " + std::string(strerror(errno)));
+
+         dropped = true;
+      }
+
+      void mergeInto(Set& other)
+      {
+         for(FragmentIter it = openFragments.begin(), end = openFragments.end(); it != end; ++it)
+            other.acceptFragment(it->second);
+
+         openFragments.clear();
+         saveConfig();
+      }
+
+      void makeUnique()
+      {
+         sort();
+
+         Fragment* input = openFragments.begin()->second;
+         Cursor cursor(*input);
+         typename Data::KeyType key;
+
+         if(!cursor.step() )
+            return;
+
+         Fragment* unique = newFragment();
+         unique->append(*cursor.get() );
+         key = cursor.get()->pkey();
+
+         while(cursor.step() )
+         {
+            if(key == cursor.get()->pkey() )
+               continue;
+
+            unique->append(*cursor.get() );
+            key = cursor.get()->pkey();
+         }
+
+         unique->flush();
+
+         removeFragment(input->filename() );
+         saveConfig();
+      }
+};
+
+#endif

fsck/source/database/SetFragment.h

@@ -0,0 +1,325 @@
+#ifndef SETFRAGMENT_H_
+#define SETFRAGMENT_H_
+
+#include <algorithm>
+#include <cerrno>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+#include <boost/scoped_array.hpp>
+#include <fcntl.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+struct FragmentDoesNotExist : public std::runtime_error
+{
+   FragmentDoesNotExist(const std::string& file)
+      : runtime_error(file)
+   {}
+};
+
+template<typename Data>
+class SetFragment {
+   public:
+      static const unsigned CONFIG_AREA_SIZE = 4096;
+      static const size_t BUFFER_SIZE = 4ULL * 1024 * 1024;
+
+   private:
+      std::string file;
+      int fd;
+      size_t itemCount;
+      bool sorted;
+
+      std::vector<Data> buffer;
+      size_t firstBufferedItem;
+      ssize_t firstDirtyItem;
+      ssize_t lastDirtyItem;
+
+      typename Data::KeyType lastItemKey;
+
+      SetFragment(const SetFragment&);
+      SetFragment& operator=(const SetFragment&);
+
+      size_t readBlock(Data* into, size_t count, size_t from)
+      {
+         size_t total = 0;
+         count *= sizeof(Data);
+         from *= sizeof(Data);
+
+         from += CONFIG_AREA_SIZE;
+
+         char* buf = (char*) into;
+
+         while(total < count)
+         {
+            ssize_t current = ::pread(fd, buf + total, count - total, from + total);
+            total += current;
+
+            if (current < 0)
+               throw std::runtime_error("read failed: " + std::string(strerror(errno)));
+
+            if (current == 0)
+               break;
+         }
+
+         return total / sizeof(Data);
+      }
+
+      size_t writeBlock(const Data* source, size_t count, size_t from)
+      {
+         size_t total = 0;
+         count *= sizeof(Data);
+         from *= sizeof(Data);
+
+         from += CONFIG_AREA_SIZE;
+
+         const char* buf = (const char*) source;
+
+         while(total < count)
+         {
+            ssize_t current = ::pwrite(fd, buf + total, count - total, from + total);
+            total += current;
+
+            if (current < 0)
+               throw std::runtime_error("write failed: " + std::string(strerror(errno)));
+
+            if (current == 0)
+               break;
+         }
+
+         return total / sizeof(Data);
+      }
+
+      void flushBuffer()
+      {
+         if(firstDirtyItem < 0)
+            return;
+
+         const Data* first = &buffer[firstDirtyItem - firstBufferedItem];
+
+         writeBlock(first, lastDirtyItem - firstDirtyItem + 1, firstDirtyItem);
+
+         buffer.clear();
+         firstDirtyItem = -1;
+      }
+
+      void bufferFileRange(size_t begin, size_t end)
+      {
+         flushBuffer();
+
+         const size_t firstAddress = begin;
+         const size_t windowSize = end == size_t(-1) ? end : (end - begin + 1);
+         const size_t totalRead = std::min<size_t>(windowSize, BUFFER_SIZE / sizeof(Data) );
+
+         buffer.resize(totalRead);
+
+         size_t read = readBlock(&buffer[0], totalRead, firstAddress);
+
+         firstBufferedItem = begin;
+         buffer.resize(read);
+      }
+
+   public:
+      SetFragment(const std::string& file, bool allowCreate = true)
+         : file(file),
+           itemCount(0),
+           firstBufferedItem(0), firstDirtyItem(-1), lastDirtyItem(0)
+      {
+         fd = ::open(file.c_str(), O_RDWR | (allowCreate ? O_CREAT : 0), 0660);
+         if(fd < 0)
+         {
+            int eno = errno;
+            if(!allowCreate && errno == ENOENT)
+               throw FragmentDoesNotExist(file);
+            else
+               throw std::runtime_error("could not open fragment file " + file + ": " + strerror(eno));
+         }
+
+         off_t totalSize = ::lseek(fd, 0, SEEK_END);
+
+         if(totalSize > 0)
+         {
+            int eno = errno;
+            if (totalSize < (off_t) CONFIG_AREA_SIZE)
+               throw std::runtime_error("error while opening fragment file " + file + ": " + strerror(eno));
+            else
+            if ( (totalSize - CONFIG_AREA_SIZE) % sizeof(Data) )
+               throw std::runtime_error("error while opening fragment file " + file + ": " + strerror(eno));
+         }
+
+         if(totalSize == 0)
+         {
+            sorted = true;
+            itemCount = 0;
+         }
+         else
+         {
+            if (::pread(fd, &sorted, sizeof(sorted), 0) != sizeof(sorted)) {
+               int eno = errno;
+               throw std::runtime_error("error while opening fragment file " + file + ": " + strerror(eno));
+            }
+
+            itemCount = (totalSize - CONFIG_AREA_SIZE) / sizeof(Data);
+         }
+
+         if(itemCount > 0)
+            lastItemKey = (*this)[itemCount - 1].pkey();
+      }
+
+      ~SetFragment()
+      {
+         if(fd >= 0)
+         {
+            flush();
+            close(fd);
+         }
+      }
+
+      const std::string filename() const { return file; }
+      size_t size() const { return itemCount; }
+
+      void append(const Data& data)
+      {
+         if(itemCount > firstBufferedItem + buffer.size() )
+         {
+            flushBuffer();
+            firstBufferedItem = itemCount;
+         }
+
+         if(firstDirtyItem < 0)
+            firstDirtyItem = itemCount;
+
+         lastDirtyItem = itemCount;
+
+         if(itemCount > 0 && sorted)
+            sorted = lastItemKey < data.pkey();
+
+         buffer.push_back(data);
+         itemCount++;
+         lastItemKey = data.pkey();
+
+         if(buffer.size() * sizeof(Data) >= BUFFER_SIZE)
+            flushBuffer();
+      }
+
+      Data& operator[](size_t offset)
+      {
+         if(offset < firstBufferedItem || offset >= firstBufferedItem + buffer.size() )
+            bufferFileRange(offset == 0 ? 0 : offset - 1, -1);
+
+         return buffer[offset - firstBufferedItem];
+      }
+
+      void flush()
+      {
+         if(firstDirtyItem < 0)
+            return;
+
+         flushBuffer();
+         buffer = std::vector<Data>();
+
+         if (::pwrite(fd, &sorted, sizeof(sorted), 0) < 0) {
+            int eno = errno;
+            throw std::runtime_error("error in flush of " + file + ": " + strerror(eno));
+         }
+
+         // truncate to CONFIG_AREA_SIZE (for reopen)
+         if (itemCount == 0 && ::ftruncate(fd, CONFIG_AREA_SIZE) < 0) {
+            int eno = errno;
+            throw std::runtime_error("error in flush of " + file + ": " + strerror(eno));
+         }
+      }
+
+      void sort()
+      {
+         flush();
+
+         if(sorted)
+            return;
+
+         boost::scoped_array<Data> data(new Data[size()]);
+
+         if(readBlock(data.get(), size(), 0) < size() )
+            throw 42;
+
+         struct ops
+         {
+            static bool compare(const Data& l, const Data& r)
+            {
+               return l.pkey() < r.pkey();
+            }
+         };
+
+         std::sort(data.get(), data.get() + size(), ops::compare);
+
+         writeBlock(data.get(), size(), 0);
+
+         sorted = true;
+
+         flush();
+      }
+
+      void drop()
+      {
+         flush();
+         if (::unlink(file.c_str()) < 0) {
+            int eno = errno;
+            throw std::runtime_error("could not unlink fragment file " + file + ": " + strerror(eno));
+         }
+
+         close(fd);
+         fd = -1;
+      }
+
+      void rename(const std::string& to)
+      {
+         if (::rename(file.c_str(), to.c_str()) < 0) {
+            int eno = errno;
+            throw std::runtime_error("could not rename fragment file " + file + ": " + strerror(eno));
+         }
+
+         file = to;
+      }
+
+      template<typename Key, typename Fn>
+      std::pair<bool, Data> getByKeyProjection(const Key& key, Fn fn)
+      {
+         if(size() == 0)
+            return std::make_pair(false, Data() );
+
+         size_t first = 0;
+         size_t last = size() - 1;
+
+         while(first != last)
+         {
+            size_t midIdx = first + (last - first) / 2;
+
+            bufferFileRange(midIdx, midIdx + 1);
+            Data& mid = (*this)[midIdx];
+
+            if (fn(mid.pkey() ) < key)
+               first = midIdx + 1;
+            else
+               last = midIdx;
+         }
+
+         if(fn( (*this)[first].pkey() ) == key)
+            return std::make_pair(true, (*this)[first]);
+         else
+            return std::make_pair(false, Data() );
+      }
+
+      std::pair<bool, Data> getByKey(const typename Data::KeyType& key)
+      {
+         struct ops
+         {
+            static typename Data::KeyType key(const typename Data::KeyType& key) { return key; }
+         };
+
+         return getByKeyProjection(key, ops::key);
+      }
+};
+
+#endif

fsck/source/database/SetFragmentCursor.h

@@ -0,0 +1,47 @@
+#ifndef SETFRAGMENTCURSOR_H_
+#define SETFRAGMENTCURSOR_H_
+
+#include <database/SetFragment.h>
+
+template<typename Data>
+class SetFragmentCursor {
+   public:
+      typedef Data ElementType;
+      typedef size_t MarkerType;
+
+   private:
+      SetFragment<Data>* fragment;
+      size_t currentGetIndex;
+
+   public:
+      explicit SetFragmentCursor(SetFragment<Data>& fragment)
+         : fragment(&fragment), currentGetIndex(-1)
+      {}
+
+      bool step()
+      {
+         if (currentGetIndex + 1 >= fragment->size() )
+            return false;
+
+         currentGetIndex++;
+         return true;
+      }
+
+      Data* get()
+      {
+         return &(*fragment)[currentGetIndex];
+      }
+
+      MarkerType mark() const
+      {
+         return currentGetIndex;
+      }
+
+      void restore(MarkerType mark)
+      {
+         currentGetIndex = mark - 1;
+         step();
+      }
+};
+
+#endif

fsck/source/database/StripeTargets.h

@@ -0,0 +1,45 @@
+#ifndef STRIPETARGETS_H_
+#define STRIPETARGETS_H_
+
+#include <database/EntryID.h>
+
+#include <inttypes.h>
+
+namespace db {
+
+struct StripeTargets
+{
+   EntryID id;                 /* 12 */
+
+   enum {
+      NTARGETS = 7
+   };
+   uint16_t targets[NTARGETS]; /* 26 */
+   uint16_t firstTargetIndex;  /* 28 */
+
+   typedef EntryID KeyType;
+
+   EntryID pkey() const { return id; }
+
+   friend std::ostream& operator<<(std::ostream& os, StripeTargets const& obj)
+   {
+      os << "-------------" << "\n";
+      os << "StripeTargets" << "\n";
+      os << "-------------" << "\n";
+      os << "EntryID: " << obj.id.str() << "\n";
+      os << "Stripe targets: [ ";
+
+      for (int i=0; i<obj.NTARGETS; i++)
+      {
+         os <<  obj.targets[i] << " ";
+      }
+      os << "]\n";
+
+      os << "FirstTargetsIndex: " << obj.firstTargetIndex << "\n";
+      return os;
+   }
+};
+
+}
+
+#endif

fsck/source/database/Table.h

@@ -0,0 +1,430 @@
+#ifndef TABLE_H_
+#define TABLE_H_
+
+#include <database/Buffer.h>
+#include <database/Filter.h>
+#include <database/LeftJoinEq.h>
+#include <database/Select.h>
+#include <database/Set.h>
+
+#include <boost/make_shared.hpp>
+#include <boost/scoped_ptr.hpp>
+#include <boost/shared_ptr.hpp>
+#include <boost/ref.hpp>
+#include <boost/weak_ptr.hpp>
+
+template<typename Data, bool AllowDuplicateKeys = false>
+class Table {
+   private:
+      struct Key
+      {
+         typedef typename Data::KeyType KeyType;
+
+         KeyType value;
+
+         Key() {}
+
+         Key(const KeyType& key) : value(key) {}
+
+         KeyType pkey() const { return value; }
+
+         bool operator<(const Key& r) const { return value < r.value; }
+         bool operator==(const Key& r) const { return value == r.value; }
+      };
+
+      struct GetKey
+      {
+         typedef typename Data::KeyType result_type;
+
+         result_type operator()(const Data& data) const { return data.pkey(); }
+         result_type operator()(const Key& data) const { return data.value; }
+      };
+
+      struct SecondIsNull
+      {
+         typedef bool result_type;
+
+         bool operator()(std::pair<Data, Key*>& p) const { return p.second == NULL; }
+      };
+
+      struct First
+      {
+         typedef Data result_type;
+
+         Data operator()(std::pair<Data, Key*>& p) const { return p.first; }
+      };
+
+      struct HasMatchingKey
+      {
+         HasMatchingKey(const typename Data::KeyType& k) : key(k) {}
+         typename Data::KeyType key;
+
+         bool operator()(std::pair<Data, Key*>& p) const
+         {
+            return (p.first.pkey() == key);
+         }
+      };
+
+   private:
+      std::string basename;
+      size_t fragmentSize;
+
+      Set<Data> base;
+      Set<Data> inserts;
+      Set<Key> deletes;
+
+      typename Data::KeyType lastChangeKey;
+      Set<Data> insertsPending;
+      Set<Key> deletesPending;
+
+      boost::scoped_ptr<Buffer<Data> > insertBuffer;
+      boost::scoped_ptr<Buffer<Key> > deleteBuffer;
+
+      enum {
+         ms_none,
+         ms_ordered_only_insert,
+         ms_ordered_only_delete,
+         ms_ordered_at_insert,
+         ms_ordered_at_delete,
+         ms_unordered_insert,
+         ms_unordered_delete,
+         ms_bulk_insert,
+      } modificationState;
+
+      bool dropped;
+      bool insertsIntoBase;
+
+      std::vector<boost::weak_ptr<Buffer<Data> > > bulkInserters;
+
+      void assertNoBulkInsert()
+      {
+         for(size_t i = 0; i < bulkInserters.size(); i++)
+         {
+            if(!bulkInserters[i].expired() )
+               throw std::runtime_error("bulk insert still running");
+         }
+
+         bulkInserters.clear();
+      }
+
+      void collapseChanges()
+      {
+         deleteBuffer.reset();
+         insertBuffer.reset();
+         insertsIntoBase = false;
+
+         if(deletesPending.size() )
+         {
+            Set<Data> insertsTemp(basename + ".it");
+
+            LeftJoinEq<
+                  SetFragmentCursor<Data>,
+                  SetFragmentCursor<Key>,
+                  GetKey>
+               cleanedInserts = db::leftJoinBy(
+                  GetKey(),
+                  inserts.cursor(),
+                  deletesPending.cursor() );
+
+            // sequence will always be sorted, no need to fragment and sort again
+            SetFragment<Data>* fragment = insertsTemp.newFragment();
+            while(cleanedInserts.step() )
+            {
+               if (cleanedInserts.get()->second == nullptr)
+                  fragment->append(cleanedInserts.get()->first);
+            }
+
+            inserts.clear();
+            insertsTemp.mergeInto(inserts);
+            insertsTemp.drop();
+
+            deletesPending.mergeInto(deletes);
+            deletes.makeUnique();
+         }
+
+         if(insertsPending.size() )
+            insertsPending.mergeInto(inserts);
+
+         modificationState = ms_none;
+      }
+
+      void setupBaseBuffer()
+      {
+         insertBuffer.reset(new Buffer<Data>(base, fragmentSize));
+         insertsIntoBase = true;
+      }
+
+   public:
+      Table(const std::string& basename, size_t fragmentSize, bool allowCreate = true)
+         : basename(basename), fragmentSize(fragmentSize),
+           base(basename + ".b", allowCreate),
+           inserts(basename + ".i", allowCreate),
+           deletes(basename + ".d", allowCreate),
+           insertsPending(basename + ".ip", true),
+           deletesPending(basename + ".dp", true),
+           modificationState(ms_none),
+           dropped(false), insertsIntoBase(false)
+      {
+         if (base.size() == 0)
+            setupBaseBuffer();
+      }
+
+      ~Table()
+      {
+         if(dropped)
+            return;
+
+         collapseChanges();
+
+         deletesPending.drop();
+         insertsPending.drop();
+      }
+
+      boost::shared_ptr<Buffer<Data> > bulkInsert()
+      {
+         if(inserts.size() || deletes.size() || insertsPending.size() || deletesPending.size() )
+            throw std::runtime_error("unordered operation");
+
+         switch(modificationState)
+         {
+         case ms_none:
+         case ms_ordered_only_insert:
+         case ms_bulk_insert:
+            break;
+
+         default:
+            throw std::runtime_error("unordered operation");
+         }
+
+         modificationState = ms_bulk_insert;
+         boost::shared_ptr<Buffer<Data> > buffer = boost::make_shared<Buffer<Data> >(
+            boost::ref(base), fragmentSize);
+
+         bulkInserters.push_back(buffer);
+         return buffer;
+      }
+
+      void insert(const Data& data)
+      {
+         using namespace std::rel_ops;
+
+         switch(modificationState)
+         {
+         case ms_bulk_insert:
+            assertNoBulkInsert();
+            BEEGFS_FALLTHROUGH;
+
+         case ms_none:
+            modificationState = ms_ordered_only_insert;
+            break;
+
+         case ms_ordered_only_insert:
+            if(lastChangeKey >= data.pkey() )
+               modificationState = ms_unordered_insert;
+
+            break;
+
+         case ms_unordered_delete:
+            throw std::runtime_error("unordered operation");
+
+         case ms_unordered_insert:
+            break;
+
+         case ms_ordered_at_delete:
+         case ms_ordered_only_delete:
+            if(lastChangeKey <= data.pkey() )
+               modificationState = ms_ordered_at_insert;
+            else
+               throw std::runtime_error("unordered operation");
+
+            break;
+
+         case ms_ordered_at_insert:
+            if(lastChangeKey < data.pkey()
+                  || (!AllowDuplicateKeys && lastChangeKey == data.pkey() ) )
+               throw std::runtime_error("unordered operation");
+
+            break;
+         }
+
+         lastChangeKey = data.pkey();
+
+         if(!insertBuffer)
+            insertBuffer.reset(new Buffer<Data>(insertsPending, fragmentSize) );
+
+         insertBuffer->append(data);
+      }
+
+      void remove(const typename Data::KeyType& key)
+      {
+         using namespace std::rel_ops;
+
+         if(modificationState == ms_bulk_insert)
+            assertNoBulkInsert();
+
+         // if deletes happen during initial insert, complete the base set. delete tracking
+         // cannot cope otherwise
+         if(insertsIntoBase)
+         {
+            insertBuffer.reset();
+            insertsIntoBase = false;
+            modificationState = ms_none;
+         }
+
+         switch(modificationState)
+         {
+         case ms_none:
+         case ms_bulk_insert:
+            modificationState = ms_ordered_only_delete;
+            break;
+
+         case ms_ordered_only_delete:
+            if(lastChangeKey >= key)
+               modificationState = ms_unordered_delete;
+
+            break;
+
+         case ms_unordered_delete:
+            break;
+
+         case ms_unordered_insert:
+            throw std::runtime_error("unordered operation");
+
+         case ms_ordered_at_delete:
+         case ms_ordered_at_insert:
+         case ms_ordered_only_insert:
+            if(key <= lastChangeKey)
+               throw std::runtime_error("unordered operation");
+
+            modificationState = ms_ordered_at_delete;
+            break;
+         }
+
+         lastChangeKey = key;
+
+         if(!deleteBuffer)
+            deleteBuffer.reset(new Buffer<Key>(deletesPending, fragmentSize) );
+
+         deleteBuffer->append(key);
+      }
+
+      void commitChanges()
+      {
+         collapseChanges();
+         base.sort();
+         inserts.sort();
+         deletes.sort();
+      }
+
+      typedef Union<
+         Select<
+            Filter<
+               LeftJoinEq<
+                  SetFragmentCursor<Data>,
+                  SetFragmentCursor<Key>,
+                  GetKey>,
+               SecondIsNull>,
+            First>,
+         SetFragmentCursor<Data>,
+         GetKey> QueryType;
+
+      QueryType cursor()
+      {
+         return db::unionBy(
+            GetKey(),
+            db::leftJoinBy(
+               GetKey(),
+               base.cursor(),
+               deletes.cursor() )
+            | db::where(SecondIsNull() )
+            | db::select(First() ),
+            inserts.cursor() );
+      }
+
+      typedef Union<
+         Select<
+            Filter<
+               Filter<
+                  LeftJoinEq<
+                     SetFragmentCursor<Data>,
+                     SetFragmentCursor<Key>,
+                     GetKey>,
+                  SecondIsNull>,
+               HasMatchingKey>,
+            First>,
+         SetFragmentCursor<Data>,
+         GetKey> MultiRowResultType;
+
+      MultiRowResultType getAllByKey(const typename Data::KeyType& key)
+      {
+         return db::unionBy(
+            GetKey(),
+            db::leftJoinBy(
+               GetKey(),
+               base.cursor(),
+               deletes.cursor() )
+            | db::where(SecondIsNull() )
+            | db::where(HasMatchingKey(key) )
+            | db::select(First() ),
+            inserts.cursor() );
+      }
+
+      template<typename Key, typename Fn>
+      std::pair<bool, Data> getByKeyProjection(const Key& key, Fn fn)
+      {
+         Data dummy = {};
+
+         std::pair<bool, Data> insIdx = inserts.getByKeyProjection(key, fn);
+         if(insIdx.first)
+            return insIdx;
+
+         std::pair<bool, Data> baseIdx = base.getByKeyProjection(key, fn);
+         if(!baseIdx.first)
+            return std::make_pair(false, dummy);
+
+         std::pair<bool, typename Table::Key> delIdx = deletes.getByKey(baseIdx.second.pkey() );
+         if(delIdx.first)
+            return std::make_pair(false, dummy);
+
+         return baseIdx;
+      }
+
+      std::pair<bool, Data> getByKey(const typename Data::KeyType& key)
+      {
+         struct ops
+         {
+            static typename Data::KeyType key(const typename Data::KeyType& key) { return key; }
+         };
+
+         return getByKeyProjection(key, ops::key);
+      }
+
+      void drop()
+      {
+         insertBuffer.reset();
+         deleteBuffer.reset();
+
+         base.drop();
+         inserts.drop();
+         deletes.drop();
+         insertsPending.drop();
+         deletesPending.drop();
+         dropped = true;
+      }
+
+      void clear()
+      {
+         insertBuffer.reset();
+         deleteBuffer.reset();
+
+         base.clear();
+         inserts.clear();
+         deletes.clear();
+         insertsPending.clear();
+         deletesPending.clear();
+
+         setupBaseBuffer();
+      }
+};
+
+#endif

fsck/source/database/Union.h

@@ -0,0 +1,130 @@
+#ifndef UNION_H_
+#define UNION_H_
+
+#include <utility>
+
+template<typename Left, typename Right, typename KeyExtract>
+class Union
+{
+   public:
+      typedef typename Left::ElementType ElementType;
+
+      struct MarkerType
+      {
+         typename Left::MarkerType leftMark;
+         typename Right::MarkerType rightMark;
+         bool leftEnded;
+         bool rightEnded;
+         bool nextStepLeft;
+      };
+
+   public:
+      Union(Left left, Right right, KeyExtract keyExtract):
+         left(left), right(right), keyExtract(keyExtract), leftEnded(false), nextStepLeft(true),
+         currentAtLeft(false)
+      {
+         this->rightEnded = !this->right.step();
+      }
+
+      bool step()
+      {
+         if(this->leftEnded && this->rightEnded)
+            return false;
+
+         if(this->leftEnded)
+         {
+            this->rightEnded = !this->right.step();
+            return resetCurrent();
+         }
+
+         if(this->rightEnded)
+         {
+            this->leftEnded = !this->left.step();
+            return resetCurrent();
+         }
+
+         if(this->nextStepLeft)
+            this->leftEnded = !this->left.step();
+         else
+            this->rightEnded = !this->right.step();
+
+         return resetCurrent();
+      }
+
+      ElementType* get()
+      {
+         if(this->currentAtLeft)
+            return this->left.get();
+         else
+            return this->right.get();
+      }
+
+      MarkerType mark() const
+      {
+         MarkerType result = { this->left.mark(), this->right.mark(), this->leftEnded,
+            this->rightEnded, this->nextStepLeft };
+         return result;
+      }
+
+      void restore(MarkerType mark)
+      {
+         this->leftEnded = mark.leftEnded;
+         this->rightEnded = mark.rightEnded;
+         this->nextStepLeft = mark.nextStepLeft;
+
+         if(!this->leftEnded)
+            this->left.restore(mark.leftMark);
+
+         if(!this->rightEnded)
+            this->right.restore(mark.rightMark);
+
+         resetCurrent();
+      }
+
+   private:
+      bool resetCurrent()
+      {
+         if(this->leftEnded && this->rightEnded)
+            return false;
+
+         if(this->leftEnded)
+            this->currentAtLeft = false;
+         else
+         if(this->rightEnded)
+            this->currentAtLeft = true;
+         else
+         if(this->keyExtract(*this->left.get() ) < this->keyExtract(*this->right.get() ) )
+         {
+            this->currentAtLeft = true;
+            this->nextStepLeft = true;
+         }
+         else
+         {
+            this->currentAtLeft = false;
+            this->nextStepLeft = false;
+         }
+
+         return true;
+      }
+
+   private:
+      Left left;
+      Right right;
+      KeyExtract keyExtract;
+      bool leftEnded;
+      bool rightEnded;
+      bool nextStepLeft;
+      bool currentAtLeft;
+};
+
+namespace db {
+
+template<typename KeyExtract, typename Left, typename Right>
+inline Union<Left, Right, KeyExtract> unionBy(KeyExtract keyExtract, Left left, Right right)
+{
+   return Union<Left, Right, KeyExtract>(left, right, keyExtract);
+}
+
+}
+
+#endif

fsck/source/database/UsedTarget.h

@@ -0,0 +1,25 @@
+#ifndef USEDTARGET_H_
+#define USEDTARGET_H_
+
+#include <common/fsck/FsckTargetID.h>
+
+namespace db {
+
+struct UsedTarget
+{
+   uint32_t id;
+   uint32_t type;
+
+   typedef std::pair<uint32_t, uint32_t> KeyType;
+
+   KeyType pkey() const { return std::make_pair(id, type); }
+
+   operator FsckTargetID() const
+   {
+      return FsckTargetID(id, FsckTargetIDType(type) );
+   }
+};
+
+}
+
+#endif

fsck/source/database/VectorSource.h

@@ -0,0 +1,52 @@
+#ifndef VECTORSOURCE_H_
+#define VECTORSOURCE_H_
+
+#include <vector>
+
+#include <boost/shared_ptr.hpp>
+#include <boost/make_shared.hpp>
+
+template<typename Obj>
+class VectorSource
+{
+   public:
+      typedef Obj ElementType;
+      typedef typename std::vector<Obj>::const_iterator MarkerType;
+
+   public:
+      explicit VectorSource(std::vector<Obj>& data)
+         : content(boost::make_shared<std::vector<Obj> >() ), index(-1)
+      {
+         data.swap(*content);
+      }
+
+      bool step()
+      {
+         if(this->index == this->content->size() - 1)
+            return false;
+
+         this->index += 1;
+         return true;
+      }
+
+      ElementType* get()
+      {
+         return &(*this->content)[this->index];
+      }
+
+      MarkerType mark() const
+      {
+         return this->content->begin() + this->index;
+      }
+
+      void restore(MarkerType mark)
+      {
+         this->index = mark - this->content->begin();
+      }
+
+   private:
+      boost::shared_ptr<std::vector<Obj> > content;
+      size_t index;
+};
+
+#endif