New upstream version 8.1.0

storage/source/session/Session.cpp

@@ -0,0 +1,24 @@
+#include <common/toolkit/serialization/Serialization.h>
+#include "Session.h"
+
+
+/* Merges the SessionLocalFiles of the given session into this session, only not existing
+ * SessionLocalFiles will be added to the existing session
+ * @param session the session which will be merged with this session
+ */
+void Session::mergeSessionLocalFiles(Session* session)
+{
+   this->getLocalFiles()->mergeSessionLocalFiles(session->getLocalFiles());
+}
+
+void Session::serializeForTarget(Serializer& ser, uint16_t targetID)
+{
+   ser % sessionID;
+   localFiles.serializeForTarget(ser, targetID);
+}
+
+void Session::deserializeForTarget(Deserializer& des, uint16_t targetID)
+{
+   des % sessionID;
+   localFiles.deserializeForTarget(des, targetID);
+}

storage/source/session/Session.h

@@ -0,0 +1,44 @@
+#pragma once
+
+#include <common/Common.h>
+#include <common/nodes/NumNodeID.h>
+#include "SessionLocalFileStore.h"
+
+/*
+ * A session always belongs to a client ID, therefore the session ID is always the nodeID of the
+ * corresponding client
+ */
+class Session
+{
+   public:
+      Session(const NumNodeID sessionID) : sessionID(sessionID) {}
+
+      /*
+       * For deserialization only
+       */
+      Session() {};
+
+      void mergeSessionLocalFiles(Session* session);
+
+      void serializeForTarget(Serializer& ser, uint16_t targetID);
+      void deserializeForTarget(Deserializer& des, uint16_t targetID);
+
+   private:
+      NumNodeID sessionID;
+
+      SessionLocalFileStore localFiles;
+
+   public:
+      // getters & setters
+      NumNodeID getSessionID() const
+      {
+         return sessionID;
+      }
+
+      SessionLocalFileStore* getLocalFiles()
+      {
+         return &localFiles;
+      }
+};
+
+

storage/source/session/SessionLocalFile.cpp

@@ -0,0 +1,168 @@
+#include <common/toolkit/serialization/Serialization.h>
+#include <common/storage/quota/ExceededQuotaStore.h>
+#include <net/msghelpers/MsgHelperIO.h>
+#include <program/Program.h>
+#include <storage/ChunkStore.h>
+
+#include "SessionLocalFile.h"
+
+bool SessionLocalFile::Handle::close()
+{
+   if (!fd.valid())
+      return true;
+
+   if (const int err = fd.close())
+   {
+      LOG(GENERAL, ERR, "Unable to close local file.", sysErr(err), id);
+      return false;
+   }
+   else
+   {
+      LOG(GENERAL, DEBUG, "Local file closed.", id);
+      return true;
+   }
+}
+
+void SessionLocalFile::serializeNodeID(SessionLocalFile* obj, Deserializer& des)
+{
+   uint16_t mirrorNodeID;
+
+   des % mirrorNodeID;
+   if (unlikely(!des.good()))
+      return;
+
+   if(mirrorNodeID)
+   {
+      NodeStoreServers* nodeStore = Program::getApp()->getStorageNodes();
+      auto node = nodeStore->referenceNode(NumNodeID(mirrorNodeID) );
+
+      if(!node)
+         des.setBad();
+
+      obj->mirrorNode = node;
+   }
+}
+
+/**
+ * Open a chunkFile for this session
+ *
+ * @param quotaInfo may be NULL if not isWriteOpen (i.e. if the file will not be created).
+ * @param isWriteOpen if set to true, the file will be created if it didn't exist.
+ */
+FhgfsOpsErr SessionLocalFile::openFile(int targetFD, const PathInfo* pathInfo,
+   bool isWriteOpen, const SessionQuotaInfo* quotaInfo)
+{
+   FhgfsOpsErr retVal = FhgfsOpsErr_SUCCESS;
+
+   App* app = Program::getApp();
+   Logger* log = Logger::getLogger();
+   const char* logContext = "SessionLocalFile (open)";
+
+
+   if (handle->fd.valid()) // no lock here as optimization, with lock below
+      return FhgfsOpsErr_SUCCESS; // file already open
+
+
+   std::lock_guard<Mutex> const lock(sessionMutex);
+
+
+   if (handle->fd.valid())
+   {
+      // file already open (race with another thread) => nothing more to do here
+   }
+   else
+   {  // open chunk file (and create dir if necessary)...
+
+      std::string entryID = getFileID();
+      Path chunkDirPath;
+      std::string chunkFilePathStr;
+      bool hasOrigFeature = pathInfo->hasOrigFeature();
+
+      StorageTk::getChunkDirChunkFilePath(pathInfo, entryID, hasOrigFeature, chunkDirPath,
+         chunkFilePathStr);
+
+      ChunkStore* chunkDirStore = app->getChunkDirStore();
+
+      int fd = -1;
+
+      if (isWriteOpen)
+      {  // chunk needs to be created if not exists
+         int openFlags = O_CREAT | this->openFlags;
+
+         const ExceededQuotaStorePtr exceededQuotaStore =
+               app->getExceededQuotaStores()->get(getTargetID());
+
+         FhgfsOpsErr openChunkRes = chunkDirStore->openChunkFile(
+            targetFD, &chunkDirPath, chunkFilePathStr, hasOrigFeature, openFlags, &fd, quotaInfo,
+            exceededQuotaStore);
+
+         // fix chunk path permissions
+         if (unlikely(openChunkRes == FhgfsOpsErr_NOTOWNER && quotaInfo->useQuota) )
+         {
+            // it already logs a message, so need to further check this ret value
+            chunkDirStore->chmodV2ChunkDirPath(targetFD, &chunkDirPath, entryID);
+
+            openChunkRes = chunkDirStore->openChunkFile(
+               targetFD, &chunkDirPath, chunkFilePathStr, hasOrigFeature, openFlags, &fd,
+               quotaInfo, exceededQuotaStore);
+         }
+
+         if (openChunkRes != FhgfsOpsErr_SUCCESS)
+         {
+            if (openChunkRes == FhgfsOpsErr_INTERNAL) // only log unhandled errors
+               LogContext(logContext).logErr("Failed to open chunkFile: " + chunkFilePathStr);
+
+            retVal = openChunkRes;
+         }
+      }
+      else
+      {  // just reading the file, no create
+         mode_t openMode = S_IRWXU|S_IRWXG|S_IRWXO;
+
+         fd = MsgHelperIO::openat(targetFD, chunkFilePathStr.c_str(), this->openFlags,
+            openMode);
+
+         if(fd == -1)
+         { // not exists or error
+            int errCode = errno;
+
+            // we ignore ENOENT (file does not exist), as that's not an error
+            if(errCode != ENOENT)
+            {
+               LOG(SESSIONS, ERR, "Unable to open file.",
+                     ("Chunk File Path", chunkFilePathStr),
+                     ("SysErr", System::getErrString(errCode)));
+
+               retVal = FhgfsOpsErrTk::fromSysErr(errCode);
+
+            }
+         }
+      }
+
+      // prepare session data...
+      handle->fd = FDHandle(fd);
+      offset = 0;
+
+      log->log(Log_DEBUG, logContext, "File created. ID: " + getFileID() );
+   }
+
+   return retVal;
+}
+
+/**
+ * To avoid races with other writers in same session, apply new mirrorNode only if it was
+ * NULL before. Otherwise release given mirrorNode and return the existing one.
+ *
+ * @mirrorNode will be released if a mirrorNode was set already.
+ * @return existing mirrorNode if it was not NULL, given mirrorNode otherwise.
+ */
+NodeHandle SessionLocalFile::setMirrorNodeExclusive(NodeHandle mirrorNode)
+{
+   std::lock_guard<Mutex> const lock(sessionMutex);
+
+   if (!this->mirrorNode)
+      this->mirrorNode = mirrorNode;
+
+   return this->mirrorNode;
+}
+

storage/source/session/SessionLocalFile.h

@@ -0,0 +1,284 @@
+#pragma once
+
+#include <common/Common.h>
+#include <common/nodes/Node.h>
+#include <common/storage/Path.h>
+#include <common/storage/PathInfo.h>
+#include <common/storage/quota/QuotaData.h>
+#include <common/threading/Mutex.h>
+#include <common/toolkit/FDHandle.h>
+
+#include <atomic>
+
+/**
+ * Represents the client session information for an open chunk file.
+ */
+class SessionLocalFile
+{
+   public:
+      class Handle
+      {
+         friend class SessionLocalFile;
+
+         public:
+            Handle() = default;
+
+            Handle(const std::string& id, FDHandle fd):
+               id(id), fd(std::move(fd)), claimed(0)
+            {
+            }
+
+            bool close();
+
+            const FDHandle& getFD() const { return fd; }
+            const std::string& getID() const { return id; }
+
+         private:
+            std::string id;
+            FDHandle fd;
+            // for use by SessionLocalFile::releaseLastReference. only one caller may receive the
+            // handle if multiple threads try to release the last reference concurrently. we could
+            // also do this under a lock in SessionLocalFileStore but don't since we don't expect
+            // contention on the release path.
+            std::atomic<bool> claimed;
+      };
+
+   public:
+      /**
+       * @param fileHandleID format: <ownerID>#<fileID>
+       * @param openFlags system flags for open()
+       * @param serverCrashed true if session was created after a server crash, mark session as
+       * dirty
+       */
+      SessionLocalFile(const std::string& fileHandleID, uint16_t targetID, std::string fileID,
+            int openFlags, bool serverCrashed) :
+         handle(std::make_shared<Handle>(fileHandleID, FDHandle())), targetID(targetID),
+         fileID(fileID)
+      {
+         this->openFlags = openFlags;
+         this->offset = -1; // initialize as invalid offset (will be set on file open)
+
+         this->isMirrorSession = false;
+
+         this->writeCounter = 0;
+         this->readCounter = 0;
+         this->lastReadAheadTrigger = 0;
+
+         this->serverCrashed = serverCrashed;
+      }
+
+      /**
+       * For dezerialisation only
+       */
+      SessionLocalFile():
+         handle(std::make_shared<Handle>())
+      {
+         this->offset = -1; // initialize as invalid offset (will be set on file open)
+      }
+
+      template<typename This, typename Ctx>
+      static void serialize(This obj, Ctx& ctx)
+      {
+         ctx
+            % obj->handle->id
+            % obj->targetID
+            % obj->fileID
+            % obj->openFlags
+            % obj->offset
+            % obj->isMirrorSession
+            % obj->serverCrashed;
+
+         serializeNodeID(obj, ctx);
+
+         ctx
+            % serdes::atomicAs<int64_t>(obj->writeCounter)
+            % serdes::atomicAs<int64_t>(obj->readCounter)
+            % serdes::atomicAs<int64_t>(obj->lastReadAheadTrigger);
+      }
+
+      FhgfsOpsErr openFile(int targetFD, const PathInfo* pathInfo, bool isWriteOpen,
+         const SessionQuotaInfo* quotaInfo);
+
+      NodeHandle setMirrorNodeExclusive(NodeHandle mirrorNode);
+
+   private:
+      static void serializeNodeID(const SessionLocalFile* obj, Serializer& ser)
+      {
+         if (!obj->mirrorNode)
+            ser % uint16_t(0);
+         else
+            ser % uint16_t(obj->mirrorNode->getNumID().val());
+      }
+
+      static void serializeNodeID(SessionLocalFile* obj, Deserializer& des);
+
+   private:
+      // holds information about the underlying filesystem state this session file refers to for
+      // clients. this handle may be referenced by outsiders to separate removal of session files
+      // from their respective stores and closing of underlying fs objects. the protocol for such
+      // a close operation should be as follows:
+      //  1. remove the session file from its store, retain a shared_ptr
+      //  2. copy this handle to a temporary
+      //  3. move the session file reference to a weak_ptr. if the weak_ptr is expired after the
+      //     move, the handle may be closed
+      //  4. claim the handle, and close() the handle if claiming succeeded
+      std::shared_ptr<Handle> handle;
+
+      uint16_t targetID;
+      std::string fileID;
+      int32_t openFlags; // system flags for open()
+
+      int64_t offset; // negative value for unspecified/invalid offset
+
+      NodeHandle mirrorNode; // the node to which all writes should be mirrored
+      bool isMirrorSession; // true if this is the mirror session of a file
+
+      AtomicInt64 writeCounter; // how much sequential data we have written after open/sync_file_range
+      AtomicInt64 readCounter; // how much sequential data we have read since open / last seek
+      AtomicInt64 lastReadAheadTrigger; // last readCounter value at which read-ahead was triggered
+
+      Mutex sessionMutex;
+
+      bool serverCrashed; // true if session was created after a server crash, mark session as dirty
+
+      static std::shared_ptr<Handle> releaseLastReference(std::shared_ptr<SessionLocalFile> file)
+      {
+         auto handle = file->handle;
+         std::weak_ptr<SessionLocalFile> weak(file);
+
+         // see Handle::claimed for explanation
+         file.reset();
+         if (weak.expired() && !handle->claimed.exchange(true))
+            return handle;
+         else
+            return nullptr;
+      }
+
+   public:
+      bool close()
+      {
+         std::lock_guard<Mutex> lock(sessionMutex);
+
+         return handle->close();
+      }
+
+      friend std::shared_ptr<Handle> releaseLastReference(std::shared_ptr<SessionLocalFile>&& file)
+      {
+         return SessionLocalFile::releaseLastReference(std::move(file));
+      }
+
+      std::string getFileHandleID() const
+      {
+         return handle->id;
+      }
+
+      uint16_t getTargetID() const
+      {
+         return targetID;
+      }
+
+      std::string getFileID() const
+      {
+         return fileID;
+      }
+
+      const FDHandle& getFD()
+      {
+         if (handle->fd.valid()) // optimization: try without a lock first
+            return handle->fd;
+
+         std::lock_guard<Mutex> const lock(sessionMutex);
+
+         return handle->fd;
+      }
+
+      int getOpenFlags() const
+      {
+         return openFlags;
+      }
+
+      bool getIsDirectIO() const
+      {
+         return (this->openFlags & (O_DIRECT | O_SYNC) ) != 0;
+      }
+
+      int64_t getOffset()
+      {
+         std::lock_guard<Mutex> const lock(sessionMutex);
+
+         return offset;
+      }
+
+      void setOffset(int64_t offset)
+      {
+         std::lock_guard<Mutex> const lock(sessionMutex);
+
+         this->offset = offset;
+      }
+
+      NodeHandle getMirrorNode() const
+      {
+         return mirrorNode;
+      }
+
+      bool getIsMirrorSession() const
+      {
+         return isMirrorSession;
+      }
+
+      void setIsMirrorSession(bool isMirrorSession)
+      {
+         this->isMirrorSession = isMirrorSession;
+      }
+
+      void resetWriteCounter()
+      {
+         this->writeCounter = 0;
+      }
+
+      void incWriteCounter(int64_t size)
+      {
+         this->writeCounter.increase(size);
+      }
+
+      int64_t getWriteCounter()
+      {
+         return this->writeCounter.read();
+      }
+
+      int64_t getReadCounter()
+      {
+         return this->readCounter.read();
+      }
+
+      void resetReadCounter()
+      {
+         this->readCounter.setZero();
+      }
+
+      void incReadCounter(int64_t size)
+      {
+         this->readCounter.increase(size);
+      }
+
+      int64_t getLastReadAheadTrigger()
+      {
+         return this->lastReadAheadTrigger.read();
+      }
+
+      void resetLastReadAheadTrigger()
+      {
+         this->lastReadAheadTrigger.setZero();
+      }
+
+      void setLastReadAheadTrigger(int64_t lastReadAheadTrigger)
+      {
+         this->lastReadAheadTrigger.set(lastReadAheadTrigger);
+      }
+
+      bool isServerCrashed()
+      {
+         return this->serverCrashed;
+      }
+};
+

storage/source/session/SessionLocalFileStore.cpp

@@ -0,0 +1,191 @@
+#include <app/App.h>
+#include <program/Program.h>
+#include "SessionLocalFileStore.h"
+
+
+/**
+ * Add a new session to the store (if it doesn't exist yet) and return a referenced version
+ * of the session with this ID.
+ *
+ * @param session belongs to the store after calling this method
+ */
+std::shared_ptr<SessionLocalFile> SessionLocalFileStore::addAndReferenceSession(
+   std::unique_ptr<SessionLocalFile> session)
+{
+   std::string fileHandleID(session->getFileHandleID() );
+   uint16_t targetID = session->getTargetID();
+   bool isMirrorSession = session->getIsMirrorSession();
+
+   std::lock_guard<Mutex> const lock(mutex);
+
+   // try to insert the new session
+   auto insertRes = sessions.insert(
+         {Key{fileHandleID, targetID, isMirrorSession}, std::move(session)});
+
+   // reference session (note: insertRes.first is an iterator to the inserted/existing session)
+   return insertRes.first->second;
+}
+
+/**
+ * @param targetID really targetID (not buddy group ID for mirrors, because both buddies can be
+ *        attached to the same server).
+ * @param isMirrorSession true if this is a session for a mirrored chunk file.
+ * @return NULL if no such session exists.
+ */
+std::shared_ptr<SessionLocalFile> SessionLocalFileStore::referenceSession(
+   const std::string& fileHandleID, uint16_t targetID, bool isMirrorSession) const
+{
+   std::lock_guard<Mutex> const lock(mutex);
+
+   auto iter = sessions.find({fileHandleID, targetID, isMirrorSession});
+   if (iter != sessions.end())
+      return iter->second;
+
+   return nullptr;
+}
+
+/**
+ * @param isMirrorSession true if this is a session for a mirror chunk file
+ * @return filesystem state of the session if it was unused, nullptr otherwise.
+ */
+std::shared_ptr<SessionLocalFile::Handle> SessionLocalFileStore::removeSession(
+   const std::string& fileHandleID, uint16_t targetID, bool isMirrorSession)
+{
+   std::shared_ptr<SessionLocalFile> file;
+
+   {
+      std::lock_guard<Mutex> const lock(mutex);
+
+      auto iter = sessions.find({fileHandleID, targetID, isMirrorSession});
+      if (iter == sessions.end())
+         return nullptr;
+
+      file = std::move(iter->second);
+      sessions.erase(iter);
+   }
+
+   return releaseLastReference(std::move(file));
+}
+
+size_t SessionLocalFileStore::removeAllSessions()
+{
+   std::lock_guard<Mutex> const lock(mutex);
+
+   size_t total = sessions.size();
+
+   sessions.clear();
+
+   return total;
+}
+
+/**
+ * Removes all mirror sessions on a specific target.
+ */
+void SessionLocalFileStore::removeAllMirrorSessions(uint16_t targetID)
+{
+   std::lock_guard<Mutex> const lock(mutex);
+
+   for (auto iter = sessions.begin(); iter != sessions.end(); )
+   {
+      auto& session = iter->second;
+      ++iter;
+
+      if (session->getTargetID() == targetID && session->getIsMirrorSession())
+         sessions.erase(std::prev(iter));
+   }
+}
+
+/*
+ * Intended to be used for cleanup if deserialization failed, no locking is used
+ */
+void SessionLocalFileStore::deleteAllSessions()
+{
+   sessions.clear();
+}
+
+size_t SessionLocalFileStore::getSize() const
+{
+   std::lock_guard<Mutex> const lock(mutex);
+
+   return sessions.size();
+}
+
+/* Merges the SessionLocalFiles of the given SessionLocalFileStore into this SessionLocalFileStore.
+ * Only not existing SessionLocalFiles will be added to the existing SessionLocalFileStore
+ *
+ * @param sessionLocalFileStore the sessionLocalFileStore which will be merged with this
+ * sessionLocalFileStore
+ */
+void SessionLocalFileStore::mergeSessionLocalFiles(SessionLocalFileStore* sessionLocalFileStore)
+{
+   for (auto sessionIter = sessionLocalFileStore->sessions.begin();
+         sessionIter != sessionLocalFileStore->sessions.end();
+         ++sessionIter)
+   {
+      if (sessions.count(sessionIter->first))
+      {
+         const auto& id = sessionIter->first;
+
+         LOG(GENERAL, WARNING, "Found SessionLocalFile with same ID, merge not possible, may be a bug?",
+               id.fileHandleID, id.targetID, id.isMirrored);
+         continue;
+      }
+
+      sessions[sessionIter->first] = std::move(sessionIter->second);
+   }
+}
+
+void SessionLocalFileStore::serializeForTarget(Serializer& ser, uint16_t targetID)
+{
+   Serializer atStart = ser.mark();
+
+   uint32_t elemCount = 0;
+   ser % elemCount; // needs fixup
+
+   for (auto it = sessions.begin(), end = sessions.end(); it != end; ++it)
+   {
+      if (it->first.targetID != targetID)
+         continue;
+
+      ser
+         % it->first
+         % *it->second;
+      elemCount++;
+   }
+
+   atStart % elemCount;
+
+   LOG_DEBUG("SessionLocalFileStore serialize", Log_DEBUG, "count of serialized "
+      "SessionLocalFiles: " + StringTk::uintToStr(elemCount) + " of " +
+      StringTk::uintToStr(sessions.size()));
+}
+
+void SessionLocalFileStore::deserializeForTarget(Deserializer& des, uint16_t targetID)
+{
+   uint32_t elemCount;
+
+   des % elemCount;
+   if (unlikely(!des.good()))
+      return;
+
+   for (uint32_t i = 0; i < elemCount; i++)
+   {
+      Key key;
+      auto sessionLocalFile = boost::make_unique<SessionLocalFile>();
+
+      des
+         % key
+         % *sessionLocalFile;
+
+      if (unlikely(!des.good() || key.targetID != targetID))
+      {
+         des.setBad();
+         return;
+      }
+
+      this->sessions.insert({key, std::move(sessionLocalFile)});
+   }
+
+   LOG_DEBUG("SessionLocalFileStore deserialize", Log_DEBUG, "count of deserialized "
+      "SessionLocalFiles: " + StringTk::uintToStr(elemCount));
+}

storage/source/session/SessionLocalFileStore.h

@@ -0,0 +1,62 @@
+#pragma once
+
+#include <common/toolkit/ObjectReferencer.h>
+#include <common/threading/Mutex.h>
+#include <common/Common.h>
+#include "SessionLocalFile.h"
+
+
+
+class SessionLocalFileStore
+{
+   public:
+      SessionLocalFileStore() {}
+
+      std::shared_ptr<SessionLocalFile> addAndReferenceSession(
+         std::unique_ptr<SessionLocalFile> session);
+      std::shared_ptr<SessionLocalFile> referenceSession(const std::string& fileHandleID,
+         uint16_t targetID, bool isMirrorSession) const;
+      std::shared_ptr<SessionLocalFile::Handle> removeSession(
+         const std::string& fileHandleID, uint16_t targetID, bool isMirrorSession);
+      size_t removeAllSessions();
+      void removeAllMirrorSessions(uint16_t targetID);
+      void deleteAllSessions();
+      void mergeSessionLocalFiles(SessionLocalFileStore*
+         sessionLocalFileStore);
+
+      size_t getSize() const;
+
+      void serializeForTarget(Serializer& ser, uint16_t targetID);
+      void deserializeForTarget(Deserializer& des, uint16_t targetID);
+
+   private:
+      struct Key
+      {
+         std::string fileHandleID;
+         uint16_t targetID; // really targetID (not buddy group ID for mirrors, because both buddies
+                            // can be attached to the same server)
+         bool isMirrored; // true if this is a session for a mirror chunk file (has an influence on
+                          // the map key to avoid conflicts with the original session in rotated
+                          // mirrors mode).
+
+         template<typename This, typename Ctx>
+         static void serialize(This obj, Ctx& ctx)
+         {
+            ctx
+               % obj->fileHandleID
+               % obj->targetID
+               % obj->isMirrored;
+         }
+
+         friend bool operator<(const Key& a, const Key& b)
+         {
+            return std::tie(a.fileHandleID, a.targetID, a.isMirrored)
+                 < std::tie(b.fileHandleID, b.targetID, b.isMirrored);
+         }
+      };
+
+      std::map<Key, std::shared_ptr<SessionLocalFile>> sessions;
+
+      mutable Mutex mutex;
+};
+

storage/source/session/SessionStore.cpp

@@ -0,0 +1,301 @@
+#include "SessionStore.h"
+
+#include <boost/scoped_array.hpp>
+
+#include <mutex>
+
+/**
+ * @return NULL if no such session exists
+ */
+std::shared_ptr<Session> SessionStore::referenceSession(NumNodeID sessionID) const
+{
+   std::lock_guard<Mutex> const lock(mutex);
+
+   auto iter = sessions.find(sessionID);
+   if (iter != sessions.end())
+      return iter->second;
+
+   return nullptr;
+}
+
+std::shared_ptr<Session> SessionStore::referenceOrAddSession(NumNodeID sessionID)
+{
+   std::lock_guard<Mutex> lock(mutex);
+
+   auto iter = sessions.find(sessionID);
+   if (iter != sessions.end())
+      return iter->second;
+
+   // add as new session and reference it
+   LogContext log("SessionStore (ref)");
+   log.log(Log_DEBUG, std::string("Creating a new session. SessionID: ") + sessionID.str());
+
+   auto session = std::make_shared<Session>(sessionID);
+   sessions[sessionID] = session;
+   return session;
+}
+
+/**
+ * @param masterList must be ordered; contained nodes will be removed and may no longer be
+ * accessed after calling this method.
+ * @return contained sessions must be cleaned up by the caller
+ */
+std::list<std::shared_ptr<Session>> SessionStore::syncSessions(
+   const std::vector<NodeHandle>& masterList)
+{
+   std::lock_guard<Mutex> const lock(mutex);
+
+   std::list<std::shared_ptr<Session>> result;
+
+   auto sessionIter = sessions.begin();
+   auto masterIter = masterList.begin();
+
+   while (sessionIter != sessions.end() && masterIter != masterList.end())
+   {
+      NumNodeID currentSession = sessionIter->first;
+      NumNodeID currentMaster = (*masterIter)->getNumID();
+
+      if(currentMaster < currentSession)
+      { // session doesn't exist locally
+         // note: we add sessions only on demand
+         masterIter++;
+      }
+      else
+      if(currentSession < currentMaster)
+      { // session is removed
+         auto session = std::move(sessionIter->second);
+         sessionIter++; // (removal invalidates iterator)
+
+         result.push_back(std::move(session));
+         sessions.erase(std::prev(sessionIter));
+      }
+      else
+      { // session unchanged
+         masterIter++;
+         sessionIter++;
+      }
+   }
+
+   // remaining sessions are removed
+   while(sessionIter != sessions.end() )
+   {
+      auto session = std::move(sessionIter->second);
+      sessionIter++; // (removal invalidates iterator)
+
+      result.push_back(std::move(session));
+      sessions.erase(std::prev(sessionIter));
+   }
+
+   return result;
+}
+
+/**
+ * @return number of sessions
+ */
+size_t SessionStore::getAllSessionIDs(NumNodeIDList* outSessionIDs) const
+{
+   std::lock_guard<Mutex> const lock(mutex);
+
+   size_t retVal = sessions.size();
+
+   for (auto iter = sessions.begin(); iter != sessions.end(); iter++)
+      outSessionIDs->push_back(iter->first);
+
+   return retVal;
+}
+
+size_t SessionStore::getSize() const
+{
+   std::lock_guard<Mutex> const lock(mutex);
+
+   return sessions.size();
+}
+
+void SessionStore::serializeForTarget(Serializer& ser, uint16_t targetID) const
+{
+   ser % uint32_t(sessions.size());
+
+   for (auto it = sessions.begin(), end = sessions.end(); it != end; ++it)
+   {
+      ser % it->first;
+      it->second->serializeForTarget(ser, targetID);
+   }
+
+   LOG_DEBUG("SessionStore serialize", Log_DEBUG, "count of serialized Sessions: " +
+      StringTk::uintToStr(sessions.size()));
+}
+
+void SessionStore::deserializeForTarget(Deserializer& des, uint16_t targetID)
+{
+   uint32_t elemCount;
+
+   des % elemCount;
+   if (unlikely(!des.good()))
+      return;
+
+   for(unsigned i = 0; i < elemCount; i++)
+   {
+      NumNodeID key;
+
+      des % key;
+      if (unlikely(!des.good()))
+         return;
+
+      auto session = boost::make_unique<Session>();
+      session->deserializeForTarget(des, targetID);
+      if (unlikely(!des.good()))
+      {
+         session->getLocalFiles()->deleteAllSessions();
+         return;
+      }
+
+      auto searchResult = this->sessions.find(key);
+      if (searchResult == this->sessions.end())
+      {
+         this->sessions.insert({key, std::move(session)});
+      }
+      else
+      { // exist so local files will merged
+         searchResult->second->mergeSessionLocalFiles(session.get());
+      }
+   }
+
+   LOG_DEBUG("SessionStore deserialize", Log_DEBUG, "count of deserialized Sessions: " +
+         StringTk::uintToStr(elemCount));
+}
+
+bool SessionStore::loadFromFile(std::string filePath, uint16_t targetID)
+{
+   LogContext log("SessionStore (load)");
+   log.log(Log_DEBUG,"load sessions of target: " + StringTk::uintToStr(targetID));
+
+   bool retVal = false;
+   boost::scoped_array<char> buf;
+   int readRes;
+
+   struct stat statBuf;
+   int retValStat;
+
+   if(!filePath.length() )
+      return false;
+
+   std::lock_guard<Mutex> const lock(mutex);
+
+   int fd = open(filePath.c_str(), O_RDONLY, 0);
+   if(fd == -1)
+   { // open failed
+      log.log(Log_DEBUG, "Unable to open session file: " + filePath + ". " +
+         "SysErr: " + System::getErrString() );
+
+      goto err_unlock;
+   }
+
+   retValStat = fstat(fd, &statBuf);
+   if(retValStat)
+   { // stat failed
+      log.log(Log_WARNING, "Unable to stat session file: " + filePath + ". " +
+         "SysErr: " + System::getErrString() );
+
+      goto err_stat;
+   }
+
+   buf.reset(new char[statBuf.st_size]);
+   readRes = read(fd, buf.get(), statBuf.st_size);
+   if(readRes <= 0)
+   { // reading failed
+      log.log(Log_WARNING, "Unable to read session file: " + filePath + ". " +
+         "SysErr: " + System::getErrString() );
+   }
+   else
+   { // parse contents
+      Deserializer des(buf.get(), readRes);
+      deserializeForTarget(des, targetID);
+      retVal = des.good();
+   }
+
+   if (!retVal)
+      log.logErr("Could not deserialize SessionStore from file: " + filePath);
+
+err_stat:
+   close(fd);
+
+err_unlock:
+   return retVal;
+}
+
+/**
+ * Note: setStorePath must be called before using this.
+ */
+bool SessionStore::saveToFile(std::string filePath, uint16_t targetID) const
+{
+   LogContext log("SessionStore (save)");
+   log.log(Log_DEBUG,"save sessions of target: " + StringTk::uintToStr(targetID));
+
+   bool retVal = false;
+
+   boost::scoped_array<char> buf;
+   unsigned bufLen;
+   ssize_t writeRes;
+
+   if(!filePath.length() )
+      return false;
+
+   std::lock_guard<Mutex> const lock(mutex);
+
+   // create/trunc file
+   int openFlags = O_CREAT|O_TRUNC|O_WRONLY;
+
+   int fd = open(filePath.c_str(), openFlags, 0666);
+   if(fd == -1)
+   { // error
+      log.logErr("Unable to create session file: " + filePath + ". " +
+         "SysErr: " + System::getErrString() );
+
+      goto err_unlock;
+   }
+
+   // file created => store data
+   {
+      Serializer lengthCalc;
+      serializeForTarget(lengthCalc, targetID);
+
+      bufLen = lengthCalc.size();
+      buf.reset(new (std::nothrow) char[bufLen]);
+      if (!buf)
+      {
+         LOG(SESSIONS, ERR, "Unable to allocate serializer memory", filePath);
+         goto err_closefile;
+      }
+   }
+
+   {
+      Serializer ser(buf.get(), bufLen);
+      serializeForTarget(ser, targetID);
+
+      if (!ser.good())
+      {
+         log.logErr("Unable to serialize session file: " + filePath + ".");
+         goto err_closefile;
+      }
+   }
+
+   writeRes = write(fd, buf.get(), bufLen);
+   if(writeRes != (ssize_t)bufLen)
+   {
+      log.logErr("Unable to store session file: " + filePath + ". " +
+         "SysErr: " + System::getErrString() );
+
+      goto err_closefile;
+   }
+
+   retVal = true;
+
+   LOG_DEBUG_CONTEXT(log, Log_DEBUG, "Session file stored: " + filePath);
+
+   // error compensation
+err_closefile:
+   close(fd);
+
+err_unlock:
+   return retVal;
+}

storage/source/session/SessionStore.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include <common/nodes/Node.h>
+#include <common/toolkit/ObjectReferencer.h>
+#include <common/threading/Mutex.h>
+#include <common/Common.h>
+#include "Session.h"
+
+/*
+ * A session always belongs to a client ID, therefore the session ID is always the nodeID of the
+ * corresponding client
+ */
+class SessionStore
+{
+   public:
+      SessionStore() {}
+
+      std::shared_ptr<Session> referenceSession(NumNodeID sessionID) const;
+      std::shared_ptr<Session> referenceOrAddSession(NumNodeID sessionID);
+      std::list<std::shared_ptr<Session>> syncSessions(const std::vector<NodeHandle>& masterList);
+
+      size_t getAllSessionIDs(NumNodeIDList* outSessionIDs) const;
+      size_t getSize() const;
+
+      void serializeForTarget(Serializer& ser, uint16_t targetID) const;
+      void deserializeForTarget(Deserializer& des, uint16_t targetID);
+
+      bool loadFromFile(std::string filePath, uint16_t targetID);
+      bool saveToFile(std::string filePath, uint16_t targetID) const;
+
+   private:
+      std::map<NumNodeID, std::shared_ptr<Session>> sessions;
+
+      mutable Mutex mutex;
+};
+

storage/source/session/ZfsSession.cpp

@@ -0,0 +1,148 @@
+#include <toolkit/QuotaTk.h>
+#include <program/Program.h>
+#include "ZfsSession.h"
+
+#include <dlfcn.h>
+
+
+
+/**
+ * dummy constructor, constructs a invalid ZfsSession, this constructor doesn't require the libzfs,
+ * the method initZfsSession can make the session valid if the installed libzfs is working with this
+ * implementation. The session must be initialized if a zfs block device is detected.
+ */
+ZfsSession::ZfsSession()
+{
+   this->dlOpenHandleLibZfs = NULL;
+   this->libZfsHandle = NULL;
+   this->zfs_open = NULL;
+   this->zfs_prop_get_userquota_int = NULL;
+   this->libzfs_error_description = NULL;
+   this->libzfs_error_action = NULL;
+
+   this->isValid = false;
+}
+
+/**
+ * destructor
+ */
+ZfsSession::~ZfsSession()
+{
+   QuotaTk::uninitLibZfs(this->dlOpenHandleLibZfs, this->libZfsHandle);
+}
+
+/**
+ * initialize the ZfsSession, sets the valid flag if all zfs handels and function pointers are
+ * successful created
+ *
+ * @param dlOpenHandleLib the handle from dlopen to the libzfs
+ * @return true if all zfs handels and function pointers are successful created
+ */
+bool ZfsSession::initZfsSession(void* dlOpenHandleLib)
+{
+   App* app = Program::getApp();
+
+   if(this->isValid)
+      return true;
+
+   if(!dlOpenHandleLib)
+      return false;
+
+
+   char* dlErrorString;
+
+   this->dlOpenHandleLibZfs = dlOpenHandleLib;
+   this->isValid = true;
+
+
+   this->libZfsHandle = QuotaTk::initLibZfs(this->dlOpenHandleLibZfs);
+   if(!this->libZfsHandle)
+   {
+      this->isValid = false;
+      return false;
+   }
+
+
+   this->zfs_open = (void* (*)(void*, const char*, int))dlsym(this->dlOpenHandleLibZfs, "zfs_open");
+   if ( (dlErrorString = dlerror() ) != NULL)
+   {
+      LOG(QUOTA, ERR, "Error during dynamic load of function zfs_open.", dlErrorString);
+      app->setLibZfsErrorReported(true);
+      this->isValid = false;
+      return false;
+   }
+
+
+   this->zfs_prop_get_userquota_int = (int (*)(void*, const char*, uint64_t*))dlsym(
+      this->dlOpenHandleLibZfs, "zfs_prop_get_userquota_int");
+   if ( (dlErrorString = dlerror() ) != NULL)
+   {
+      LOG(QUOTA, ERR, "Error during dynamic load of function zfs_prop_get_userquota_int.",
+            dlErrorString);
+      app->setLibZfsErrorReported(true);
+      this->isValid = false;
+      return false;
+   }
+
+
+   this->libzfs_error_description = (char* (*)(void*))dlsym(this->dlOpenHandleLibZfs,
+      "libzfs_error_description");
+   if ( (dlErrorString = dlerror() ) != NULL)
+   {
+      LOG(QUOTA, ERR, "Error during dynamic load of function libzfs_error_description.",
+            dlErrorString);
+      app->setLibZfsErrorReported(true);
+      this->isValid = false;
+      return false;
+   }
+
+
+   this->libzfs_error_action = (char* (*)(void*))dlsym(this->dlOpenHandleLibZfs,
+      "libzfs_error_action");
+   if ( (dlErrorString = dlerror() ) != NULL)
+   {
+      LOG(QUOTA, ERR, "Error during dynamic load of function libzfs_error_action.",
+         dlErrorString);
+      app->setLibZfsErrorReported(true);
+      this->isValid = false;
+      return false;
+   }
+
+   return this->isValid;
+}
+
+/**
+ * checks for a existing blockdevice/pool handle (zfs_handle_t*) or creates a new zfs
+ * blockdevice/pool handle (zfs_handle_t*) for the given path an targetNumID
+ *
+ * @param targetNumID the targetNumID of the storage target
+ * @param path the path of the blockdevice/poolname
+ * @return returns a zfs blockdevice/pool handle (zfs_handle_t*) or NULL if a error occurs
+ */
+void* ZfsSession::getZfsDeviceHandle(uint16_t targetNumID, std::string path)
+{
+   if(!this->isValid)
+      return NULL;
+
+   ZfsPoolHandleMapIter iter = fsHandles.find(targetNumID);
+
+   if(iter != this->fsHandles.end() )
+      return iter->second;
+   else
+   {
+      void* newFsHandle = (*this->zfs_open)(this->libZfsHandle, path.c_str(), ZFSSESSION_ZFS_TYPE);
+      if(!newFsHandle)
+      {
+         LOG(QUOTA, ERR, "Error during create of zfs pool handle.",
+               ("ErrorAction", (*this->libzfs_error_action)(this->libZfsHandle)),
+               ("ErrorDescription", (*this->libzfs_error_description)(this->libZfsHandle)));
+      }
+      else
+      {
+         fsHandles.insert(ZfsPoolHandleMapMapVal(targetNumID, newFsHandle) );
+         return newFsHandle;
+      }
+   }
+
+   return NULL;
+}

storage/source/session/ZfsSession.h

@@ -0,0 +1,54 @@
+#pragma once
+
+
+
+
+
+
+#define ZFSSESSION_ZFS_TYPE            1 // must be same as ZFS_TYPE_FILESYSTEM from libzfs
+
+typedef std::map<uint16_t, void*> ZfsPoolHandleMap; // targetNumID => zfs_handle_t*
+typedef ZfsPoolHandleMap::iterator ZfsPoolHandleMapIter;
+typedef ZfsPoolHandleMap::value_type ZfsPoolHandleMapMapVal;
+
+
+class ZfsSession
+{
+   public:
+      ZfsSession();
+      virtual ~ZfsSession();
+      bool initZfsSession(void* dlOpenHandleLib);
+
+      void* getZfsDeviceHandle(uint16_t targetNumID, std::string path);
+
+      int (*zfs_prop_get_userquota_int)(void*, const char*, uint64_t*); // fp to get quota data
+      char* (*libzfs_error_description)(void*); // fp to get error description
+      char* (*libzfs_error_action)(void*); // fp to get action during the error occurs
+
+
+   private:
+      void* dlOpenHandleLibZfs;     // handle of dlOpen from the libzfs
+      void* libZfsHandle;           // handle of the libzfs from libzfs_init()
+
+      ZfsPoolHandleMap fsHandles;   // handle of a ZFS pool, the type is zfs_handle_t*
+
+      void* (*zfs_open)(void*, const char*, int); // fp to open zfs pool
+
+      bool isValid;
+
+
+   public:
+      /**
+       * getter and setter
+       */
+      void* getlibZfsHandle()
+      {
+         return this->libZfsHandle;
+      }
+
+      bool isSessionValid()
+      {
+         return this->isValid;
+      }
+};
+