deb/beegfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

New upstream version 8.1.0

Browse Source
This commit is contained in:
geos_one
2025-08-10 01:34:16 +02:00
commit c891bb7105
4398 changed files with 838833 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
meta/source/session/EntryLock.h Normal file
View File

@@ -0,0 +1,127 @@
#pragma once
#include "EntryLockStore.h"
template<typename LockDataT>
class UniqueEntryLockBase
{
public:
~UniqueEntryLockBase()
{
if (lockData)
entryLockStore->unlock(lockData);
}
UniqueEntryLockBase(const UniqueEntryLockBase&) = delete;
UniqueEntryLockBase& operator=(const UniqueEntryLockBase&) = delete;
UniqueEntryLockBase(UniqueEntryLockBase&& src)
: entryLockStore(NULL), lockData(NULL)
{
swap(src);
}
UniqueEntryLockBase& operator=(UniqueEntryLockBase&& src)
{
UniqueEntryLockBase(std::move(src)).swap(*this);
return *this;
}
void swap(UniqueEntryLockBase& other)
{
std::swap(entryLockStore, other.entryLockStore);
std::swap(lockData, other.lockData);
}
protected:
typedef UniqueEntryLockBase BaseType;
template<typename... ArgsT>
UniqueEntryLockBase(EntryLockStore* entryLockStore, const ArgsT&... args)
: entryLockStore(entryLockStore)
{
lockData = entryLockStore->lock(args...);
}
UniqueEntryLockBase()
: entryLockStore(NULL), lockData(NULL)
{
}
private:
EntryLockStore* entryLockStore;
LockDataT* lockData;
};
template<typename LockDataT>
inline void swap(UniqueEntryLockBase<LockDataT>& a, UniqueEntryLockBase<LockDataT>& b)
{
a.swap(b);
}
class FileIDLock : UniqueEntryLockBase<FileIDLockData>
{
public:
FileIDLock() = default;
FileIDLock(const FileIDLock&) = delete;
FileIDLock& operator=(const FileIDLock&) = delete;
FileIDLock(FileIDLock&& src) : BaseType(std::move(src)) {}
FileIDLock& operator=(FileIDLock&& src)
{
BaseType::operator=(std::move(src));
return *this;
}
FileIDLock(EntryLockStore* entryLockStore, const std::string& fileID, const bool writeLock)
: UniqueEntryLockBase<FileIDLockData>(entryLockStore, fileID, writeLock)
{
}
};
class ParentNameLock : UniqueEntryLockBase<ParentNameLockData>
{
public:
ParentNameLock() = default;
ParentNameLock(const ParentNameLock&) = delete;
ParentNameLock& operator=(const ParentNameLock&) = delete;
ParentNameLock(ParentNameLock&& src) : BaseType(std::move(src)) {}
ParentNameLock& operator=(ParentNameLock&& src)
{
BaseType::operator=(std::move(src));
return *this;
}
ParentNameLock(EntryLockStore* entryLockStore, const std::string& parentID,
const std::string& name)
: UniqueEntryLockBase<ParentNameLockData>(entryLockStore, parentID, name)
{
}
};
class HashDirLock : UniqueEntryLockBase<HashDirLockData>
{
public:
HashDirLock() = default;
HashDirLock(const HashDirLock&) = delete;
HashDirLock& operator=(const HashDirLock&) = delete;
HashDirLock(HashDirLock&& src) : BaseType(std::move(src)) {}
HashDirLock& operator=(HashDirLock&& src)
{
BaseType::operator=(std::move(src));
return *this;
}
HashDirLock(EntryLockStore* entryLockStore, std::pair<unsigned, unsigned> hashDir)
: UniqueEntryLockBase<HashDirLockData>(entryLockStore, hashDir)
{
}
};

45
meta/source/session/EntryLockStore.cpp Normal file
View File

@@ -0,0 +1,45 @@
#include "EntryLockStore.h"
ParentNameLockData* EntryLockStore::lock(const std::string& parentID, const std::string& name)
{
ParentNameLockData& lock = parentNameLocks.getLockFor(
std::pair<const std::string&, const std::string&>(parentID, name) );
lock.getLock().lock();
return &lock;
}
FileIDLockData* EntryLockStore::lock(const std::string& fileID, const bool writeLock)
{
FileIDLockData& lock = fileLocks.getLockFor(fileID);
if(writeLock)
lock.getLock().writeLock();
else
lock.getLock().readLock();
return &lock;
}
HashDirLockData* EntryLockStore::lock(std::pair<unsigned, unsigned> hashDir)
{
HashDirLockData& lock = hashDirLocks.getLockFor(hashDir);
lock.getLock().lock();
return &lock;
}
void EntryLockStore::unlock(ParentNameLockData* parentNameLockData)
{
parentNameLockData->getLock().unlock();
parentNameLocks.putLock(*parentNameLockData);
}
void EntryLockStore::unlock(FileIDLockData* fileIDLockData)
{
fileIDLockData->getLock().unlock();
fileLocks.putLock(*fileIDLockData);
}
void EntryLockStore::unlock(HashDirLockData* hashDirLockData)
{
hashDirLockData->getLock().unlock();
hashDirLocks.putLock(*hashDirLockData);
}

231
meta/source/session/EntryLockStore.h Normal file
View File

@@ -0,0 +1,231 @@
#pragma once
#include <common/Common.h>
#include <common/app/log/LogContext.h>
#include <common/threading/Mutex.h>
#include <common/threading/RWLock.h>
template<typename Value>
struct ValueLockHash;
// This class implements a hashmap Value -> Lock of size HashSize. To avoid memory allocations
// and deallocations, each bucket may keep up to BufferSize lock objects that are not currently
// used.
//
// Hashes for values are computed by ValueLockHash<Value>, so an appropriate specialization of this
// template must exists in order to use this class.
//
// Currently the Lock argument is not restricted in any way, but it is intended to only ever be set
// to types of locking primitives.
//
// This implementation does not use boost::unordered_map because we want a fixed-size table and
// control over the per-bucket value lists, since these lists required memory allocations and
// insert and deallocations on erase. The allocation overhead per list entry could also be
// optimized away by using intrusive lists for bucket contents, but currently the lock buffers
// are sufficient to amortize almost all allocator traffic.
template<typename Value, typename Lock, unsigned HashSize, unsigned BufferSize = 32>
class ValueLockStore
{
private:
struct LockBucket;
public:
class ValueLock
{
friend class ValueLockStore;
public:
Lock& getLock()
{
return lock;
}
private:
Lock lock;
Value value;
LockBucket* bucket;
typename std::list<ValueLock*>::iterator iter;
unsigned references;
ValueLock(const Value& value, LockBucket& bucket,
typename std::list<ValueLock*>::iterator iter)
: value(value), bucket(&bucket), iter(iter), references(0)
{}
ValueLock(const ValueLock&);
ValueLock& operator=(const ValueLock&);
};
private:
struct LockBucket
{
Mutex mtx;
std::list<ValueLock*> locks;
std::list<ValueLock*> lockBuffer;
unsigned lockBufferSize;
LockBucket()
: lockBufferSize(0)
{}
};
public:
// Acquires a lock descriptor for `value`. The descriptor is only acquired, not locked;
// locking and unlocking is left to the user. Increments the reference count of the returned
// descriptor. The descriptor must be released with `putLock` when done.
ValueLock& getLockFor(const Value& value)
{
const uint32_t valueHash = ValueLockHash<Value>()(value);
LockBucket& bucket = buckets[valueHash % HashSize];
ValueLock* lock = NULL;
bucket.mtx.lock();
{
for(typename std::list<ValueLock*>::iterator it = bucket.locks.begin(),
end = bucket.locks.end(); it != end; ++it)
{
if(value == (*it)->value)
{
lock = *it;
break;
}
}
if(!lock)
{
if(bucket.lockBufferSize == 0)
{
bucket.locks.push_front(NULL);
lock = new ValueLock(value, bucket, bucket.locks.begin() );
bucket.locks.front() = lock;
}
else
{
bucket.locks.splice(
bucket.locks.begin(),
bucket.lockBuffer,
bucket.lockBuffer.begin() );
bucket.lockBufferSize--;
lock = bucket.locks.front();
lock->value = value;
}
}
lock->references++;
}
bucket.mtx.unlock();
return *lock;
}
// Releases a lock descriptor acquired by `getLockFor` and decreases its reference counter.
// When the reference count reaches 0, `lock` is invalidated.
void putLock(ValueLock& lock)
{
LockBucket& bucket = *lock.bucket;
bucket.mtx.lock();
do {
lock.references--;
if(lock.references > 0)
break;
if(bucket.lockBufferSize < BufferSize)
{
bucket.lockBuffer.splice(
bucket.lockBuffer.begin(),
bucket.locks,
lock.iter);
bucket.lockBufferSize++;
}
else
{
bucket.locks.erase(lock.iter);
delete &lock;
}
} while (0);
bucket.mtx.unlock();
}
private:
LockBucket buckets[HashSize];
};
template<>
struct ValueLockHash<std::string>
{
// this is the jenkins hash function (http://www.burtleburtle.net/bob/hash/doobs.html).
// it was chosen for this use case because it is fast for small inputs, has no alignment
// requirements and exhibits good mixing.
uint32_t operator()(const std::string& str) const
{
uint32_t hash, i;
for(hash = i = 0; i < str.size(); ++i)
{
hash += str[i];
hash += (hash << 10);
hash ^= (hash >> 6);
}
hash += (hash << 3);
hash ^= (hash >> 11);
hash += (hash << 15);
return hash;
}
};
template<>
struct ValueLockHash<std::pair<std::string, std::string> >
{
uint32_t operator()(std::pair<const std::string&, const std::string&> pair) const
{
ValueLockHash<std::string> hash;
return hash(pair.first) ^ hash(pair.second);
}
};
template<>
struct ValueLockHash<std::pair<unsigned, unsigned>>
{
uint32_t operator()(std::pair<unsigned, unsigned> pair) const
{
// optimized for the inode/dentries hash dir structure of 2 fragments of 7 bits each
return std::hash<unsigned>()((pair.first << 8) | pair.second);
}
};
typedef ValueLockStore<std::pair<std::string, std::string>, Mutex, 1024> ParentNameLockStore;
typedef ParentNameLockStore::ValueLock ParentNameLockData;
typedef ValueLockStore<std::string, RWLock, 1024> FileIDLockStore;
typedef FileIDLockStore::ValueLock FileIDLockData;
typedef ValueLockStore<std::pair<unsigned, unsigned>, Mutex, 1024> HashDirLockStore;
typedef HashDirLockStore::ValueLock HashDirLockData;
class EntryLockStore
{
public:
ParentNameLockData* lock(const std::string& parentID, const std::string& name);
//FileIDLock is used for both files and directories
FileIDLockData* lock(const std::string& fileID, const bool writeLock);
HashDirLockData* lock(std::pair<unsigned, unsigned> hashDir);
void unlock(ParentNameLockData* parentNameLockData);
void unlock(FileIDLockData* fileIDLockData);
void unlock(HashDirLockData* hashDirLockData);
private:
ParentNameLockStore parentNameLocks;
FileIDLockStore fileLocks;
HashDirLockStore hashDirLocks;
};

40
meta/source/session/LockingNotifier.cpp Normal file
View File

@@ -0,0 +1,40 @@
#include <components/worker/LockEntryNotificationWork.h>
#include <components/worker/LockRangeNotificationWork.h>
#include <program/Program.h>
#include "LockingNotifier.h"
/**
* @param notifyList will be owned and freed by the LockingNotifier, so do not use or free it after
* calling this.
*/
void LockingNotifier::notifyWaitersEntryLock(LockEntryNotifyType lockType,
const std::string& parentEntryID, const std::string& entryID, bool isBuddyMirrored,
LockEntryNotifyList notifyList)
{
// just delegate to comm slaves
MultiWorkQueue* slaveQ = Program::getApp()->getCommSlaveQueue();
Work* work = new LockEntryNotificationWork(lockType, parentEntryID, entryID, isBuddyMirrored,
std::move(notifyList));
slaveQ->addDirectWork(work);
}
/**
* @param notifyList will be owned and freed by the LockingNotifier, so do not use or free it after
* calling this.
*/
void LockingNotifier::notifyWaitersRangeLock(const std::string& parentEntryID,
const std::string& entryID, bool isBuddyMirrored, LockRangeNotifyList notifyList)
{
// just delegate to comm slaves
MultiWorkQueue* slaveQ = Program::getApp()->getCommSlaveQueue();
Work* work = new LockRangeNotificationWork(parentEntryID, entryID, isBuddyMirrored,
std::move(notifyList));
slaveQ->addDirectWork(work);
}

26
meta/source/session/LockingNotifier.h Normal file
View File

@@ -0,0 +1,26 @@
#pragma once
#include <common/Common.h>
#include <storage/Locking.h>
#include <components/worker/LockEntryNotificationWork.h>
#include <components/worker/LockRangeNotificationWork.h>
/**
* Creates work packages to notify waiting clients/processes about file lock grants.
*/
class LockingNotifier
{
public:
static void notifyWaitersEntryLock(LockEntryNotifyType lockType,
const std::string& parentEntryID, const std::string& entryID, bool isBuddyMirrored,
LockEntryNotifyList notifyList);
static void notifyWaitersRangeLock(const std::string& parentEntryID,
const std::string& entryID, bool isBuddyMirrored, LockRangeNotifyList notifyList);
private:
LockingNotifier() {}
};

82
meta/source/session/MirrorMessageResponseState.cpp Normal file
View File

@@ -0,0 +1,82 @@
#include "MirrorMessageResponseState.h"
#include <net/message/session/AckNotifyMsgEx.h>
#include <net/message/session/BumpFileVersionMsgEx.h>
#include <net/message/session/locking/FLockEntryMsgEx.h>
#include <net/message/session/locking/FLockRangeMsgEx.h>
#include <net/message/session/opening/CloseFileMsgEx.h>
#include <net/message/session/opening/OpenFileMsgEx.h>
#include <net/message/storage/attribs/RefreshEntryInfoMsgEx.h>
#include <net/message/storage/attribs/RemoveXAttrMsgEx.h>
#include <net/message/storage/attribs/SetAttrMsgEx.h>
#include <net/message/storage/attribs/SetDirPatternMsgEx.h>
#include <net/message/storage/attribs/SetXAttrMsgEx.h>
#include <net/message/storage/attribs/UpdateDirParentMsgEx.h>
#include <net/message/storage/creating/HardlinkMsgEx.h>
#include <net/message/storage/creating/MkDirMsgEx.h>
#include <net/message/storage/creating/MkFileMsgEx.h>
#include <net/message/storage/creating/MkFileWithPatternMsgEx.h>
#include <net/message/storage/creating/MkLocalDirMsgEx.h>
#include <net/message/storage/creating/RmDirMsgEx.h>
#include <net/message/storage/creating/RmLocalDirMsgEx.h>
#include <net/message/storage/creating/UnlinkFileMsgEx.h>
#include <net/message/storage/lookup/LookupIntentMsgEx.h>
#include <net/message/storage/moving/MovingDirInsertMsgEx.h>
#include <net/message/storage/moving/MovingFileInsertMsgEx.h>
#include <net/message/storage/moving/RenameV2MsgEx.h>
#include <net/message/storage/TruncFileMsgEx.h>
void MirroredMessageResponseState::serialize(Serializer& ser) const
{
ser % serializerTag();
serializeContents(ser);
}
std::unique_ptr<MirroredMessageResponseState> MirroredMessageResponseState::deserialize(
Deserializer& des)
{
uint32_t tag;
des % tag;
if (!des.good())
return {};
#define HANDLE_TAG(value, type) \
case value: return boost::make_unique<type::ResponseState>(des);
switch (tag)
{
HANDLE_TAG(NETMSGTYPE_OpenFile, OpenFileMsgEx)
HANDLE_TAG(NETMSGTYPE_CloseFile, CloseFileMsgEx)
HANDLE_TAG(NETMSGTYPE_TruncFile, TruncFileMsgEx)
HANDLE_TAG(NETMSGTYPE_MkFileWithPattern, MkFileWithPatternMsgEx)
HANDLE_TAG(NETMSGTYPE_Hardlink, HardlinkMsgEx)
HANDLE_TAG(NETMSGTYPE_MkLocalDir, MkLocalDirMsgEx)
HANDLE_TAG(NETMSGTYPE_UnlinkFile, UnlinkFileMsgEx)
HANDLE_TAG(NETMSGTYPE_RmLocalDir, RmLocalDirMsgEx)
HANDLE_TAG(NETMSGTYPE_MkDir, MkDirMsgEx)
HANDLE_TAG(NETMSGTYPE_MkFile, MkFileMsgEx)
HANDLE_TAG(NETMSGTYPE_RmDir, RmDirMsgEx)
HANDLE_TAG(NETMSGTYPE_UpdateDirParent, UpdateDirParentMsgEx)
HANDLE_TAG(NETMSGTYPE_RemoveXAttr, RemoveXAttrMsgEx)
HANDLE_TAG(NETMSGTYPE_SetXAttr, SetXAttrMsgEx)
HANDLE_TAG(NETMSGTYPE_SetDirPattern, SetDirPatternMsgEx)
HANDLE_TAG(NETMSGTYPE_SetAttr, SetAttrMsgEx)
HANDLE_TAG(NETMSGTYPE_RefreshEntryInfo, RefreshEntryInfoMsgEx)
HANDLE_TAG(NETMSGTYPE_MovingFileInsert, MovingFileInsertMsgEx)
HANDLE_TAG(NETMSGTYPE_MovingDirInsert, MovingDirInsertMsgEx)
HANDLE_TAG(NETMSGTYPE_Rename, RenameV2MsgEx)
HANDLE_TAG(NETMSGTYPE_LookupIntent, LookupIntentMsgEx)
HANDLE_TAG(NETMSGTYPE_AckNotify, AckNotifiyMsgEx)
HANDLE_TAG(NETMSGTYPE_FLockEntry, FLockEntryMsgEx)
HANDLE_TAG(NETMSGTYPE_FLockRange, FLockRangeMsgEx)
HANDLE_TAG(NETMSGTYPE_BumpFileVersion, BumpFileVersionMsgEx)
default:
LOG(MIRRORING, ERR, "bad mirror response state tag.", tag);
des.setBad();
break;
}
return {};
}

115
meta/source/session/MirrorMessageResponseState.h Normal file
View File

@@ -0,0 +1,115 @@
#pragma once
#include <common/net/message/control/GenericResponseMsg.h>
#include <common/storage/EntryInfo.h>
class MirroredMessageResponseState
{
public:
virtual ~MirroredMessageResponseState()
{
}
virtual void sendResponse(NetMessage::ResponseContext& ctx) = 0;
virtual bool changesObservableState() const = 0;
void serialize(Serializer& ser) const;
static std::unique_ptr<MirroredMessageResponseState> deserialize(Deserializer& des);
protected:
virtual uint32_t serializerTag() const = 0;
virtual void serializeContents(Serializer& ser) const = 0;
};
template<typename RespMsgT, uint32_t SerializerTag>
class ErrorCodeResponseState : public MirroredMessageResponseState
{
public:
explicit ErrorCodeResponseState(Deserializer& des)
{
des % serdes::as<int32_t>(result);
}
explicit ErrorCodeResponseState(FhgfsOpsErr result) : result(result) {}
void sendResponse(NetMessage::ResponseContext& ctx) override
{
if (result == FhgfsOpsErr_COMMUNICATION)
ctx.sendResponse(
GenericResponseMsg(
GenericRespMsgCode_INDIRECTCOMMERR,
"Communication with storage targets failed"));
else
ctx.sendResponse(RespMsgT(result));
}
bool changesObservableState() const override
{
return result == FhgfsOpsErr_SUCCESS;
}
FhgfsOpsErr getResult() const { return result; }
protected:
uint32_t serializerTag() const override { return SerializerTag; }
void serializeContents(Serializer& ser) const override
{
ser % serdes::as<int32_t>(result);
}
private:
FhgfsOpsErr result;
};
template<typename RespMsgT, uint32_t SerializerTag>
class ErrorAndEntryResponseState : public MirroredMessageResponseState
{
public:
explicit ErrorAndEntryResponseState(Deserializer& des)
{
des
% serdes::as<int32_t>(result)
% info;
}
ErrorAndEntryResponseState(FhgfsOpsErr result, EntryInfo info)
: result(result), info(std::move(info))
{
}
void sendResponse(NetMessage::ResponseContext& ctx) override
{
if (result == FhgfsOpsErr_COMMUNICATION)
ctx.sendResponse(
GenericResponseMsg(
GenericRespMsgCode_INDIRECTCOMMERR,
"Communication with storage targets failed"));
else
ctx.sendResponse(RespMsgT(result, &info));
}
bool changesObservableState() const override
{
return result == FhgfsOpsErr_SUCCESS;
}
FhgfsOpsErr getResult() const { return result; }
protected:
uint32_t serializerTag() const override { return SerializerTag; }
void serializeContents(Serializer& ser) const override
{
ser
% serdes::as<int32_t>(result)
% info;
}
private:
FhgfsOpsErr result;
EntryInfo info;
};

16
meta/source/session/Session.cpp Normal file
View File

@@ -0,0 +1,16 @@
#include "Session.h"
/* Merges the SessionFiles of the given session into this session, only not existing
* SessionFiles will be added to the existing session
* @param session the session which will be merged with this session
*/
void Session::mergeSessionFiles(Session* session)
{
this->getFiles()->mergeSessionFiles(session->getFiles() );
}
bool Session::operator==(const Session& other) const
{
return sessionID == other.sessionID
&& files == other.files;
}

179
meta/source/session/Session.h Normal file
View File

@@ -0,0 +1,179 @@
#pragma once
#include <common/Common.h>
#include <common/threading/Mutex.h>
#include <session/MirrorMessageResponseState.h>
#include "SessionFileStore.h"
#include <mutex>
struct MirrorStateSlot
{
std::unique_ptr<MirroredMessageResponseState> response;
friend Serializer& operator%(Serializer& ser, const std::shared_ptr<MirrorStateSlot>& obj)
{
ser % bool(obj->response);
if (obj->response)
obj->response->serialize(ser);
return ser;
}
friend Deserializer& operator%(Deserializer& des, std::shared_ptr<MirrorStateSlot>& obj)
{
bool hasContent;
obj.reset(new MirrorStateSlot);
des % hasContent;
if (des.good() && hasContent)
obj->response = MirroredMessageResponseState::deserialize(des);
return des;
}
};
/*
* A session always belongs to a client ID, therefore the session ID is always the nodeID of the
* corresponding client
*/
class Session
{
public:
Session(NumNodeID sessionID) : sessionID(sessionID) {}
/*
* For deserialization only
*/
Session() {};
void mergeSessionFiles(Session* session);
template<typename This, typename Ctx>
static void serialize(This obj, Ctx& ctx)
{
ctx
% obj->sessionID
% obj->files
% obj->mirrorProcessState;
obj->dropEmptyStateSlots(ctx);
}
bool operator==(const Session& other) const;
bool operator!=(const Session& other) const { return !(*this == other); }
private:
NumNodeID sessionID;
SessionFileStore files;
Mutex mirrorProcessStateLock;
// mirror state slots are shared_ptrs for two reasons: ordering of memory operations, and
// misbehaving clients.
// * memory ordering turns into a very easy problem if every user of an object holds a
// reference that will keep the object alive, even if it is removed from this map for
// some reason.
// * clients may reuse sequences numbers in violation of the protocol, or clear out sequence
// numbers that are still processing on the server. in the first case, we will want an
// ephemeral response state that will immediatly go away, in the second we want to keep
// the response state alive as long as some thread is still operating on it.
std::map<uint64_t, std::shared_ptr<MirrorStateSlot>> mirrorProcessState;
void dropEmptyStateSlots(Serializer&) const
{
}
// when the session state for any given client is loaded, the server must discard empty state
// slots. if empty slots are not discarded, requests for any such sequence number would be
// answered with "try again later" - forever, because the slot will never be filled.
// this is not an issue for resync, because during session resync, all slots are filled.
// empty slots can only occur if a server was shut down while messages were still being
// processed.
void dropEmptyStateSlots(Deserializer&)
{
auto it = mirrorProcessState.begin();
auto end = mirrorProcessState.end();
while (it != end)
{
if (it->second->response)
{
++it;
continue;
}
auto slot = it;
++it;
mirrorProcessState.erase(slot);
}
}
public:
// getters & setters
NumNodeID getSessionID()
{
return sessionID;
}
SessionFileStore* getFiles()
{
return &files;
}
bool relinkInodes(MetaStore& store)
{
return files.relinkInodes(store);
}
void freeMirrorStateSlot(uint64_t seqNo)
{
std::lock_guard<Mutex> lock(mirrorProcessStateLock);
mirrorProcessState.erase(seqNo);
}
std::pair<std::shared_ptr<MirrorStateSlot>, bool> acquireMirrorStateSlot(uint64_t endSeqno,
uint64_t thisSeqno)
{
std::lock_guard<Mutex> lock(mirrorProcessStateLock);
// the client sets endSeqno to the largest sequence number it knows to be fully processed,
// with no sequence numbers below it still being active somewhere. we can remove all states
// with smaller sequence numbers and endSeqno itself
mirrorProcessState.erase(
mirrorProcessState.begin(),
mirrorProcessState.lower_bound(endSeqno + 1));
auto inserted = mirrorProcessState.insert(
{thisSeqno, std::make_shared<MirrorStateSlot>()});
return {inserted.first->second, inserted.second};
}
std::pair<std::shared_ptr<MirrorStateSlot>, bool> acquireMirrorStateSlotSelective(
uint64_t finishedSeqno, uint64_t thisSeqno)
{
std::lock_guard<Mutex> lock(mirrorProcessStateLock);
mirrorProcessState.erase(finishedSeqno);
auto inserted = mirrorProcessState.insert(
{thisSeqno, std::make_shared<MirrorStateSlot>()});
return {inserted.first->second, inserted.second};
}
uint64_t getSeqNoBase()
{
std::lock_guard<Mutex> lock(mirrorProcessStateLock);
if (!mirrorProcessState.empty())
return mirrorProcessState.rbegin()->first + 1;
else
return 1;
}
};

27
meta/source/session/SessionFile.cpp Normal file
View File

@@ -0,0 +1,27 @@
#include <program/Program.h>
#include <storage/MetaStore.h>
#include "SessionFile.h"
bool SessionFile::operator==(const SessionFile& other) const
{
return accessFlags == other.accessFlags
&& sessionID == other.sessionID
&& entryInfo == other.entryInfo
&& useAsyncCleanup == other.useAsyncCleanup;
}
bool SessionFile::relinkInode(MetaStore& store)
{
// Bypass access checks during session recovery to ensure locked files in disposal directory
// can be properly recovered after system crashes or unclean shutdowns. This preserves session
// continuity regardless of file state locks. We may revisit this approach if anything changes.
auto openRes = store.openFile(&entryInfo, accessFlags, /* bypassAccessCheck */ true, inode, true);
if (openRes == FhgfsOpsErr_SUCCESS)
return true;
LOG(SESSIONS, ERR, "Could not relink session for inode.",
("inode", entryInfo.getParentEntryID() + "/" + entryInfo.getEntryID()));
return false;
}

121
meta/source/session/SessionFile.h Normal file
View File

@@ -0,0 +1,121 @@
#pragma once
#include <common/storage/Path.h>
#include <common/Common.h>
#include <storage/DirInode.h>
#include <storage/FileInode.h>
#include <storage/MetaFileHandle.h>
class MetaStore;
class SessionFile
{
public:
/**
* @param accessFlags OPENFILE_ACCESS_... flags
*/
SessionFile(MetaFileHandle openInode, unsigned accessFlags, EntryInfo* entryInfo):
inode(std::move(openInode)), accessFlags(accessFlags), sessionID(0),
useAsyncCleanup(false)
{
this->entryInfo.set(entryInfo);
}
/**
* For dezerialisation only
*/
SessionFile():
accessFlags(0), sessionID(0), useAsyncCleanup(false)
{
}
template<typename This, typename Ctx>
static void serialize(This obj, Ctx& ctx)
{
ctx
% obj->accessFlags
% obj->sessionID
% obj->entryInfo
% obj->useAsyncCleanup;
}
bool relinkInode(MetaStore& store);
bool operator==(const SessionFile& other) const;
bool operator!=(const SessionFile& other) const { return !(*this == other); }
private:
MetaFileHandle inode;
uint32_t accessFlags; // OPENFILE_ACCESS_... flags
uint32_t sessionID;
bool useAsyncCleanup; // if session was busy (=> referenced) while client sent close
EntryInfo entryInfo;
// getters & setters
/**
* Returns lock of internal FileInode object.
*/
RWLock* getInodeLock()
{
return &inode->rwlock;
}
public:
// getters & setters
unsigned getAccessFlags()
{
return accessFlags;
}
unsigned getSessionID()
{
return sessionID;
}
void setSessionID(unsigned sessionID)
{
this->sessionID = sessionID;
}
const MetaFileHandle& getInode() const
{
return inode;
}
MetaFileHandle releaseInode()
{
return std::move(inode);
}
bool getUseAsyncCleanup()
{
// note: unsynced, because it can only become true and stays true then
return this->useAsyncCleanup;
}
void setUseAsyncCleanup()
{
// note: unsynced, because it can only become true and stays true then
this->useAsyncCleanup = true;
}
EntryInfo* getEntryInfo(void)
{
return &this->entryInfo;
}
/**
* Update the parentEntryID
*/
void setParentEntryID(const std::string& newParentID)
{
this->entryInfo.setParentEntryID(newParentID);
}
};

369
meta/source/session/SessionFileStore.cpp Normal file
View File

@@ -0,0 +1,369 @@
#include <program/Program.h>
#include "SessionFileStore.h"
#include <mutex>
/**
* @param session belongs to the store after calling this method - so do not free it and don't
* use it any more afterwards (re-get it from this store if you need it)
* @return assigned sessionID
*/
unsigned SessionFileStore::addSession(SessionFile* session)
{
const std::lock_guard<Mutex> lock(mutex);
unsigned sessionID = generateNewSessionID();
session->setSessionID(sessionID);
sessions.insert(SessionFileMapVal(sessionID, new SessionFileReferencer(session) ) );
return sessionID;
}
/**
* @param session belongs to the store after calling this method - so do not free it and don't
* use it any more afterwards (re-get it from this store if you need it)
* @param sessionFileID use this ID to store session file
* @return true if successfully added, false otherwise (most likely ID conflict)
*/
bool SessionFileStore::addSession(SessionFile* session, unsigned sessionFileID)
{
const std::lock_guard<Mutex> lock(mutex);
session->setSessionID(sessionFileID);
std::pair<SessionFileMapIter, bool> insertRes =
sessions.insert(SessionFileMapVal(sessionFileID, new SessionFileReferencer(session) ) );
return insertRes.second;
}
/**
* Insert a session with a pre-defined sessionID and reference it. Will fail if given ID existed
* already.
*
* Note: This is a special method intended for recovery purposes only!
* Note: remember to call releaseSession() if the new session was actually inserted.
*
* @param session belongs to the store if it was inserted - so do not free it and don't
* use it any more afterwards (reference it if you still need it)
* @return NULL if sessionID existed already (and new session not inserted or touched in any way),
* pointer to referenced session file otherwise.
*/
SessionFile* SessionFileStore::addAndReferenceRecoverySession(SessionFile* session)
{
const std::lock_guard<Mutex> lock(mutex);
SessionFile* retVal = NULL;
unsigned sessionID = session->getSessionID();
SessionFileMapIter iter = sessions.find(sessionID);
if(iter == sessions.end() )
{ // session with this ID didn't exist (this is the normal case) => insert and reference it
/* note: we do the sessions.find() first to avoid allocating SessionRefer if it's not
necessary */
SessionFileReferencer* sessionFileRefer = new SessionFileReferencer(session);
sessions.insert(SessionFileMapVal(sessionID, sessionFileRefer) );
retVal = sessionFileRefer->reference();
}
return retVal;
}
/**
* Note: remember to call releaseSession()
*
* @return NULL if no such session exists
*/
SessionFile* SessionFileStore::referenceSession(unsigned sessionID)
{
const std::lock_guard<Mutex> lock(mutex);
SessionFileMapIter iter = sessions.find(sessionID);
if(iter == sessions.end() )
{ // not found
return NULL;
}
else
{
SessionFileReferencer* sessionRefer = iter->second;
return sessionRefer->reference();
}
}
void SessionFileStore::releaseSession(SessionFile* session, EntryInfo* entryInfo)
{
unsigned sessionID = session->getSessionID();
bool asyncCleanup = false;
unsigned asyncCleanupAccessFlags = 0; // only for asyncCleanup
MetaFileHandle asyncCleanupFile; // only for asyncCleanup
{
const std::lock_guard<Mutex> lock(mutex);
SessionFileMapIter iter = sessions.find(sessionID);
if(iter != sessions.end() )
{ // session exists => decrease refCount
SessionFileReferencer* sessionRefer = iter->second;
SessionFile* sessionNonRef = sessionRefer->getReferencedObject();
if(unlikely(sessionNonRef->getUseAsyncCleanup() ) )
{ // marked for async cleanup => check whether we're dropping the last reference
if(sessionRefer->getRefCount() == 1)
{ // we're dropping the last reference => save async cleanup data and trigger cleanup
asyncCleanup = true;
asyncCleanupFile = sessionNonRef->releaseInode();
asyncCleanupAccessFlags = sessionNonRef->getAccessFlags();
sessionRefer->release();
sessions.erase(iter);
delete(sessionRefer);
}
}
else
{ // the normal case: just release this reference
sessionRefer->release();
}
}
}
if(unlikely(asyncCleanup) )
performAsyncCleanup(entryInfo, std::move(asyncCleanupFile), asyncCleanupAccessFlags);
}
/**
* @return false if session could not be removed, because it was still in use; it will be marked
* for async cleanup when the last reference is dropped
*/
bool SessionFileStore::removeSession(unsigned sessionID)
{
bool delErr = true;
const std::lock_guard<Mutex> lock(mutex);
SessionFileMapIter iter = sessions.find(sessionID);
if(iter != sessions.end() )
{
SessionFileReferencer* sessionRefer = iter->second;
if(sessionRefer->getRefCount() )
{ // session still in use => mark for async cleanup (on last reference drop)
SessionFile* fileNonRef = sessionRefer->getReferencedObject();
fileNonRef->setUseAsyncCleanup();
delErr = true;
}
else
{ // no references => delete
sessions.erase(iter);
delete(sessionRefer);
delErr = false;
}
}
return !delErr;
}
/**
* @return might be NULL if the session is in use
*/
SessionFile* SessionFileStore::removeAndGetSession(unsigned sessionID)
{
// note: this method is currently unused
SessionFile* session = NULL;
const std::lock_guard<Mutex> lock(mutex);
SessionFileMapIter iter = sessions.find(sessionID);
if(iter != sessions.end() )
{
SessionFileReferencer* sessionRefer = iter->second;
if(!sessionRefer->getRefCount() )
{ // no references => allow deletion
sessions.erase(iter);
sessionRefer->setOwnReferencedObject(false);
session = sessionRefer->getReferencedObject();
delete(sessionRefer);
}
}
return session;
}
/**
* Removes all sessions and additionally adds those that had a reference count to the StringList.
*
* @outRemovedSessions caller is responsible for clean up of contained objects
*/
void SessionFileStore::removeAllSessions(SessionFileList* outRemovedSessions,
UIntList* outReferencedSessions)
{
const std::lock_guard<Mutex> lock(mutex);
for(SessionFileMapIter iter = sessions.begin(); iter != sessions.end(); iter++)
{
SessionFileReferencer* sessionRefer = iter->second;
SessionFile* session = sessionRefer->getReferencedObject();
outRemovedSessions->push_back(session);
if(unlikely(sessionRefer->getRefCount() ) )
outReferencedSessions->push_back(iter->first);
sessionRefer->setOwnReferencedObject(false);
delete(sessionRefer);
}
sessions.clear();
}
/*
* Intended to be used for cleanup if deserialization failed, no locking is used
*/
void SessionFileStore::deleteAllSessions()
{
for(SessionFileMapIter iter = sessions.begin(); iter != sessions.end(); iter++)
{
SessionFileReferencer* sessionRefer = iter->second;
delete(sessionRefer);
}
sessions.clear();
}
size_t SessionFileStore::getSize()
{
const std::lock_guard<Mutex> lock(mutex);
return sessions.size();
}
unsigned SessionFileStore::generateNewSessionID()
{
SessionFileMapIter iter;
// note: we assume here that there always is at least one free sessionID.
do
{
// generate new ID
lastSessionID++;
// check whether this ID is being used already
iter = sessions.find(lastSessionID);
} while(iter != sessions.end() );
// we found an available sessionID => return it
return lastSessionID;
}
/**
* Performs local cleanup tasks for file sessions, which are marked for async cleanup.
*/
void SessionFileStore::performAsyncCleanup(EntryInfo* entryInfo,
MetaFileHandle inode, unsigned accessFlags)
{
const char* logContext = __func__;
App* app = Program::getApp();
MetaStore* metaStore = app->getMetaStore();
unsigned numHardlinks; // ignored here
unsigned numInodeRefs; // ignored here
bool lastWriterClosed; // ignored here
LOG_DEBUG(logContext, Log_NOTICE, "Performing async cleanup of file session");
IGNORE_UNUSED_VARIABLE(logContext);
metaStore->closeFile(entryInfo, std::move(inode), accessFlags, &numHardlinks, &numInodeRefs,
lastWriterClosed);
/* note: we ignore closing storage server files here (e.g. because we don't have the sessionID
and fileHandleID at hand) and unlinking of disposable files (disposal can still be triggered
by fhgfs_online_cfg).
this is something that we should change in the future (but maybe rather indirectly by syncing
open files between clients and servers at regular intervals). */
}
SessionFileMap* SessionFileStore::getSessionMap()
{
return &this->sessions;
}
/* Merges the SessionFiles of the given SessionFileStore into this SessionFileStore.
* Only not existing SessionFiles will be added to the existing SessionFileStore
*
* @param sessionFileStore the sessionFileStore which will be merged with this sessionFileStore
*/
void SessionFileStore::mergeSessionFiles(SessionFileStore* sessionFileStore)
{
Logger* log = Logger::getLogger();
SessionFileMapIter sessionIter = sessionFileStore->getSessionMap()->begin();
while(sessionIter != sessionFileStore->getSessionMap()->end() )
{
bool sessionFound = false;
SessionFileMapIter destSessionIter = this->sessions.find(sessionIter->first);
if (destSessionIter != this->sessions.end())
{
sessionFound = true;
log->log(Log_WARNING, "SessionFileStore merge", "found SessionFile with same "
"ID: " + StringTk::uintToStr(sessionIter->first) +
" , merge not possible, may be a bug?");
}
if (!sessionFound)
{
bool success = this->sessions.insert(SessionFileMapVal(sessionIter->first,
sessionIter->second)).second;
if (!success)
{
log->log(Log_WARNING, "SessionFileStore merge", "could not merge: " +
StringTk::uintToStr(sessionIter->first) );
delete(sessionIter->second);
}
}
else
{
delete(sessionIter->second);
}
sessionIter++;
}
}
bool SessionFileStore::operator==(const SessionFileStore& other) const
{
struct ops {
static bool cmp(const SessionFileMapVal& lhs, const SessionFileMapVal& rhs)
{
return lhs.first == rhs.first
&& *lhs.second->getReferencedObject() == *rhs.second->getReferencedObject();
}
};
return lastSessionID == other.lastSessionID
&& sessions.size() == other.sessions.size()
&& std::equal(
sessions.begin(), sessions.end(),
other.sessions.begin(),
ops::cmp);
}

132
meta/source/session/SessionFileStore.h Normal file
View File

@@ -0,0 +1,132 @@
#pragma once
#include <common/toolkit/ObjectReferencer.h>
#include <common/threading/Mutex.h>
#include <common/toolkit/Random.h>
#include <common/Common.h>
#include "SessionFile.h"
typedef ObjectReferencer<SessionFile*> SessionFileReferencer;
typedef std::map<uint32_t, SessionFileReferencer*> SessionFileMap;
typedef SessionFileMap::iterator SessionFileMapIter;
typedef SessionFileMap::const_iterator SessionFileMapCIter;
typedef SessionFileMap::value_type SessionFileMapVal;
typedef std::list<SessionFile*> SessionFileList;
typedef SessionFileList::iterator SessionFileListIter;
class SessionStore;
class SessionFileStore
{
friend class SessionStore;
public:
SessionFileStore()
{
// note: randomize to make it more unlikely that after a meta server shutdown the same
// client instance can open a file and gets a colliding (with the old meta server
// instance) sessionID
this->lastSessionID = Random().getNextInt();
}
unsigned addSession(SessionFile* session);
bool addSession(SessionFile* session, unsigned sessionFileID);
SessionFile* addAndReferenceRecoverySession(SessionFile* session);
SessionFile* referenceSession(unsigned sessionID);
void releaseSession(SessionFile* session, EntryInfo* entryInfo);
bool removeSession(unsigned sessionID);
void removeAllSessions(SessionFileList* outRemovedSessions,
UIntList* outReferencedSessions);
void deleteAllSessions();
void mergeSessionFiles(SessionFileStore* sessionFileStore);
size_t getSize();
bool relinkInodes(MetaStore& store)
{
bool result = true;
for (auto it = sessions.begin(); it != sessions.end(); )
{
const bool relinkRes = it->second->getReferencedObject()->relinkInode(store);
if (!relinkRes)
sessions.erase(it++);
else
++it;
result &= relinkRes;
}
return result;
}
static void serialize(const SessionFileStore* obj, Serializer& ser)
{
ser
% obj->lastSessionID
% uint32_t(obj->sessions.size());
for (SessionFileMapCIter it = obj->sessions.begin(); it != obj->sessions.end(); ++it)
{
ser
% it->first
% *it->second->getReferencedObject();
}
LOG_DEBUG("SessionFileStore serialize", Log_DEBUG, "count of serialized "
"SessionFiles: " + StringTk::uintToStr(obj->sessions.size()) );
}
static void serialize(SessionFileStore* obj, Deserializer& des)
{
uint32_t elemCount;
des
% obj->lastSessionID
% elemCount;
for(unsigned i = 0; i < elemCount; i++)
{
uint32_t key;
des % key;
if (!des.good())
return;
SessionFile* sessionFile = new SessionFile();
des % *sessionFile;
if (!des.good())
{
delete(sessionFile);
return;
}
obj->sessions.insert(SessionFileMapVal(key, new SessionFileReferencer(sessionFile)));
}
LOG_DEBUG("SessionFileStore deserialize", Log_DEBUG, "count of deserialized "
"SessionFiles: " + StringTk::uintToStr(elemCount));
}
bool operator==(const SessionFileStore& other) const;
bool operator!=(const SessionFileStore& other) const { return !(*this == other); }
protected:
SessionFileMap* getSessionMap();
private:
SessionFileMap sessions;
uint32_t lastSessionID;
Mutex mutex;
SessionFile* removeAndGetSession(unsigned sessionID); // actually not needed now
unsigned generateNewSessionID();
void performAsyncCleanup(EntryInfo* entryInfo, MetaFileHandle inode, unsigned accessFlags);
};

625
meta/source/session/SessionStore.cpp Normal file
View File

@@ -0,0 +1,625 @@
#include <program/Program.h>
#include "SessionStore.h"
#include <mutex>
#include <boost/scoped_array.hpp>
// these two u32 values are the first eight bytes of a session dump file.
// in 2015.03, the first four bytes were the number of sessions in the dump, so here we use
// 0xffffffff to indicate that we are not using the 2015.03 format - it couldn't store that many
// sessions anyway.
#define SESSION_FORMAT_HEADER uint32_t(0xffffffffUL)
#define SESSION_FORMAT_VERSION uint32_t(1)
/**
* @param session belongs to the store after calling this method - so do not free it and don't
* use it any more afterwards (re-get it from this store if you need it)
*/
void SessionStore::addSession(Session* session)
{
NumNodeID sessionID = session->getSessionID();
std::lock_guard<Mutex> mutexLock(mutex);
// is session in the store already?
SessionMapIter iter = sessions.find(sessionID);
if(iter != sessions.end() )
{
delete(session);
}
else
{ // session not in the store yet
sessions.insert(SessionMapVal(sessionID, new SessionReferencer(session) ) );
}
}
/**
* Note: remember to call releaseSession()
*
* @return NULL if no such session exists
*/
Session* SessionStore::referenceSession(NumNodeID sessionID, bool addIfNotExists)
{
Session* session;
std::lock_guard<Mutex> mutexLock(mutex);
SessionMapIter iter = sessions.find(sessionID);
if(iter == sessions.end() )
{ // not found
if(!addIfNotExists)
session = NULL;
else
{ // add as new session and reference it
LogContext log("SessionStore (ref)");
log.log(Log_DEBUG, std::string("Creating a new session. SessionID: ") + sessionID.str());
Session* newSession = new Session(sessionID);
SessionReferencer* sessionRefer = new SessionReferencer(newSession);
sessions.insert(SessionMapVal(sessionID, sessionRefer) );
session = sessionRefer->reference();
}
}
else
{
SessionReferencer* sessionRefer = iter->second;
session = sessionRefer->reference();
}
return session;
}
void SessionStore::releaseSession(Session* session)
{
NumNodeID sessionID = session->getSessionID();
std::lock_guard<Mutex> mutexLock(mutex);
SessionMapIter iter = sessions.find(sessionID);
if(iter != sessions.end() )
{ // session exists => decrease refCount
SessionReferencer* sessionRefer = iter->second;
sessionRefer->release();
}
}
bool SessionStore::removeSession(NumNodeID sessionID)
{
bool delErr = true;
std::lock_guard<Mutex> mutexLock(mutex);
SessionMapIter iter = sessions.find(sessionID);
if(iter != sessions.end() )
{
SessionReferencer* sessionRefer = iter->second;
if(sessionRefer->getRefCount() )
delErr = true;
else
{ // no references => delete
sessions.erase(sessionID);
delete(sessionRefer);
delErr = false;
}
}
return !delErr;
}
/**
* @return NULL if session is referenced, otherwise the sesion must be cleaned up by the caller
*/
Session* SessionStore::removeSessionUnlocked(NumNodeID sessionID)
{
Session* retVal = NULL;
SessionMapIter iter = sessions.find(sessionID);
if(iter != sessions.end() )
{
SessionReferencer* sessionRefer = iter->second;
if(!sessionRefer->getRefCount() )
{ // no references => return session to caller
retVal = sessionRefer->getReferencedObject();
sessionRefer->setOwnReferencedObject(false);
delete(sessionRefer);
sessions.erase(sessionID);
}
}
return retVal;
}
/**
* Removes all sessions from the session store which are not contained in the masterList.
* @param masterList must be ordered; contained nodes will be removed and may no longer be
* accessed after calling this method.
* @param outRemovedSessions contained sessions must be cleaned up by the caller
* note: list will NOT be cleared, new elements will be appended
* @param outUnremovableSesssions contains sessions that would have been removed but are currently
* referenced
* note: list will NOT be cleared, new elements will be appended
*/
void SessionStore::syncSessions(const std::vector<NodeHandle>& masterList,
SessionList& outRemovedSessions, NumNodeIDList& outUnremovableSessions)
{
std::lock_guard<Mutex> mutexLock(mutex);
SessionMapIter sessionIter = sessions.begin();
auto masterIter = masterList.begin();
while( (sessionIter != sessions.end() ) && (masterIter != masterList.end() ) )
{
NumNodeID currentSession = sessionIter->first;
NumNodeID currentMaster = (*masterIter)->getNumID() ;
if(currentMaster < currentSession)
{ // session is added
// note: we add sessions only on demand, so no operation here
masterIter++;
}
else
if(currentSession < currentMaster)
{ // session is removed
sessionIter++; // (removal invalidates iterator)
Session* session = removeSessionUnlocked(currentSession);
if(session)
outRemovedSessions.push_back(session);
else
outUnremovableSessions.push_back(currentSession); // session was referenced
}
else
{ // session unchanged
masterIter++;
sessionIter++;
}
}
// remaining sessions are removed
while(sessionIter != sessions.end() )
{
NumNodeID currentSession = sessionIter->first;
sessionIter++; // (removal invalidates iterator)
Session* session = removeSessionUnlocked(currentSession);
if(session)
outRemovedSessions.push_back(session);
else
outUnremovableSessions.push_back(currentSession); // session was referenced
}
}
/**
* @return number of sessions
*/
NumNodeIDList SessionStore::getAllSessionIDs()
{
std::lock_guard<Mutex> mutexLock(mutex);
NumNodeIDList res;
for(SessionMapIter iter = sessions.begin(); iter != sessions.end(); iter++)
res.push_back(iter->first);
return res;
}
size_t SessionStore::getSize()
{
std::lock_guard<Mutex> mutexLock(mutex);
return sessions.size();
}
void SessionStore::serialize(Serializer& ser) const
{
ser
% SESSION_FORMAT_HEADER
% SESSION_FORMAT_VERSION
% uint32_t(sessions.size());
for (SessionMap::const_iterator it = sessions.begin(); it != sessions.end(); ++it)
{
ser
% it->first
% *it->second->getReferencedObject();
}
{
Serializer lenPos = ser.mark();
ser % uint32_t(0);
std::set<const FileInode*> inodesSeen;
for (auto sessionIter = sessions.begin(); sessionIter != sessions.end(); ++sessionIter)
{
Session& session = *sessionIter->second->getReferencedObject();
const auto& fileMap = session.getFiles()->sessions;
for (auto fileMapIter = fileMap.begin(); fileMapIter != fileMap.end(); ++fileMapIter)
{
SessionFile& file = *fileMapIter->second->getReferencedObject();
const auto& inode = file.getInode();
if (!inodesSeen.insert(inode.get()).second)
continue;
ser
% *file.getEntryInfo()
% inode->lockState();
}
}
lenPos % uint32_t(inodesSeen.size());
}
LOG_DEBUG("SessionStore serialize", Log_DEBUG, "Serialized session store. Session count: "
+ StringTk::uintToStr(sessions.size()));
}
void SessionStore::deserialize(Deserializer& des)
{
uint32_t sessionFormatHeader;
uint32_t sessionFormatVersion;
uint32_t elemCount;
des
% sessionFormatHeader
% sessionFormatVersion;
if (sessionFormatHeader != SESSION_FORMAT_HEADER ||
sessionFormatVersion != SESSION_FORMAT_VERSION)
{
des.setBad();
return;
}
des % elemCount;
// sessions
for(unsigned i = 0; i < elemCount; i++)
{
NumNodeID key;
des % key;
if (!des.good())
return;
Session* session = new Session();
des % *session;
if (!des.good())
{
session->getFiles()->deleteAllSessions();
delete(session);
return;
}
SessionMapIter searchResult = this->sessions.find(key);
if (searchResult == this->sessions.end() )
{
this->sessions.insert(SessionMapVal(key, new SessionReferencer(session)));
}
else
{ // exist so local files will merged
searchResult->second->getReferencedObject()->mergeSessionFiles(session);
delete(session);
}
}
LOG_DEBUG("SessionStore deserialize", Log_DEBUG, "Deserialized session store. Session count: "
+ StringTk::uintToStr(elemCount));
}
void SessionStore::deserializeLockStates(Deserializer& des, MetaStore& metaStore)
{
uint32_t elemCount;
des % elemCount;
// file inode lock states
while (elemCount > 0)
{
elemCount--;
EntryInfo info;
des % info;
if (!des.good())
break;
auto [inode, referenceRes] = metaStore.referenceFile(&info);
des % inode->lockState();
metaStore.releaseFile(info.getParentEntryID(), inode);
if (!des.good())
break;
}
if (!des.good())
{
LOG(SESSIONS, ERR, "Could not restore file inode lock states.");
return;
}
}
/**
* Note: Caller must either hold a lock on the SessionStore, or otherwise ensure that it is not
* accessed from anywhere.
*/
bool SessionStore::deserializeFromBuf(const char* buf, size_t bufLen, MetaStore& metaStore)
{
Deserializer des(buf, bufLen);
deserialize(des);
if (!des.good())
LOG(SESSIONS, ERR, "Unable to deserialize session store from buffer.");
const bool relinkRes = relinkInodes(metaStore);
if (!relinkRes)
LOG(SESSIONS, ERR, "Unable to relink inodes.");
if (!des.good() || !relinkRes)
return false;
deserializeLockStates(des, metaStore);
if (!des.good())
{
LOG(SESSIONS, ERR, "Unable to load lock states.");
return false;
}
return true;
}
bool SessionStore::loadFromFile(const std::string& filePath, MetaStore& metaStore)
{
LogContext log("SessionStore (load)");
log.log(Log_DEBUG,"load sessions from file: " + filePath);
bool retVal = false;
boost::scoped_array<char> buf;
int readRes;
struct stat statBuf;
int retValStat;
if(filePath.empty())
return false;
std::lock_guard<Mutex> mutexLock(mutex);
int fd = open(filePath.c_str(), O_RDONLY, 0);
if(fd == -1)
{ // open failed
log.log(Log_DEBUG, "Unable to open session file: " + filePath + ". " +
"SysErr: " + System::getErrString() );
return false;
}
retValStat = fstat(fd, &statBuf);
if(retValStat)
{ // stat failed
log.log(Log_WARNING, "Unable to stat session file: " + filePath + ". " +
"SysErr: " + System::getErrString() );
goto err_stat;
}
if (statBuf.st_size == 0)
{
LOG(SESSIONS, WARNING, "Session file exists, but is empty.", filePath);
return false;
}
buf.reset(new (std::nothrow) char[statBuf.st_size]);
if (!buf)
{
LOG(SESSIONS, ERR, "Could not allocate memory to read session file", filePath);
goto err_stat;
}
readRes = read(fd, buf.get(), statBuf.st_size);
if(readRes <= 0)
{ // reading failed
log.log(Log_WARNING, "Unable to read session file: " + filePath + ". " +
"SysErr: " + System::getErrString() );
}
else
{ // parse contents
retVal = deserializeFromBuf(buf.get(), readRes, metaStore);
}
if (!retVal)
log.logErr("Could not deserialize SessionStore from file: " + filePath);
err_stat:
close(fd);
return retVal;
}
/*
* @returns pair of buffer and buffer size, { nullptr, 0 } on error.
*/
std::pair<std::unique_ptr<char[]>, size_t> SessionStore::serializeToBuf()
{
size_t bufLen;
std::unique_ptr<char[]> buf;
{
Serializer ser;
serialize(ser);
bufLen = ser.size();
buf.reset(new (std::nothrow)char[bufLen]);
if (!buf)
{
LOG(SESSIONS, ERR, "Could not allocate memory to serialize sessions.");
return { std::unique_ptr<char[]>(), 0 };
}
}
{
Serializer ser(buf.get(), bufLen);
serialize(ser);
if (!ser.good())
{
LOG(SESSIONS, ERR, "Unable to serialize sessions.");
return { std::unique_ptr<char[]>(), 0 };
}
}
return { std::move(buf), bufLen };
}
/**
* Note: setStorePath must be called before using this.
*/
bool SessionStore::saveToFile(const std::string& filePath)
{
LogContext log("SessionStore (save)");
log.log(Log_DEBUG,"save sessions to file: " + filePath);
bool retVal = false;
if(!filePath.length() )
return false;
std::lock_guard<Mutex> mutexLock(mutex);
// create/trunc file
int openFlags = O_CREAT|O_TRUNC|O_WRONLY;
int fd = open(filePath.c_str(), openFlags, 0666);
if(fd == -1)
{ // error
log.logErr("Unable to create session file: " + filePath + ". " +
"SysErr: " + System::getErrString() );
return false;
}
// file created => store data
{
ssize_t writeRes;
auto buf = serializeToBuf();
if (!buf.first)
{
LOG(SESSIONS, ERR, "Unable to save session file", filePath);
goto err_closefile;
}
writeRes = write(fd, buf.first.get(), buf.second);
if(writeRes != (ssize_t)buf.second)
{
log.logErr("Unable to store session file: " + filePath + ". " +
"SysErr: " + System::getErrString() );
goto err_closefile;
}
}
retVal = true;
LOG_DEBUG_CONTEXT(log, Log_DEBUG, "Session file stored: " + filePath);
// error compensation
err_closefile:
close(fd);
return retVal;
}
/**
* Clears out all sessions from session store.
* Caller must hold lock on session store or otherwise ensure no access to the session store takes
* place.
* Intended for using prior to a session store resync on the secondary of a mirror buddy group.
* This method *does not* unlink disposed files or remove their chunk files on storage servers,
* since this may invalidate results of previous resync activity or conflict with the role of the
* current server as secondary in a mirror group.
* @returns true if all sessions could be removed, false if there are sessions still referenced and
* can't be removed.
*/
bool SessionStore::clear()
{
MetaStore* metaStore = Program::getApp()->getMetaStore();
for (SessionMapIter sessionIt = sessions.begin(); sessionIt != sessions.end(); /* inc in body */)
{
// Iter is incremented here because removeSessionUnlocked invalidates it.
Session* session = removeSessionUnlocked((sessionIt++)->first);
if (session)
{
// session was removed - clean it up
SessionFileStore* sessionFiles = session->getFiles();
SessionFileList removedSessionFiles;
UIntList referencedSessionFiles;
sessionFiles->removeAllSessions(&removedSessionFiles, &referencedSessionFiles);
// referencedSessionFiles should always be empty, because otherwise, the session itself
// would also still be referenced somewhere, so we wouldn't end up here. Nevertheless,
// check + log.
if (!referencedSessionFiles.empty())
{
LOG(SESSIONS, ERR, "Session file still references in unreferenced session.");
continue;
}
for (SessionFileListIter fileIt = removedSessionFiles.begin();
fileIt != removedSessionFiles.end(); ++fileIt)
{
SessionFile* sessionFile = *fileIt;
unsigned accessFlags = sessionFile->getAccessFlags();
unsigned numHardlinks; // ignored here
unsigned numInodeRefs; // ignored here
bool lastWriterClosed; // ignored here
MetaFileHandle inode = sessionFile->releaseInode();
EntryInfo* entryInfo = sessionFile->getEntryInfo();
metaStore->closeFile(entryInfo, std::move(inode), accessFlags, &numHardlinks,
&numInodeRefs, lastWriterClosed);
delete sessionFile;
}
delete session;
}
else
{
LOG(SESSIONS, ERR, "Session still referenced", ("ID", sessionIt->first));
}
}
return true;
}
bool SessionStore::operator==(const SessionStore& other) const
{
struct ops {
static bool cmp(const SessionMapVal& lhs, const SessionMapVal& rhs)
{
return lhs.first == rhs.first
&& *lhs.second->getReferencedObject() == *rhs.second->getReferencedObject();
}
};
return sessions.size() == other.sessions.size()
&& std::equal(
sessions.begin(), sessions.end(),
other.sessions.begin(),
ops::cmp);
}

97
meta/source/session/SessionStore.h Normal file
View File

@@ -0,0 +1,97 @@
#pragma once
#include <common/nodes/Node.h>
#include <common/toolkit/ObjectReferencer.h>
#include <common/threading/Mutex.h>
#include <common/Common.h>
#include <session/EntryLockStore.h>
#include "Session.h"
typedef ObjectReferencer<Session*> SessionReferencer;
typedef std::map<NumNodeID, SessionReferencer*> SessionMap;
typedef SessionMap::iterator SessionMapIter;
typedef SessionMap::value_type SessionMapVal;
typedef std::list<Session*> SessionList;
typedef SessionList::iterator SessionListIter;
/*
* A session always belongs to a client ID, therefore the session ID is always the nodeID of the
* corresponding client
*/
class SessionStore
{
friend class TestSerialization; // for testing
public:
SessionStore() {}
Session* referenceSession(NumNodeID sessionID, bool addIfNotExists=true);
void releaseSession(Session* session);
void syncSessions(const std::vector<NodeHandle>& masterList, SessionList& outRemovedSessions,
NumNodeIDList& outUnremovableSessions);
NumNodeIDList getAllSessionIDs();
size_t getSize();
void serialize(Serializer& ser) const;
void deserialize(Deserializer& des);
void deserializeLockStates(Deserializer& des, MetaStore& metaStore);
std::pair<std::unique_ptr<char[]>, size_t> serializeToBuf();
bool deserializeFromBuf(const char* buf, size_t bufLen, MetaStore& metaStore);
bool loadFromFile(const std::string& filePath, MetaStore& metaStore);
bool saveToFile(const std::string& filePath);
bool clear();
EntryLockStore* getEntryLockStore()
{
return &this->entryLockStore;
}
bool operator==(const SessionStore& other) const;
bool operator!=(const SessionStore& other) const { return !(*this == other); }
private:
SessionMap sessions;
Mutex mutex;
EntryLockStore entryLockStore; // Locks on entryIDs to synchronize with mirror buddy.
void addSession(Session* session); // actually not needed (maybe later one day...)
bool removeSession(NumNodeID sessionID); // actually not needed (maybe later one day...)
Session* removeSessionUnlocked(NumNodeID sessionID);
bool relinkInodes(MetaStore& store)
{
bool result = true;
for (auto it = sessions.begin(); it != sessions.end(); )
{
const bool relinkRes = it->second->getReferencedObject()->relinkInodes(store);
if (!relinkRes)
{
// Relinking failed on at least one SessionFile in this Session -> remove if empty
if (it->second->getReferencedObject()->getFiles()->getSize() == 0)
sessions.erase(it++);
else
++it;
}
else
{
++it;
}
result &= relinkRes;
}
return result;
}
};