deb/beegfs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
beegfs/meta/source/components/InternodeSyncer.cpp
geos_one c891bb7105 New upstream version 8.1.0
2025-08-10 01:34:16 +02:00

1344 lines
43 KiB
C++
Raw Blame History

#include <common/net/message/nodes/HeartbeatMsg.h>
#include <common/net/message/nodes/ChangeTargetConsistencyStatesMsg.h>
#include <common/net/message/nodes/ChangeTargetConsistencyStatesRespMsg.h>
#include <common/net/message/nodes/RefreshCapacityPoolsMsg.h>
#include <common/net/message/storage/SetStorageTargetInfoMsg.h>
#include <common/net/message/storage/SetStorageTargetInfoRespMsg.h>
#include <common/net/message/storage/quota/RequestExceededQuotaMsg.h>
#include <common/net/message/storage/quota/RequestExceededQuotaRespMsg.h>
#include <common/toolkit/MessagingTk.h>
#include <common/toolkit/NodesTk.h>
#include <common/toolkit/SessionTk.h>
#include <common/toolkit/Time.h>
#include <common/toolkit/ZipIterator.h>
#include <common/nodes/NodeStore.h>
#include <common/nodes/TargetCapacityPools.h>
#include <app/App.h>
#include <app/config/Config.h>
#include <net/msghelpers/MsgHelperClose.h>
#include <program/Program.h>
#include <toolkit/BuddyCommTk.h>
#include "InternodeSyncer.h"
#include <boost/lexical_cast.hpp>
// forward declaration
namespace UUID {
std::string getMachineUUID();
}
InternodeSyncer::InternodeSyncer(TargetConsistencyState initialConsistencyState)
: PThread("XNodeSync"),
log("XNodeSync"),
forcePoolsUpdate(true),
forceTargetStatesUpdate(true),
forcePublishCapacities(true),
forceStoragePoolsUpdate(true),
offlineWait(Program::getApp()->getConfig() ),
nodeConsistencyState(initialConsistencyState),
buddyResyncInProgress(false)
{
MirrorBuddyGroupMapper* mbg = Program::getApp()->getMetaBuddyGroupMapper();
MirrorBuddyState buddyState = mbg->getBuddyState(Program::getApp()->getLocalNodeNumID().val() );
if ((buddyState == BuddyState_PRIMARY)
&& (nodeConsistencyState == TargetConsistencyState_NEEDS_RESYNC))
offlineWait.startTimer();
}
void InternodeSyncer::run()
{
try
{
registerSignalHandler();
syncLoop();
log.log(Log_DEBUG, "Component stopped.");
}
catch(std::exception& e)
{
PThread::getCurrentThreadApp()->handleComponentException(e);
}
}
void InternodeSyncer::syncLoop()
{
const App* app = Program::getApp();
const Config* cfg = app->getConfig();
const int sleepIntervalMS = 3*1000; // 3sec
// If (undocumented) sysUpdateTargetStatesSecs is set in config, use that value, otherwise
// default to 1/6 sysTargetOfflineTimeoutSecs.
const unsigned updateTargetStatesMS =
(cfg->getSysUpdateTargetStatesSecs() != 0)
? cfg->getSysUpdateTargetStatesSecs() * 1000
: cfg->getSysTargetOfflineTimeoutSecs() * 166;
const unsigned updateCapacityPoolsMS = 4 * updateTargetStatesMS;
const unsigned metaCacheSweepNormalMS = 5*1000; // 5sec
const unsigned metaCacheSweepStressedMS = 2*1000; // 2sec
const unsigned idleDisconnectIntervalMS = 70*60*1000; /* 70 minutes (must be less than half the
streamlis idle disconnect interval to avoid cases where streamlis disconnects first) */
const unsigned updateIDTimeMS = 60 * 1000; // 1 min
const unsigned downloadNodesIntervalMS = 300000; // 5 min
const unsigned updateStoragePoolsMS = downloadNodesIntervalMS;
const unsigned checkNetworkIntervalMS = 60*1000; // 1 minute
Time lastCapacityUpdateT;
Time lastMetaCacheSweepT;
Time lastIdleDisconnectT;
Time lastTimeIDSet;
Time lastTargetStatesUpdateT;
Time lastDownloadNodesT;
Time lastStoragePoolsUpdateT;
Time lastCapacityPublishedT;
Time lastCheckNetworkT;
bool doRegisterLocalNode = false;
unsigned currentCacheSweepMS = metaCacheSweepNormalMS; // (adapted inside the loop below)
while(!waitForSelfTerminateOrder(sleepIntervalMS) )
{
const bool capacityPoolsUpdateForced = forcePoolsUpdate.exchange(false);
const bool doCapacityPoolsUpdate = capacityPoolsUpdateForced
|| (lastCapacityUpdateT.elapsedMS() > updateCapacityPoolsMS);
const bool doTargetStatesUpdate = forceTargetStatesUpdate.exchange(false)
|| (lastTargetStatesUpdateT.elapsedMS() > updateTargetStatesMS);
const bool doPublishCapacities = forcePublishCapacities.exchange(false)
|| (lastCapacityPublishedT.elapsedMS() > updateTargetStatesMS);
const bool doStoragePoolsUpdate = forceStoragePoolsUpdate.exchange(false)
|| (lastStoragePoolsUpdateT.elapsedMS() > updateStoragePoolsMS);
const bool doCheckNetwork = forceCheckNetwork.exchange(false)
|| (lastCheckNetworkT.elapsedMS() > checkNetworkIntervalMS);
if (doCheckNetwork)
{
if (checkNetwork())
doRegisterLocalNode = true;
lastCheckNetworkT.setToNow();
}
if (doRegisterLocalNode)
doRegisterLocalNode = !registerNode(app->getDatagramListener());
// download & sync nodes
if (lastDownloadNodesT.elapsedMS() > downloadNodesIntervalMS)
{
downloadAndSyncNodes();
downloadAndSyncTargetMappings();
lastDownloadNodesT.setToNow();
}
if (doStoragePoolsUpdate)
{
downloadAndSyncStoragePools();
lastStoragePoolsUpdateT.setToNow();
}
if(doCapacityPoolsUpdate)
{
updateMetaCapacityPools();
updateMetaBuddyCapacityPools();
if ( (capacityPoolsUpdateForced) && (!doStoragePoolsUpdate) )
{ // capacity pools changed, but storage pools were not downloaded (i.e. no update on
// storage capactity pools was made)
updateStorageCapacityPools();
updateTargetBuddyCapacityPools();
}
lastCapacityUpdateT.setToNow();
}
if(lastMetaCacheSweepT.elapsedMS() > currentCacheSweepMS)
{
bool flushTriggered = app->getMetaStore()->cacheSweepAsync();
currentCacheSweepMS = (flushTriggered ? metaCacheSweepStressedMS : metaCacheSweepNormalMS);
lastMetaCacheSweepT.setToNow();
}
if(lastIdleDisconnectT.elapsedMS() > idleDisconnectIntervalMS)
{
dropIdleConns();
lastIdleDisconnectT.setToNow();
}
if(lastTimeIDSet.elapsedMS() > updateIDTimeMS)
{
StorageTk::resetIDCounterToNow();
lastTimeIDSet.setToNow();
}
if(doTargetStatesUpdate)
{
if (this->offlineWait.hasTimeout() )
{
// if we're waiting to be offlined, set our local state to needs-resync and don't report
// anything to the mgmtd
setNodeConsistencyState(TargetConsistencyState_NEEDS_RESYNC);
}
else
{
TargetConsistencyState newConsistencyState;
if (updateMetaStatesAndBuddyGroups(newConsistencyState, true))
setNodeConsistencyState(newConsistencyState);
downloadAndSyncTargetStatesAndBuddyGroups();
}
lastTargetStatesUpdateT.setToNow();
}
if (doPublishCapacities)
{
publishNodeCapacity();
lastCapacityPublishedT.setToNow();
}
}
}
/**
* Inspect the available and allowed network interfaces for any changes.
*/
bool InternodeSyncer::checkNetwork()
{
App* app = Program::getApp();
NicAddressList newLocalNicList;
bool res = false;
app->findAllowedInterfaces(newLocalNicList);
app->findAllowedRDMAInterfaces(newLocalNicList);
if (!std::equal(newLocalNicList.begin(), newLocalNicList.end(), app->getLocalNicList().begin()))
{
log.log(Log_NOTICE, "checkNetwork: local interfaces have changed");
app->updateLocalNicList(newLocalNicList);
res = true;
}
return res;
}
bool InternodeSyncer::updateMetaCapacityPools()
{
NodeCapacityPools* pools = Program::getApp()->getMetaCapacityPools();
auto downloadRes = downloadCapacityPools(CapacityPoolQuery_META);
if(!downloadRes.first)
return false;
// poolsMap does only contain one element with INVALID_POOL_ID in this case
const auto& capacityPoolLists = downloadRes.second[StoragePoolStore::INVALID_POOL_ID];
pools->syncPoolsFromLists(capacityPoolLists);
return true;
}
bool InternodeSyncer::updateMetaBuddyCapacityPools()
{
NodeCapacityPools* pools = Program::getApp()->getMetaBuddyCapacityPools();
auto downloadRes = downloadCapacityPools(CapacityPoolQuery_METABUDDIES);
if(!downloadRes.first)
return false;
// poolsMap does only contain one element with INVALID_POOL_ID in this case
const auto& capacityPoolLists = downloadRes.second[StoragePoolStore::INVALID_POOL_ID];
pools->syncPoolsFromLists(capacityPoolLists);
return true;
}
bool InternodeSyncer::updateStorageCapacityPools()
{
const App* app = Program::getApp();
const Config* cfg = app->getConfig();
const TargetMapper* targetMapper = app->getTargetMapper();
const auto downloadRes = downloadCapacityPools(CapacityPoolQuery_STORAGE);
if(!downloadRes.first)
return false;
TargetMap targetMap;
if(cfg->getSysTargetAttachmentMap() )
targetMap = *cfg->getSysTargetAttachmentMap(); // user-provided custom assignment map
else
targetMap = targetMapper->getMapping();
const GetNodeCapacityPoolsRespMsg::PoolsMap& poolsMap = downloadRes.second;
bool failed = false;
for (auto iter = poolsMap.begin(); iter != poolsMap.end(); iter++)
{
StoragePoolPtr storagePool = app->getStoragePoolStore()->getPool(iter->first);
if (!storagePool)
{
LOG(CAPACITY, ERR, "Received capacity pools for unknown storage pool.",
("storagePoolId", iter->first));
failed = true;
continue;
}
storagePool->getTargetCapacityPools()->syncPoolsFromLists(iter->second, targetMap);
}
return !failed;
}
bool InternodeSyncer::updateTargetBuddyCapacityPools()
{
const App* app = Program::getApp();
auto downloadRes = downloadCapacityPools(CapacityPoolQuery_STORAGEBUDDIES);
if(!downloadRes.first)
return false;
const GetNodeCapacityPoolsRespMsg::PoolsMap& poolsMap = downloadRes.second;
bool failed = false;
for (auto iter = poolsMap.begin(); iter != poolsMap.end(); iter++)
{
StoragePoolPtr storagePool = app->getStoragePoolStore()->getPool(iter->first);
if (!storagePool)
{
LOG(CAPACITY, ERR, "Received capacity pools for unknown storage pool.",
("storagePoolId", iter->first));
failed = true;
continue;
}
storagePool->getBuddyCapacityPools()->syncPoolsFromLists(iter->second);
}
return !failed;
}
/**
* @return a pair, with first being false on error/true on success and second being the downloaded
* map of capacity pools sorted by storage pool
*/
std::pair<bool, GetNodeCapacityPoolsRespMsg::PoolsMap> InternodeSyncer::downloadCapacityPools(
CapacityPoolQueryType poolType)
{
LOG(STATES, DEBUG, "Downloading capacity pools.",
("Pool type", GetNodeCapacityPoolsMsg::queryTypeToStr(poolType)));
NodeStore* mgmtNodes = Program::getApp()->getMgmtNodes();
auto node = mgmtNodes->referenceFirstNode();
if(!node)
return std::make_pair(false, GetNodeCapacityPoolsRespMsg::PoolsMap());
GetNodeCapacityPoolsMsg msg(poolType);
RequestResponseArgs rrArgs(node.get(), &msg, NETMSGTYPE_GetNodeCapacityPoolsResp);
GetNodeCapacityPoolsRespMsg* respMsgCast;
#ifndef BEEGFS_DEBUG
rrArgs.logFlags |= REQUESTRESPONSEARGS_LOGFLAG_CONNESTABLISHFAILED
| REQUESTRESPONSEARGS_LOGFLAG_RETRY;
#endif // BEEGFS_DEBUG
// connect & communicate
bool commRes = MessagingTk::requestResponse(&rrArgs);
if(!commRes)
return std::make_pair(false, GetNodeCapacityPoolsRespMsg::PoolsMap());
// handle result
respMsgCast = (GetNodeCapacityPoolsRespMsg*)rrArgs.outRespMsg.get();
GetNodeCapacityPoolsRespMsg::PoolsMap poolsMap = respMsgCast->getPoolsMap();
return std::make_pair(true, poolsMap);
}
/**
* @return true if an ack was received for the heartbeat, false otherwise
*/
bool InternodeSyncer::registerNode(AbstractDatagramListener* dgramLis)
{
const char* logContext = "Register node";
static bool registrationFailureLogged = false; // to avoid log spamming
App* app = Program::getApp();
NodeStore* mgmtNodes = app->getMgmtNodes();
Config* cfg = app->getConfig();
auto mgmtNode = mgmtNodes->referenceFirstNode();
if(!mgmtNode)
return false;
Node& localNode = Program::getApp()->getLocalNode();
NumNodeID localNodeNumID = localNode.getNumID();
NumNodeID rootNodeID = app->getMetaRoot().getID();
bool rootIsBuddyMirrored = app->getMetaRoot().getIsMirrored();
NicAddressList nicList(localNode.getNicList());
HeartbeatMsg msg(localNode.getAlias(), localNodeNumID, NODETYPE_Meta, &nicList);
msg.setRootNumID(rootNodeID);
msg.setRootIsBuddyMirrored(rootIsBuddyMirrored);
msg.setPorts(cfg->getConnMetaPort(), cfg->getConnMetaPort() );
auto uuid = UUID::getMachineUUID();
if (uuid.empty()) {
LogContext(logContext).log(Log_CRITICAL,
"Couldn't determine UUID for machine. Node registration not possible.");
return false;
}
msg.setMachineUUID(uuid);
bool nodeRegistered = dgramLis->sendToNodeUDPwithAck(mgmtNode, &msg);
if(nodeRegistered)
LogContext(logContext).log(Log_WARNING, "Node registration successful.");
else
if(!registrationFailureLogged)
{
LogContext(logContext).log(Log_CRITICAL,
"Node registration not successful. Management node offline? Will keep on trying...");
registrationFailureLogged = true;
}
return nodeRegistered;
}
/**
* Download and sync metadata server target states and mirror buddy groups.
*
* @param outConsistencyState The new node consistency state.
*/
bool InternodeSyncer::updateMetaStatesAndBuddyGroups(TargetConsistencyState& outConsistencyState,
bool publish)
{
LOG(STATES, DEBUG, "Starting state update.");
App* app = Program::getApp();
NodeStore* mgmtNodes = app->getMgmtNodes();
TargetStateStore* metaStateStore = app->getMetaStateStore();
MirrorBuddyGroupMapper* buddyGroupMapper = app->getMetaBuddyGroupMapper();
static bool downloadFailedLogged = false; // to avoid log spamming
static bool publishFailedLogged = false;
NumNodeID localNodeID = app->getLocalNodeNumID();
auto node = mgmtNodes->referenceFirstNode();
if(!node)
{
LOG(STATES, ERR, "Management node not defined.");
return false;
}
unsigned numRetries = 10; // If publishing states fails 10 times, give up (-> POFFLINE).
// Note: Publishing fails if between downloadStatesAndBuddyGroups and
// publishLocalTargetStateChanges, a state on the mgmtd is changed (e.g. because the primary
// sets NEEDS_RESYNC for the secondary). In that case, we will retry.
LOG(STATES, DEBUG, "Beginning target state update...");
bool publishSuccess = false;
while (!publishSuccess && (numRetries--) )
{
MirrorBuddyGroupMap buddyGroups;
TargetStateMap states;
bool downloadRes = NodesTk::downloadStatesAndBuddyGroups(*node, NODETYPE_Meta, buddyGroups,
states, true);
if (!downloadRes)
{
if (!downloadFailedLogged)
{
LOG(STATES, WARNING,
"Downloading target states from management node failed. "
"Setting all target states to probably-offline.");
downloadFailedLogged = true;
}
metaStateStore->setAllStates(TargetReachabilityState_POFFLINE);
break;
}
downloadFailedLogged = false;
// Sync buddy groups here, because decideResync depends on it.
metaStateStore->syncStatesAndGroups(buddyGroupMapper, states,
std::move(buddyGroups), localNodeID);
CombinedTargetState newStateFromMgmtd;
// Find local state which was sent by mgmtd
for (const auto& state : states)
{
if (state.first == localNodeID.val())
{
newStateFromMgmtd = CombinedTargetState(state.second.reachabilityState,
state.second.consistencyState);
}
}
TargetConsistencyState localChangedState = decideResync(newStateFromMgmtd);
outConsistencyState = localChangedState;
if (!publish)
{
metaStateStore->setState(localNodeID.val(),
CombinedTargetState(TargetReachabilityState_ONLINE, localChangedState) );
return true;
}
// Note: In this case "old" means "before we changed it locally".
TargetConsistencyState oldState = newStateFromMgmtd.consistencyState;
publishSuccess = publishNodeStateChange(oldState, localChangedState);
if (publishSuccess)
{
metaStateStore->setState(localNodeID.val(),
CombinedTargetState(TargetReachabilityState_ONLINE, localChangedState) );
BuddyCommTk::checkBuddyNeedsResync();
}
}
if (!publishSuccess)
{
if (!publishFailedLogged)
{
LOG(STATES, WARNING, "Pushing local state to management node failed.");
publishFailedLogged = true;
}
}
else
publishFailedLogged = false;
return true;
}
/**
* Synchronize local client sessions with registered mgmtd clients to release orphaned sessions.
*
* @param clientsList must be ordered; contained nodes will be removed and may no longer be
* accessed after calling this method.
* @param allowRemoteComm usually true; setting this to false is only useful when called during
* app shutdown to avoid communication; if false, unlocking of user locks, closing of storage server
* files and disposal of unlinked files won't be performed
*/
void InternodeSyncer::syncClients(const std::vector<NodeHandle>& clientsList, bool allowRemoteComm)
{
const char* logContext = "Sync clients";
App* app = Program::getApp();
MetaStore* metaStore = Program::getApp()->getMetaStore();
SessionStore* sessions = app->getSessions();
SessionStore* mirroredSessions = app->getMirroredSessions();
SessionList removedSessions;
NumNodeIDList unremovableSessions;
sessions->syncSessions(clientsList, removedSessions, unremovableSessions);
mirroredSessions->syncSessions(clientsList, removedSessions, unremovableSessions);
// print client session removal results (upfront)
if(!removedSessions.empty() || !unremovableSessions.empty())
{
std::ostringstream logMsgStream;
logMsgStream << "Removing " << removedSessions.size() << " client sessions.";
if(unremovableSessions.empty() )
LogContext(logContext).log(Log_DEBUG, logMsgStream.str() ); // no unremovable sessions
else
{ // unremovable sessions found => log warning
logMsgStream << " (" << unremovableSessions.size() << " are unremovable)";
LogContext(logContext).log(Log_WARNING, logMsgStream.str() );
}
}
// walk over all removed sessions (to cleanup the contained files)
SessionListIter sessionIter = removedSessions.begin();
for( ; sessionIter != removedSessions.end(); sessionIter++)
{ // walk over all client sessions: cleanup each session
Session* session = *sessionIter;
NumNodeID sessionID = session->getSessionID();
SessionFileStore* sessionFiles = session->getFiles();
SessionFileList removedSessionFiles;
UIntList referencedSessionFiles;
sessionFiles->removeAllSessions(&removedSessionFiles, &referencedSessionFiles);
/* note: referencedSessionFiles should always be empty, because otherwise the reference holder
would also hold a reference to the client session (and we woudn't be here if the client
session had any references) */
// print session files results (upfront)
if (!removedSessionFiles.empty() || !referencedSessionFiles.empty())
{
std::ostringstream logMsgStream;
logMsgStream << "Removing " << removedSessionFiles.size() << " file sessions. ("
<< referencedSessionFiles.size() << " are unremovable). clientNumID: " << sessionID;
if (referencedSessionFiles.empty())
LogContext(logContext).log(Log_SPAM, logMsgStream.str() );
else
LogContext(logContext).log(Log_NOTICE, logMsgStream.str() );
}
// walk over all files in the current session (to clean them up)
SessionFileListIter fileIter = removedSessionFiles.begin();
for( ; fileIter != removedSessionFiles.end(); fileIter++)
{ // walk over all files: unlock user locks, close meta, close local, dispose unlinked
SessionFile* sessionFile = *fileIter;
unsigned ownerFD = sessionFile->getSessionID();
unsigned accessFlags = sessionFile->getAccessFlags();
unsigned numHardlinks;
unsigned numInodeRefs;
bool lastWriterClosed; // ignored here!
MetaFileHandle inode = sessionFile->releaseInode();
std::string fileID = inode->getEntryID();
std::string fileHandleID = SessionTk::generateFileHandleID(ownerFD, fileID);
// save nodeIDs for later
StripePattern* pattern = inode->getStripePattern();
int maxUsedNodesIndex = pattern->getStripeTargetIDs()->size() - 1;
// unlock all user locks
auto appendGranted = inode->flockAppendCancelByClientID(sessionID);
auto flockGranted = inode->flockEntryCancelByClientID(sessionID);
auto rangeGranted = inode->flockRangeCancelByClientID(sessionID);
if(allowRemoteComm)
{
LockingNotifier::notifyWaitersEntryLock(LockEntryNotifyType_APPEND,
inode->getReferenceParentID(), inode->getEntryID(), inode->getIsBuddyMirrored(),
std::move(appendGranted));
LockingNotifier::notifyWaitersEntryLock(LockEntryNotifyType_FLOCK,
inode->getReferenceParentID(), inode->getEntryID(), inode->getIsBuddyMirrored(),
std::move(flockGranted));
LockingNotifier::notifyWaitersRangeLock(inode->getReferenceParentID(),
inode->getEntryID(), inode->getIsBuddyMirrored(), std::move(rangeGranted));
}
EntryInfo* entryInfo = sessionFile->getEntryInfo();
FileIDLock lock(sessions->getEntryLockStore(), entryInfo->getEntryID(), true);
if(allowRemoteComm)
MsgHelperClose::closeChunkFile(sessionID, fileHandleID.c_str(),
maxUsedNodesIndex, *inode, entryInfo, NETMSG_DEFAULT_USERID);
LogContext(logContext).log(Log_NOTICE, "closing file. ParentID: " +
entryInfo->getParentEntryID() + " FileName: " + entryInfo->getFileName() );
metaStore->closeFile(entryInfo, std::move(inode), accessFlags, &numHardlinks,
&numInodeRefs, lastWriterClosed);
if(allowRemoteComm && !numHardlinks && !numInodeRefs)
MsgHelperClose::unlinkDisposableFile(fileID, NETMSG_DEFAULT_USERID,
entryInfo->getIsBuddyMirrored());
delete sessionFile;
} // end of files loop
delete(session);
} // end of client sessions loop
}
bool InternodeSyncer::downloadAndSyncNodes()
{
LOG(STATES, DEBUG, "Starting node list sync.");
App* app = Program::getApp();
NodeStore* mgmtNodes = app->getMgmtNodes();
auto mgmtNode = mgmtNodes->referenceFirstNode();
if(!mgmtNode)
return false;
Node& localNode = app->getLocalNode();
NodeStore* metaNodes = app->getMetaNodes();
NodeStore* storageNodes = app->getStorageNodes();
// metadata nodes
std::vector<NodeHandle> metaNodesList;
NumNodeIDList addedMetaNodes;
NumNodeIDList removedMetaNodes;
NumNodeID rootNodeID;
bool rootIsBuddyMirrored;
if(NodesTk::downloadNodes(*mgmtNode, NODETYPE_Meta, metaNodesList, true, &rootNodeID,
&rootIsBuddyMirrored) )
{
metaNodes->syncNodes(metaNodesList, &addedMetaNodes, &removedMetaNodes, &localNode);
if (app->getMetaRoot().setIfDefault(rootNodeID, rootIsBuddyMirrored))
{
LOG(STATES, CRITICAL,
std::string("Root node ID (from sync results): ") + rootNodeID.str());
app->getRootDir()->setOwnerNodeID(rootNodeID);
}
printSyncResults(NODETYPE_Meta, &addedMetaNodes, &removedMetaNodes);
}
// storage nodes
std::vector<NodeHandle> storageNodesList;
NumNodeIDList addedStorageNodes;
NumNodeIDList removedStorageNodes;
if(NodesTk::downloadNodes(*mgmtNode, NODETYPE_Storage, storageNodesList, true) )
{
storageNodes->syncNodes(storageNodesList, &addedStorageNodes, &removedStorageNodes,
&localNode);
printSyncResults(NODETYPE_Storage, &addedStorageNodes, &removedStorageNodes);
}
return true;
}
void InternodeSyncer::downloadAndSyncClients(bool requeue)
{
App* app = Program::getApp();
NodeStore* mgmtNodes = app->getMgmtNodes();
NodeStoreClients* clientNodes = app->getClientNodes();
TimerQueue* timerQ = app->getTimerQueue();
auto mgmtNode = mgmtNodes->referenceFirstNode();
if(!mgmtNode)
return;
std::vector<NodeHandle> clientNodesList;
NumNodeIDList addedClientNodes;
NumNodeIDList removedClientNodes;
if(NodesTk::downloadNodes(*mgmtNode, NODETYPE_Client, clientNodesList, true) )
{
clientNodes->syncNodes(clientNodesList, &addedClientNodes, &removedClientNodes);
printSyncResults(NODETYPE_Client, &addedClientNodes, &removedClientNodes);
syncClients(clientNodesList, true); // sync client sessions
}
if (requeue)
timerQ->enqueue(std::chrono::minutes(5),
[] { InternodeSyncer::downloadAndSyncClients(true); });
}
bool InternodeSyncer::downloadAndSyncTargetMappings()
{
LOG(STATES, DEBUG, "Syncing target mappings.");
App* app = Program::getApp();
TargetMapper* targetMapper = app->getTargetMapper();
NodeStore* mgmtNodes = app->getMgmtNodes();
auto mgmtNode = mgmtNodes->referenceFirstNode();
if(!mgmtNode)
return false;
auto mappings = NodesTk::downloadTargetMappings(*mgmtNode, true);
if (mappings.first)
targetMapper->syncTargets(std::move(mappings.second));
return true;
}
bool InternodeSyncer::downloadAndSyncStoragePools()
{
LOG(STORAGEPOOLS, DEBUG, "Syncing storage pools.");
const App* app = Program::getApp();
const NodeStore* mgmtNodes = app->getMgmtNodes();
StoragePoolStore* storagePoolStore = app->getStoragePoolStore();
const auto mgmtNode = mgmtNodes->referenceFirstNode();
if(!mgmtNode)
return false;
StoragePoolPtrVec storagePools;
// note: storage pool download does include capacity pools
const bool downloadStoragePoolsRes =
NodesTk::downloadStoragePools(*mgmtNode, storagePools, true);
if (!downloadStoragePoolsRes)
return false;
storagePoolStore->syncFromVector(storagePools);
return true;
}
/**
* Download and sync storage target states and mirror buddy groups.
*/
bool InternodeSyncer::downloadAndSyncTargetStatesAndBuddyGroups()
{
App* app = Program::getApp();
NodeStore* mgmtNodes = app->getMgmtNodes();
MirrorBuddyGroupMapper* buddyGroupMapper = app->getStorageBuddyGroupMapper();
TargetStateStore* targetStates = app->getTargetStateStore();
LOG(STATES, DEBUG, "Downloading target states and buddy groups");
static bool downloadFailedLogged = false;
auto mgmtNode = mgmtNodes->referenceFirstNode();
if(!mgmtNode)
return false;
TargetStateMap states;
MirrorBuddyGroupMap buddyGroups;
bool downloadRes = NodesTk::downloadStatesAndBuddyGroups(*mgmtNode, NODETYPE_Storage,
buddyGroups, states, true);
if ( downloadRes )
{
targetStates->syncStatesAndGroups(buddyGroupMapper, states,
std::move(buddyGroups), app->getLocalNode().getNumID());
downloadFailedLogged = false;
}
else
{ // download failed, so we don't know actual status => carefully set all to poffline
if(!downloadFailedLogged)
{
LOG(STATES, WARNING, "Download from management node failed. "
"Setting all targets to probably-offline.");
downloadFailedLogged = true;
}
targetStates->setAllStates(TargetReachabilityState_POFFLINE);
}
return true;
}
void InternodeSyncer::printSyncResults(NodeType nodeType, NumNodeIDList* addedNodes,
NumNodeIDList* removedNodes)
{
Logger* const logger = Logger::getLogger();
const int logLevel = nodeType == NODETYPE_Client ? Log_DEBUG : Log_WARNING;
if (!addedNodes->empty())
logger->log(LogTopic_STATES, logLevel, __func__,
std::string("Nodes added (sync results): ") +
StringTk::uintToStr(addedNodes->size() ) +
" (Type: " + boost::lexical_cast<std::string>(nodeType) + ")");
if (!removedNodes->empty())
logger->log(LogTopic_STATES, logLevel, __func__,
std::string("Nodes removed (sync results): ") +
StringTk::uintToStr(removedNodes->size() ) +
" (Type: " + boost::lexical_cast<std::string>(nodeType) + ")");
}
/**
* Decides new consistency state based on local state, buddy group membership and state fetched
* from management node.
*
* @param newState New state from the management node.
* @returns The new target consistency state.
*/
TargetConsistencyState InternodeSyncer::decideResync(const CombinedTargetState newState)
{
App* app = Program::getApp();
MirrorBuddyGroupMapper* metaBuddyGroupMapper = app->getMetaBuddyGroupMapper();
InternodeSyncer* internodeSyncer = app->getInternodeSyncer();
NumNodeID localNodeID = app->getLocalNodeNumID();
MirrorBuddyState buddyState = metaBuddyGroupMapper->getBuddyState(localNodeID.val());
if (buddyState == BuddyState_UNMAPPED)
return TargetConsistencyState_GOOD;
// If the consistency state is BAD, it stays BAD until admin intervenes.
if (internodeSyncer && // during early startup, INS is not constructed yet.
internodeSyncer->getNodeConsistencyState() == TargetConsistencyState_BAD)
return TargetConsistencyState_BAD;
const bool isResyncing = newState.consistencyState == TargetConsistencyState_NEEDS_RESYNC;
const bool isBad = newState.consistencyState == TargetConsistencyState_BAD;
if (internodeSyncer &&
internodeSyncer->getNodeConsistencyState() == TargetConsistencyState_NEEDS_RESYNC)
{
// If we're already (locally) maked as needs resync, this state can only be left when our
// (primary) buddy tells us the resync is finished.
return TargetConsistencyState_NEEDS_RESYNC;
}
else if (isResyncing || isBad)
{
return TargetConsistencyState_NEEDS_RESYNC;
}
else
{
// If mgmtd reports the target is (P)OFFLINE, then the meta server knows better and we set our
// state to GOOD / ONLINE. Otherwise we accept the state reported by the mgmtd.
if ( (newState.reachabilityState == TargetReachabilityState_OFFLINE)
|| (newState.reachabilityState == TargetReachabilityState_POFFLINE) )
return TargetConsistencyState_GOOD;
else
return newState.consistencyState;
}
}
bool InternodeSyncer::publishNodeStateChange(const TargetConsistencyState oldState,
const TargetConsistencyState newState)
{
const char* logContext = "Publish node state";
App* app = Program::getApp();
NodeStore* mgmtNodes = app->getMgmtNodes();
const NumNodeID localNodeID = app->getLocalNodeNumID();
auto mgmtNode = mgmtNodes->referenceFirstNode();
if (!mgmtNode)
{
LogContext(logContext).logErr("Management node not defined.");
return false;
}
bool res;
UInt8List oldStateList(1, oldState);
UInt8List newStateList(1, newState);
UInt16List nodeIDList(1, localNodeID.val());
ChangeTargetConsistencyStatesMsg msg(NODETYPE_Meta, &nodeIDList, &oldStateList, &newStateList);
RequestResponseArgs rrArgs(mgmtNode.get(), &msg, NETMSGTYPE_ChangeTargetConsistencyStatesResp);
#ifndef BEEGFS_DEBUG
rrArgs.logFlags |= REQUESTRESPONSEARGS_LOGFLAG_CONNESTABLISHFAILED
| REQUESTRESPONSEARGS_LOGFLAG_RETRY;
#endif // BEEGFS_DEBUG
bool sendRes = MessagingTk::requestResponse(&rrArgs);
static bool failureLogged = false;
if (!sendRes)
{
if (!failureLogged)
LogContext(logContext).log(Log_CRITICAL, "Pushing node state to management node failed.");
res = false;
failureLogged = true;
}
else
{
const auto respMsgCast =
static_cast<const ChangeTargetConsistencyStatesRespMsg*>(rrArgs.outRespMsg.get());
if ( (FhgfsOpsErr)respMsgCast->getValue() != FhgfsOpsErr_SUCCESS)
{
LogContext(logContext).log(Log_CRITICAL, "Management node did not accept node state change.");
res = false;
}
else
{
res = true;
}
failureLogged = false;
}
return res;
}
/**
* Send local node free space / free inode info to management node.
*/
void InternodeSyncer::publishNodeCapacity()
{
App* app = Program::getApp();
NodeStore* mgmtNodes = app->getMgmtNodes();
auto mgmtNode = mgmtNodes->referenceFirstNode();
if (!mgmtNode)
{
log.logErr("Management node not defined.");
return;
}
int64_t sizeTotal = 0;
int64_t sizeFree = 0;
int64_t inodesTotal = 0;
int64_t inodesFree = 0;
std::string metaPath = app->getMetaPath();
getStatInfo(&sizeTotal, &sizeFree, &inodesTotal, &inodesFree);
StorageTargetInfo targetInfo(app->getLocalNodeNumID().val(), metaPath, sizeTotal, sizeFree,
inodesTotal, inodesFree, getNodeConsistencyState());
// Note: As long as we don't have meta-HA, consistency state will always be GOOD.
StorageTargetInfoList targetInfoList(1, targetInfo);
SetStorageTargetInfoMsg msg(NODETYPE_Meta, &targetInfoList);
RequestResponseArgs rrArgs(mgmtNode.get(), &msg, NETMSGTYPE_SetStorageTargetInfoResp);
#ifndef BEEGFS_DEBUG
rrArgs.logFlags |= REQUESTRESPONSEARGS_LOGFLAG_CONNESTABLISHFAILED
| REQUESTRESPONSEARGS_LOGFLAG_RETRY;
#endif // BEEGFS_DEBUG
bool sendRes = MessagingTk::requestResponse(&rrArgs);
static bool failureLogged = false;
if (!sendRes)
{
if (!failureLogged)
log.log(Log_CRITICAL, "Pushing node free space to management node failed.");
failureLogged = true;
return;
}
else
{
const auto respMsgCast = (const SetStorageTargetInfoRespMsg*)rrArgs.outRespMsg.get();
failureLogged = false;
if ( (FhgfsOpsErr)respMsgCast->getValue() != FhgfsOpsErr_SUCCESS)
{
log.log(Log_CRITICAL, "Management node did not accept free space info message.");
return;
}
}
// If we were just started and are publishing our capacity for the first time, force a pool
// refresh on the mgmtd so we're not stuck in the emergency pool until the first regular
// pool refresh.
static bool firstTimePubilished = true;
if (firstTimePubilished)
{
forceMgmtdPoolsRefresh();
firstTimePubilished = false;
}
}
/**
* Drop/reset idle conns from all server stores.
*/
void InternodeSyncer::dropIdleConns()
{
App* app = Program::getApp();
unsigned numDroppedConns = 0;
numDroppedConns += dropIdleConnsByStore(app->getMgmtNodes() );
numDroppedConns += dropIdleConnsByStore(app->getMetaNodes() );
numDroppedConns += dropIdleConnsByStore(app->getStorageNodes() );
if(numDroppedConns)
{
log.log(Log_DEBUG, "Dropped idle connections: " + StringTk::uintToStr(numDroppedConns) );
}
}
/**
* Walk over all nodes in the given store and drop/reset idle connections.
*
* @return number of dropped connections
*/
unsigned InternodeSyncer::dropIdleConnsByStore(NodeStoreServers* nodes)
{
App* app = Program::getApp();
unsigned numDroppedConns = 0;
for (const auto& node : nodes->referenceAllNodes())
{
/* don't do any idle disconnect stuff with local node
(the LocalNodeConnPool doesn't support and doesn't need this kind of treatment) */
if (node.get() != &app->getLocalNode())
{
NodeConnPool* connPool = node->getConnPool();
numDroppedConns += connPool->disconnectAndResetIdleStreams();
}
}
return numDroppedConns;
}
void InternodeSyncer::getStatInfo(int64_t* outSizeTotal, int64_t* outSizeFree,
int64_t* outInodesTotal, int64_t* outInodesFree)
{
const char* logContext = "GetStorageTargetInfoMsg (stat path)";
std::string targetPathStr = Program::getApp()->getMetaPath();
bool statSuccess = StorageTk::statStoragePath(targetPathStr, outSizeTotal, outSizeFree,
outInodesTotal, outInodesFree);
if(unlikely(!statSuccess) )
{ // error
LogContext(logContext).logErr("Unable to statfs() storage path: " + targetPathStr +
" (SysErr: " + System::getErrString() );
*outSizeTotal = -1;
*outSizeFree = -1;
*outInodesTotal = -1;
*outInodesFree = -1;
}
// read and use value from manual free space override file (if it exists)
StorageTk::statStoragePathOverride(targetPathStr, outSizeFree, outInodesFree);
}
/**
* Tell mgmtd to update its capacity pools.
*/
void InternodeSyncer::forceMgmtdPoolsRefresh()
{
App* app = Program::getApp();
DatagramListener* dgramLis = app->getDatagramListener();
NodeStoreServers* mgmtNodes = app->getMgmtNodes();
auto mgmtNode = mgmtNodes->referenceFirstNode();
if (!mgmtNode)
{
log.log(Log_DEBUG, "Management node not defined.");
return;
}
RefreshCapacityPoolsMsg msg;
bool ackReceived = dgramLis->sendToNodeUDPwithAck(mgmtNode, &msg);
if (!ackReceived)
log.log(Log_DEBUG, "Management node did not accept pools refresh request.");
}
/**
* @return false on error
*/
bool InternodeSyncer::downloadExceededQuotaList(StoragePoolId storagePoolId, QuotaDataType idType,
QuotaLimitType exType, UIntList* outIDList, FhgfsOpsErr& error)
{
App* app = Program::getApp();
NodeStoreServers* mgmtNodes = app->getMgmtNodes();
bool retVal = false;
auto mgmtNode = mgmtNodes->referenceFirstNode();
if(!mgmtNode)
return false;
RequestExceededQuotaMsg msg(idType, exType, storagePoolId);
RequestExceededQuotaRespMsg* respMsgCast = NULL;
const auto respMsg = MessagingTk::requestResponse(*mgmtNode, msg,
NETMSGTYPE_RequestExceededQuotaResp);
if (!respMsg)
goto err_exit;
// handle result
respMsgCast = (RequestExceededQuotaRespMsg*)respMsg.get();
respMsgCast->getExceededQuotaIDs()->swap(*outIDList);
error = respMsgCast->getError();
retVal = true;
err_exit:
return retVal;
}
bool InternodeSyncer::downloadAllExceededQuotaLists(const StoragePoolPtrVec& storagePools)
{
bool retVal = true;
// note: this is fairly inefficient, but it is done only once on startup
for (auto iter = storagePools.begin(); iter != storagePools.end(); iter++)
{
if (!downloadAllExceededQuotaLists(*iter))
retVal = false;
}
return retVal;
}
/**
* @return false on error
*/
bool InternodeSyncer::downloadAllExceededQuotaLists(const StoragePoolPtr storagePool)
{
const char* logContext = "Exceeded quota sync";
App* app = Program::getApp();
Config* cfg = app->getConfig();
UInt16Set targets = storagePool->getTargets();
bool retVal = true;
UIntList tmpExceededUIDsSize;
UIntList tmpExceededGIDsSize;
UIntList tmpExceededUIDsInode;
UIntList tmpExceededGIDsInode;
FhgfsOpsErr error;
if (downloadExceededQuotaList(storagePool->getId(), QuotaDataType_USER, QuotaLimitType_SIZE,
&tmpExceededUIDsSize, error) )
{
// update exceeded store for every target in the pool
for (auto iter = targets.begin(); iter != targets.end(); iter++)
{
uint16_t targetId = *iter;
ExceededQuotaStorePtr exceededQuotaStore = app->getExceededQuotaStores()->get(targetId);
if (!exceededQuotaStore)
{
LOG(STORAGEPOOLS, ERR, "Could not access exceeded quota store in file size quota for users.", targetId);
retVal = false;
break;
}
exceededQuotaStore->updateExceededQuota(&tmpExceededUIDsSize, QuotaDataType_USER,
QuotaLimitType_SIZE);
}
// enable or disable quota enforcement
if (error == FhgfsOpsErr_NOTSUPP)
{
if (cfg->getQuotaEnableEnforcement())
{
LogContext(logContext).log(Log_DEBUG,
"Quota enforcement is enabled in the configuration of this metadata server, "
"but not on the management daemon. "
"The configuration from the management daemon overrides the local setting.");
}
else
{
LogContext(logContext).log(Log_DEBUG, "Quota enforcement disabled by management daemon.");
}
cfg->setQuotaEnableEnforcement(false);
return true;
}
else
{
if (!cfg->getQuotaEnableEnforcement())
{
LogContext(logContext).log(Log_DEBUG,
"Quota enforcement is enabled on the management daemon, "
"but not in the configuration of this metadata server. "
"The configuration from the management daemon overrides the local setting.");
}
else
{
LogContext(logContext).log(Log_DEBUG, "Quota enforcement enabled by management daemon.");
}
cfg->setQuotaEnableEnforcement(true);
}
}
else
{ // error
LogContext(logContext).logErr("Unable to download exceeded file size quota for users.");
retVal = false;
}
if (downloadExceededQuotaList(storagePool->getId(), QuotaDataType_GROUP, QuotaLimitType_SIZE,
&tmpExceededGIDsSize, error))
{
for (auto iter = targets.begin(); iter != targets.end(); iter++)
{
uint16_t targetId = *iter;
ExceededQuotaStorePtr exceededQuotaStore = app->getExceededQuotaStores()->get(targetId);
if (!exceededQuotaStore)
{
LOG(STORAGEPOOLS, ERR, "Could not access exceeded quota store in file size quota for groups.", targetId);
retVal = false;
break;
}
exceededQuotaStore->updateExceededQuota(&tmpExceededGIDsSize, QuotaDataType_GROUP,
QuotaLimitType_SIZE);
}
}
else
{ // error
LogContext(logContext).logErr("Unable to download exceeded file size quota for groups.");
retVal = false;
}
if (downloadExceededQuotaList(storagePool->getId(), QuotaDataType_USER, QuotaLimitType_INODE,
&tmpExceededUIDsInode, error))
{
for (auto iter = targets.begin(); iter != targets.end(); iter++)
{
uint16_t targetId = *iter;
ExceededQuotaStorePtr exceededQuotaStore = app->getExceededQuotaStores()->get(targetId);
if (!exceededQuotaStore)
{
LOG(STORAGEPOOLS, ERR, "Could not access exceeded quota store in file number quota for users.", targetId);
retVal = false;
break;
}
exceededQuotaStore->updateExceededQuota(&tmpExceededUIDsInode, QuotaDataType_USER,
QuotaLimitType_INODE);
}
}
else
{ // error
LogContext(logContext).logErr("Unable to download exceeded file number quota for users.");
retVal = false;
}
if (downloadExceededQuotaList(storagePool->getId(), QuotaDataType_USER, QuotaLimitType_INODE,
&tmpExceededGIDsInode, error))
{
for (auto iter = targets.begin(); iter != targets.end(); iter++)
{
uint16_t targetId = *iter;
ExceededQuotaStorePtr exceededQuotaStore = app->getExceededQuotaStores()->get(targetId);
if (!exceededQuotaStore)
{
LOG(STORAGEPOOLS, ERR, "Could not access exceeded quota store in file number quota for groups.", targetId);
retVal = false;
break;
}
exceededQuotaStore->updateExceededQuota(&tmpExceededGIDsInode, QuotaDataType_GROUP,
QuotaLimitType_INODE);
}
}
else
{ // error
LogContext(logContext).logErr("Unable to download exceeded file number quota for groups.");
retVal = false;
}
return retVal;
}
Reference in New Issue View Git Blame Copy Permalink