New upstream version 8.1.0

client_module/source/nodes/NodeStoreEx.c

@@ -0,0 +1,588 @@
+#include <common/net/sock/NetworkInterfaceCard.h>
+#include <common/nodes/MirrorBuddyGroupMapper.h>
+#include <common/nodes/NodeListIter.h>
+#include <common/system/System.h>
+#include <os/OsCompat.h>
+#include "NodeStoreEx.h"
+
+#define NODESTORE_WARN_REFNUM 2000
+
+/**
+ * @param storeType will be applied to nodes on addOrUpdate()
+ */
+void NodeStoreEx_init(NodeStoreEx* this, App* app, NodeType storeType)
+{
+   this->app = app;
+
+   RWLock_init(&this->rwLock);
+
+   NodeTree_init(&this->nodeTree);
+
+   this->newNodeAppeared = NULL;
+
+   this->_rootOwner = NodeOrGroup_fromGroup(0); // 0 means undefined/invalid
+
+   this->storeType = storeType;
+}
+
+NodeStoreEx* NodeStoreEx_construct(App* app, NodeType storeType)
+{
+   NodeStoreEx* this = (NodeStoreEx*)os_kmalloc(sizeof(*this) );
+
+   NodeStoreEx_init(this, app, storeType);
+
+   return this;
+}
+
+void NodeStoreEx_uninit(NodeStoreEx* this)
+{
+   NodeTree_uninit(&this->nodeTree);
+}
+
+void NodeStoreEx_destruct(NodeStoreEx* this)
+{
+   NodeStoreEx_uninit(this);
+
+   kfree(this);
+}
+
+
+/**
+ * @param node belongs to the store after calling this method; this method will set (*node=NULL);
+ * so do not free it and don't use it any more afterwards (reference it from this store if you need
+ * it)
+ * @return true if the node was not in the store yet, false otherwise
+ */
+bool NodeStoreEx_addOrUpdateNode(NodeStoreEx* this, Node** node)
+{
+   const char* logContext = __func__;
+   Logger* log = App_getLogger(this->app);
+
+   NumNodeID nodeNumID = Node_getNumID(*node);
+   NodeString incomingAlias;
+   NodeString activeAlias;
+   bool setAliasResult;
+   Node* active;
+   NicAddressList nicList;
+   Node_copyAlias(*node, &incomingAlias);
+
+   // check if numeric ID is defined
+
+   if(unlikely(!nodeNumID.value) )
+   { // undefined numeric ID should never happen
+      Logger_logErrFormatted(log, logContext,
+         "Rejecting node with undefined numeric ID: %s; Type: %s",
+         incomingAlias.buf, Node_nodeTypeToStr(this->storeType) );
+      Node_put(*node);
+      *node = NULL;
+      return false;
+   }
+
+   RWLock_writeLock(&this->rwLock); // L O C K
+
+   // is node in any of the stores already?
+
+   active = NodeTree_find(&this->nodeTree, nodeNumID);
+
+   if(active)
+   { // node was in the store already => update it
+      Node_copyAlias(active, &activeAlias);
+      // If the string IDs differ, update the current active node ID from the incoming node ID.
+      // Ignore if the incomingNodeID is empty, this can happen when a node first starts up
+      // because it has to download its own alias from the mgmtd.
+      if(incomingAlias.buf[0] != '\0' && strcmp(incomingAlias.buf, activeAlias.buf))
+      {
+         // Before 8.0 BeeGFS logged "numeric ID collision for two different node string IDs".
+         // Starting in 8.0 string IDs are considered aliases and can be updated as needed.
+         Logger_logFormatted(log, 3, logContext,
+            "Updating alias for node: %s -> %s; Type: %s",
+            activeAlias.buf, incomingAlias.buf, Node_nodeTypeToStr(this->storeType) );
+         // The node type should not be updated this way so we set it to invalid (no update).
+         setAliasResult = Node_setNodeAliasAndType(active, incomingAlias.buf, NODETYPE_Invalid);
+
+         if (!setAliasResult) {
+            NodeString nodeAndType;
+            Node_copyAliasWithTypeStr(active, &nodeAndType);
+
+            Logger_logErrFormatted(log, logContext,
+               // Partial updates should never happen. Print what is set for both the alias and node
+               // alias with type string. We don't know what may be helpful for debugging.
+               "Error updating alias for node: %s : %s (ignoring)",
+               activeAlias.buf, nodeAndType.buf);
+         }
+      }
+
+      // update heartbeat time of existing node
+      Node_cloneNicList(*node, &nicList);
+      Node_updateLastHeartbeatT(active);
+      Node_updateInterfaces(active, Node_getPortUDP(*node), Node_getPortTCP(*node),
+         &nicList);
+      ListTk_kfreeNicAddressListElems(&nicList);
+      NicAddressList_uninit(&nicList);
+
+      Node_put(*node);
+   }
+   else
+   { // node is not currently active => insert it
+      NodeTree_insert(&this->nodeTree, nodeNumID, *node);
+
+      #ifdef BEEGFS_DEBUG
+      // check whether this node type and store type differ
+      if( (Node_getNodeType(*node) != NODETYPE_Invalid) &&
+          (Node_getNodeType(*node) != this->storeType) )
+      {
+         Logger_logErrFormatted(log, logContext,
+            "Node type and store type differ. Node: %s %s; Store: %s",
+            Node_getNodeTypeStr(*node), incomingAlias.buf, Node_nodeTypeToStr(this->storeType) );
+      }
+      #endif // BEEGFS_DEBUG
+
+      Node_setIsActive(*node, true);
+      Node_setNodeAliasAndType(*node, NULL, this->storeType);
+
+      __NodeStoreEx_handleNodeVersion(this, *node);
+
+      if(this->newNodeAppeared)
+         complete(this->newNodeAppeared);
+   }
+
+   RWLock_writeUnlock(&this->rwLock); // U N L O C K
+
+   *node = NULL;
+
+   return !active;
+}
+
+/**
+ * Note: remember to call releaseNode()
+ *
+ * @return NULL if no such node exists
+ */
+Node* NodeStoreEx_referenceNode(NodeStoreEx* this, NumNodeID id)
+{
+   Logger* log = App_getLogger(this->app);
+   Node* node = NULL;
+
+   // check for invalid id 0
+   #ifdef BEEGFS_DEBUG
+   if(!id.value)
+   {
+      Logger_log(log, Log_CRITICAL, __func__, "BUG?: Attempt to reference numeric node ID '0'");
+      dump_stack();
+   }
+   #endif // BEEGFS_DEBUG
+
+   IGNORE_UNUSED_VARIABLE(log);
+
+   RWLock_readLock(&this->rwLock); // L O C K
+
+   node = NodeTree_find(&this->nodeTree, id);
+   if (likely(node))
+   { // found it
+      unsigned refs;
+      Node_get(node);
+
+      (void) refs;
+      // check for unusually high reference count
+#ifdef BEEGFS_DEBUG
+# ifdef KERNEL_HAS_KREF_READ
+      refs = kref_read(&node->references);
+# else
+      refs = atomic_read(&node->references.refcount);
+#endif
+
+      if (refs > NODESTORE_WARN_REFNUM) {
+         NodeString alias;
+         Node_copyAlias(node, &alias);
+         Logger_logFormatted(log, Log_CRITICAL, __func__,
+            "WARNING: Lots of references to node (=> leak?): %s %s; ref count: %d",
+            Node_getNodeTypeStr(node), alias.buf, refs); // G E T
+      }
+#endif // BEEGFS_DEBUG
+   }
+
+   RWLock_readUnlock(&this->rwLock); // U N L O C K
+
+   return node;
+}
+
+/**
+ * Note: remember to call releaseNode()
+ *
+ * @return NULL if no such node exists
+ */
+Node* NodeStoreEx_referenceRootNode(NodeStoreEx* this, NodeOrGroup* rootID)
+{
+   Node* node = NULL;
+   MirrorBuddyGroupMapper* metaBuddyGroupMapper = App_getMetaBuddyGroupMapper(this->app);
+   NumNodeID nodeID;
+
+   RWLock_readLock(&this->rwLock); // L O C K
+
+   *rootID = this->_rootOwner;
+   if (this->_rootOwner.isGroup)
+      nodeID.value = MirrorBuddyGroupMapper_getPrimaryTargetID(metaBuddyGroupMapper,
+            this->_rootOwner.node.value);
+   else
+      nodeID = this->_rootOwner.node;
+
+   node = NodeTree_find(&this->nodeTree, nodeID);
+   if(likely(node) )
+      Node_get(node);
+
+   RWLock_readUnlock(&this->rwLock); // U N L O C K
+
+   return node;
+}
+
+/**
+ * This is a helper to have only one call for the typical targetMapper->getNodeID() and following
+ * referenceNode() calls.
+ *
+ * Note: remember to call releaseNode().
+ *
+ * @param targetMapper where to resolve the given targetID
+ * @param outErr will be set to FhgfsOpsErr_UNKNOWNNODE, _UNKNOWNTARGET, _SUCCESS (may be NULL)
+ * @return NULL if targetID is not mapped or if the mapped node does not exist in the store.
+ */
+Node* NodeStoreEx_referenceNodeByTargetID(NodeStoreEx* this, uint16_t targetID,
+   TargetMapper* targetMapper, FhgfsOpsErr* outErr)
+{
+   NumNodeID nodeID;
+   Node* node;
+
+   nodeID = TargetMapper_getNodeID(targetMapper, targetID);
+   if(!nodeID.value)
+   {
+      SAFE_ASSIGN(outErr, FhgfsOpsErr_UNKNOWNTARGET);
+      return NULL;
+   }
+
+   node = NodeStoreEx_referenceNode(this, nodeID);
+
+   if(!node)
+   {
+      SAFE_ASSIGN(outErr, FhgfsOpsErr_UNKNOWNNODE);
+      return NULL;
+   }
+
+   SAFE_ASSIGN(outErr, FhgfsOpsErr_SUCCESS);
+   return node;
+}
+
+/**
+ * @return true if node existed as active node
+ */
+bool NodeStoreEx_deleteNode(NodeStoreEx* this, NumNodeID nodeID)
+{
+   const char* logContext = __func__;
+   Logger* log = App_getLogger(this->app);
+
+   bool nodeWasActive;
+
+   #ifdef BEEGFS_DEBUG
+   if(unlikely(!nodeID.value) ) // should never happen
+      Logger_logFormatted(log, Log_CRITICAL, logContext, "Called with invalid node ID '0'");
+   #endif // BEEGFS_DEBUG
+
+   IGNORE_UNUSED_VARIABLE(logContext);
+   IGNORE_UNUSED_VARIABLE(log);
+
+   RWLock_writeLock(&this->rwLock); // L O C K
+
+   nodeWasActive = NodeTree_erase(&this->nodeTree, nodeID);
+
+   RWLock_writeUnlock(&this->rwLock); // U N L O C K
+
+   return nodeWasActive;
+}
+
+unsigned NodeStoreEx_getSize(NodeStoreEx* this)
+{
+   unsigned nodesSize;
+
+   RWLock_readLock(&this->rwLock); // L O C K
+
+   nodesSize = this->nodeTree.size;
+
+   RWLock_readUnlock(&this->rwLock); // U N L O C K
+
+   return nodesSize;
+}
+
+/**
+ * This is used to iterate over all stored nodes.
+ * Start with this and then use referenceNextNode() until it returns NULL.
+ *
+ * Note: remember to call releaseNode()
+ *
+ * @return can be NULL
+ */
+Node* NodeStoreEx_referenceFirstNode(NodeStoreEx* this)
+{
+   NodeTreeIter iter;
+   Node* resultNode = NULL;
+
+   RWLock_readLock(&this->rwLock); // L O C K
+
+   NodeTreeIter_init(&iter, &this->nodeTree);
+   if (!NodeTreeIter_end(&iter) )
+   {
+      resultNode = NodeTreeIter_value(&iter);
+      Node_get(resultNode);
+   }
+
+   RWLock_readUnlock(&this->rwLock); // U N L O C K
+
+   return resultNode;
+}
+
+/**
+ * Note: remember to call releaseNode()
+ *
+ * @return NULL if nodeID was the last node
+ */
+Node* NodeStoreEx_referenceNextNodeAndReleaseOld(NodeStoreEx* this, Node* oldNode)
+{
+   Node* result = NULL;
+
+   RWLock_readLock(&this->rwLock); // L O C K
+
+   result = NodeTree_getNext(&this->nodeTree, oldNode);
+   if (result)
+      Node_get(result);
+
+   Node_put(oldNode);
+
+   RWLock_readUnlock(&this->rwLock); // U N L O C K
+
+   return result;
+}
+
+/**
+ * @return 0 if no root node is known
+ */
+NodeOrGroup NodeStoreEx_getRootOwner(NodeStoreEx* this)
+{
+   NodeOrGroup owner;
+
+   RWLock_readLock(&this->rwLock); // L O C K
+
+   owner = this->_rootOwner;
+
+   RWLock_readUnlock(&this->rwLock); // U N L O C K
+
+   return owner;
+}
+
+/**
+ * Set internal root node ID.
+ *
+ * @return false if the new ID was rejected (e.g. because we already had an id set and
+ * ignoreExistingRoot was false).
+ */
+bool NodeStoreEx_setRootOwner(NodeStoreEx* this, NodeOrGroup owner,
+   bool ignoreExistingRoot)
+{
+   bool setRootRes = true;
+
+   // don't allow invalid id 0 (if not forced to do so)
+   if(!owner.group && !ignoreExistingRoot)
+      return false;
+
+   RWLock_writeLock(&this->rwLock); // L O C K
+
+   if (!NodeOrGroup_valid(this->_rootOwner))
+   { // rootID empty => set the new root
+      this->_rootOwner = owner;
+   }
+   else if (!ignoreExistingRoot)
+   { // root defined already, reject new root
+      setRootRes = false;
+   }
+   else
+   { // root defined already, but shall be ignored
+      this->_rootOwner = owner;
+   }
+
+   RWLock_writeUnlock(&this->rwLock); // U N L O C K
+
+   return setRootRes;
+}
+
+/**
+ * Waits for the first node that is added to the store.
+ *
+ * @return true when a new node was added to the store before the timeout expired
+ */
+bool NodeStoreEx_waitForFirstNode(NodeStoreEx* this, int waitTimeoutMS)
+{
+   bool retVal = false;
+   struct completion cond;
+
+   RWLock_readLock(&this->rwLock); // L O C K
+
+   retVal = this->nodeTree.size > 0;
+
+   RWLock_readUnlock(&this->rwLock); // U N L O C K
+
+   if(retVal)
+      return retVal;
+
+   RWLock_writeLock(&this->rwLock); // L O C K
+
+   WARN_ON(this->newNodeAppeared);
+   init_completion(&cond);
+   this->newNodeAppeared = &cond;
+
+   RWLock_writeUnlock(&this->rwLock); // U N L O C K
+
+   /* may time out or not, we don't care. activeCount is what's important */
+   wait_for_completion_timeout(&cond, TimeTk_msToJiffiesSchedulable(waitTimeoutMS) );
+
+   RWLock_writeLock(&this->rwLock); // L O C K
+
+   this->newNodeAppeared = NULL;
+   retVal = this->nodeTree.size > 0;
+
+   RWLock_writeUnlock(&this->rwLock); // U N L O C K
+
+   return retVal;
+}
+
+
+/**
+ * @param masterList must be ordered; contained nodes will be removed and may no longer be
+ * accessed after calling this method.
+ * @param appLocalNode just what you get from app->getLocalNode(), to determine NIC capabilities
+ */
+void NodeStoreEx_syncNodes(NodeStoreEx* this, NodeList* masterList, NumNodeIDList* outAddedIDs,
+   NumNodeIDList* outRemovedIDs, Node* appLocalNode)
+{
+   // Note: We have two phases here:
+   //    Phase 1 (locked): Identify added/removed nodes.
+   //    Phase 2 (unlocked): Add/remove nodes from store.
+   // This separation is required to not break compatibility with virtual overwritten add/remove
+   // methods in derived classes (e.g. fhgfs_mgmtd).
+
+
+   // P H A S E 1 (Identify added/removed nodes.)
+
+   NodeTreeIter activeIter;
+   NodeListIter masterIter;
+
+   NumNodeIDListIter removedIDsIter;
+   NodeList addLaterNodes; // nodes to be added in phase 2
+   NodeListIter addLaterIter;
+
+   NicListCapabilities localNicCaps;
+
+   NodeList_init(&addLaterNodes);
+
+
+   RWLock_writeLock(&this->rwLock); // L O C K
+
+   NodeTreeIter_init(&activeIter, &this->nodeTree);
+   NodeListIter_init(&masterIter, masterList);
+
+   while(!NodeTreeIter_end(&activeIter) && !NodeListIter_end(&masterIter) )
+   {
+      Node* active = NodeTreeIter_value(&activeIter);
+      NumNodeID currentActive = Node_getNumID(active);
+      NumNodeID currentMaster = Node_getNumID(NodeListIter_value(&masterIter) );
+
+      if(currentMaster.value < currentActive.value)
+      { // currentMaster is added
+         NumNodeIDList_append(outAddedIDs, currentMaster);
+         NodeList_append(&addLaterNodes, NodeListIter_value(&masterIter) );
+
+         NodeList_removeHead(masterList);
+         NodeListIter_init(&masterIter, masterList);
+      }
+      else
+      if(currentActive.value < currentMaster.value)
+      { // currentActive is removed
+         NumNodeIDList_append(outRemovedIDs, currentActive);
+         NodeTreeIter_next(&activeIter);
+      }
+      else
+      { // node unchanged
+         NodeList_append(&addLaterNodes, NodeListIter_value(&masterIter) );
+
+         NodeTreeIter_next(&activeIter);
+
+         NodeList_removeHead(masterList);
+         NodeListIter_init(&masterIter, masterList);
+      }
+   }
+
+   // remaining masterList nodes are added
+   while(!NodeListIter_end(&masterIter) )
+   {
+      NumNodeID currentMaster = Node_getNumID(NodeListIter_value(&masterIter) );
+
+      NumNodeIDList_append(outAddedIDs, currentMaster);
+      NodeList_append(&addLaterNodes, NodeListIter_value(&masterIter) );
+
+      NodeList_removeHead(masterList);
+      NodeListIter_init(&masterIter, masterList);
+   }
+
+   // remaining active nodes are removed
+   for(; !NodeTreeIter_end(&activeIter); NodeTreeIter_next(&activeIter) )
+   {
+      Node* active = NodeTreeIter_value(&activeIter);
+
+      NumNodeIDList_append(outRemovedIDs, Node_getNumID(active) );
+   }
+
+   RWLock_writeUnlock(&this->rwLock); // U N L O C K
+
+
+   // P H A S E 2 (Add/remove nodes from store.)
+
+   // remove nodes
+   NumNodeIDListIter_init(&removedIDsIter, outRemovedIDs);
+
+   while(!NumNodeIDListIter_end(&removedIDsIter) )
+   {
+      NumNodeID nodeID = NumNodeIDListIter_value(&removedIDsIter);
+      NumNodeIDListIter_next(&removedIDsIter); // (removal invalidates iter)
+
+      NodeStoreEx_deleteNode(this, nodeID);
+   }
+
+   // set local nic capabilities
+   if(appLocalNode)
+   {
+      NodeConnPool* connPool = Node_getConnPool(appLocalNode);
+      NodeConnPool_lock(connPool);
+      NIC_supportedCapabilities(NodeConnPool_getNicListLocked(connPool), &localNicCaps);
+      NodeConnPool_unlock(connPool);
+   }
+
+   // add nodes
+   NodeListIter_init(&addLaterIter, &addLaterNodes);
+
+   for(; !NodeListIter_end(&addLaterIter); NodeListIter_next(&addLaterIter) )
+   {
+      Node* node = NodeListIter_value(&addLaterIter);
+
+      if(appLocalNode)
+         NodeConnPool_setLocalNicCaps(Node_getConnPool(node), &localNicCaps);
+
+      NodeStoreEx_addOrUpdateNode(this, &node);
+   }
+
+   NodeList_uninit(&addLaterNodes);
+}
+
+/**
+ * Take special actions based on version of a (typically new) node, e.g. compat flags deactivation.
+ *
+ * Note: Caller must hold lock.
+ */
+void __NodeStoreEx_handleNodeVersion(NodeStoreEx* this, Node* node)
+{
+   // nothing to be done here currently
+}

client_module/source/nodes/NodeStoreEx.h

@@ -0,0 +1,82 @@
+#ifndef NODESTOREEX_H_
+#define NODESTOREEX_H_
+
+#include <app/log/Logger.h>
+#include <app/App.h>
+#include <common/toolkit/list/NumNodeIDList.h>
+#include <common/toolkit/list/UInt16List.h>
+#include <common/storage/EntryInfo.h>
+#include <common/storage/StorageErrors.h>
+#include <common/threading/Condition.h>
+#include <common/threading/RWLock.h>
+#include <common/toolkit/StringTk.h>
+#include <common/nodes/Node.h>
+#include <common/nodes/NodeList.h>
+#include <common/nodes/NodeTree.h>
+#include <common/nodes/TargetMapper.h>
+#include <common/Common.h>
+#include <linux/completion.h>
+#include <linux/rwsem.h>
+
+
+struct NodeStoreEx;
+typedef struct NodeStoreEx NodeStoreEx;
+
+
+extern void NodeStoreEx_init(NodeStoreEx* this, App* app, NodeType storeType);
+extern NodeStoreEx* NodeStoreEx_construct(App* app, NodeType storeType);
+extern void NodeStoreEx_uninit(NodeStoreEx* this);
+extern void NodeStoreEx_destruct(NodeStoreEx* this);
+
+extern bool NodeStoreEx_addOrUpdateNode(NodeStoreEx* this, Node** node);
+extern Node* NodeStoreEx_referenceNode(NodeStoreEx* this, NumNodeID id);
+extern Node* NodeStoreEx_referenceRootNode(NodeStoreEx* this, NodeOrGroup* rootID);
+extern Node* NodeStoreEx_referenceNodeByTargetID(NodeStoreEx* this, uint16_t targetID,
+   TargetMapper* targetMapper, FhgfsOpsErr* outErr);
+extern bool NodeStoreEx_deleteNode(NodeStoreEx* this, NumNodeID nodeID);
+extern unsigned NodeStoreEx_getSize(NodeStoreEx* this);
+
+extern Node* NodeStoreEx_referenceFirstNode(NodeStoreEx* this);
+extern Node* NodeStoreEx_referenceNextNode(NodeStoreEx* this, NumNodeID nodeID);
+extern Node* NodeStoreEx_referenceNextNodeAndReleaseOld(NodeStoreEx* this, Node* oldNode);
+
+extern NodeOrGroup NodeStoreEx_getRootOwner(NodeStoreEx* this);
+extern bool NodeStoreEx_setRootOwner(NodeStoreEx* this, NodeOrGroup owner,
+   bool ignoreExistingRoot);
+
+extern bool NodeStoreEx_waitForFirstNode(NodeStoreEx* this, int waitTimeoutMS);
+
+extern void NodeStoreEx_syncNodes(NodeStoreEx* this, NodeList* masterList, NumNodeIDList* outAddedIDs,
+   NumNodeIDList* outRemovedIDs, Node* localNode);
+
+
+extern void __NodeStoreEx_handleNodeVersion(NodeStoreEx* this, Node* node);
+
+
+// getters & setters
+static inline NodeType NodeStoreEx_getStoreType(NodeStoreEx* this);
+
+
+/**
+ * This is the corresponding class for NodeStoreServers in userspace.
+ */
+struct NodeStoreEx
+{
+   App* app;
+
+   RWLock rwLock;
+   struct completion* newNodeAppeared; /* signalled when nodes are added (NULL without waiters) */
+
+   NodeTree nodeTree;
+
+   NodeOrGroup _rootOwner;
+
+   NodeType storeType; // will be applied to nodes on addOrUpdate()
+};
+
+NodeType NodeStoreEx_getStoreType(NodeStoreEx* this)
+{
+   return this->storeType;
+}
+
+#endif /*NODESTOREEX_H_*/