New upstream version 8.1.0

mon/source/components/worker/GetNodesWork.cpp

@@ -0,0 +1,40 @@
+#include "GetNodesWork.h"
+
+#include <common/toolkit/NodesTk.h>
+
+void GetNodesWork::process(char* bufIn, unsigned bufInLen, char* bufOut, unsigned bufOutLen)
+{
+   std::vector<std::shared_ptr<Node>> nodesList;
+   std::list<NumNodeID> addedNodes;
+   std::list<NumNodeID> removedNodes;
+
+
+
+   if (NodesTk::downloadNodes(*mgmtdNode, nodeType, nodesList, false))
+   {
+      // sync the downloaded list with the node store
+      nodes->syncNodes(nodesList, &addedNodes, &removedNodes, localNode.get());
+
+      if (!addedNodes.empty())
+         LOG(GENERAL, WARNING, "Nodes added.", ("addedNodes", addedNodes.size()), nodeType);
+
+      if (!removedNodes.empty())
+         LOG(GENERAL, WARNING, "Nodes removed.", ("removedNodes", removedNodes.size()), nodeType);
+   }
+   else
+   {
+      LOG(GENERAL, ERR, "Couldn't download server list from management daemon.", nodeType);
+   }
+
+   std::list<uint16_t> buddyGroupIDList;
+   std::list<uint16_t> primaryTargetIDList;
+   std::list<uint16_t> secondaryTargetIDList;
+
+   // update the storage buddy groups
+   if (NodesTk::downloadMirrorBuddyGroups(*mgmtdNode, nodeType, &buddyGroupIDList,
+       &primaryTargetIDList, &secondaryTargetIDList, false) )
+   {
+      buddyGroupMapper->syncGroupsFromLists(buddyGroupIDList, primaryTargetIDList,
+            secondaryTargetIDList, NumNodeID());
+   }
+}

mon/source/components/worker/GetNodesWork.h

@@ -0,0 +1,32 @@
+#ifndef GETNODESWORK_H_
+#define GETNODESWORK_H_
+
+#include <common/components/worker/Work.h>
+#include <common/nodes/MirrorBuddyGroupMapper.h>
+#include <common/nodes/NodeType.h>
+#include <common/nodes/NodeStoreServers.h>
+
+class GetNodesWork : public Work
+{
+   public:
+      GetNodesWork(std::shared_ptr<Node> mgmtdNode, NodeStoreServers *nodes, NodeType nodeType,
+         MirrorBuddyGroupMapper* buddyGroupMapper, std::shared_ptr<Node> localNode)
+          : mgmtdNode(std::move(mgmtdNode)),
+            nodes(nodes),
+            nodeType(nodeType),
+            buddyGroupMapper(buddyGroupMapper),
+            localNode(localNode)
+      {}
+
+      virtual void process(char* bufIn, unsigned bufInLen,
+            char* bufOut, unsigned bufOutLen) override;
+
+   private:
+      std::shared_ptr<Node> mgmtdNode;
+      NodeStoreServers* nodes;
+      NodeType nodeType;
+      MirrorBuddyGroupMapper* buddyGroupMapper;
+      std::shared_ptr<Node> localNode;
+};
+
+#endif /*GETNODESWORK_H_*/

mon/source/components/worker/RequestMetaDataWork.cpp

@@ -0,0 +1,69 @@
+#include "RequestMetaDataWork.h"
+
+#include <common/toolkit/MessagingTk.h>
+#include <common/net/message/nodes/HeartbeatRequestMsg.h>
+#include <common/net/message/mon/RequestMetaDataMsg.h>
+#include <common/net/message/mon/RequestMetaDataRespMsg.h>
+#include <components/StatsCollector.h>
+
+void RequestMetaDataWork::process(char* bufIn, unsigned bufInLen, char* bufOut, unsigned bufOutLen)
+{
+
+   if (!node->getIsResponding())
+   {
+      HeartbeatRequestMsg heartbeatRequestMsg;
+      if(MessagingTk::requestResponse(*node, heartbeatRequestMsg,
+            NETMSGTYPE_Heartbeat))
+      {
+         LOG(GENERAL, DEBUG, "Node is responding again.",
+               ("NodeID", node->getNodeIDWithTypeStr()));
+         node->setIsResponding(true);
+      }
+   }
+
+   Result result = {};
+   result.data.isResponding = false;
+
+   if (node->getIsResponding())
+   {
+      // generate the RequestDataMsg with the lastStatsTime
+      RequestMetaDataMsg requestDataMsg(node->getLastStatRequestTime().count());
+      auto respMsg = MessagingTk::requestResponse(*node, requestDataMsg,
+            NETMSGTYPE_RequestMetaDataResp);
+
+      if (!respMsg)
+      {
+         LOG(GENERAL, DEBUG, "Node is not responding.", ("NodeID", node->getNodeIDWithTypeStr()));
+         node->setIsResponding(false);
+      }
+      else
+      {
+         // get response and process it
+         auto metaRspMsg = static_cast<RequestMetaDataRespMsg*>(respMsg.get());
+         result.highResStatsList = std::move(metaRspMsg->getStatsList());
+
+         result.data.isResponding = true;
+         result.data.indirectWorkListSize = metaRspMsg->getIndirectWorkListSize();
+         result.data.directWorkListSize = metaRspMsg->getDirectWorkListSize();
+         result.data.sessionCount = metaRspMsg->getSessionCount();
+         result.data.hostnameid = metaRspMsg->gethostnameid();
+
+         if (!result.highResStatsList.empty())
+         {
+            auto lastStatsRequestTime = std::chrono::milliseconds(
+                  result.highResStatsList.front().rawVals.statsTimeMS);
+            node->setLastStatRequestTime(lastStatsRequestTime);
+         }
+
+         if (collectClientOpsByNode)
+            result.ipOpsUnorderedMap = ClientOpsRequestor::request(*node, false);
+
+         if (collectClientOpsByUser)
+            result.userOpsUnorderedMap = ClientOpsRequestor::request(*node, true);
+      }
+   }
+
+   result.node = std::move(node);
+
+   statsCollector->insertMetaData(std::move(result));
+}

mon/source/components/worker/RequestMetaDataWork.h

@@ -0,0 +1,42 @@
+#ifndef REQUESTMETADATAWORK_H_
+#define REQUESTMETADATAWORK_H_
+
+#include <common/components/worker/Work.h>
+#include <common/nodes/ClientOps.h>
+#include <misc/TSDatabase.h>
+#include <nodes/MetaNodeEx.h>
+
+class StatsCollector;
+
+class RequestMetaDataWork : public Work
+{
+   public:
+      struct Result
+      {
+         std::shared_ptr<MetaNodeEx> node;
+         MetaNodeDataContent data;
+         HighResStatsList highResStatsList;
+         ClientOpsRequestor::IdOpsUnorderedMap ipOpsUnorderedMap;
+         ClientOpsRequestor::IdOpsUnorderedMap userOpsUnorderedMap;
+      };
+
+      RequestMetaDataWork(std::shared_ptr<MetaNodeEx> node,
+            StatsCollector* statsCollector,
+            bool collectClientOpsByNode, bool collectClientOpsByUser) :
+            node(std::move(node)),
+            statsCollector(statsCollector),
+            collectClientOpsByNode(collectClientOpsByNode),
+            collectClientOpsByUser(collectClientOpsByUser)
+      {}
+
+      virtual void process(char* bufIn, unsigned bufInLen,
+            char* bufOut, unsigned bufOutLen) override;
+
+   private:
+      std::shared_ptr<MetaNodeEx> node;
+      StatsCollector* statsCollector;
+      bool collectClientOpsByNode;
+      bool collectClientOpsByUser;
+};
+
+#endif /*REQUESTMETADATAWORK_H_*/

mon/source/components/worker/RequestStorageDataWork.cpp

@@ -0,0 +1,74 @@
+#include "RequestStorageDataWork.h"
+
+#include <common/toolkit/MessagingTk.h>
+#include <common/net/message/nodes/HeartbeatRequestMsg.h>
+#include <common/net/message/mon/RequestStorageDataMsg.h>
+#include <common/net/message/mon/RequestStorageDataRespMsg.h>
+#include <components/StatsCollector.h>
+
+void RequestStorageDataWork::process(char* bufIn, unsigned bufInLen,
+      char* bufOut, unsigned bufOutLen)
+{
+
+   if (!node->getIsResponding())
+   {
+      HeartbeatRequestMsg heartbeatRequestMsg;
+
+      if(MessagingTk::requestResponse(*node, heartbeatRequestMsg,
+            NETMSGTYPE_Heartbeat))
+      {
+         LOG(GENERAL, DEBUG, "Node is responding again.",
+               ("NodeID", node->getNodeIDWithTypeStr()));
+         node->setIsResponding(true);
+      }
+   }
+
+   Result result = {};
+   result.data.isResponding = false;
+
+   if (node->getIsResponding())
+   {
+      // generate the RequestStorageDataMsg with the lastStatsTime
+      RequestStorageDataMsg requestDataMsg(node->getLastStatRequestTime().count());
+      auto respMsg = MessagingTk::requestResponse(*node, requestDataMsg,
+            NETMSGTYPE_RequestStorageDataResp);
+
+      if (!respMsg)
+      {
+         LOG(GENERAL, DEBUG, "Node is not responding.", ("NodeID", node->getNodeIDWithTypeStr()));
+         node->setIsResponding(false);
+      }
+      else
+      {
+         // get response and process it
+         auto storageRspMsg = static_cast<RequestStorageDataRespMsg*>(respMsg.get());
+         result.highResStatsList = std::move(storageRspMsg->getStatsList());
+         result.storageTargetList = std::move(storageRspMsg->getStorageTargets());
+
+         result.data.isResponding = true;
+         result.data.indirectWorkListSize = storageRspMsg->getIndirectWorkListSize();
+         result.data.directWorkListSize = storageRspMsg->getDirectWorkListSize();
+         result.data.diskSpaceTotal = storageRspMsg->getDiskSpaceTotalMiB();
+         result.data.diskSpaceFree = storageRspMsg->getDiskSpaceFreeMiB();
+         result.data.sessionCount = storageRspMsg->getSessionCount();
+         result.data.hostnameid = storageRspMsg->gethostnameid();
+
+         if (!result.highResStatsList.empty())
+         {
+            auto lastStatsRequestTime = std::chrono::milliseconds(
+                  result.highResStatsList.front().rawVals.statsTimeMS);
+            node->setLastStatRequestTime(lastStatsRequestTime);
+         }
+
+         if (collectClientOpsByNode)
+            result.ipOpsUnorderedMap = ClientOpsRequestor::request(*node, false);
+
+         if (collectClientOpsByUser)
+            result.userOpsUnorderedMap = ClientOpsRequestor::request(*node, true);
+      }
+   }
+
+   result.node = std::move(node);
+
+   statsCollector->insertStorageData(std::move(result));
+}

mon/source/components/worker/RequestStorageDataWork.h

@@ -0,0 +1,44 @@
+#ifndef REQUESTSTORAGEDATAWORK_H_
+#define REQUESTSTORAGEDATAWORK_H_
+
+#include <common/components/worker/Work.h>
+#include <common/nodes/ClientOps.h>
+#include <common/storage/StorageTargetInfo.h>
+#include <misc/TSDatabase.h>
+#include <nodes/StorageNodeEx.h>
+
+class StatsCollector;
+
+class RequestStorageDataWork : public Work
+{
+   public:
+      struct Result
+      {
+         std::shared_ptr<StorageNodeEx> node;
+         StorageNodeDataContent data;
+         HighResStatsList highResStatsList;
+         StorageTargetInfoList storageTargetList;
+         ClientOpsRequestor::IdOpsUnorderedMap ipOpsUnorderedMap;
+         ClientOpsRequestor::IdOpsUnorderedMap userOpsUnorderedMap;
+      };
+
+      RequestStorageDataWork(std::shared_ptr<StorageNodeEx> node,
+            StatsCollector* statsCollector, bool collectClientOpsByNode,
+            bool collectClientOpsByUser) :
+            node(std::move(node)),
+            statsCollector(statsCollector),
+            collectClientOpsByNode(collectClientOpsByNode),
+            collectClientOpsByUser(collectClientOpsByUser)
+      {}
+
+      void process(char* bufIn, unsigned bufInLen, char* bufOut,
+            unsigned bufOutLen);
+
+   private:
+      std::shared_ptr<StorageNodeEx> node;
+      StatsCollector* statsCollector;
+      bool collectClientOpsByNode;
+      bool collectClientOpsByUser;
+};
+
+#endif /*REQUESTSTORAGEDATAWORK_H_*/