New upstream version 8.1.0

storage/source/app/App.h

@@ -0,0 +1,424 @@
+#pragma once
+
+#include <app/config/Config.h>
+#include <common/app/log/LogContext.h>
+#include <common/app/log/Logger.h>
+#include <common/app/AbstractApp.h>
+#include <common/components/streamlistenerv2/ConnAcceptor.h>
+#include <common/components/streamlistenerv2/StreamListenerV2.h>
+#include <common/components/worker/queue/MultiWorkQueue.h>
+#include <common/components/worker/Worker.h>
+#include <common/components/TimerQueue.h>
+#include <common/nodes/MirrorBuddyGroupMapper.h>
+#include <common/nodes/NodeStoreServers.h>
+#include <common/nodes/TargetStateStore.h>
+#include <common/storage/Path.h>
+#include <common/storage/Storagedata.h>
+#include <common/toolkit/AcknowledgmentStore.h>
+#include <common/storage/quota/ExceededQuotaStore.h>
+#include <common/toolkit/NetFilter.h>
+#include <common/Common.h>
+#include <components/benchmarker/StorageBenchOperator.h>
+#include <components/buddyresyncer/BuddyResyncer.h>
+#include <components/chunkfetcher/ChunkFetcher.h>
+#include <components/DatagramListener.h>
+#include <components/InternodeSyncer.h>
+#include <components/StorageStatsCollector.h>
+#include <net/message/NetMessageFactory.h>
+#include <nodes/StorageNodeOpStats.h>
+#include <session/SessionStore.h>
+#include <storage/ChunkLockStore.h>
+#include <storage/ChunkStore.h>
+#include <storage/SyncedStoragePaths.h>
+#include <storage/StorageTargets.h>
+#include <toolkit/QuotaTk.h>
+
+
+#ifndef BEEGFS_VERSION
+   #error BEEGFS_VERSION undefined
+#endif
+
+// program return codes
+#define APPCODE_NO_ERROR               0
+#define APPCODE_INVALID_CONFIG         1
+#define APPCODE_INITIALIZATION_ERROR   2
+#define APPCODE_RUNTIME_ERROR          3
+
+
+typedef std::list<Worker*> WorkerList;
+typedef WorkerList::iterator WorkerListIter;
+
+typedef std::vector<StreamListenerV2*> StreamLisVec;
+typedef StreamLisVec::iterator StreamLisVecIter;
+
+
+// forward declarations
+class LogContext;
+
+class App : public AbstractApp
+{
+   public:
+      App(int argc, char** argv);
+      virtual ~App();
+
+      virtual void run() override;
+
+      virtual void stopComponents() override;
+      virtual void handleComponentException(std::exception& e) override;
+      virtual void handleNetworkInterfaceFailure(const std::string& devname) override;
+
+      void handleNetworkInterfacesChanged(NicAddressList nicList);
+
+   private:
+      int appResult;
+      int argc;
+      char** argv;
+
+      Config* cfg;
+      LogContext* log;
+      std::list<std::string> allowedInterfaces;
+
+      LockFD pidFileLockFD;
+      std::vector<LockFD> storageTargetLocks;
+
+      NetFilter* netFilter; // empty filter means "all nets allowed"
+      NetFilter* tcpOnlyFilter; // for IPs that allow only plain TCP (no RDMA etc)
+      std::shared_ptr<Node> localNode;
+
+      NodeStoreServers* mgmtNodes;
+      NodeStoreServers* metaNodes; // needed for backward communication introduced with GAM integration
+      NodeStoreServers* storageNodes;
+
+      TargetMapper* targetMapper;
+      MirrorBuddyGroupMapper* mirrorBuddyGroupMapper; // maps targets to mirrorBuddyGroups
+      TargetStateStore* targetStateStore; // map storage targets to a state
+
+      MultiWorkQueueMap workQueueMap; // maps targetIDs to WorkQueues
+      SessionStore* sessions;
+      StorageNodeOpStats* nodeOperationStats; // file system operation statistics
+      AcknowledgmentStore* ackStore;
+      NetMessageFactory* netMessageFactory;
+
+      StorageTargets* storageTargets; // target IDs and corresponding storage paths
+      SyncedStoragePaths* syncedStoragePaths; // serializes access to paths (=> entryIDs)
+      StorageBenchOperator* storageBenchOperator; // benchmark for the storage
+
+      DatagramListener* dgramListener;
+      ConnAcceptor* connAcceptor;
+      StatsCollector* statsCollector;
+      InternodeSyncer* internodeSyncer;
+      TimerQueue* timerQueue;
+
+      ChunkFetcher* chunkFetcher;
+
+      unsigned numStreamListeners; // value copied from cfg (for performance)
+      StreamLisVec streamLisVec;
+
+      WorkerList workerList;
+      bool workersRunning;
+      Mutex mutexWorkersRunning;
+
+      ChunkStore* chunkDirStore;
+
+      unsigned nextNumaBindTarget; // the numa node to which we will bind the next component thread
+
+      ExceededQuotaPerTarget exceededQuotaStores;
+
+      BuddyResyncer* buddyResyncer;
+      ChunkLockStore* chunkLockStore;
+
+      std::unique_ptr<StoragePoolStore> storagePoolStore;
+
+      void* dlOpenHandleLibZfs;  // handle of the libzfs from dlopen
+      bool libZfsErrorReported;
+
+      void runNormal();
+
+      void streamListenersInit();
+      void streamListenersStart();
+      void streamListenersStop();
+      void streamListenersDelete();
+      void streamListenersJoin();
+
+      void workersInit();
+      void workersStart();
+      void workersStop();
+      void workersDelete();
+      void workersJoin();
+
+      void initLogging();
+      void initDataObjects();
+      void initBasicNetwork();
+      void initLocalNodeIDs(NumNodeID& outLocalNodeNumID);
+      void initLocalNode(NumNodeID localNodeNumID);
+      void initLocalNodeNumIDFile(NumNodeID localNodeNumID) ;
+      void preinitStorage();
+      void checkTargetsUUIDs();
+      void initStorage();
+      void initPostTargetRegistration();
+      void initComponents();
+
+      void startComponents();
+      void joinComponents();
+
+      bool waitForMgmtNode();
+      bool preregisterNode(NumNodeID& outLocalNodeNumID);
+      boost::optional<std::map<uint16_t, std::unique_ptr<StorageTarget>>> preregisterTargets(
+            const NumNodeID localNodeNumID);
+      bool preregisterTarget(Node& mgmtNode, std::string targetID, uint16_t targetNumID,
+         uint16_t* outNewTargetNumID);
+      bool registerAndDownloadMgmtInfo();
+
+      void logInfos();
+
+      void setUmask();
+
+      void daemonize();
+
+      void registerSignalHandler();
+      static void signalHandler(int sig);
+
+      bool restoreSessions();
+      bool storeSessions();
+      bool deleteSessionFiles();
+
+      bool openLibZfs();
+      bool closeLibZfs();
+
+
+   public:
+      /**
+       * Get one of the available stream listeners based on the socket file descriptor number.
+       * This is to load-balance the sockets over all available stream listeners and ensure that
+       * sockets are not bouncing between different stream listeners.
+       *
+       * Note that IB connections eat two fd numbers, so 2 and multiples of 2 might not be a good
+       * value for number of stream listeners.
+       */
+      virtual StreamListenerV2* getStreamListenerByFD(int fd) override
+      {
+         return streamLisVec[fd % numStreamListeners];
+      }
+
+      // getters & setters
+      virtual const ICommonConfig* getCommonConfig() const override
+      {
+         return cfg;
+      }
+
+      virtual const NetFilter* getNetFilter() const override
+      {
+         return netFilter;
+      }
+
+      virtual const NetFilter* getTcpOnlyFilter() const override
+      {
+         return tcpOnlyFilter;
+      }
+
+      virtual const AbstractNetMessageFactory* getNetMessageFactory() const override
+      {
+         return netMessageFactory;
+      }
+
+      AcknowledgmentStore* getAckStore() const
+      {
+         return ackStore;
+      }
+
+      Config* getConfig() const
+      {
+         return cfg;
+      }
+
+      void updateLocalNicList(NicAddressList& localNicList);
+
+      Node& getLocalNode() const
+      {
+         return *localNode;
+      }
+
+      NodeStoreServers* getMgmtNodes() const
+      {
+         return mgmtNodes;
+      }
+
+      NodeStoreServers* getMetaNodes() const
+      {
+         return metaNodes;
+      }
+
+      NodeStoreServers* getStorageNodes() const
+      {
+         return storageNodes;
+      }
+
+      TargetMapper* getTargetMapper() const
+      {
+         return targetMapper;
+      }
+
+      MirrorBuddyGroupMapper* getMirrorBuddyGroupMapper() const
+      {
+         return mirrorBuddyGroupMapper;
+      }
+
+      TargetStateStore* getTargetStateStore() const
+      {
+         return targetStateStore;
+      }
+
+      MultiWorkQueue* getWorkQueue(uint16_t targetID) const
+      {
+         MultiWorkQueueMapCIter iter = workQueueMap.find(targetID);
+
+         if(iter != workQueueMap.end() )
+            return iter->second;
+
+         /* note: it's not unusual to not find given targetID, e.g.
+               - when per-target queues are disabled
+               - or when server restarted without one of its targets (and clients don't know that)
+               - or if client couldn't provide targetID because it's not a target message */
+
+         return workQueueMap.begin()->second;
+      }
+
+      MultiWorkQueueMap* getWorkQueueMap()
+      {
+         return &workQueueMap;
+      }
+
+      SessionStore* getSessions() const
+      {
+         return sessions;
+      }
+
+      StorageNodeOpStats* getNodeOpStats() const
+      {
+         return nodeOperationStats;
+      }
+
+      StorageTargets* getStorageTargets() const
+      {
+         return storageTargets;
+      }
+
+      SyncedStoragePaths* getSyncedStoragePaths() const
+      {
+         return syncedStoragePaths;
+      }
+
+      StorageBenchOperator* getStorageBenchOperator() const
+      {
+         return this->storageBenchOperator;
+      }
+
+      DatagramListener* getDatagramListener() const
+      {
+         return dgramListener;
+      }
+
+      const StreamLisVec* getStreamListenerVec() const
+      {
+         return &streamLisVec;
+      }
+
+      StatsCollector* getStatsCollector() const
+      {
+         return statsCollector;
+      }
+
+      InternodeSyncer* getInternodeSyncer() const
+      {
+         return internodeSyncer;
+      }
+
+      TimerQueue* getTimerQueue() const
+      {
+         return timerQueue;
+      }
+
+      int getAppResult() const
+      {
+         return appResult;
+      }
+
+      bool getWorkersRunning()
+      {
+         const std::lock_guard<Mutex> lock(mutexWorkersRunning);
+         return this->workersRunning;
+      }
+
+      ChunkStore* getChunkDirStore() const
+      {
+         return this->chunkDirStore;
+      }
+
+      ChunkFetcher* getChunkFetcher() const
+      {
+         return this->chunkFetcher;
+      }
+
+      const ExceededQuotaPerTarget* getExceededQuotaStores() const
+      {
+         return &exceededQuotaStores;
+      }
+
+      BuddyResyncer* getBuddyResyncer() const
+      {
+         return this->buddyResyncer;
+      }
+
+      ChunkLockStore* getChunkLockStore() const
+      {
+         return chunkLockStore;
+      }
+
+      WorkerList* getWorkers()
+      {
+         return &workerList;
+      }
+
+      StoragePoolStore* getStoragePoolStore() const
+      {
+         return storagePoolStore.get();
+      }
+
+      void setLibZfsErrorReported(bool isReported)
+      {
+         libZfsErrorReported = isReported;
+      }
+
+      void* getDlOpenHandleLibZfs()
+      {
+         if(dlOpenHandleLibZfs)
+            return dlOpenHandleLibZfs;
+         else
+         if(cfg->getQuotaDisableZfsSupport() )
+         {
+            if(!libZfsErrorReported)
+            {
+               LOG(QUOTA, ERR,  "Quota support for ZFS is disabled.");
+               libZfsErrorReported = true;
+            }
+         }
+         else
+         if(!libZfsErrorReported)
+            openLibZfs();
+
+         return dlOpenHandleLibZfs;
+      }
+
+      bool isDlOpenHandleLibZfsValid()
+      {
+         if(dlOpenHandleLibZfs)
+            return true;
+
+         return false;
+      }
+
+      void findAllowedInterfaces(NicAddressList& outList) const;
+      void findAllowedRDMAInterfaces(NicAddressList& outList) const;
+
+
+};

storage/source/app/config/Config.cpp

@@ -0,0 +1,256 @@
+#include <common/toolkit/StringTk.h>
+#include <common/toolkit/UnitTk.h>
+#include "Config.h"
+
+#define CONFIG_DEFAULT_CFGFILENAME "/etc/beegfs/beegfs-storage.conf"
+
+#define CONFIG_STORAGETARGETS_DELIMITER   ','
+
+
+Config::Config(int argc, char** argv) :
+      AbstractConfig(argc, argv)
+{
+   initConfig(argc, argv, true);
+}
+
+/**
+ * Sets the default values for each configurable in the configMap.
+ *
+ * @param addDashes currently unused
+ */
+void Config::loadDefaults(bool addDashes)
+{
+   AbstractConfig::loadDefaults();
+
+   // re-definitions
+   configMapRedefine("cfgFile",                  "");
+
+   // own definitions
+   configMapRedefine("connInterfacesFile",       "");
+   configMapRedefine("connInterfacesList",       "");
+
+   configMapRedefine("storeStorageDirectory",    "");
+   configMapRedefine("storeFsUUID",              "");
+   configMapRedefine("storeAllowFirstRunInit",   "true");
+
+   configMapRedefine("tuneNumStreamListeners",        "1");
+   configMapRedefine("tuneNumWorkers",                "8");
+   configMapRedefine("tuneWorkerBufSize",             "4m");
+   configMapRedefine("tuneProcessFDLimit",            "50000");
+   configMapRedefine("tuneWorkerNumaAffinity",        "false");
+   configMapRedefine("tuneListenerNumaAffinity",      "false");
+   configMapRedefine("tuneListenerPrioShift",         "-1");
+   configMapRedefine("tuneBindToNumaZone",            "");
+   configMapRedefine("tuneFileReadSize",              "32k");
+   configMapRedefine("tuneFileReadAheadTriggerSize",  "4m");
+   configMapRedefine("tuneFileReadAheadSize",         "0");
+   configMapRedefine("tuneFileWriteSize",             "64k");
+   configMapRedefine("tuneFileWriteSyncSize",         "0");
+   configMapRedefine("tuneUsePerUserMsgQueues",       "false");
+   configMapRedefine("tuneDirCacheLimit",             "1024");
+   configMapRedefine("tuneEarlyStat",                 "false");
+   configMapRedefine("tuneNumResyncSlaves",           "12");
+   configMapRedefine("tuneNumResyncGatherSlaves",     "6");
+   configMapRedefine("tuneUseAggressiveStreamPoll",   "false");
+   configMapRedefine("tuneUsePerTargetWorkers",       "true");
+
+   configMapRedefine("quotaEnableEnforcement",        "false");
+   configMapRedefine("quotaDisableZfsSupport",        "false");
+
+   configMapRedefine("sysResyncSafetyThresholdMins",  "10");
+   configMapRedefine("sysTargetOfflineTimeoutSecs",   "180");
+
+   configMapRedefine("runDaemonized",            "false");
+
+   configMapRedefine("pidFile",                  "");
+}
+
+/**
+ * @param addDashes currently usused
+ */
+void Config::applyConfigMap(bool enableException, bool addDashes)
+{
+   AbstractConfig::applyConfigMap(false);
+
+   for (StringMapIter iter = configMap.begin(); iter != configMap.end();)
+   {
+      bool unknownElement = false;
+
+      if (iter->first == std::string("logType"))
+      {
+         if (iter->second == "syslog")
+         {
+            logType = LogType_SYSLOG;
+         }
+         else if (iter->second == "logfile")
+         {
+            logType =  LogType_LOGFILE;
+         }
+         else
+         {
+            throw InvalidConfigException("The value of config argument logType is invalid.");
+         }
+      }
+      else if (iter->first == std::string("connInterfacesFile"))
+         connInterfacesFile = iter->second;
+      else if (iter->first == std::string("connInterfacesList"))
+         connInterfacesList = iter->second;
+      else if (iter->first == std::string("storeStorageDirectory"))
+      {
+         storageDirectories.clear();
+
+         std::list<std::string> split;
+
+         StringTk::explode(iter->second, CONFIG_STORAGETARGETS_DELIMITER, &split);
+
+         std::transform(
+               split.begin(), split.end(),
+               std::back_inserter(storageDirectories),
+               [] (const std::string& p) {
+                  return Path(StringTk::trim(p));
+               });
+         storageDirectories.remove_if(std::mem_fn(&Path::empty));
+      }
+      else if (iter->first == std::string("storeFsUUID"))
+      {
+         storeFsUUID.clear();
+
+         std::list<std::string> split;
+
+         StringTk::explode(iter->second, CONFIG_STORAGETARGETS_DELIMITER, &split);
+
+         std::transform(
+               split.begin(), split.end(),
+               std::back_inserter(storeFsUUID),
+               [] (const std::string& p) {
+                  return StringTk::trim(p);
+               });
+         storeFsUUID.remove_if(std::mem_fn(&std::string::empty));
+      }
+      else if (iter->first == std::string("storeAllowFirstRunInit"))
+         storeAllowFirstRunInit = StringTk::strToBool(iter->second);
+      else if (iter->first == std::string("tuneNumStreamListeners"))
+         tuneNumStreamListeners = StringTk::strToUInt(iter->second);
+      else if (iter->first == std::string("tuneNumWorkers"))
+         tuneNumWorkers = StringTk::strToUInt(iter->second);
+      else if (iter->first == std::string("tuneWorkerBufSize"))
+         tuneWorkerBufSize = UnitTk::strHumanToInt64(iter->second);
+      else if (iter->first == std::string("tuneProcessFDLimit"))
+         tuneProcessFDLimit = StringTk::strToUInt(iter->second);
+      else if (iter->first == std::string("tuneWorkerNumaAffinity"))
+         tuneWorkerNumaAffinity = StringTk::strToBool(iter->second);
+      else if (iter->first == std::string("tuneListenerNumaAffinity"))
+         tuneListenerNumaAffinity = StringTk::strToBool(iter->second);
+      else if (iter->first == std::string("tuneBindToNumaZone"))
+      {
+         if (iter->second.empty()) // not defined => disable
+            tuneBindToNumaZone = -1; // -1 means disable binding
+         else
+            tuneBindToNumaZone = StringTk::strToInt(iter->second);
+      }
+      else if (iter->first == std::string("tuneListenerPrioShift"))
+         tuneListenerPrioShift = StringTk::strToInt(iter->second);
+      else if (iter->first == std::string("tuneFileReadSize"))
+         tuneFileReadSize = UnitTk::strHumanToInt64(iter->second);
+      else if (iter->first == std::string("tuneFileReadAheadTriggerSize"))
+         tuneFileReadAheadTriggerSize = UnitTk::strHumanToInt64(iter->second);
+      else if (iter->first == std::string("tuneFileReadAheadSize"))
+         tuneFileReadAheadSize = UnitTk::strHumanToInt64(iter->second);
+      else if (iter->first == std::string("tuneFileWriteSize"))
+         tuneFileWriteSize = UnitTk::strHumanToInt64(iter->second);
+      else if (iter->first == std::string("tuneFileWriteSyncSize"))
+         tuneFileWriteSyncSize = UnitTk::strHumanToInt64(iter->second);
+      else if (iter->first == std::string("tuneUsePerUserMsgQueues"))
+         tuneUsePerUserMsgQueues = StringTk::strToBool(iter->second);
+      else if (iter->first == std::string("tuneDirCacheLimit"))
+         tuneDirCacheLimit = StringTk::strToUInt(iter->second);
+      else if (iter->first == std::string("tuneEarlyStat"))
+         this->tuneEarlyStat = StringTk::strToBool(iter->second);
+      else if (iter->first == std::string("tuneNumResyncGatherSlaves"))
+         this->tuneNumResyncGatherSlaves = StringTk::strToUInt(iter->second);
+      else if (iter->first == std::string("tuneNumResyncSlaves"))
+         this->tuneNumResyncSlaves = StringTk::strToUInt(iter->second);
+      else if (iter->first == std::string("tuneUseAggressiveStreamPoll"))
+         tuneUseAggressiveStreamPoll = StringTk::strToBool(iter->second);
+      else if (iter->first == std::string("tuneUsePerTargetWorkers"))
+         tuneUsePerTargetWorkers = StringTk::strToBool(iter->second);
+      else if (iter->first == std::string("quotaEnableEnforcement"))
+         quotaEnableEnforcement = StringTk::strToBool(iter->second);
+      else if (iter->first == std::string("quotaDisableZfsSupport"))
+         quotaDisableZfsSupport = StringTk::strToBool(iter->second);
+      else if (iter->first == std::string("sysResyncSafetyThresholdMins"))
+         sysResyncSafetyThresholdMins = StringTk::strToInt64(iter->second);
+      else if (iter->first == std::string("sysTargetOfflineTimeoutSecs"))
+      {
+         sysTargetOfflineTimeoutSecs = StringTk::strToUInt(iter->second);
+
+         if (sysTargetOfflineTimeoutSecs < 30)
+         {
+            throw InvalidConfigException("Invalid sysTargetOfflineTimeoutSecs value "
+                  + iter->second + " (must be at least 30)");
+         }
+      }
+      else if (iter->first == std::string("runDaemonized"))
+         runDaemonized = StringTk::strToBool(iter->second);
+      else if (iter->first == std::string("pidFile"))
+         pidFile = iter->second;
+      else
+      {
+         // unknown element occurred
+         unknownElement = true;
+
+         if (enableException)
+         {
+            throw InvalidConfigException("The config argument '" + iter->first + "' is invalid");
+         }
+      }
+
+      if (unknownElement)
+      {
+         // just skip the unknown element
+         iter++;
+      }
+      else
+      {
+         // remove this element from the map
+         iter = eraseFromConfigMap(iter);
+      }
+   }
+}
+
+void Config::initImplicitVals()
+{
+   // tuneFileReadAheadTriggerSize (should be ">= tuneFileReadAheadSize")
+   if(tuneFileReadAheadTriggerSize < tuneFileReadAheadSize)
+      tuneFileReadAheadTriggerSize = tuneFileReadAheadSize;
+
+   // connInterfacesList(/File)
+   AbstractConfig::initInterfacesList(connInterfacesFile, connInterfacesList);
+
+   AbstractConfig::initSocketBufferSizes();
+
+   // check if sync_file_range was enabled on a distro that doesn't support it
+   #ifndef CONFIG_DISTRO_HAS_SYNC_FILE_RANGE
+      if(tuneFileWriteSyncSize)
+      {
+         throw InvalidConfigException(
+            "Config option is not supported for this distribution: 'tuneFileWriteSyncSize'");
+      }
+   #endif
+
+   // connAuthHash
+   AbstractConfig::initConnAuthHash(connAuthFile, &connAuthHash);
+}
+
+std::string Config::createDefaultCfgFilename() const
+{
+   struct stat statBuf;
+
+   const int statRes = stat(CONFIG_DEFAULT_CFGFILENAME, &statBuf);
+
+   if(!statRes && S_ISREG(statBuf.st_mode) )
+      return CONFIG_DEFAULT_CFGFILENAME; // there appears to be a config file
+
+   return ""; // no default file otherwise
+}
+

storage/source/app/config/Config.h

@@ -0,0 +1,229 @@
+#pragma once
+
+#include <common/app/config/AbstractConfig.h>
+
+/**
+ * Find out whether this distro hash sync_file_range() support (added in linux-2.6.17, glibc 2.6).
+ * Note: Problem is that RHEL 5 defines SYNC_FILE_RANGE_WRITE, but uses glibc 2.5 which has no
+ * sync_file_range support, so linker complains about undefined reference.
+ */
+#ifdef __GNUC__
+   #include <features.h>
+   #include <fcntl.h>
+   #if __GLIBC_PREREQ(2, 6) && defined(SYNC_FILE_RANGE_WRITE)
+      #define CONFIG_DISTRO_HAS_SYNC_FILE_RANGE
+   #endif
+#endif
+
+
+class Config : public AbstractConfig
+{
+   public:
+      Config(int argc, char** argv);
+
+   private:
+
+      // configurables
+
+      std::string connInterfacesFile; // implicitly generates connInterfacesList
+      std::string connInterfacesList; // comma-separated list
+
+      std::list<Path> storageDirectories;
+      std::list<std::string> storeFsUUID;
+      bool        storeAllowFirstRunInit;
+
+      unsigned    tuneNumStreamListeners;
+      unsigned    tuneNumWorkers;
+      unsigned    tuneWorkerBufSize;
+      unsigned    tuneProcessFDLimit; // 0 means "don't touch limit"
+      bool        tuneWorkerNumaAffinity;
+      bool        tuneListenerNumaAffinity;
+      int         tuneBindToNumaZone; // bind all threads to this zone, -1 means no binding
+      int         tuneListenerPrioShift;
+      ssize_t     tuneFileReadSize;
+      ssize_t     tuneFileReadAheadTriggerSize; // after how much seq read to start read-ahead
+      ssize_t     tuneFileReadAheadSize; // read-ahead with posix_fadvise(..., POSIX_FADV_WILLNEED)
+      ssize_t     tuneFileWriteSize;
+      ssize_t     tuneFileWriteSyncSize; // after how many of per session data to sync_file_range()
+      bool        tuneUsePerUserMsgQueues; // true to use UserWorkContainer for MultiWorkQueue
+      unsigned    tuneDirCacheLimit;
+      bool        tuneEarlyStat;          // stat the chunk file before closing it
+      unsigned    tuneNumResyncGatherSlaves;
+      unsigned    tuneNumResyncSlaves;
+      bool        tuneUseAggressiveStreamPoll; // true to not sleep on epoll in streamlisv2
+      bool        tuneUsePerTargetWorkers; // true to have tuneNumWorkers separate for each target
+
+      bool        quotaEnableEnforcement;
+      bool        quotaDisableZfsSupport;
+
+      int64_t     sysResyncSafetyThresholdMins; // minutes to add to last buddy comm timestamp
+      unsigned    sysTargetOfflineTimeoutSecs;
+
+      bool        runDaemonized;
+
+      std::string pidFile;
+
+
+      // internals
+
+      virtual void loadDefaults(bool addDashes) override;
+      virtual void applyConfigMap(bool enableException, bool addDashes) override;
+      virtual void initImplicitVals() override;
+      std::string createDefaultCfgFilename() const;
+
+   public:
+      // getters & setters
+      const std::string& getConnInterfacesList() const
+      {
+         return connInterfacesList;
+      }
+
+      const std::list<Path>& getStorageDirectories() const { return storageDirectories; }
+
+      const std::list<std::string>& getStoreFsUUID() const
+      {
+         return storeFsUUID;
+      }
+
+      bool getStoreAllowFirstRunInit() const
+      {
+         return storeAllowFirstRunInit;
+      }
+
+      unsigned getTuneNumStreamListeners() const
+      {
+         return tuneNumStreamListeners;
+      }
+
+      unsigned getTuneNumWorkers() const
+      {
+         return tuneNumWorkers;
+      }
+
+      unsigned getTuneWorkerBufSize() const
+      {
+         return tuneWorkerBufSize;
+      }
+
+      unsigned getTuneProcessFDLimit() const
+      {
+         return tuneProcessFDLimit;
+      }
+
+      bool getTuneWorkerNumaAffinity() const
+      {
+         return tuneWorkerNumaAffinity;
+      }
+
+      bool getTuneListenerNumaAffinity() const
+      {
+         return tuneListenerNumaAffinity;
+      }
+
+      int getTuneBindToNumaZone() const
+      {
+         return tuneBindToNumaZone;
+      }
+
+      int getTuneListenerPrioShift() const
+      {
+         return tuneListenerPrioShift;
+      }
+
+      ssize_t getTuneFileReadSize() const
+      {
+         return tuneFileReadSize;
+      }
+
+      ssize_t getTuneFileReadAheadTriggerSize() const
+      {
+         return tuneFileReadAheadTriggerSize;
+      }
+
+      ssize_t getTuneFileReadAheadSize() const
+      {
+         return tuneFileReadAheadSize;
+      }
+
+      ssize_t getTuneFileWriteSize() const
+      {
+         return tuneFileWriteSize;
+      }
+
+      ssize_t getTuneFileWriteSyncSize() const
+      {
+         return this->tuneFileWriteSyncSize;
+      }
+
+      bool getTuneUsePerUserMsgQueues() const
+      {
+         return tuneUsePerUserMsgQueues;
+      }
+
+      bool getRunDaemonized() const
+      {
+         return runDaemonized;
+      }
+
+      const std::string& getPIDFile() const
+      {
+         return pidFile;
+      }
+
+      unsigned getTuneDirCacheLimit() const
+      {
+         return tuneDirCacheLimit;
+      }
+
+      bool getTuneEarlyStat() const
+      {
+         return this->tuneEarlyStat;
+      }
+
+      bool getQuotaEnableEnforcement() const
+      {
+         return quotaEnableEnforcement;
+      }
+
+      void setQuotaEnableEnforcement(bool doQuotaEnforcement)
+      {
+         quotaEnableEnforcement = doQuotaEnforcement;
+      }
+
+      bool getQuotaDisableZfsSupport() const
+      {
+         return quotaDisableZfsSupport;
+      }
+
+      unsigned getTuneNumResyncGatherSlaves() const
+      {
+         return tuneNumResyncGatherSlaves;
+      }
+
+      unsigned getTuneNumResyncSlaves() const
+      {
+         return tuneNumResyncSlaves;
+      }
+
+      bool getTuneUseAggressiveStreamPoll() const
+      {
+         return tuneUseAggressiveStreamPoll;
+      }
+
+      bool getTuneUsePerTargetWorkers() const
+      {
+         return tuneUsePerTargetWorkers;
+      }
+
+      int64_t getSysResyncSafetyThresholdMins() const
+      {
+         return sysResyncSafetyThresholdMins;
+      }
+
+      unsigned getSysTargetOfflineTimeoutSecs() const
+      {
+         return sysTargetOfflineTimeoutSecs;
+      }
+
+};
+