New upstream version 8.1.0

storage/source/net/message/fsck/DeleteChunksMsgEx.cpp

@@ -0,0 +1,60 @@
+#include "DeleteChunksMsgEx.h"
+
+#include <program/Program.h>
+#include <toolkit/StorageTkEx.h>
+
+bool DeleteChunksMsgEx::processIncoming(ResponseContext& ctx)
+{
+   const char* logContext = "DeleteChunksMsg incoming";
+
+   App* app = Program::getApp();
+   ChunkStore* chunkDirStore = app->getChunkDirStore();
+
+   FsckChunkList& chunks = getChunks();
+   FsckChunkList failedDeletes;
+
+   for ( FsckChunkListIter iter = chunks.begin(); iter != chunks.end(); iter++ )
+   {
+      std::string chunkDirRelative;
+      std::string delPathStrRelative;
+      bool isMirrorFD = iter->getBuddyGroupID();
+
+      chunkDirRelative = iter->getSavedPath()->str();
+
+      delPathStrRelative = chunkDirRelative + "/" + iter->getID();
+
+      auto* const target = app->getStorageTargets()->getTarget(iter->getTargetID());
+
+      if (!target)
+      { // unknown targetID
+         LogContext(logContext).logErr(std::string("Unknown targetID: ") +
+            StringTk::uintToStr(iter->getTargetID()));
+         failedDeletes.push_back(*iter);
+      }
+      else
+      { // valid targetID
+         int targetFD = isMirrorFD ? *target->getMirrorFD() : *target->getChunkFD();
+         int unlinkRes = unlinkat(targetFD, delPathStrRelative.c_str(), 0);
+         if ( (unlinkRes == -1) && (errno != ENOENT) )
+         { // error
+            LogContext(logContext).logErr(
+               "Unable to unlink file: " + delPathStrRelative + ". " + "SysErr: "
+                  + System::getErrString());
+
+            failedDeletes.push_back(*iter);
+         }
+
+         // Now try to rmdir chunkDirPath (checks if it is empty)
+         if (unlinkRes == 0)
+         {
+            Path chunkDirRelativeVec(chunkDirRelative);
+            chunkDirStore->rmdirChunkDirPath(targetFD, &chunkDirRelativeVec);
+         }
+
+      }
+   }
+
+   ctx.sendResponse(DeleteChunksRespMsg(&failedDeletes) );
+
+   return true;
+}

storage/source/net/message/fsck/DeleteChunksMsgEx.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include <common/net/message/NetMessage.h>
+#include <common/net/message/fsck/DeleteChunksMsg.h>
+#include <common/net/message/fsck/DeleteChunksRespMsg.h>
+
+class DeleteChunksMsgEx : public DeleteChunksMsg
+{
+   public:
+      virtual bool processIncoming(ResponseContext& ctx);
+};
+

storage/source/net/message/fsck/FetchFsckChunkListMsgEx.cpp

@@ -0,0 +1,53 @@
+#include "FetchFsckChunkListMsgEx.h"
+
+#include <program/Program.h>
+
+bool FetchFsckChunkListMsgEx::processIncoming(ResponseContext& ctx)
+{
+   App* app = Program::getApp();
+   ChunkFetcher* chunkFetcher = app->getChunkFetcher();
+
+   FetchFsckChunkListStatus status;
+   FsckChunkList chunkList;
+
+   if (getLastStatus() == FetchFsckChunkListStatus_NOTSTARTED)
+   {
+      // This is the first message of a new Fsck run
+      if (chunkFetcher->getNumRunning() != 0 || !chunkFetcher->isQueueEmpty())
+      {
+         // another fsck is already in progress
+         if (!getForceRestart())
+         {
+            LOG(GENERAL, NOTICE, "Received request to start fsck although previous run is not finished. "
+                  "Not starting.", ("From", ctx.peerName()));
+
+            ctx.sendResponse(FetchFsckChunkListRespMsg(&chunkList,
+                     FetchFsckChunkListStatus_NOTSTARTED));
+            return true;
+         }
+         else
+         {
+            LOG(GENERAL, NOTICE, "Aborting previous fsck chunk fetcher run by user request.",
+                  ("From", ctx.peerName()));
+
+            chunkFetcher->stopFetching();
+            chunkFetcher->waitForStopFetching();
+         }
+      }
+
+      chunkFetcher->startFetching();
+   }
+
+
+   if(chunkFetcher->getIsBad())
+      status = FetchFsckChunkListStatus_READERROR;
+   else if (chunkFetcher->getNumRunning() == 0)
+      status = FetchFsckChunkListStatus_FINISHED;
+   else
+      status = FetchFsckChunkListStatus_RUNNING;
+
+   chunkFetcher->getAndDeleteChunks(chunkList, getMaxNumChunks());
+
+   ctx.sendResponse(FetchFsckChunkListRespMsg(&chunkList, status));
+   return true;
+}

storage/source/net/message/fsck/FetchFsckChunkListMsgEx.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <common/net/message/fsck/FetchFsckChunkListMsg.h>
+#include <common/net/message/fsck/FetchFsckChunkListRespMsg.h>
+
+class FetchFsckChunkListMsgEx : public FetchFsckChunkListMsg
+{
+   public:
+      virtual bool processIncoming(ResponseContext& ctx);
+};
+

storage/source/net/message/fsck/MoveChunkFileMsgEx.cpp

@@ -0,0 +1,88 @@
+#include "MoveChunkFileMsgEx.h"
+
+#include <program/Program.h>
+
+bool MoveChunkFileMsgEx::processIncoming(ResponseContext& ctx)
+{
+   ctx.sendResponse(MoveChunkFileRespMsg(moveChunk()));
+   return true;
+}
+
+unsigned MoveChunkFileMsgEx::moveChunk()
+{
+   const char* logContext = "MoveChunkFileMsg incoming";
+
+   App* app = Program::getApp();
+
+   std::string chunkName = this->getChunkName();
+   std::string oldPath = this->getOldPath(); // relative path to chunks dir
+   std::string newPath = this->getNewPath(); // relative path to chunks dir
+   uint16_t targetID = this->getTargetID();
+   bool overwriteExisting = this->getOverwriteExisting();
+
+   int renameRes;
+
+   std::string moveFrom = oldPath + "/" + chunkName;
+   std::string moveTo = newPath + "/" + chunkName;
+
+   auto* const target = app->getStorageTargets()->getTarget(targetID);
+
+   if (!target)
+   {
+      LogContext(logContext).log(Log_CRITICAL, "Could not open path for target ID; targetID: "
+         + StringTk::uintToStr(targetID));
+      return 1;
+   }
+
+   const auto targetPath = getIsMirrored()
+      ? target->getPath() / CONFIG_BUDDYMIRROR_SUBDIR_NAME
+      : target->getPath() / CONFIG_CHUNK_SUBDIR_NAME;
+
+   const int targetFD = getIsMirrored() ? *target->getMirrorFD() : *target->getChunkFD();
+
+   // if overwriteExisting set to false, make sure, that output file does not exist
+   if (!overwriteExisting)
+   {
+      bool pathExists = StorageTk::pathExists(targetFD, moveTo);
+      if (pathExists)
+      {
+         LogContext(logContext).log(Log_CRITICAL,
+            "Could not move chunk file. Destination file does already exist; chunkID: " + chunkName
+               + "; targetID: " + StringTk::uintToStr(targetID) + "; oldChunkPath: " + oldPath
+               + "; newChunkPath: " + newPath);
+         return 1;
+      }
+   }
+
+   {
+      // create the parent directory (perhaps it didn't exist)
+      // can be more efficient if we write a createPathOnDisk that uses mkdirat
+      const Path moveToPath = targetPath / moveTo;
+      mode_t dirMode = S_IRWXU | S_IRWXG | S_IRWXO;
+      bool mkdirRes = StorageTk::createPathOnDisk(moveToPath, true, &dirMode);
+
+      if(!mkdirRes)
+      {
+         LogContext(logContext).log(Log_CRITICAL,
+            "Could not create parent directory for chunk; chunkID: " + chunkName + "; targetID: "
+               + StringTk::uintToStr(targetID) + "; oldChunkPath: " + oldPath + "; newChunkPath: "
+               + newPath);
+         return 1;
+      }
+   }
+
+   // perform the actual move
+   renameRes = renameat(targetFD, moveFrom.c_str(), targetFD, moveTo.c_str() );
+   if ( renameRes != 0 )
+   {
+      LogContext(logContext).log(Log_CRITICAL,
+         "Could not perform move; chunkID: " + chunkName + "; targetID: "
+            + StringTk::uintToStr(targetID) + "; oldChunkPath: " + oldPath + "; newChunkPath: "
+            + newPath + "; SysErr: " + System::getErrString());
+      return 1;
+   }
+   else if (getIsMirrored())
+      target->setBuddyNeedsResync(true);
+
+   return 0;
+}

storage/source/net/message/fsck/MoveChunkFileMsgEx.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <common/net/message/NetMessage.h>
+#include <common/net/message/fsck/MoveChunkFileMsg.h>
+#include <common/net/message/fsck/MoveChunkFileRespMsg.h>
+
+class MoveChunkFileMsgEx : public MoveChunkFileMsg
+{
+   public:
+      virtual bool processIncoming(ResponseContext& ctx);
+
+   private:
+      unsigned moveChunk();
+};
+