New upstream version 8.1.0

fsck/source/modes/Mode.cpp

@@ -0,0 +1,23 @@
+#include "Mode.h"
+
+
+/**
+ * Check given config for a remaining arg and print it as invalid.
+ *
+ * @return true if a remaining invalid argument was found in the given config.
+ */
+bool Mode::checkInvalidArgs(const StringMap* cfg)
+{
+   if(cfg->empty() )
+      return false;
+
+   if(cfg->size() == 1)
+      std::cerr << "Invalid argument: " << cfg->begin()->first << std::endl;
+   else
+   { // multiple invalid args
+      std::cerr << "Found " << cfg->size() << " invalid arguments. One of them is: " <<
+         cfg->begin()->first << std::endl;
+   }
+
+   return true;
+}

fsck/source/modes/Mode.h

@@ -0,0 +1,24 @@
+#ifndef MODE_H_
+#define MODE_H_
+
+#include <common/nodes/Node.h>
+#include <common/Common.h>
+
+class Mode
+{
+   public:
+      virtual ~Mode() {}
+
+      /**
+       * Executes the mode inside the calling thread.
+       *
+       * @return APPCODE_...
+       */
+      virtual int execute() = 0;
+
+   protected:
+      Mode() {}
+      bool checkInvalidArgs(const StringMap* cfg);
+};
+
+#endif /*MODE_H_*/

fsck/source/modes/ModeCheckFS.h

@@ -0,0 +1,170 @@
+#ifndef MODECHECKFS_H
+#define MODECHECKFS_H
+
+#include <boost/scoped_ptr.hpp>
+#include <vector>
+
+#include <database/FsckDB.h>
+#include <database/FsckDBException.h>
+#include <modes/Mode.h>
+
+class UserPrompter
+{
+   public:
+      template<unsigned Actions>
+      UserPrompter(const FsckRepairAction (&possibleActions)[Actions],
+         FsckRepairAction defaultRepairAction);
+
+      FsckRepairAction chooseAction(const std::string& prompt);
+
+   private:
+      bool askForAction;
+      std::vector<FsckRepairAction> possibleActions;
+      FsckRepairAction repairAction;
+};
+
+struct RepairChunkState
+{
+   UserPrompter* prompt;
+   std::string lastID;
+   FsckRepairAction lastChunkAction;
+};
+
+struct FsckErrCount
+{
+   FsckErrCount() : unfixableErrors(0), fixableErrors(0)
+   {
+   }
+
+   FsckErrCount operator+(const FsckErrCount& other)
+   {
+      unfixableErrors += other.unfixableErrors;
+      fixableErrors += other.fixableErrors;
+      return *this;
+   }
+
+   void operator+=(const FsckErrCount& other)
+   {
+      *this = *this + other;
+   }
+
+   uint64_t getTotalErrors() const
+   {
+      return unfixableErrors + fixableErrors;
+   }
+
+   uint64_t unfixableErrors;
+   uint64_t fixableErrors;
+};
+
+class ModeCheckFS : public Mode
+{
+   public:
+      ModeCheckFS();
+      virtual ~ModeCheckFS();
+
+      static void printHelp();
+
+      virtual int execute();
+
+   private:
+      boost::scoped_ptr<FsckDB> database;
+
+      LogContext log;
+
+      NodeHandle lostAndFoundNode;
+      EntryInfo lostAndFoundInfo;
+      boost::shared_ptr<FsckDirInode> lostAndFoundInode;
+
+      std::set<NumNodeID> secondariesSetBad;
+
+      int initDatabase();
+      void printHeaderInformation();
+      void disposeUnusedFiles();
+      FhgfsOpsErr gatherData(bool forceRestart);
+
+      template<typename Obj, typename State>
+      FsckErrCount checkAndRepairGeneric(Cursor<Obj> cursor,
+         void (ModeCheckFS::*repair)(Obj&, FsckErrCount&, State&), State& state);
+
+      FsckErrCount checkAndRepairDanglingDentry();
+      FsckErrCount checkAndRepairWrongInodeOwner();
+      FsckErrCount checkAndRepairWrongOwnerInDentry();
+      FsckErrCount checkAndRepairOrphanedContDir();
+      FsckErrCount checkAndRepairOrphanedDirInode();
+      FsckErrCount checkAndRepairOrphanedFileInode();
+      FsckErrCount checkAndRepairDuplicateInodes();
+      FsckErrCount checkAndRepairOrphanedChunk();
+      FsckErrCount checkAndRepairMissingContDir();
+      FsckErrCount checkAndRepairWrongFileAttribs();
+      FsckErrCount checkAndRepairWrongDirAttribs();
+      FsckErrCount checkAndRepairFilesWithMissingTargets();
+      FsckErrCount checkAndRepairDirEntriesWithBrokeByIDFile();
+      FsckErrCount checkAndRepairOrphanedDentryByIDFiles();
+      FsckErrCount checkAndRepairChunksWithWrongPermissions();
+      FsckErrCount checkMissingMirrorChunks();
+      FsckErrCount checkMissingPrimaryChunks();
+      FsckErrCount checkDifferingChunkAttribs();
+      FsckErrCount checkAndRepairChunksInWrongPath();
+      FsckErrCount checkAndUpdateOldStyledHardlinks();
+
+      void logDuplicateInodeID(checks::DuplicatedInode& dups, int&);
+
+      FsckErrCount checkDuplicateChunks();
+      void logDuplicateChunk(std::list<FsckChunk>& dups, FsckErrCount& errCount, int&);
+
+      FsckErrCount checkDuplicateContDirs();
+      void logDuplicateContDir(std::list<db::ContDir>& dups, FsckErrCount& errCount, int&);
+
+      FsckErrCount checkMismirroredDentries();
+      void logMismirroredDentry(db::DirEntry& entry, FsckErrCount& errCount, int&);
+
+      FsckErrCount checkMismirroredDirectories();
+      void logMismirroredDirectory(db::DirInode& dir, FsckErrCount& errCount, int&);
+
+      FsckErrCount checkMismirroredFiles();
+      void logMismirroredFile(db::FileInode& file, FsckErrCount& errCount, int&);
+
+      FsckErrCount checkAndRepairMalformedChunk();
+      void repairMalformedChunk(FsckChunk& chunk, FsckErrCount& errCount, UserPrompter& prompt);
+
+      void checkAndRepair();
+
+      void repairDanglingDirEntry(db::DirEntry& entry, FsckErrCount& errCount,
+         std::pair<UserPrompter*, UserPrompter*>& prompt);
+      void repairWrongInodeOwner(FsckDirInode& inode, FsckErrCount& errCount,
+         UserPrompter& prompt);
+      void repairWrongInodeOwnerInDentry(std::pair<db::DirEntry, NumNodeID>& error,
+         FsckErrCount& errCount, UserPrompter& prompt);
+      void repairOrphanedDirInode(FsckDirInode& inode, FsckErrCount& errCount,
+         UserPrompter& prompt);
+      void repairOrphanedFileInode(FsckFileInode& inode, FsckErrCount& errCount,
+         UserPrompter& prompt);
+      void repairDuplicateInode(checks::DuplicatedInode& dupInode, FsckErrCount& errCount,
+         UserPrompter& prompt);
+      void repairOrphanedChunk(FsckChunk& chunk, FsckErrCount& errCount, RepairChunkState& state);
+      void repairMissingContDir(FsckDirInode& inode, FsckErrCount& errCount, UserPrompter& prompt);
+      void repairOrphanedContDir(FsckContDir& dir, FsckErrCount& errCount, UserPrompter& prompt);
+      void repairWrongFileAttribs(std::pair<FsckFileInode, checks::OptionalInodeAttribs>& error,
+         FsckErrCount& errCount, UserPrompter& prompt);
+      void repairWrongDirAttribs(std::pair<FsckDirInode, checks::InodeAttribs>& error,
+         FsckErrCount& errCount, UserPrompter& prompt);
+      void repairFileWithMissingTargets(db::DirEntry& entry, FsckErrCount& errCount,
+         UserPrompter& prompt);
+      void repairDirEntryWithBrokenByIDFile(db::DirEntry& entry, FsckErrCount& errCount,
+         UserPrompter& prompt);
+      void repairOrphanedDentryByIDFile(FsckFsID& id, FsckErrCount& errCount, UserPrompter& prompt);
+      void repairChunkWithWrongPermissions(std::pair<FsckChunk, FsckFileInode>& error,
+         FsckErrCount& errCount, UserPrompter& prompt);
+      void repairWrongChunkPath(std::pair<FsckChunk, FsckFileInode>& error, FsckErrCount& errCount,
+         UserPrompter& prompt);
+      void updateOldStyledHardlinks(db::FileInode& inode, FsckErrCount& errCount, UserPrompter& prompt);
+
+      void deleteFsIDsFromDB(FsckDirEntryList& dentries);
+      void deleteFilesFromDB(FsckDirEntryList& dentries);
+
+      bool ensureLostAndFoundExists();
+      void releaseLostAndFound();
+};
+
+#endif /* MODECHECKFS_H */

fsck/source/modes/ModeEnableQuota.cpp

@@ -0,0 +1,116 @@
+#include "ModeEnableQuota.h"
+
+#include <common/toolkit/ListTk.h>
+#include <common/toolkit/UnitTk.h>
+#include <components/worker/AdjustChunkPermissionsWork.h>
+#include <net/msghelpers/MsgHelperRepair.h>
+#include <toolkit/FsckTkEx.h>
+
+#include <program/Program.h>
+
+ModeEnableQuota::ModeEnableQuota()
+ : log("ModeEnableQuota")
+{
+}
+
+void ModeEnableQuota::printHelp()
+{
+   std::cout << "MODE ARGUMENTS:" << std::endl;
+   std::cout << " Optional:" << std::endl;
+   std::cout << "  --databasePath=<path>  Path to store the database files." << std::endl;
+   std::cout << "                         (Default: " << CONFIG_DEFAULT_DBPATH << ")" << std::endl;
+   std::cout << "  --overwriteDbFile      Overwrite an existing database file without prompt." << std::endl;
+   std::cout << "  --logOutFile=<path>    Path to the fsck output file, which contains a copy of" << std::endl;
+   std::cout << "                         the console output." << std::endl;
+   std::cout << "                         (Default: " << CONFIG_DEFAULT_OUTFILE << ")" << std::endl;
+   std::cout << "  --logStdFile=<path>    Path to the program error log file, which contains e.g." << std::endl;
+   std::cout << "                         network error messages." << std::endl;
+   std::cout << "                         (Default: " << CONFIG_DEFAULT_LOGFILE << ")" << std::endl;
+   std::cout << std::endl;
+   std::cout << "USAGE:" << std::endl;
+   std::cout << " This mode sets quota information on an existing file system." << std::endl;
+   std::cout << std::endl;
+   std::cout << " This is useful when quota support is being enabled on a file system instance" << std::endl;
+   std::cout << " that was previously used without quota support." << std::endl;
+   std::cout << std::endl;
+   std::cout << " Example: Set quota information" << std::endl;
+   std::cout << "  $ beegfs-fsck --enablequota" << std::endl;
+}
+
+int ModeEnableQuota::execute()
+{
+   App* app = Program::getApp();
+   Config *cfg = app->getConfig();
+
+   if(this->checkInvalidArgs(cfg->getUnknownConfigArgs()))
+      return APPCODE_INVALID_CONFIG;
+
+   FsckTkEx::printVersionHeader(false);
+   printHeaderInformation();
+
+   if ( !FsckTkEx::checkReachability() )
+      return APPCODE_COMMUNICATION_ERROR;
+
+   fixPermissions();
+
+   return APPCODE_NO_ERROR;
+}
+
+void ModeEnableQuota::printHeaderInformation()
+{
+   Config* cfg = Program::getApp()->getConfig();
+
+   // get the current time and create some nice output, so the user can see at which time the run
+   // was started (especially nice to find older runs in the log file)
+   time_t t;
+   time(&t);
+   std::string timeStr = std::string(ctime(&t));
+   FsckTkEx::fsckOutput(
+      "Started BeeGFS fsck in enableQuota mode [" + timeStr.substr(0, timeStr.length() - 1)
+         + "]\nLog will be written to " + cfg->getLogStdFile()
+         + "\nDatabase files will be saved in " + cfg->getDatabasePath(),
+         OutputOptions_LINEBREAK | OutputOptions_HEADLINE);
+}
+
+void ModeEnableQuota::fixPermissions()
+{
+   SynchronizedCounter finishedWork;
+   unsigned generatedWork = 0;
+
+   AtomicUInt64 fileCount;
+   AtomicUInt64 errorCount;
+
+   NodeStore* metaNodes = Program::getApp()->getMetaNodes();
+
+   auto nodes = metaNodes->referenceAllNodes();
+
+   for (auto iter = nodes.begin(); iter != nodes.end(); iter++)
+   {
+      generatedWork++;
+      Program::getApp()->getWorkQueue()->addIndirectWork(new AdjustChunkPermissionsWork(**iter,
+         &finishedWork, &fileCount, &errorCount));
+   }
+
+   FsckTkEx::fsckOutput("Processed 0 entries", OutputOptions_ADDLINEBREAKBEFORE |
+      OutputOptions_LINEDELETE | OutputOptions_NOLOG);
+
+   // wait for all packages to finish, because we cannot proceed if not all data was fetched
+   // BUT : update output each OUTPUT_INTERVAL_MS ms
+   while (! finishedWork.timedWaitForCount(generatedWork, MODEENABLEQUOTA_OUTPUT_INTERVAL_MS) )
+   {
+      FsckTkEx::fsckOutput("Processed " + StringTk::uint64ToStr(fileCount.read()) + " entries",
+         OutputOptions_LINEDELETE | OutputOptions_NOLOG);
+   }
+
+   FsckTkEx::fsckOutput("Processed " + StringTk::uint64ToStr(fileCount.read()) + " entries",
+      OutputOptions_LINEDELETE | OutputOptions_NOLOG | OutputOptions_LINEBREAK);
+
+   if ( errorCount.read() > 0 )
+   {
+      FsckTkEx::fsckOutput("", OutputOptions_LINEBREAK);
+      FsckTkEx::fsckOutput(
+         StringTk::uint64ToStr(errorCount.read())
+            + " errors occurred. Please see Metadata server logs for more details.",
+         OutputOptions_ADDLINEBREAKBEFORE | OutputOptions_LINEBREAK);
+   }
+}

fsck/source/modes/ModeEnableQuota.h

@@ -0,0 +1,27 @@
+#ifndef MODEENABLEQUOTA_H
+#define MODEENABLEQUOTA_H
+
+#include <app/config/Config.h>
+#include <database/FsckDB.h>
+#include <modes/Mode.h>
+
+#define MODEENABLEQUOTA_OUTPUT_INTERVAL_MS 5000
+
+class ModeEnableQuota : public Mode
+{
+   public:
+      ModeEnableQuota();
+
+      static void printHelp();
+
+      virtual int execute();
+
+   private:
+      LogContext log;
+
+      void printHeaderInformation();
+
+      void fixPermissions();
+};
+
+#endif /* MODEENABLEQUOTA_H */

fsck/source/modes/ModeHelp.cpp

@@ -0,0 +1,99 @@
+#include "ModeHelp.h"
+
+#include <program/Program.h>
+
+#include <modes/ModeCheckFS.h>
+#include <modes/ModeEnableQuota.h>
+
+int ModeHelp::execute()
+{
+   App* app = Program::getApp();
+   Config* cfg = app->getConfig();
+
+   RunMode runMode = cfg->determineRunMode();
+
+   if(runMode == RunMode_INVALID)
+      printGeneralHelp();
+   else
+      printSpecificHelp(runMode);
+
+   std::cout << std::endl;
+
+   return APPCODE_NO_ERROR;
+}
+
+void ModeHelp::printGeneralHelp()
+{
+   std::cout << "BeeGFS File System Check (http://www.beegfs.com)" << std::endl;
+   std::cout << "Version: " << BEEGFS_VERSION << std::endl;
+   std::cout << std::endl;
+   std::cout << "GENERAL USAGE:" << std::endl;
+   std::cout << " $ beegfs-fsck --<modename> --help" << std::endl;
+   std::cout << " $ beegfs-fsck --<modename> [mode_arguments] [client_arguments]" << std::endl;
+   std::cout << std::endl;
+   std::cout << "MODES:" << std::endl;
+   std::cout << " --checkfs     => Perform a full check and optional repair of " << std::endl;
+   std::cout << "                  a BeeGFS instance." << std::endl;
+   std::cout << " --enablequota => Set attributes needed for quota support in FhGFS." << std::endl;
+   std::cout << "                  Can be used to enable quota support on an existing" << std::endl;
+   std::cout << "                  system." << std::endl;
+   std::cout << std::endl;
+   std::cout << "USAGE:" << std::endl;
+   std::cout << " This is the BeeGFS file system consistency check and repair tool." << std::endl;
+   std::cout << std::endl;
+   std::cout << " Choose a mode from the list above and use the parameter \"--help\"" << std::endl;
+   std::cout << " to show arguments and usage examples for that particular mode." << std::endl;
+   std::cout << std::endl;
+   std::cout << " (Running beegfs-fsck requires root privileges.)" << std::endl;
+   std::cout << std::endl;
+   std::cout << " Example: Show help for mode \"--checkfs\"" << std::endl;
+   std::cout << "  $ beegfs-fsck --checkfs --help" << std::endl;
+}
+
+
+/**
+ * Note: don't call this with RunMode_INVALID.
+ */
+void ModeHelp::printSpecificHelp(RunMode runMode)
+{
+   printSpecificHelpHeader(); // print general usage and client options info
+
+   switch(runMode)
+   {
+      case RunMode_CHECKFS:
+      {
+         ModeCheckFS::printHelp();
+      } break;
+
+      case RunMode_ENABLEQUOTA:
+      {
+         ModeEnableQuota::printHelp();
+      } break;
+
+      default:
+      {
+         std::cerr << "Error: Unhandled mode specified. Mode number: " << runMode << std::endl;
+         std::cout << std::endl;
+
+         printGeneralHelp();
+      } break;
+   }
+
+}
+
+/**
+ * Print the help message header that applies to any specific mode help. Contains general usage
+ * and client options info.
+ */
+void ModeHelp::printSpecificHelpHeader()
+{
+   std::cout << "GENERAL USAGE:" << std::endl;
+   std::cout << "  $ beegfs-fsck --<modename> [mode_arguments] [client_arguments]" << std::endl;
+   std::cout << std::endl;
+   std::cout << "CLIENT ARGUMENTS:" << std::endl;
+   std::cout << " --cfgFile=<path>         Path to BeeGFS client config file." << std::endl;
+   std::cout << "                          (Default: " CONFIG_DEFAULT_CFGFILENAME ")" << std::endl;
+   std::cout << " --<key>=<value>          Any setting from the client config file to override" << std::endl;
+   std::cout << "                          the config file values (e.g. \"--logLevel=5\")." << std::endl;
+   std::cout << std::endl;
+}

fsck/source/modes/ModeHelp.h

@@ -0,0 +1,24 @@
+#ifndef MODEHELP_H_
+#define MODEHELP_H_
+
+#include <app/config/Config.h>
+#include <common/Common.h>
+
+#include "Mode.h"
+
+
+class ModeHelp : public Mode
+{
+   public:
+      ModeHelp() {};
+
+      virtual int execute();
+
+   private:
+      void printGeneralHelp();
+      void printSpecificHelp(RunMode runMode);
+      void printSpecificHelpHeader();
+};
+
+
+#endif /*MODEHELP_H_*/