New upstream version 8.1.0

beeond/CMakeLists.txt

@@ -0,0 +1,11 @@
+install(
+	PROGRAMS "beegfs-ondemand-stoplocal" "beeond-lib"
+	DESTINATION "usr/share/beeond"
+	COMPONENT "beeond"
+)
+
+install(
+	PROGRAMS "beeond" "beeond-cp"
+	DESTINATION "usr/bin"
+	COMPONENT "beeond"
+)

beeond/scripts/lib/beegfs-ondemand-stoplocal

@@ -0,0 +1,400 @@
+#!/bin/bash
+
+# beegfs-ondemand-stoplocal
+# This file contains helper functions to stop BeeOND services locally on one node.
+# This is meant to be sourced from another script (i.e. beeond)
+
+
+# Checks the return code of the last command that has been executed. If the code is !=0, indicating
+# an error, it prints a message and sets an error flag.
+# Parameters:
+#     * The return code of the last command
+#     * A string containing a hint on what was being done that could have caused the error. It is
+#       used for the error message.
+# Modifies:
+#     ERROR: Is set to "true" when an error was encountered.
+sl_checkerror()
+{
+   if [ "${1}" != 0 ]
+   then
+      echo "ERROR: There was a problem ${2} on host $(hostname)"
+      ERROR="true"
+   fi
+}
+
+# Prints an info message if the QUIET variable is not set.
+# Parameter:
+#     A string (the message). It is prefixed with INFO when printed.
+# Checks:
+#     QUIET: If "true", nothing is printed.
+sl_print_info()
+{
+   local MESSAGE=${1}
+   if [ "${QUIET}" != "true" ]
+   then
+      echo "INFO: ${MESSAGE}"
+   fi
+}
+
+# unmounts tmpfs mounts listed in the status file
+sl_unmount_tmpfs()
+{
+   local SERVICE MOUNTPOINT _
+   IFS=,
+   while read -r _ SERVICE MOUNTPOINT _ _
+   do
+      if [ "${SERVICE}" != "tmpfs" ]
+      then
+         continue
+      fi
+
+      sl_print_info "Unmounting tmpfs at ${MOUNTPOINT}"
+
+      if [ "${CLEANUP}" != "true" ]
+      then
+         fuser -k "${MOUNTPOINT}"
+         umount -l "${MOUNTPOINT}"
+
+         sl_checkerror $? "unmounting tmpfs"
+      else
+         fuser -k "${MOUNTPOINT}" 2>/dev/null
+         umount -l "${MOUNTPOINT}" 2>/dev/null
+         true
+      fi
+   done < "${STATUSFILE}"
+   unset IFS
+}
+
+# Unmounts all local mounts listed in the status file
+sl_unmount_local_mounts()
+{
+   local SERVICE MOUNTPOINT _
+   IFS=,
+   while read -r _ SERVICE MOUNTPOINT _ _
+   do
+      if [ "${SERVICE}" != "${CLIENTSERVICE}" ]
+      then
+         continue
+      fi
+
+      sl_print_info "Unmounting ${MOUNTPOINT}"
+      if [ "${CLEANUP}" != "true" ]
+      then
+         fuser -k "${MOUNTPOINT}" # no "sl_checkerror" after this, becuase fuser also returns
+                                  # non-zero when there are no processes accessing the file system
+         umount -l "${MOUNTPOINT}"
+         sl_checkerror $? "unmounting the ondemand file system"
+      else
+         fuser -k "${MOUNTPOINT}" 2>/dev/null
+         umount -l "${MOUNTPOINT}" 2>/dev/null
+         true # reset error code before next invocation of sl_checkerror
+      fi
+   done < "${STATUSFILE}"
+   unset IFS
+
+   # try to remove the client module - this is allowed to fail, because we might have a "normal"
+   # beegfs mount somewhere in the system.
+   rmmod beegfs 2>/dev/null || true
+}
+
+# sends a SIGTERM to a process, then waits until the process is stopped or appriximately 10 seconds
+# have passed.
+# Parameter:
+#     The PID of the proces
+# Returns:
+#     0 if process was stopped within 10 seconds, 1 if it wasn't, 255 if initial kill returned an
+#     error.
+sl_kill_check()
+{
+   local PID=$1
+
+   if ! kill "$PID"
+   then
+      return 255
+   fi
+
+   for ((i=0; i<100; i++))
+   do
+      if kill -0 "$PID" 2>/dev/null
+      then
+         sleep 0.1
+      else
+         return 0
+      fi
+   done
+
+   return 1
+}
+
+# stops all services listed in the status file except for clients
+sl_stop_services()
+{
+   local SERVICE DATAPATH PIDFILE _
+   IFS=,
+   while read -r _ SERVICE DATAPATH _ PIDFILE
+   do
+      if [ "${PIDFILE}" != "-" ] # pidfile is "-" for beegfs-client and tmpfs, because it is not
+                                 # a process
+      then
+         if [ -e "${PIDFILE}" ]
+         then
+            PID=$(cat "${PIDFILE}")
+            sl_kill_check "${PID}"
+            RES=$?
+            if [ $RES -eq 1 ]
+            then
+               echo "ERROR: ${SERVICE} did not stop within 10 seconds (PID ${PID})."
+               ERROR="true"
+            elif [ $RES -eq 255 ]
+            then
+               echo "ERROR: ${SERVICE} does not seem to be running any more (PID ${PID})."
+            fi
+         else
+            if [ "${CLEANUP}" != "true" ]
+            then
+               echo "ERROR: PID file ${PIDFILE} does not exist on host $(hostname)"
+               ERROR="true"
+            fi
+         fi
+
+         # delete data...
+         if [ "${DELETE_DATA}" = "true" ]
+         then
+            if [ "${DATAPATH}" != "-" ]
+            then
+               sl_print_info "Deleting stored data; Data path: ${DATAPATH}"
+               rm -rf "${DATAPATH}"
+               sl_checkerror $? "deleting ${DATAPATH}"
+            fi
+         fi
+
+      # delete preferredMds and preferredTarget files
+      rm -f "${PREFERRED_MDS_FILE}"
+      sl_checkerror $? "deleting ${PREFERRED_MDS_FILE}"
+      rm -f "${PREFERRED_TARGET_FILE}"
+      sl_checkerror $? "deleting ${PREFERRED_TARGET_FILE}"
+      fi
+   done < "${STATUSFILE}"
+   unset IFS
+
+   # unmount tempfs if it was used
+   sl_unmount_tmpfs
+}
+
+# deletes the logfiles listed in the status file if ERROR is set to false
+# If the log directory is empty afterwards, it is also deleted
+sl_delete_logfiles()
+{
+   local LOGFILE # declare it here, because the last LOGFILE path is needed to delete the directory
+                 # after the loop
+
+   # delete log files
+   if [ "${ERROR}" != "true" ] # if we haven't encountered an error yet.
+   then
+      # delete log files
+      local SERVICE LOGFILE _
+      IFS=,
+      while read -r _ SERVICE _ LOGFILE _
+      do
+         if [ "${ONLY_UNMOUNT}" = "true" ] && [ "${SERVICE}" != "${CLIENTSERVICE}" ]
+            then continue; fi
+         if [ "${ONLY_STOP_SERVER}" = "true" ] && [ "${SERVICE}" = "${CLIENTSERVICE}" ]
+            then continue; fi
+         if [ "${LOGFILE}" != "-" ]
+         then
+            sl_print_info "Deleting log file ${LOGFILE}"
+            rm -f "${LOGFILE}" 2>/dev/null  # beegfs-client does not (always) generate a logfile.
+                                          # in this case rm gives an error message, but we don't
+                                          # want to see it. - for the same reason no sl_checkerror
+                                          # here
+         fi
+      done < "${STATUSFILE}"
+      unset IFS
+
+      # delete log directory if empty
+      local LOG_DIR
+      LOG_DIR=$(dirname "${LOGFILE}")
+      if [ "${LOG_DIR}" != "." ] && [ ! "$(ls -A "${LOG_DIR}")" ]
+      then
+         echo "Deleting log directory ${LOG_DIR}"
+         rmdir "${LOG_DIR}"
+         sl_checkerror $? "deleting ${LOG_DIR}"
+      fi
+   else
+      sl_print_info "Not deleting log files because of a previous error."
+   fi
+}
+
+# The "main" stoplocal function. From here, the functions to unmount the file system and stop the
+# services are called. If there was no error, sl_delete_logfiles is called, and the status file is
+# also removed.
+# Checks the following variables:
+#     STATUSFILE        The location of the status file
+#     ONLY_STOP_SERVER  If "true", the umount_local_mounts step is skipped, and status file is not
+#                       removed.
+#     ONLY_UNMOUNT      If "true", the stop_services step is skipped, and status file is not
+#                       removed.
+# Modifies:
+#     ERROR             Is set to "true" (and an error message is printed to %2) if an error is
+#                       encountered in any step.
+stoplocal()
+{
+   sl_print_info "Using status file ${STATUSFILE}"
+
+   # do the actual shutdown process
+
+   # unmount the file system (skip this step if we only want to stop the server)
+   if [ "${ONLY_STOP_SERVER}" != "true" ]
+   then
+      sl_unmount_local_mounts
+   fi
+
+   # stop the services (skip this step if we only got asked to unmount the file system)
+   if [ "${ONLY_UNMOUNT}" != "true" ]
+   then
+      sl_stop_services
+   fi
+
+   # delete the logfiles
+   if [ "${ERROR}" != "true" ] && [ "${DELETE_LOGS}" = "true" ]
+   then
+      sl_delete_logfiles
+   fi
+
+
+   # delete the status file (only if a full shutdown was requested)
+   if [ "${ONLY_UNMOUNT}" != "true" ] && [ "${ONLY_STOP_SERVER}" != "true" ]
+   then
+      rm -f "${STATUSFILE}"
+      sl_checkerror $? "deleting the status file"
+   fi
+}
+
+# the user interface / main entry point to stoplocal
+# Options:
+#   -i FILENAME => Status information filename
+#                  (DEFAULT: ${DEFAULT_STATUSFILE})
+#   -d          => Delete BeeGFS data on disks
+#   -L          => Delete log files after successful shutdown
+#   -q          => Suppress \"INFO\" messages, only print \"ERROR\"s
+#   -c          => "Cleanup": Remove remaining processes and directories of a
+#                  potentially unsuccessful shutdown of an earlier beeond
+#                  instance. This switch silences the error message when a status
+#                  information file is not found or an unmount command fails;
+#                  instead, a message is printed (if \"INFO\" messages are not
+#                  suppressed) when a status file DOES exist, because this means
+#                  there actually was an instance before that is now being
+#                  cleaned up.
+#   -u          => ONLY unmount the file systems(*)
+#   -s          => ONLY stop non-client services(*)
+#
+#                  (*) Options -u and -s are mutually exclusive
+#                      If -u or -s are given, the status file is not deleted.
+do_stoplocal()
+{
+   local DEFAULT_STATUSFILE=/tmp/beeond.tmp
+   local CLIENTSERVICE=beegfs-client
+   local DELETE_DATA="false"
+   local DELETE_LOGS="false"
+   local ONLY_UNMOUNT="false"
+   local ONLY_STOP_SERVER="false"
+   local PREFERRED_MDS_FILE=/tmp/preferredMds.fod
+   local PREFERRED_TARGET_FILE=/tmp/preferredTarget.fod
+   local QUIET="false"
+
+   local ERROR="false"
+   local STATUSFILE="${DEFAULT_STATUSFILE}"
+
+   local OPTIND=1
+   local OPTARG=""
+   while getopts ":i:dLusqc" opt "$@"
+   do
+      case $opt in
+         i)
+            STATUSFILE=${OPTARG}
+         ;;
+         d)
+            DELETE_DATA="true"
+         ;;
+         L)
+            DELETE_LOGS="true"
+         ;;
+         u)
+            if [ "${ONLY_STOP_SERVER}" = "true" ]
+            then
+               echo "ERROR: Options -s and -${OPTARG} are mutually exclusive" >&2
+               if declare -f -F print_usage_and_exit >/dev/null
+                  then print_usage_and_exit; fi
+               return 1
+            fi
+            ONLY_UNMOUNT="true"
+         ;;
+         s)
+            if [ "${ONLY_UNMOUNT}" = "true" ]
+            then
+               echo "ERROR: Options -u and -${OPTARG} are mutually exclusive" >&2
+               if declare -f -F print_usage_and_exit >/dev/null
+                  then print_usage_and_exit; fi
+               return 1
+            fi
+            ONLY_STOP_SERVER="true"
+         ;;
+         q)
+            QUIET="true"
+         ;;
+         c)
+            CLEANUP="true"
+         ;;
+         \?)
+            echo "ERROR: invalid option -${OPTARG}" >&2
+            if declare -f -F print_usage_and_exit >/dev/null
+               then print_usage_and_exit; fi
+            return 1
+         ;;
+         :)
+            echo "ERROR: Option -${OPTARG} requires an argument" >&2
+            if declare -f -F print_usage_and_exit >/dev/null
+               then print_usage_and_exit; fi
+            return 1
+         ;;
+      esac
+   done
+
+   # if statusfile can't be found, print a message and exit.
+   if [ ! -f ${STATUSFILE} ]
+   then
+      # only print message when we're not doing a cleanup run.
+      if [ "${CLEANUP}" != "true" ]
+      then
+         echo "ERROR: Status file ${STATUSFILE} not found." >&2
+
+         # If the user has specified a status file, just give a brief error message and exit.
+         # If the user has not specified a status file, give the full usage info - maybe the user
+         # didn't know how to specify a status file.
+         if [ "${STATUSFILE}" = "${DEFAULT_STATUSFILE}" ]
+         then
+            if declare -f -F "print_usage_and_exit" >/dev/null
+               then print_usage_and_exit; fi
+         fi
+
+         return 1
+      else
+         return 0 # return 0 if we're doing a cleanup so that pdsh doesn't complain
+      fi
+   fi
+
+   # if we're doing a cleanup run, inform the user that a status file was found.
+   if [ "${CLEANUP}" = "true" ]
+   then
+      sl_print_info "Status file found."
+   fi
+
+   stoplocal
+
+   if [ "${ERROR}" = "true" ]
+   then
+      return 1
+   else
+      return 0
+   fi
+}

beeond/scripts/lib/beeond-lib

@@ -0,0 +1,392 @@
+#!/bin/bash
+
+# This file contains some functions used across all of the BeeOND scripts.
+
+BEEOND_FILENAME_PREFIX=".beeond_"
+BEEOND_COPY_FILE_LIST="${BEEOND_FILENAME_PREFIX}files_copy"
+BEEOND_COPY_SCAN_LIST="${BEEOND_FILENAME_PREFIX}scan_list" # List of dirs that have to be scanned.
+BEEOND_COPY_DIR_LIST="${BEEOND_FILENAME_PREFIX}dirs_copy"
+BEEOND_START_FILE_LIST="${BEEOND_FILENAME_PREFIX}files_start"
+BEEOND_END_FILE_LIST="${BEEOND_FILENAME_PREFIX}files_end"
+BEEOND_END_UPDATED_FILE_LIST="${BEEOND_FILENAME_PREFIX}files_end_updated"
+BEEOND_START_DIR_LIST="${BEEOND_FILENAME_PREFIX}dirs_start"
+BEEOND_END_DIR_LIST="${BEEOND_FILENAME_PREFIX}dirs_end"
+BEEOND_END_UPDATED_DIR_LIST="${BEEOND_FILENAME_PREFIX}dirs_end_updated"
+BEEOND_SESSION_FILE="${BEEOND_FILENAME_PREFIX}session"
+
+BEEOND_BATCH_SIZE=20
+
+beeond_print_error()
+{
+   echo "ERROR: ${1}" >&2
+   echo ""
+}
+
+beeond_print_error_and_exit()
+{
+   beeond_print_error "${1}" >&2
+   exit 1
+}
+
+beeond_print_info()
+{
+   local MESSAGE="${1}"
+   if [ "${QUIET}" != "true" ]
+   then
+      echo "INFO: ${MESSAGE}"
+   fi
+}
+
+# Saves the session info file which contains the paths of the global store and the node file.
+beeond_save_session_info()
+{
+   local NODEFILE="${1}"
+   local GLOBAL_PATH="${2}"
+
+   if ! printf "NodeFile=%q\nGlobalPath=%q\n" "${NODEFILE}" "${GLOBAL_PATH}" \
+         > "${LOCAL_PATH}/${BEEOND_SESSION_FILE}"
+   then
+      beeond_print_error_and_exit "Could not write to session file."
+   fi
+}
+
+# Generate the list of files in the beeond folder.
+beeond_generate_file_list()
+{
+   local LOCAL_PATH="${1}"
+   local LISTFILE="${2}"
+   local REFERENCE_FILE="${3}"
+
+
+   pushd "${LOCAL_PATH}"
+   if [ "${REFERENCE_FILE}" = "" ]
+   then     # No reference file - just generate the full list (e.g. on startup).
+
+      beeond_print_info "Generating file list ${LISTFILE}..."
+
+      find . ! -path ./${BEEOND_FILENAME_PREFIX}\* \( -type f -or -type l \)\
+            -exec bash -c 'printf "%q\n" "$0"' {} \; \
+            | grep -v ^\\$ | sort > "${LISTFILE}"
+
+      # The grep statement filters out file names with newlines in them. While they are technically
+      # legal they would cause problems later on due to the way the script run by parallel handles
+      # the arguments.
+
+   else     # Reference file given: Compare timestamps.
+
+      beeond_print_info \
+            "Generating file list ${LISTFILE}. Timestamp reference: ${REFERENCE_FILE}..."
+
+      find . ! -path ./${BEEOND_FILENAME_PREFIX}\* \( -type f -or -type l \) \
+            \( -cnewer "${REFERENCE_FILE}" -or -newer "${REFERENCE_FILE}" \) \
+            -exec bash -c 'printf "%q\n" "$0"' {} \; \
+            | grep -v ^\\$ | sort > "${LISTFILE}"
+
+   fi
+   popd
+}
+
+# Generate list of directories - this is necessary in case directories were created during the
+# session and need to be created on the global store as well.
+beeond_generate_directory_list()
+{
+   local LOCAL_PATH="${1}"
+   local LISTFILE="${2}"
+   local REFERENCE_FILE="${3}"
+
+   pushd "${LOCAL_PATH}"
+   if [ "${REFERENCE_FILE}" = "" ]
+   then     # No reference file - just generate the full list (e.g. on startup).
+
+      beeond_print_info "Generating directory list ${LISTFILE}..."
+
+      find . ! -path . -type d \
+            -exec bash -c 'printf "%q\n" "$0"' {} \; \
+            | grep -v ^\\$ | sort > "${LISTFILE}"
+
+   else     # Reference file given: Compare timestamps.
+
+      beeond_print_info \
+            "Generating directory list ${LISTFILE}. Timestamp reference: ${REFERENCE_FILE}..."
+
+      find . ! -path . -type d \
+            \( -cnewer "${REFERENCE_FILE}" -or -newer "${REFERENCE_FILE}" \) \
+            -exec bash -c 'printf "%q\n" "$0"' {} \; \
+            | grep -v ^\\$ | sort > "${LISTFILE}"
+
+   fi
+   popd
+}
+
+# Generate copy file list. If we do a parallel copy, we can't just list the paths to all the files
+# to be copied, because we have to "flatten" the folder hierarchy (e.g. when the user says
+# "copy dir/subfir/file_a anotherdir/file_b target_dir" we want to end up with
+# target_dir/file_a and target_dir/file_b. To achieve this, we just save an explicit target path
+# to each source file. We also have to keep a list of directories we encounter because we want to
+# create them before we start copying.
+beeond_generate_copy_lists()
+{
+   local TARGET="${1}"
+   local NODE_LIST="${2}"
+   local CONCURRENCY="${3}"
+   shift 3
+
+   # Note: We do relative path expansion here, (and not directly in the do_... functions)
+   # because this is the first time we iterate over the source list.
+
+   # Expand target path.
+   if [ ! "${TARGET:0:1}" = "/" ]
+   then
+      TARGET="${PWD}/${TARGET}"
+   fi
+
+   # Delete possibly left over file lists.
+   rm -f "${TARGET}/${BEEOND_COPY_SCAN_LIST}" \
+         "${TARGET}/${BEEOND_COPY_DIR_LIST}" \
+         "${TARGET}/${BEEOND_COPY_FILE_LIST}"
+
+   # Generate lists: A list of files which can be used directly, and a list of directories which
+   # have to be scanned first.
+   for ENTRY in "$@"
+   do
+      # Expand path if it's relative.
+      if [ ! "${ENTRY:0:1}" = "/" ]
+      then
+         ENTRY="${PWD}/${ENTRY}"
+      fi
+
+      beeond_print_info "Path to scan: ${ENTRY}"
+      if [ -d "${ENTRY}" ]; then
+         printf "%q\n" "${ENTRY}" >> "${TARGET}/${BEEOND_COPY_SCAN_LIST}"
+      elif [ -f "${ENTRY}" ]; then
+         printf "%q\n" >> "${TARGET}/${BEEOND_COPY_FILE_LIST}"
+      else
+         beeond_print_error_and_exit "File or directory does not exist: ${ENTRY}"
+      fi
+   done
+
+   beeond_print_info "Scanning sources..."
+
+   < "${TARGET}/${BEEOND_COPY_SCAN_LIST}" \
+         ${PARALLEL} -S "${NODE_LIST}" -j"${CONCURRENCY}" --pipe --recend "\n" \
+            -N${BEEOND_BATCH_SIZE} --controlmaster --will-cite \
+         " \
+            while read DIR; do \
+               cd \"\${DIR}\"; \
+               find . -type d -exec bash -c \\\'printf \"%s/%q\n\" \"\$0\" \"\$1\"\' \"\`basename \"\${DIR}\"\`\" \{\} \; \
+               | grep -v ^\\$; \
+            done;
+         " \
+   | sort > "${TARGET}/${BEEOND_COPY_DIR_LIST}"
+
+   < "${TARGET}/${BEEOND_COPY_SCAN_LIST}" \
+         ${PARALLEL} -S "${NODE_LIST}" -j"${CONCURRENCY}" --pipe --recend "\n" \
+            -N${BEEOND_BATCH_SIZE} --controlmaster --will-cite \
+         " \
+            while read DIR; do \
+               cd \"\${DIR}\"; \
+               find . \( -type f -or -type l \) -exec bash -c \
+                  \\\'printf \"%q %q\n\" \"\${PWD}/\$0\" \"${TARGET}/\`basename \"\${PWD}\"\`/\$0\"\' \{\} \; \
+               | grep -v ^\\$; \
+            done; \
+         " \
+   | sort > "${TARGET}/${BEEOND_COPY_FILE_LIST}"
+}
+
+# Parallel copy of the files from a previously generated file list to the target directory.
+# First, the directory structure is generated, then the files are copied into it.
+beeond_parallel_copy()
+{
+   local TARGET="${1}"
+   local NODE_LIST="${2}"
+   local CONCURRENCY="${3}"
+
+   # Expand target path.
+   if [ ! "${TARGET:0:1}" = "/" ]
+   then
+      TARGET="${PWD}/${TARGET}"
+   fi
+
+   beeond_print_info "Generating target directory structure..."
+
+   < "${TARGET}/${BEEOND_COPY_DIR_LIST}" \
+         ${PARALLEL} -S "${NODE_LIST}" -j"${CONCURRENCY}" --pipe --recend "\n" \
+            -N${BEEOND_BATCH_SIZE} --controlmaster --will-cite \
+         " \
+            while read DIR; do \
+               mkdir -pv \"${TARGET}/\${DIR}\"; \
+            done; \
+         "
+
+   beeond_print_info "Copying files..."
+
+   < "${TARGET}/${BEEOND_COPY_FILE_LIST}" \
+      ${PARALLEL} -S "${NODE_LIST}" -j"${CONCURRENCY}" --pipe --recend "\n" \
+         -N${BEEOND_BATCH_SIZE} --controlmaster --will-cite \
+      " \
+         while read -a LINE; do \
+            cp -av \"\${LINE[0]}\" \"\${LINE[1]}\"; \
+         done; \
+      "
+
+   # Delete temporary files.
+#  rm "${TARGET}/${BEEOND_COPY_SCAN_LIST}" \
+#        "${TARGET}/${BEEOND_COPY_DIR_LIST}" \
+#        "${TARGET}/${BEEOND_COPY_FILE_LIST}"
+}
+
+# Remove all files from the global store that have been deleted during the session.
+beeond_remove_removed_files()
+{
+   local GLOBAL_PATH="${1}"
+   local LOCAL_PATH="${2}"
+   local NODE_LIST="${3}"
+   local CONCURRENCY="${4}"
+
+   beeond_print_info "Deleting files:"
+   comm -23 "${LOCAL_PATH}/${BEEOND_START_FILE_LIST}" "${LOCAL_PATH}/${BEEOND_END_FILE_LIST}" | \
+         ${PARALLEL} -S "${NODE_LIST}" -j"${CONCURRENCY}" --pipe --recend "\n" \
+            -N${BEEOND_BATCH_SIZE} --controlmaster --will-cite \
+         " \
+            while read FILE; do \
+               rm -v \"${GLOBAL_PATH}/\${FILE}\"; \
+            done; \
+         "
+
+   beeond_print_info "Deleting directories:"
+   comm -23 "${LOCAL_PATH}/${BEEOND_START_DIR_LIST}" "${LOCAL_PATH}/${BEEOND_END_DIR_LIST}" | \
+         tac | \
+         xargs -I{} rmdir -v "${GLOBAL_PATH}/{}"
+   # Not being done in parallel to avoid deleting a subdirectory before its parent (this is also
+   # the reason the list is inverted (tac)).
+}
+
+# Copy back all files to the global store that have been updated during the session.
+beeond_copy_updated_files()
+{
+   local GLOBAL_PATH="${1}"
+   local LOCAL_PATH="${2}"
+   local NODE_LIST="${3}"
+   local CONCURRENCY="${4}"
+
+   beeond_print_info "Creating new directories:"
+
+   < "${LOCAL_PATH}/${BEEOND_END_DIR_LIST}" \
+         ${PARALLEL} -S "${NODE_LIST}" -j"${CONCURRENCY}" --pipe --recend "\n" \
+            -N${BEEOND_BATCH_SIZE} --controlmaster --will-cite \
+         " \
+            while read DIR; do \
+               mkdir -pv \"${GLOBAL_PATH}/\${DIR}\"; \
+            done; \
+         "
+
+   beeond_print_info "Copying back changed files:"
+
+   < "${LOCAL_PATH}/${BEEOND_END_UPDATED_FILE_LIST}" \
+         ${PARALLEL} -S "${NODE_LIST}" -j"${CONCURRENCY}" --pipe --recend "\n" \
+            -N${BEEOND_BATCH_SIZE} --controlmaster --will-cite \
+         " \
+            while read FILE; do \
+               cp -uv \"${LOCAL_PATH}/\${FILE}\" \"${GLOBAL_PATH}/\${FILE}\"; \
+            done; \
+         "
+
+   # Copy files into updated directories. (When a directory is renamed or files
+   # are moved to a directory, the files in it don't have their timestamp
+   # updated. Therefore, we need to check all the updated directories again).
+
+   beeond_print_info "Copying back changed files (updated dirs):"
+
+   pushd "${LOCAL_PATH}"
+   < "${LOCAL_PATH}/${BEEOND_END_UPDATED_DIR_LIST}" \
+         xargs -I{} find {} -maxdepth 1 \( -type f -or -type l \) | \
+         ${PARALLEL} -S "${NODE_LIST}" -j"${CONCURRENCY}" --pipe --recend "\n" \
+            -N${BEEOND_BATCH_SIZE} --controlmaster --will-cite \
+         " \
+            while read FILE; do \
+               cp -uv \"${LOCAL_PATH}/\${FILE}\" \"\`dirname \"${GLOBAL_PATH}/\${FILE}\"\`\"; \
+            done; \
+         "
+   popd
+}
+
+# Stage in process: Copy all files from the global store to the local store.
+beeond_stage_in()
+{
+   local GLOBAL_PATH="${1}"
+   local LOCAL_PATH="${2}"
+   local NODE_LIST="${3}"
+   local CONCURRENCY="${4}"
+
+   # Generate list of files that have to be copied.
+   beeond_generate_file_list "${GLOBAL_PATH}" "${LOCAL_PATH}/${BEEOND_START_FILE_LIST}"
+   beeond_generate_directory_list "${GLOBAL_PATH}" "${LOCAL_PATH}/${BEEOND_START_DIR_LIST}"
+
+   beeond_print_info "Creating directory structure..."
+   < "${LOCAL_PATH}/${BEEOND_START_DIR_LIST}" \
+         ${PARALLEL} -S "${NODE_LIST}" -j"${CONCURRENCY}" --pipe --recend "\n" \
+            -N${BEEOND_BATCH_SIZE} --controlmaster --will-cite \
+         " \
+            while read DIR; do \
+               mkdir -pv \"${LOCAL_PATH}/\${DIR}\"; \
+               touch --reference=\"${GLOBAL_PATH}/\${DIR}\" \"${LOCAL_PATH}/\${DIR}\"; \
+            done; \
+         "
+
+   beeond_print_info "Copying files to local directory..."
+   if ! < "${LOCAL_PATH}/${BEEOND_START_FILE_LIST}" \
+         ${PARALLEL} -S "${NODE_LIST}" -j"${CONCURRENCY}" --pipe --recend "\n" \
+            -N${BEEOND_BATCH_SIZE} --controlmaster --will-cite \
+         " \
+            while read FILE; do \
+               cp -av \"${GLOBAL_PATH}/\${FILE}\" \"\`dirname \"${LOCAL_PATH}/\${FILE}\"\`\"/; \
+            done;
+         "
+   then
+      beeond_print_error "Stage-in copy did not succeed. Data is incompletely staged in."
+   fi
+
+   # Generate list of files and dirs that were actually copied to keep track if the user deletes
+   # files during a session. (re-generate list here, so that if something went wrong during the
+   # stage-in copy, we don't start deleting stuff from the global store by accident).
+   beeond_generate_file_list "${LOCAL_PATH}" "${LOCAL_PATH}/${BEEOND_START_FILE_LIST}"
+   beeond_generate_directory_list "${LOCAL_PATH}" "${LOCAL_PATH}/${BEEOND_START_DIR_LIST}"
+}
+
+# Stage out process: remove all the files that have been removed during the beeond session,
+# and copy back the files which have been changed.
+beeond_stage_out()
+{
+   local GLOBAL_PATH="${1}"
+   local LOCAL_PATH="${2}"
+   local NODE_LIST="${3}"
+   local CONCURRENCY="${4}"
+
+   beeond_print_info "Nodes for parallel stage out: ${NODE_LIST}."
+
+   beeond_generate_file_list "${LOCAL_PATH}" "${LOCAL_PATH}/${BEEOND_END_FILE_LIST}"
+   beeond_generate_file_list "${LOCAL_PATH}" "${LOCAL_PATH}/${BEEOND_END_UPDATED_FILE_LIST}" \
+         "${BEEOND_START_FILE_LIST}"
+
+   beeond_generate_directory_list "${LOCAL_PATH}" "${LOCAL_PATH}/${BEEOND_END_DIR_LIST}"
+   beeond_generate_directory_list "${LOCAL_PATH}" "${LOCAL_PATH}/${BEEOND_END_UPDATED_DIR_LIST}" \
+         "${BEEOND_START_DIR_LIST}"
+
+   beeond_remove_removed_files "${GLOBAL_PATH}" "${LOCAL_PATH}" "${NODE_LIST}" "${CONCURRENCY}"
+
+   beeond_copy_updated_files "${GLOBAL_PATH}" "${LOCAL_PATH}" "${NODE_LIST}" "${CONCURRENCY}"
+}
+
+# Parallel copy process: copy all files and directories (recursively) into the target directory.
+beeond_copy()
+{
+   local NODE_LIST="${1}"
+   local CONCURRENCY="${2}"
+   shift 2
+
+   beeond_print_info "Nodes for parallel copy: ${NODE_LIST}; Concurrency: ${CONCURRENCY}"
+
+   beeond_generate_copy_lists "${@:$#}" "${NODE_LIST}" "${CONCURRENCY}" "${@:1:$#-1}"
+   beeond_parallel_copy "${@:$#}" "${NODE_LIST}" "${CONCURRENCY}"
+
+}

beeond/source/beeond-cp

@@ -0,0 +1,354 @@
+#!/bin/bash
+
+# Source helper script.
+ABSOLUTE_PATH=$(dirname "$(readlink -e "$0")") # using readlink, because somone might be calling
+                                               # this script using a symlink
+
+if [ -e "${ABSOLUTE_PATH}/../lib/beeond-lib" ]
+then
+   BEEOND_LIB="${ABSOLUTE_PATH}/../lib/beeond-lib"
+else
+   BEEOND_LIB="${ABSOLUTE_PATH}/../scripts/lib/beeond-lib"
+fi
+
+#shellcheck source=scripts/lib/beeond-lib
+source "${BEEOND_LIB}"
+PARALLEL="${ABSOLUTE_PATH}/../thirdparty/parallel/parallel"
+
+# Print usage.
+print_usage_and_exit()
+{
+   echo ""
+   echo "BeeOND copy (http://www.beegfs.com)"
+   echo ""
+   echo "DESCRIPTION:"
+   echo "   BeeGFS OnDemand copying/staging system."
+   echo ""
+   echo "USAGE: $(basename "$0") <mode> <options>"
+   echo ""
+   echo "ACTIONS:"
+   echo "   The first argument to $(basename "$0") is considered to be an action that the script"
+   echo "   should perform."
+   echo ""
+   echo "   The following actions are available:"
+   echo ""
+   echo "   stagein: (EXPERIMENTAL)"
+   echo "      Stage a complete directory from the global storage in to BeeOND."
+   echo ""
+   echo "      Mandatory arguments:"
+   echo "         -n FILENAME  => File containing the list of nodes where the parallel"
+   echo "                         copy should be performed. All nodes must have access to"
+   echo "                         the global and local directories."
+   echo "         -g PATH      => Directory on global storage. (*)"
+   echo "         -l PATH      => Directory on local (BeeOND) storage. (*)"
+   echo ""
+   echo "      Notes:"
+   echo "         (*) Global and local directories have to be specified in form of an"
+   echo "             absolute path."
+   echo ""
+   echo "   stageout: (EXPERIMENTAL)"
+   echo "      Stage a complete directory out from BeeOND to the global storage."
+   echo "      Only changes will be staged out of the local directory and committed to"
+   echo "      the global directory."
+   echo ""
+   echo "      Mandatory arguments:"
+   echo "         -l PATH      => Local directory."
+   echo ""
+   echo "      Notes:"
+   echo "         The contents will be completely synchronized, i.e. deleted files on "
+   echo "         BeeOND will get deleted on global storage, too."
+   echo ""
+   echo "   copy:"
+   echo "      Perform a parallel copy of a set of files or folders."
+   echo "      Files will be copied into the target directory, folders will be copied"
+   echo "      recursively. The copy process is parallelized across the set of nodes"
+   echo "      specified in the nodefile."
+   echo ""
+   echo "      Mandatory arguments:"
+   echo "         -n FILENAME  => File containing the list of nodes where the parallel"
+   echo "                         copy should be performed. All nodes must have access to"
+   echo "                         the sources and the target directory."
+   echo ""
+   echo "      Notes:"
+   echo "         Further command line arguments are consdiered source directory or file"
+   echo "         names. The last command line argument specifies the target directory"
+   echo ""
+   echo "EXAMPLES:"
+   echo "   Stage data from /mnt/beegfs-global/dataset in to BeeOND mounted at /mnt/beeond,"
+   echo "   using the nodes given in /tmp/nodefile:"
+   echo "      beeond-cp stagein -n /tmp/nodefile -g /mnt/beegfs-global/dataset -l /mnt/beeond"
+   echo ""
+   echo "   Stage out modified data from BeeOND mounted at /mnt/beeond to the global "
+   echo "   storage:"
+   echo "      beeond-cp stageout -n /tmp/nodefile -g /mnt/beegfs-global/dataset -l /mnt/beeond"
+   echo "" 
+   echo "   Recursively copy the directories dir_1 and dir_2 to /mnt/beegfs, using the nodes"
+   echo "   in /tmp/nodefile:"
+   echo "      beeond-cp copy -n /tmp/nodefile dir_1 dir_2 /mnt/beegfs"
+   echo ""
+   echo "NOTE:"
+   echo "   BeeOND copy uses GNU Parallel -"
+   echo "   When using programs that use GNU Parallel to process data for publication"
+   echo "   please cite:"
+   echo "     O. Tange (2011): GNU Parallel - The Command-Line Power Tool,"
+   echo "     ;login: The USENIX Magazine, February 2011:42-47."
+   echo ""
+   echo "   SSH is used to log into the nodes specified in the nodefile. Please make"
+   echo "   sure your SSH configuration allows for enough concurrent sessions and pending"
+   echo "   logins. You might have to (ask your admin to) raise the MaxSessions and"
+   echo "   MaxStartups settings in the sshd_config file."
+   echo ""
+   echo "   Also please make sure you have the access rights needed to write to the"
+   echo "   global store. Otherwise the stage-out might fail. Note that the access rights"
+   echo "   in the BeeOND local store do not necessarily reflect those in the global"
+   echo "   store."
+
+   exit 1
+}
+
+### main functions
+do_start()
+{
+   local NODEFILE="${1}"
+   local GLOBAL_PATH="${2}"
+   local LOCAL_PATH="${3}"
+
+   beeond_print_info "BeeOND startup..."
+
+   beeond_print_info "nodefile: ${NODEFILE}"
+   beeond_print_info "global path: ${GLOBAL_PATH}"
+   beeond_print_info "local path: ${LOCAL_PATH}"
+
+   MISSING_PARAM=0
+   if [ "${NODEFILE}" = "" ]
+   then
+      beeond_print_error "No nodefile specified."
+      MISSING_PARAM=1
+   fi
+   if [ "${GLOBAL_PATH}" = "" ]
+   then
+      beeond_print_error "Global path not specified."
+      MISSING_PARAM=1
+   fi
+   if [ "${LOCAL_PATH}" = "" ]
+   then
+      beeond_print_error "Local path not specified."
+      MISSING_PARAM=1
+   fi
+
+   # Expand relative path to nodefile.
+   if [ ! "${NODEFILE:0:1}" = "/" ]
+   then
+      NODEFILE="${PWD}/${NODEFILE}"
+   fi
+
+   if [ ! -e "${NODEFILE}" ]
+   then
+      beeond_print_error_and_exit "Node file does not exist."
+   fi
+
+   # The paths to the global and local directory have to be specified as absolute paths to prevent
+   # user errors (like copying a lot of files to ~/mnt/beeond).
+   if [ ! "${GLOBAL_PATH:0:1}" = "/" ] || [ ! "${LOCAL_PATH:0:1}" = "/" ]
+   then
+      beeond_print_error_and_exit "Global path and local path have to be absolute."
+   fi
+
+   [ "${MISSING_PARAM}" = "1" ] && exit 1
+
+   # Make sure target directory is empty before starting.
+   if [ -e "${LOCAL_PATH}" ]
+   then
+      [ -d "${LOCAL_PATH}" ] \
+         || beeond_print_error_and_exit "Target path is not a directory."
+
+      find "${LOCAL_PATH}" -maxdepth 0 -type d -empty | read -r _ \
+         || beeond_print_error_and_exit "Target directory is not empty."
+   else
+      mkdir -p "${LOCAL_PATH}" \
+         || beeond_print_error_and_exit "Cannot create target directory."
+   fi
+
+   local CONCURRENCY=$(( $(wc -l < "${NODEFILE}") ))
+   beeond_print_info "Concurrency: ${CONCURRENCY}"
+
+   beeond_print_info "Writing session information."
+   beeond_save_session_info "${NODEFILE}" "${GLOBAL_PATH}"
+
+   beeond_print_info "Starting stage-in..."
+   NODES=( $(grep -v '^$' "${NODEFILE}" | uniq) ) # Store as array and ignore empty lines.
+   NODELIST=$(IFS=,; echo "${NODES[*]}") # turn argument list into comma-separated string for PDSH
+
+   beeond_stage_in "${GLOBAL_PATH}" "${LOCAL_PATH}" "${NODELIST}" ${CONCURRENCY}
+
+   beeond_print_info "Done."
+}
+
+do_stop()
+{
+   local LOCAL_PATH="${1}"
+
+   if [ "${LOCAL_PATH}" = "" ]
+   then
+      beeond_print_error_and_exit "No path specified."
+   fi
+
+   # Expand relative local path.
+   # Note: Don't have to ensure that it's an absolute path here: We confirm it's a BeeOND instance
+   #       by looking for the session info file.
+   if [ ! "${LOCAL_PATH:0:1}" = "/" ]
+   then
+      LOCAL_PATH="${PWD}/${LOCAL_PATH}"
+   fi
+
+   # Read parameters from session info file.
+   NODEFILE=$(grep NodeFile "${LOCAL_PATH}/${BEEOND_SESSION_FILE}" | cut -d = -f 2-)
+   GLOBAL_PATH=$(grep GlobalPath "${LOCAL_PATH}/${BEEOND_SESSION_FILE}" | cut -d = -f 2-)
+
+   if [ "${NODEFILE}" = "" ]
+   then
+      beeond_print_error "Error reading node file name from session file."
+      MISSING_PARAM=1
+   fi
+   if [ "${GLOBAL_PATH}" = "" ]
+   then
+      beeond_print_error "Error reading global path from session file."
+      MISSING_PARAM=1
+   fi
+
+   [ "${MISSING_PARAM}" = "1" ] && exit 1
+
+   if [ ! -e "${NODEFILE}" ]
+   then
+      beeond_print_error_and_exit "Node file does not exist."
+   fi
+
+   beeond_print_info "BeeOND shutdown..."
+
+   beeond_print_info "nodefile: ${NODEFILE}"
+   beeond_print_info "global path: ${GLOBAL_PATH}"
+   beeond_print_info "local path: ${LOCAL_PATH}"
+
+   NODES=( $(grep -v '^$' "${NODEFILE}" | uniq) ) # Store as array and ignore empty lines.
+   NODELIST=$(IFS=,; echo "${NODES[*]}")
+
+   local CONCURRENCY=$(( $(wc -l < "${NODEFILE}") ))
+   beeond_print_info "Concurrency: ${CONCURRENCY}"
+
+   beeond_stage_out "${GLOBAL_PATH}" "${LOCAL_PATH}" "${NODELIST}" ${CONCURRENCY}
+
+   beeond_print_info "Done."
+}
+
+do_copy()
+{
+   local NODEFILE="${1}"
+   shift
+
+   beeond_print_info "BeeOND copy..."
+
+   if [ "${NODEFILE}" = "" ]
+   then
+      beeond_print_error "No nodefile specified."
+   fi
+
+   # Expand relative path to nodefile.
+   if [ ! "${NODEFILE:0:1}" = "/" ]
+   then
+      NODEFILE="${PWD}/${NODEFILE}"
+   fi
+
+   if [ ! -e "${NODEFILE}" ]
+   then
+      beeond_print_error_and_exit "Node file does not exist."
+   fi
+
+   NODES=( $(grep -v '^$' "${NODEFILE}" | uniq) ) # Store as array and ignore empty lines.
+   NODELIST=$(IFS=,; echo "${NODES[*]}")
+
+   local CONCURRENCY=$(( $(wc -l < "${NODEFILE}") ))
+   beeond_print_info "Concurrency: ${CONCURRENCY}"
+
+   beeond_copy "${NODELIST}" "${CONCURRENCY}" "$@"
+}
+
+# Print help if no arguments given.
+if [ $# -eq 0 ] ; then
+   print_usage_and_exit
+fi
+
+# Do it.
+ACTION="${1}"
+
+if [ "${ACTION}" = "stagein" ]
+then
+   shift
+   while getopts ":n:g:l:" opt; do
+      case $opt in
+         n)
+            NODEFILE="${OPTARG}"
+         ;;
+         g)
+            GLOBAL_PATH="${OPTARG}"
+         ;;
+         l)
+            LOCAL_PATH="${OPTARG}"
+         ;;
+         \?)
+            beeond_print_error_and_exit "Invalid option: -${OPTARG}."
+         ;;
+         :)
+            beeond_print_error_and_exit "Option -${OPTARG} requires an argument."
+         ;;
+      esac
+   done
+
+   do_start "${NODEFILE}" "${GLOBAL_PATH}" "${LOCAL_PATH}"
+elif [ "${ACTION}" = "stageout" ]
+then
+   shift
+   while getopts ":l:" opt; do
+      case $opt in
+         l)
+            LOCAL_PATH="${OPTARG}"
+         ;;
+         \?)
+            beeond_print_error_and_exit "Invalid option: -${OPTARG}."
+         ;;
+         :)
+            beeond_print_error_and_exit "Option -${OPTARG} requires an argument."
+         ;;
+      esac
+   done
+
+   do_stop "${LOCAL_PATH}"
+elif [ "${ACTION}" = "copy" ]
+then
+   shift
+
+   # Nodefile has to be given as the only command line argument.
+   if getopts ":n:" opt
+   then
+      if [ "$opt" = "n" ]
+      then
+         NODEFILE="${OPTARG}"
+      else
+         beeond_print_error_and_exit "Invalid option: -${opt}"
+      fi
+
+   else
+      beeond_print_error_and_exit "No nodefile specified."
+   fi
+
+   # All following command line arguments are file or directory names, specifying the sources and
+   # the target of the copy ancion
+   shift # shift out -n parameter
+   shift # shift out name of node file
+
+   do_copy "${NODEFILE}" "$@"
+elif [ "${ACTION}" = "info" ]
+then
+   do_print_info
+else
+   print_usage_and_exit
+fi