Imported Upstream version 1.5.1

This commit is contained in:
Mario Fetka
2020-09-22 02:25:22 +02:00
commit 434d6067d9
2103 changed files with 928962 additions and 0 deletions

View File

@@ -0,0 +1,37 @@
#!/bin/bash -e
# This script generates the /etc/init.d/xtreemfs-{dir,mrc,osd} scripts
# based on the file 'xtreemfs-service.template'.
# Settings
services=( "dir" "mrc" "osd" )
long_service_names=( "Directory Service" "Metadata and Replica Catalog" "Object Storage Device" )
template_file="xtreemfs-service.template"
# Create files
script_directory=$(dirname "$0")
for i in $(seq 0 $((${#services[@]} - 1)))
do
service=${services[$i]}
short_service_name_lowercase=$service
short_service_name_uppercase=$(echo $service | tr [[:lower:]] [[:upper:]])
long_service_name=${long_service_names[$i]}
if [ "$service" = "dir" ]
then
should_start="\$null"
else
should_start="xtreemfs-dir"
fi
initd_file="${script_directory}/xtreemfs-${service}"
cp "${script_directory}/${template_file}" "$initd_file"
chmod a+x "$initd_file"
sed -i -e "s|@SHORT_SERVICE_NAME@|${short_service_name_uppercase}|g" "$initd_file"
sed -i -e "s|@SHORT_SERVICE_NAME_LOWERCASE@|${short_service_name_lowercase}|g" "$initd_file"
sed -i -e "s|@LONG_SERVICE_NAME@|${long_service_name}|g" "$initd_file"
sed -i -e "s|@SHOULD_START@|${should_start}|g" "$initd_file"
done

View File

@@ -0,0 +1,161 @@
#!/bin/bash
### BEGIN INIT INFO
# Provides: xtreemfs-@SHORT_SERVICE_NAME_LOWERCASE@
# Required-Start: $network $remote_fs
# Required-Stop: $network $remote_fs
# Should-Start: @SHOULD_START@
# Should-Stop: $null
# Default-Start: 3 5
# Default-Stop: 0 1 2 6
# Short-Description: XtreemFS @SHORT_SERVICE_NAME@
# Description: XtreemFS @LONG_SERVICE_NAME@ (@SHORT_SERVICE_NAME@). http://www.xtreemfs.org/
### END INIT INFO
# Source function library.
if [ -e /lib/lsb/init-functions ]; then
. /lib/lsb/init-functions
else
. /etc/init.d/functions
fi
XTREEMFS_USER=xtreemfs
PID=/var/run/xtreemfs_@SHORT_SERVICE_NAME_LOWERCASE@.pid
CONFIG=/etc/xos/xtreemfs/@SHORT_SERVICE_NAME_LOWERCASE@config.properties
LOG=/var/log/xtreemfs/@SHORT_SERVICE_NAME_LOWERCASE@.log
if [ -z $JAVA_HOME ]; then
export JAVA_HOME=/usr
fi
JAVA_CALL="$JAVA_HOME/bin/java -ea -cp /usr/share/java/XtreemFS.jar:/usr/share/java/BabuDB.jar:/usr/share/java/Flease.jar:/usr/share/java/protobuf-java-2.5.0.jar:/usr/share/java/Foundation.jar:/usr/share/java/jdmkrt.jar:/usr/share/java/jdmktk.jar:/usr/share/java/commons-codec-1.3.jar"
# For SELinux we need to use 'runuser' not 'su'
if [ -x "/sbin/runuser" ]; then
SU="/sbin/runuser"
else
SU="/bin/su"
fi
pre_check() {
exists=`grep -c $XTREEMFS_USER /etc/passwd`
if [ $exists -eq 0 ]; then
echo "User $XTREEMFS_USER does not exist. Create it first."
exit 1
fi
log_directory=`dirname $LOG`
if [ ! -e $log_directory ]; then
echo "Directory for logfiles $log_directory does not exist. Create it first."
exit 1
fi
}
start() {
if [ -f $PID ]; then
PROCPID=`cat $PID`
if [ -e /proc/$PROCPID ];then
echo "XtreemFS @LONG_SERVICE_NAME@ (@SHORT_SERVICE_NAME@) already started"
return 0
else
echo -n "Previous XtreemFS @LONG_SERVICE_NAME@ (@SHORT_SERVICE_NAME@) was not shutdown correctly (PID $PROCPID). "
fi
fi
pre_check
echo >> $LOG
date >> $LOG
echo -e "Starting XtreemFS @LONG_SERVICE_NAME@ (@SHORT_SERVICE_NAME@)...\n\n" >> $LOG
echo -n "Starting XtreemFS @LONG_SERVICE_NAME@ (@SHORT_SERVICE_NAME@)..."
$SU -s /bin/bash $XTREEMFS_USER -c "$JAVA_CALL org.xtreemfs.@SHORT_SERVICE_NAME_LOWERCASE@.@SHORT_SERVICE_NAME@ $CONFIG" >> $LOG 2>&1 &
PROCPID=$!
echo $PROCPID > $PID
sleep 1s
if [ -e /proc/$PROCPID ]; then
echo "success"
else
echo "failed"
return 1
fi
return 0
}
stop() {
result=0
if [ -f $PID ]; then
echo -n "Stopping XtreemFS @LONG_SERVICE_NAME@ (@SHORT_SERVICE_NAME@)..."
killproc -p $PID $SU
result=$?
if [ $result -eq 0 ]; then
rm -f $PID
echo "success"
else
echo "failed"
fi
else
echo "XtreemFS @LONG_SERVICE_NAME@ (@SHORT_SERVICE_NAME@) is not running"
fi
return $result
}
status() {
if [ -f $PID ]; then
PROCPID=`cat $PID`
if [ ! -e /proc/$PROCPID ]; then
echo "XtreemFS @LONG_SERVICE_NAME@ (@SHORT_SERVICE_NAME@) has crashed"
return 1
else
echo "XtreemFS @LONG_SERVICE_NAME@ (@SHORT_SERVICE_NAME@) is running"
return 0
fi
else
echo "XtreemFS @LONG_SERVICE_NAME@ (@SHORT_SERVICE_NAME@) is not running"
return 3
fi
}
# See how we were called.
case "$1" in
start)
start
result=$?
;;
stop)
stop
result=$?
;;
status)
status
result=$?
;;
reload)
result=0
;;
restart)
stop && sleep 1 && start
result=$?
;;
try-restart)
## Stop the service and if this succeeds (i.e. the
## service was running before), start it again.
$0 status >/dev/null
if [ $? -eq 0 ]; then
$0 restart
result=$?
else
result=0
fi
;;
*)
echo -e "Usage: $0 {start|stop|restart|reload|status|try-restart}\n"
result=1
;;
esac
exit $result

View File

@@ -0,0 +1,7 @@
datacenters=A,B,C
distance.A-B=10
distance.A-C=100
distance.B-C=50
A.addresses=192.168.1.1,192.168.2.0/24
B.addresses=192.168.1.2,192.168.3.0/24
C.addresses=192.168.1.3,192.168.4.0/24,192.168.10.10

View File

@@ -0,0 +1,9 @@
# This configuration file is only used by maintenance tools like xtfs_chstatus,
# xtfs_cleanup_osd, xtfs_remove_osd and xtfs_scrub if a DIR server is not
# explicitely set at the command line.
#
# Do *not* use this file to tell MRC and OSD which DIR to contact.
# Edit the corresponding configuration files mrcconfig.properties
# and osdconfig.properties instead.
dir_service.host = localhost
dir_service.port = 32638

View File

@@ -0,0 +1,154 @@
# optional debug level
# 0: emergency
# 1: alert
# 2: critical
# 3: error
# 4: warning
# 5: notice
# 6: info (default)
# 7: debug
#debug.level = 6
# optional debug categories - a space or comma-separated list of log message categories
# all (default) - enable logging for all categories
# lifecycle - log messaages pertaining to service lifecycles (threads)
# buffer - logs messages pertaining to buffers
# net - network-related log messages
# auth - authorization-related log messages
# stage - log messages pertaining to the request flow through the stages
# proc - log messages pertaining to any kind of request processing
# db - log messages pertaining storage on OSD or database access on MRC/DIR
# replication - logs messages pertaining to replication
# misc - any other log messages
#debug.categories = all
# port for the service to listen on
listen.port = 32638
# port for the status page (HTTP server)
http_port = 30638
# optional address for network device ("any" if not specified)
# listen.address = 127.0.0.1
# specify whether SSL is required
ssl.enabled = false
# server credentials for SSL handshakes
ssl.service_creds = /etc/xos/xtreemfs/truststore/certs/ds.p12
ssl.service_creds.pw = passphrase
ssl.service_creds.container = pkcs12
#ssl.trust_manager = org.xtreemfs.auth.plugin.SSLX509TrustManager
# trusted certificates for SSL handshakes
ssl.trusted_certs = /etc/xos/xtreemfs/truststore/certs/trusted.jks
ssl.trusted_certs.pw = jks_passphrase
ssl.trusted_certs.container = jks
# Enables the Grid SSL mode. This mode uses SSL only for user authentication
# and sends all data via an unencrypted TCP connection.
#ssl.grid_ssl = false
# time to wait for the directory service to become available on start-up
# before aborting
#startup.wait_for_dir = 30
# administrator password for privileged operations
#admin_password = passphrase
# Optional directory containing deployable policy implementations.
# Policies can be directly deployed as .java or .class files in this directory
# or one of its subdirectories. They will be compiled at startup time and
# loaded at runtime. Policies may have external dependencies that can be
# deployed either as .java, .class or .jar files. While Java and Class files
# may be located in subdirectories, JAR files mustn't.
policy_dir = /etc/xos/xtreemfs/policies
#monitoring = true
#monitoring.email.program = /usr/sbin/sendmail
#monitoring.email.sender = XtreemFS DIR
#monitoring.email.receiver =
#monitoring.max_warnings = 1
# If you want to monitor your XtreemFS installation through SNMP
# uncomment the following lines. You have to set snmp.enabled = true
# and provide a listen port and optional a address. Also optional
# is a path to an aclfile which controls which hosts can access the
# monitoring information via SNMP.
#snmp.enabled = true
#snmp.address = localhost
#snmp.port = 34638
#snmp.aclfile = /etc/xos/xtreemfs/snmp.acl
#####################################################################
# BabuDB configuration #
#####################################################################
# optional debug level (
# 0 = emergency,
# 1 = alert,
# 2 = critical,
# 3 = error,
# 4 = warning,
# 5 = notice,
# 6 = info,
# 7 = debug)
babudb.debug.level = 4
# optional debug category
#babudb.debug.category = all
# name for the database configuration file
#babudb.cfgFile = config.db
# base directory to store database index snapshots in
babudb.baseDir = /var/lib/xtreemfs/dir/database
# directory in which the database logs are stored
babudb.logDir = /var/lib/xtreemfs/dir/db-log
# SyncMode the synchronization mode to use for the logFile
# ASYNC - asynchronously write log entries (data is lost when system crashes).
# FSYNC - executes an fsync on the logfile before acknowledging the operation.
# FDATASYNC
# SYNC_WRITE - synchronously writes the log entry to disk before ack. Does not
# update the metadata.
# SYNC_WRITE_METADATA - synchronously writes the log entry to disk and updates
# the metadata before ack.
babudb.sync = FDATASYNC
# max queue length: if > 0, the queue for each worker is limited to maxQ
babudb.worker.maxQueueLength = 250
# number of worker threads to use
babudb.worker.numThreads = 0
# a checkpoint is generated, if maxLogfileSize is exceeded
babudb.maxLogfileSize = 16777216
# interval between two checks in seconds, 0 disables auto checkPointing
babudb.checkInterval = 300
# if set to a value > 0, operations are acknowledged immediately before
# they are written to the disk log. The disk logger will do batch writes
# and call fSync... every pseudoSyncWait seconds. This can be used to
# increase performance and emulate PostgreSQL behavior.
babudb.pseudoSyncWait = 200
# flag that determines whether the indices shall be compressed or not.
#babudb.compression = false
# maximum number of key-value pairs per block
#babudb.maxNumRecordsPerBlock = 16
# maximum size for a babudb on-disk index file
#babudb.maxBlockFileSize = 52428800
# Uncomment this line if you want to enable the DIR replication. The path points
# to the database replication configuration. Place here the hostnames of all replicas.
#babudb.plugin.0 = /etc/xos/xtreemfs/server-repl-plugin/dir.properties

View File

@@ -0,0 +1,190 @@
# optional debug level
# 0: emergency
# 1: alert
# 2: critical
# 3: error
# 4: warning
# 5: notice
# 6: info (default)
# 7: debug
#debug.level = 6
# optional debug categories - a space or comma-separated list of log message categories
# all (default) - enable logging for all categories
# lifecycle - log messaages pertaining to service lifecycles (threads)
# buffer - logs messages pertaining to buffers
# net - network-related log messages
# auth - authorization-related log messages
# stage - log messages pertaining to the request flow through the stages
# proc - log messages pertaining to any kind of request processing
# db - log messages pertaining storage on OSD or database access on MRC/DIR
# replication - logs messages pertaining to replication
# misc - any other log messages
#debug.categories = all
# port for the service to listen on
listen.port = 32636
# port for the status page (HTTP server)
http_port = 30636
# optional address for network device, "any" if not specified
# listen.address = 127.0.0.1
# optinal host name that is used to register the service at the DIR
# hostname = foo.bar.com
# interval for querying the Directory Service for new OSDs (in s)
osd_check_interval = 10
# Directory Service endpoint
dir_service.host = localhost
dir_service.port = 32638
# If you run a replicated DIR, you also have to set the addresses of the additional DIR replicas here:
#dir_service1.host = second-DIR-replica
#dir_service1.port = 32638
#dir_service2.host = third-DIR-replica
#dir_service2.port = 32638
# specify whether access time stamps are updated
no_atime = true
# granularity of the local clock (in ms) (0 disables it to always use the current system time)
local_clock_renewal = 0
# interval between two remote clock syncs (in ms)
remote_time_sync = 60000
# specify whether SSL is required
ssl.enabled = false
# server credentials for SSL handshakes
ssl.service_creds = /etc/xos/xtreemfs/truststore/certs/mrc.p12
ssl.service_creds.pw = passphrase
ssl.service_creds.container = pkcs12
#ssl.trust_manager = org.xtreemfs.auth.plugin.SSLX509TrustManager
# trusted certificates for SSL handshakes
ssl.trusted_certs = /etc/xos/xtreemfs/truststore/certs/trusted.jks
ssl.trusted_certs.pw = jks_passphrase
ssl.trusted_certs.container = jks
# Enables the Grid SSL mode. This mode uses SSL only for user authentication
# and sends all data via an unencrypted TCP connection.
#ssl.grid_ssl = false
# Authentication providers are used to retrieve the user identities
# from the client or from certificate.
# The default provider is org.xtreemfs.common.auth.NullAuthProvider, which just
# takes the information provided by the client. The name of a pluggable
# provider can be used here.
authentication_provider = org.xtreemfs.common.auth.NullAuthProvider
# Optional directory containing deployable policy implementations.
# Policies can be directly deployed as .java or .class files in this directory
# or one of its subdirectories. They will be compiled at startup time and
# loaded at runtime. Policies may have external dependencies that can be
# deployed either as .java, .class or .jar files. While Java and Class files
# may be located in subdirectories, JAR files mustn't. So far, pluggable
# policies have to inherit from either org.xtreemfs.mrc.ac.FileAccessPolicy,
# org.xtreemfs.mrc.osdstatus.OSDSelectionPolicy,
# org.xtreemfs.common.auth.AuthenticationProvider, or javax.ssl.TrustManager.
# Policies identified by policy IDs (OSDSelectionPolicy and FileAccessPolicy)
# require a public static long field called POLICY_ID that assigns the policy
# a unique number.
policy_dir = /etc/xos/xtreemfs/policies
# Shared secret between the MRC and all OSDs.
# The secret is used by the MRC to sign capabilities, i.e. security tokens for
# data access at OSDs. In turn, an OSD uses the secret to verify that the
# capability has been issued by the MRC. The shared secret will be replaced by
# a public key infrastructure in future releases.
capability_secret = secretPassphrase
# validity time span for capabilities in seconds
#capability_timeout = 600
# administrator password for privileged operations
#admin_password = passphrase
# If you want to monitor your XtreemFS installation through SNMP
# uncomment the following lines. You have to set snmp.enabled = true
# and provide a listen port and optional a address. Also optional
# is a path to an aclfile which controls which hosts can access the
# monitoring information via SNMP.
#snmp.enabled = true
#snmp.address = localhost
#snmp.port = 34636
#snmp.aclfile = /etc/xos/xtreemfs/snmp.acl
#####################################################################
# BabuDB configuration #
#####################################################################
# optional debug level (
# 0 = emergency,
# 1 = alert,
# 2 = critical,
# 3 = error,
# 4 = warning,
# 5 = notice,
# 6 = info,
# 7 = debug)
babudb.debug.level = 4
# optional debug category
#babudb.debug.category = all
# name for the database configuration file
#babudb.cfgFile = config.db
# base directory to store database index snapshots in
babudb.baseDir = /var/lib/xtreemfs/mrc/database
# directory in which the database logs are stored
babudb.logDir = /var/lib/xtreemfs/mrc/db-log
# SyncMode the synchronization mode to use for the logFile
# ASYNC - asynchronously write log entries (data is lost when system crashes).
# FSYNC - executes an fsync on the logfile before acknowledging the operation.
# FDATASYNC
# SYNC_WRITE - synchronously writes the log entry to disk before ack. Does not
# update the metadata.
# SYNC_WRITE_METADATA - synchronously writes the log entry to disk and updates
# the metadata before ack.
babudb.sync = ASYNC
# max queue length: if > 0, the queue for each worker is limited to maxQ
babudb.worker.maxQueueLength = 250
# number of worker threads to use
babudb.worker.numThreads = 0
# a checkpoint is generated, if maxLogfileSize is exceeded
babudb.maxLogfileSize = 16777216
# interval between two checks in seconds, 0 disables auto checkPointing
babudb.checkInterval = 300
# if set to a value > 0, operations are acknowledged immediately before
# they are written to the disk log. The disk logger will do batch writes
# and call fSync... every pseudoSyncWait seconds. This can be used to
# increase performance and emulate PostgreSQL behavior.
babudb.pseudoSyncWait = 0
# flag that determines whether the indices shall be compressed or not.
#babudb.compression = false
# maximum number of key-value pairs per block
#babudb.maxNumRecordsPerBlock = 16
# maximum size for a babudb on-disk index file
#babudb.maxBlockFileSize = 52428800
# Uncomment this line if you want to enable the MRC replication. The path points
# to the database replication configuration. Place here the hostnames of all replicas.
#
# Please note, if you replicate the MRC you have to change the option
# "babudb.sync" in this file from ASYNC to a synchronous one e.g., FDATASYNC.
#babudb.plugin.0 = /etc/xos/xtreemfs/server-repl-plugin/mrc.properties

View File

@@ -0,0 +1,130 @@
# optional debug level
# 0: emergency
# 1: alert
# 2: critical
# 3: error
# 4: warning
# 5: notice
# 6: info (default)
# 7: debug
#debug.level = 6
# optional debug categories - a space or comma-separated list of log message categories
# all (default) - enable logging for all categories
# lifecycle - log messaages pertaining to service lifecycles (threads)
# buffer - logs messages pertaining to buffers
# net - network-related log messages
# auth - authorization-related log messages
# stage - log messages pertaining to the request flow through the stages
# proc - log messages pertaining to any kind of request processing
# db - log messages pertaining storage on OSD or database access on MRC/DIR
# replication - logs messages pertaining to replication
# misc - any other log messages
#debug.categories = all
# port for the service to listen on
listen.port = 32640
# port for the status page (HTTP server)
http_port = 30640
# optional address for network device, "any" if not specified
# listen.address = 127.0.0.1
# optinal host name that is used to register the service at the DIR
# hostname = foo.bar.com
# Directory Service endpoint
dir_service.host = localhost
dir_service.port = 32638
# If you run a replicated DIR, you also have to set the addresses of the additional DIR replicas here:
#dir_service1.host = second-DIR-replica
#dir_service1.port = 32638
#dir_service2.host = third-DIR-replica
#dir_service2.port = 32638
# directory containing XtreemFS file content
object_dir = /var/lib/xtreemfs/objs/
# Number of storage threads. Increase it to improve concurrency in case of multiple open files.
# Set it to a value >1 only if the underlying device can cope with concurrency, e.g. an SSD.
#storage_threads = 1
# granularity of the local clock (in ms) (0 disables it to always use the current system time)
local_clock_renewal = 0
# interval between two remote clock syncs (in ms)
remote_time_sync = 60000
# specify whether SSL is required
ssl.enabled = false
# server credentials for SSL handshakes
ssl.service_creds = /etc/xos/xtreemfs/truststore/certs/osd.p12
ssl.service_creds.pw = passphrase
ssl.service_creds.container = pkcs12
#ssl.trust_manager = org.xtreemfs.auth.plugin.SSLX509TrustManager
# trusted certificates for SSL handshakes
ssl.trusted_certs = /etc/xos/xtreemfs/truststore/certs/trusted.jks
ssl.trusted_certs.pw = jks_passphrase
ssl.trusted_certs.container = jks
# Enables the Grid SSL mode. This mode uses SSL only for user authentication
# and sends all data via an unencrypted TCP connection.
#ssl.grid_ssl = false
# send and receive buffer sizes of sockets
#socket.send_buffer_size = 262144
#socket.recv_buffer_size = 262144
report_free_space = true
# specify whether internal OSD checksums are required
# if the flag is set to true, the OSD will calculate checksums for
# newly created objects, which will be checked when the object is read
checksums.enabled = false
# algorithm used for checksum calculation
# by default, Adler32, CRC32, MD5 and SHA-1 are supported
checksums.algorithm = Adler32
# Shared secret between the MRC and all OSDs.
# The secret is used by the MRC to sign capabilities, i.e. security tokens for
# data access at OSDs. In turn, an OSD uses the secret to verify that the
# capability has been issued by the MRC. The shared secret will be replaced by
# a public key infrastructure in future releases.
capability_secret = secretPassphrase
# administrator password for privileged operations
#admin_password = passphrase
# time to wait for the directory service to become available on start-up
# before aborting
#startup.wait_for_dir = 30
# Optional directory containing deployable policy implementations.
# Policies can be directly deployed as .java or .class files in this directory
# or one of its subdirectories. They will be compiled at startup time and
# loaded at runtime. Policies may have external dependencies that can be
# deployed either as .java, .class or .jar files. While Java and Class files
# may be located in subdirectories, JAR files mustn't.
policy_dir = /etc/xos/xtreemfs/policies
# If you want to monitor your XtreemFS installation through SNMP
# uncomment the following lines. You have to set snmp.enabled = true
# and provide a listen port and optional a address. Also optional
# is a path to an aclfile which controls which hosts can access the
# monitoring information via SNMP.
#snmp.enabled = true
#snmp.address = localhost
#snmp.port = 34640
#snmp.aclfile = /etc/xos/xtreemfs/snmp.acl
# Optional user-definied script to determine the health status of the OSD.
# The health status will be used in the default OSD selection policy to
# exclude OSDs with FAILED or WARNING health status.
# See the xtreemfs user guide for more information.
#health_check = /usr/share/xtreemfs/osd_health_check.sh

38
etc/xos/xtreemfs/snmp.acl Normal file
View File

@@ -0,0 +1,38 @@
# @(#)file jdmk.acl
# @(#)author Sun Microsystems, Inc.
# @(#)version 1.11
# @(#)lastedit 04/04/07
#
# Copyright 2004 Sun Microsystems, Inc. All rights reserved.
# SUN PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
# access: can take only "read-only" or "read-write" values
#
# managers can be :
# - hostname: hubble
# - ip v4 and v6 addresses: 123.456.789.12 , fe80::a00:20ff:fe9b:ea82
# - subnet mask: 123!255!255!255 (its an IPO address where "." are replaced
# by "!"). This way of expressing the subnet is deprecated, use the prefix
# notation.
# - ip v4 and v6 netmask prefix notation :
# 123.456.789.12/24, fe80::a00:20ff:fe9b:ea82/64
acl = {
{
communities = public
access = read-only
managers = localhost
}
{
communities = private
access = read-write
managers = localhost
}
}
trap = {
{
trap-community = public
hosts = localhost
}
}