hwraid/snapraid
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge tag 'upstream/11.5'

Browse Source 
Upstream version 11.5
This commit is contained in:
Mario Fetka 2020-09-11 13:42:24 +02:00
commit 1dd99167db
54 changed files with 506 additions and 234 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.version
View File

@ -1 +1 @@
11.3 11.5

18
HISTORY
View File

@ -1,6 +1,20 @@
SnapRAID HISTORY SnapRAID HISTORY
================ ================
11.5 2020/05
============
* Removed the default -march=native to allow to deploy in any machine.
* Fixed typos [Andrea Gelmini]
11.4 2020/05
============
* Fix build errors due new gcc 10 default for -fno-common.
* In fixing, if a parity is filtered out, don't attempt to recover its size,
and proceed without it if missing.
* Avoid unnecessary parity read when fixing the parity itself.
This improves the 'fix' speed when a parity file is completely missing.
* Removed a build warning about major/minor defined now in sys/sysmacros.h.
11.3 2018/11 11.3 2018/11
============ ============
* Fixed handing of Linux devices that have multiple slaves. This affects * Fixed handing of Linux devices that have multiple slaves. This affects
@ -139,7 +153,7 @@ SnapRAID HISTORY
A workaround was to just run 'sync' one time with the -N, A workaround was to just run 'sync' one time with the -N,
--force-nocopy option to disable the copy detection. --force-nocopy option to disable the copy detection.
* Restored the -O2 optimization option for Windows binaries, as -Og has * Restored the -O2 optimization option for Windows binaries, as -Og has
a too big performance penality. a too big performance penalty.
9.2 2016/01 9.2 2016/01
=========== ===========
@ -412,7 +426,7 @@ SnapRAID HISTORY
and Opteron. and Opteron.
* Faster RAID5 and RAID6 implementation for ARM 64 bit CPUs. * Faster RAID5 and RAID6 implementation for ARM 64 bit CPUs.
* If a silent error is found during a "sync" command, directly marks * If a silent error is found during a "sync" command, directly marks
the block as bad like in "scrub", without stopping the the "sync" the block as bad like in "scrub", without stopping the "sync"
process. process.
* Sort files by inode when listing the directory. This improves * Sort files by inode when listing the directory. This improves
the scanning performance. the scanning performance.

5
Makefile.am
View File

@ -83,6 +83,7 @@ noinst_HEADERS = \
cmdline/murmur3test.c \ cmdline/murmur3test.c \
cmdline/spooky2.c \ cmdline/spooky2.c \
cmdline/spooky2test.c \ cmdline/spooky2test.c \
cmdline/metro.c \
cmdline/fnmatch.h \ cmdline/fnmatch.h \
cmdline/import.h \ cmdline/import.h \
cmdline/search.h \ cmdline/search.h \
@ -158,7 +159,7 @@ maintainer-clean-local:
CHECKFLAGS_COMMON = --test-skip-device --test-skip-self --test-force-progress --no-warnings --test-parity-limit=3333333 CHECKFLAGS_COMMON = --test-skip-device --test-skip-self --test-force-progress --no-warnings --test-parity-limit=3333333
# --test-force-order-alpha # --test-force-order-alpha
# Ensures to process files always in the same order despites # Ensures to process files always in the same order despite
# the inode, physical location, and dir order assigned by the OS. # the inode, physical location, and dir order assigned by the OS.
CHECKFLAGS_ALPHA = $(CHECKFLAGS_COMMON) --test-force-order-alpha CHECKFLAGS_ALPHA = $(CHECKFLAGS_COMMON) --test-force-order-alpha
@ -499,7 +500,7 @@ endif
$(FAILENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-run "rm bench/disk1/RUN-RM" --test-expect-failure sync $(FAILENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-run "rm bench/disk1/RUN-RM" --test-expect-failure sync
echo RUN > bench/disk1/RUN-CHMOD echo RUN > bench/disk1/RUN-CHMOD
if HAVE_POSIX if HAVE_POSIX
# Doesn't run this test as root because the root user overrride permissions # Doesn't run this test as root because the root user override permissions
if [[ $$EUID -ne 0 ]]; then \ if [[ $$EUID -ne 0 ]]; then \
$(FAILENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-run "chmod a-r bench/disk1/RUN-CHMOD" --test-expect-failure sync; \ $(FAILENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-run "chmod a-r bench/disk1/RUN-CHMOD" --test-expect-failure sync; \
fi fi

5
Makefile.in
View File

@ -429,6 +429,7 @@ noinst_HEADERS = \
cmdline/murmur3test.c \ cmdline/murmur3test.c \
cmdline/spooky2.c \ cmdline/spooky2.c \
cmdline/spooky2test.c \ cmdline/spooky2test.c \
cmdline/metro.c \
cmdline/fnmatch.h \ cmdline/fnmatch.h \
cmdline/import.h \ cmdline/import.h \
cmdline/search.h \ cmdline/search.h \
@ -487,7 +488,7 @@ man_MANS = snapraid.1
CHECKFLAGS_COMMON = --test-skip-device --test-skip-self --test-force-progress --no-warnings --test-parity-limit=3333333 CHECKFLAGS_COMMON = --test-skip-device --test-skip-self --test-force-progress --no-warnings --test-parity-limit=3333333
# --test-force-order-alpha # --test-force-order-alpha
# Ensures to process files always in the same order despites # Ensures to process files always in the same order despite
# the inode, physical location, and dir order assigned by the OS. # the inode, physical location, and dir order assigned by the OS.
CHECKFLAGS_ALPHA = $(CHECKFLAGS_COMMON) --test-force-order-alpha CHECKFLAGS_ALPHA = $(CHECKFLAGS_COMMON) --test-force-order-alpha
@ -1409,7 +1410,7 @@ check-local:
@HAVE_THREAD_CHECKER_FALSE@ echo RUN > bench/disk1/RUN-RM @HAVE_THREAD_CHECKER_FALSE@ echo RUN > bench/disk1/RUN-RM
@HAVE_THREAD_CHECKER_FALSE@ $(FAILENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-run "rm bench/disk1/RUN-RM" --test-expect-failure sync @HAVE_THREAD_CHECKER_FALSE@ $(FAILENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-run "rm bench/disk1/RUN-RM" --test-expect-failure sync
@HAVE_THREAD_CHECKER_FALSE@ echo RUN > bench/disk1/RUN-CHMOD @HAVE_THREAD_CHECKER_FALSE@ echo RUN > bench/disk1/RUN-CHMOD
# Doesn't run this test as root because the root user overrride permissions # Doesn't run this test as root because the root user override permissions
@HAVE_POSIX_TRUE@@HAVE_THREAD_CHECKER_FALSE@ if [[ $$EUID -ne 0 ]]; then \ @HAVE_POSIX_TRUE@@HAVE_THREAD_CHECKER_FALSE@ if [[ $$EUID -ne 0 ]]; then \
@HAVE_POSIX_TRUE@@HAVE_THREAD_CHECKER_FALSE@ $(FAILENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-run "chmod a-r bench/disk1/RUN-CHMOD" --test-expect-failure sync; \ @HAVE_POSIX_TRUE@@HAVE_THREAD_CHECKER_FALSE@ $(FAILENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-run "chmod a-r bench/disk1/RUN-CHMOD" --test-expect-failure sync; \
@HAVE_POSIX_TRUE@@HAVE_THREAD_CHECKER_FALSE@ fi @HAVE_POSIX_TRUE@@HAVE_THREAD_CHECKER_FALSE@ fi

5
README
View File

@ -11,6 +11,8 @@ have a lot of big files that rarely change.
Beside the ability to recover from disk failures, the other Beside the ability to recover from disk failures, the other
features of SnapRAID are: features of SnapRAID are:
* You can use disk already filled with files, without the need to
reformat them. You will access them like now.
* All your data is hashed to ensure data integrity and to avoid * All your data is hashed to ensure data integrity and to avoid
silent corruption. silent corruption.
* If the failed disks are too many to allow a recovery, * If the failed disks are too many to allow a recovery,
@ -18,12 +20,11 @@ features of SnapRAID are:
All the data in the other disks is safe. All the data in the other disks is safe.
* If you accidentally delete some files in a disk, you can * If you accidentally delete some files in a disk, you can
recover them. recover them.
* You can start with already filled disks.
* The disks can have different sizes. * The disks can have different sizes.
* You can add disks at any time. * You can add disks at any time.
* It doesn't lock-in your data. You can stop using SnapRAID at any * It doesn't lock-in your data. You can stop using SnapRAID at any
time without the need to reformat or move data. time without the need to reformat or move data.
* To access a file, a single disk needs to spin, saving power and * To access a file, only a single disk needs to spin, saving power and
producing less noise. producing less noise.
The official site of SnapRAID is: The official site of SnapRAID is:

12
TODO
View File

@ -25,8 +25,8 @@ with a mono thread implementation with 100.0000 files.
* Support more parity levels * Support more parity levels
It can be done with a generic computation function, using It can be done with a generic computation function, using
intrinsic for SSSE3 and AVX instructions. intrinsic for SSSE3 and AVX instructions.
It would be intersting to compare performance with the hand-written It would be interesting to compare performance with the hand-written
assembler functions. Eventially we can convert them to use intrinsic also. assembler functions. Eventually we can convert them to use intrinsic also.
https://sourceforge.net/p/snapraid/discussion/1677233/thread/9dbd7581/ https://sourceforge.net/p/snapraid/discussion/1677233/thread/9dbd7581/
* Extend haspdeep to support the SnapRAID hash : * Extend haspdeep to support the SnapRAID hash :
@ -151,7 +151,7 @@ See: https://sourceforge.net/p/snapraid/discussion/1677233/thread/cdea773f/
* Allocate parity minimizing concurrent use of it * Allocate parity minimizing concurrent use of it
Each parity allocation should check for a parity Each parity allocation should check for a parity
range with less utilization by other disks. range with less utilization by other disks.
We need to take care do disable this meachnism when the parity space We need to take care do disable this mechanism when the parity space
is near to fillup the parity partition. is near to fillup the parity partition.
See: https://sourceforge.net/p/snapraid/discussion/1677233/thread/1797bf7d/ See: https://sourceforge.net/p/snapraid/discussion/1677233/thread/1797bf7d/
+ This increase the possibility of recovering with multiple failures with not + This increase the possibility of recovering with multiple failures with not
@ -182,7 +182,7 @@ allowing the user to choose where to put it.
- It won't work with disks of different size. - It won't work with disks of different size.
Suppose to have all disks of size N, with only one of size M>N. Suppose to have all disks of size N, with only one of size M>N.
To fully use the M space, you can allocate a full N parity in such disk, To fully use the M space, you can allocate a full N parity in such disk,
but the remaning space will also need additional parity in the other disks, but the remaining space will also need additional parity in the other disks,
in fact requiring a total of M parity for the array. in fact requiring a total of M parity for the array.
In the end, we cannot avoid that the first biggest disk added is fully In the end, we cannot avoid that the first biggest disk added is fully
dedicated to parity, even if it means to leave some space unused. dedicated to parity, even if it means to leave some space unused.
@ -199,7 +199,7 @@ But it should be only few bits for each file. So, it should be manageable.
A lot of discussions about this feature :) A lot of discussions about this feature :)
https://sourceforge.net/p/snapraid/discussion/1677233/thread/b2cd9385/ https://sourceforge.net/p/snapraid/discussion/1677233/thread/b2cd9385/
- The only benefit is to distribute better the data. This could help the recovery process, - The only benefit is to distribute better the data. This could help the recovery process,
in case of multiple failures. But no real usability or funtionality benefit in the normal in case of multiple failures. But no real usability or functionality benefit in the normal
case. case.
* https://sourceforge.net/p/snapraid/discussion/1677233/thread/2cb97e8a/ * https://sourceforge.net/p/snapraid/discussion/1677233/thread/2cb97e8a/
@ -229,7 +229,7 @@ are automatically deleted.
* Checks if splitting hash/parity computation in 4K pages * Checks if splitting hash/parity computation in 4K pages
can improve speed in sync. That should increase cache locality, can improve speed in sync. That should increase cache locality,
because we read the data two times for hash and and parity, because we read the data two times for hash and parity,
and if we manage to keep it in the cache, we should save time. and if we manage to keep it in the cache, we should save time.
- We now hash first the faster disks, and this could - We now hash first the faster disks, and this could
reduce performance as we'll have to wait for all disks. reduce performance as we'll have to wait for all disks.

4
acinclude.m4
View File

@ -1,7 +1,7 @@
dnl @synopsis AC_CHECK_CC_OPT(flag, ifyes, ifno) dnl @synopsis AC_CHECK_CC_OPT(flag, ifyes, ifno)
dnl dnl
dnl Shows a message as like "checking wether gcc accepts flag ... no" dnl Shows a message as like "checking whether gcc accepts flag ... no"
dnl and executess ifyes or ifno. dnl and executes ifyes or ifno.
AC_DEFUN([AC_CHECK_CC_OPT], AC_DEFUN([AC_CHECK_CC_OPT],
[ [

72
cmdline/check.c
View File

@ -574,7 +574,7 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
} else { } else {
log_tag("recover_unsync:%u:%u: Skipped for%s%s\n", pos, n, log_tag("recover_unsync:%u:%u: Skipped for%s%s\n", pos, n,
!something_to_recover ? " nothing to recover" : "", !something_to_recover ? " nothing to recover" : "",
!something_unsynced ? " nothing unsynched" : "" !something_unsynced ? " nothing unsynced" : ""
); );
} }
@ -1441,7 +1441,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
char path[PATH_MAX]; char path[PATH_MAX];
struct stat st; struct stat st;
struct snapraid_file* file; struct snapraid_file* file;
int unsuccesful = 0; int unsuccessful = 0;
file = node->data; file = node->data;
node = node->next; /* next node */ node = node->next; /* next node */
@ -1460,24 +1460,24 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
pathprint(path, sizeof(path), "%s%s", disk->dir, file->sub); pathprint(path, sizeof(path), "%s%s", disk->dir, file->sub);
ret = stat(path, &st); ret = stat(path, &st);
if (ret == -1) { if (ret == -1) {
unsuccesful = 1; unsuccessful = 1;
log_error("Error stating empty file '%s'. %s.\n", path, strerror(errno)); log_error("Error stating empty file '%s'. %s.\n", path, strerror(errno));
log_tag("error:%s:%s: Empty file stat error\n", disk->name, esc_tag(file->sub, esc_buffer)); log_tag("error:%s:%s: Empty file stat error\n", disk->name, esc_tag(file->sub, esc_buffer));
++error; ++error;
} else if (!S_ISREG(st.st_mode)) { } else if (!S_ISREG(st.st_mode)) {
unsuccesful = 1; unsuccessful = 1;
log_tag("error:%s:%s: Empty file error for not regular file\n", disk->name, esc_tag(file->sub, esc_buffer)); log_tag("error:%s:%s: Empty file error for not regular file\n", disk->name, esc_tag(file->sub, esc_buffer));
++error; ++error;
} else if (st.st_size != 0) { } else if (st.st_size != 0) {
unsuccesful = 1; unsuccessful = 1;
log_tag("error:%s:%s: Empty file error for size '%" PRIu64 "'\n", disk->name, esc_tag(file->sub, esc_buffer), (uint64_t)st.st_size); log_tag("error:%s:%s: Empty file error for size '%" PRIu64 "'\n", disk->name, esc_tag(file->sub, esc_buffer), (uint64_t)st.st_size);
++error; ++error;
} }
if (fix && unsuccesful) { if (fix && unsuccessful) {
int f; int f;
/* create the ancestor directories */ /* create the ancestor directories */
@ -1552,7 +1552,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
struct stat st; struct stat st;
struct stat stto; struct stat stto;
struct snapraid_link* slink; struct snapraid_link* slink;
int unsuccesful = 0; int unsuccessful = 0;
int unrecoverable = 0; int unrecoverable = 0;
slink = node->data; slink = node->data;
@ -1568,13 +1568,13 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
pathprint(path, sizeof(path), "%s%s", disk->dir, slink->sub); pathprint(path, sizeof(path), "%s%s", disk->dir, slink->sub);
ret = stat(path, &st); ret = stat(path, &st);
if (ret == -1) { if (ret == -1) {
unsuccesful = 1; unsuccessful = 1;
log_error("Error stating hardlink '%s'. %s.\n", path, strerror(errno)); log_error("Error stating hardlink '%s'. %s.\n", path, strerror(errno));
log_tag("hardlink_error:%s:%s:%s: Hardlink stat error\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt)); log_tag("hardlink_error:%s:%s:%s: Hardlink stat error\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt));
++error; ++error;
} else if (!S_ISREG(st.st_mode)) { } else if (!S_ISREG(st.st_mode)) {
unsuccesful = 1; unsuccessful = 1;
log_tag("hardlink_error:%s:%s:%s: Hardlink error for not regular file\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt)); log_tag("hardlink_error:%s:%s:%s: Hardlink error for not regular file\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt));
++error; ++error;
@ -1584,7 +1584,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
pathprint(pathto, sizeof(pathto), "%s%s", disk->dir, slink->linkto); pathprint(pathto, sizeof(pathto), "%s%s", disk->dir, slink->linkto);
ret = stat(pathto, &stto); ret = stat(pathto, &stto);
if (ret == -1) { if (ret == -1) {
unsuccesful = 1; unsuccessful = 1;
if (errno == ENOENT) { if (errno == ENOENT) {
unrecoverable = 1; unrecoverable = 1;
@ -1604,12 +1604,12 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
log_tag("hardlink_error:%s:%s:%s: Hardlink to stat error\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt)); log_tag("hardlink_error:%s:%s:%s: Hardlink to stat error\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt));
++error; ++error;
} else if (!S_ISREG(stto.st_mode)) { } else if (!S_ISREG(stto.st_mode)) {
unsuccesful = 1; unsuccessful = 1;
log_tag("hardlink_error:%s:%s:%s: Hardlink-to error for not regular file\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt)); log_tag("hardlink_error:%s:%s:%s: Hardlink-to error for not regular file\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt));
++error; ++error;
} else if (!unsuccesful && st.st_ino != stto.st_ino) { } else if (!unsuccessful && st.st_ino != stto.st_ino) {
unsuccesful = 1; unsuccessful = 1;
log_error("Mismatch hardlink '%s' and '%s'. Different inode.\n", path, pathto); log_error("Mismatch hardlink '%s' and '%s'. Different inode.\n", path, pathto);
log_tag("hardlink_error:%s:%s:%s: Hardlink mismatch for different inode\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt)); log_tag("hardlink_error:%s:%s:%s: Hardlink mismatch for different inode\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt));
@ -1620,13 +1620,13 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
pathprint(path, sizeof(path), "%s%s", disk->dir, slink->sub); pathprint(path, sizeof(path), "%s%s", disk->dir, slink->sub);
ret = readlink(path, linkto, sizeof(linkto)); ret = readlink(path, linkto, sizeof(linkto));
if (ret < 0) { if (ret < 0) {
unsuccesful = 1; unsuccessful = 1;
log_error("Error reading symlink '%s'. %s.\n", path, strerror(errno)); log_error("Error reading symlink '%s'. %s.\n", path, strerror(errno));
log_tag("symlink_error:%s:%s: Symlink read error\n", disk->name, esc_tag(slink->sub, esc_buffer)); log_tag("symlink_error:%s:%s: Symlink read error\n", disk->name, esc_tag(slink->sub, esc_buffer));
++error; ++error;
} else if (ret >= PATH_MAX) { } else if (ret >= PATH_MAX) {
unsuccesful = 1; unsuccessful = 1;
log_error("Error reading symlink '%s'. Symlink too long.\n", path); log_error("Error reading symlink '%s'. Symlink too long.\n", path);
log_tag("symlink_error:%s:%s: Symlink read error\n", disk->name, esc_tag(slink->sub, esc_buffer)); log_tag("symlink_error:%s:%s: Symlink read error\n", disk->name, esc_tag(slink->sub, esc_buffer));
@ -1635,7 +1635,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
linkto[ret] = 0; linkto[ret] = 0;
if (strcmp(linkto, slink->linkto) != 0) { if (strcmp(linkto, slink->linkto) != 0) {
unsuccesful = 1; unsuccessful = 1;
log_tag("symlink_error:%s:%s: Symlink data error '%s' instead of '%s'\n", disk->name, esc_tag(slink->sub, esc_buffer), linkto, slink->linkto); log_tag("symlink_error:%s:%s: Symlink data error '%s' instead of '%s'\n", disk->name, esc_tag(slink->sub, esc_buffer), linkto, slink->linkto);
++error; ++error;
@ -1643,7 +1643,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
} }
} }
if (fix && unsuccesful && !unrecoverable) { if (fix && unsuccessful && !unrecoverable) {
/* create the ancestor directories */ /* create the ancestor directories */
ret = mkancestor(path); ret = mkancestor(path);
if (ret != 0) { if (ret != 0) {
@ -1720,7 +1720,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
char path[PATH_MAX]; char path[PATH_MAX];
struct stat st; struct stat st;
struct snapraid_dir* dir; struct snapraid_dir* dir;
int unsuccesful = 0; int unsuccessful = 0;
dir = node->data; dir = node->data;
node = node->next; /* next node */ node = node->next; /* next node */
@ -1734,19 +1734,19 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
pathprint(path, sizeof(path), "%s%s", disk->dir, dir->sub); pathprint(path, sizeof(path), "%s%s", disk->dir, dir->sub);
ret = stat(path, &st); ret = stat(path, &st);
if (ret == -1) { if (ret == -1) {
unsuccesful = 1; unsuccessful = 1;
log_error("Error stating dir '%s'. %s.\n", path, strerror(errno)); log_error("Error stating dir '%s'. %s.\n", path, strerror(errno));
log_tag("dir_error:%s:%s: Dir stat error\n", disk->name, esc_tag(dir->sub, esc_buffer)); log_tag("dir_error:%s:%s: Dir stat error\n", disk->name, esc_tag(dir->sub, esc_buffer));
++error; ++error;
} else if (!S_ISDIR(st.st_mode)) { } else if (!S_ISDIR(st.st_mode)) {
unsuccesful = 1; unsuccessful = 1;
log_tag("dir_error:%s:%s: Dir error for not directory\n", disk->name, esc_tag(dir->sub, esc_buffer)); log_tag("dir_error:%s:%s: Dir error for not directory\n", disk->name, esc_tag(dir->sub, esc_buffer));
++error; ++error;
} }
if (fix && unsuccesful) { if (fix && unsuccessful) {
/* create the ancestor directories */ /* create the ancestor directories */
ret = mkancestor(path); ret = mkancestor(path);
if (ret != 0) { if (ret != 0) {
@ -1973,6 +1973,17 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b
} }
parity_ptr[l] = &parity[l]; parity_ptr[l] = &parity[l];
/* if the parity is excluded */
if (state->parity[l].is_excluded_by_filter) {
/* open for reading, and ignore error */
ret = parity_open(parity_ptr[l], &state->parity[l], l, state->file_mode, state->block_size, state->opt.parity_limit_size);
if (ret == -1) {
/* continue anyway */
parity_ptr[l] = 0;
}
} else {
/* open for writing */
ret = parity_create(parity_ptr[l], &state->parity[l], l, state->file_mode, state->block_size, state->opt.parity_limit_size); ret = parity_create(parity_ptr[l], &state->parity[l], l, state->file_mode, state->block_size, state->opt.parity_limit_size);
if (ret == -1) { if (ret == -1) {
/* LCOV_EXCL_START */ /* LCOV_EXCL_START */
@ -1989,6 +2000,7 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b
/* LCOV_EXCL_STOP */ /* LCOV_EXCL_STOP */
} }
} }
}
} else if (!state->opt.auditonly) { } else if (!state->opt.auditonly) {
/* if checking, open the file for reading */ /* if checking, open the file for reading */
/* it may fail if the file doesn't exist, in this case we continue to check the files */ /* it may fail if the file doesn't exist, in this case we continue to check the files */
@ -2030,16 +2042,28 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b
/* try to close only if opened */ /* try to close only if opened */
for (l = 0; l < state->level; ++l) { for (l = 0; l < state->level; ++l) {
if (parity_ptr[l]) { if (parity_ptr[l]) {
ret = parity_close(parity_ptr[l]); /* if fixing and not excluded, truncate parity not valid */
/* LCOV_EXCL_START */ if (fix && !state->parity[l].is_excluded_by_filter) {
ret = parity_truncate(parity_ptr[l]);
if (ret == -1) { if (ret == -1) {
/* LCOV_EXCL_START */
log_fatal("DANGER! Unexpected truncate error in %s disk.\n", lev_name(l));
++error;
/* continue, as we are already exiting */
/* LCOV_EXCL_STOP */
}
}
ret = parity_close(parity_ptr[l]);
if (ret == -1) {
/* LCOV_EXCL_START */
log_fatal("DANGER! Unexpected close error in %s disk.\n", lev_name(l)); log_fatal("DANGER! Unexpected close error in %s disk.\n", lev_name(l));
++error; ++error;
/* continue, as we are already exiting */ /* continue, as we are already exiting */
}
/* LCOV_EXCL_STOP */ /* LCOV_EXCL_STOP */
} }
} }
}
/* abort if error are present */ /* abort if error are present */
if (error != 0) if (error != 0)

2
cmdline/device.c
View File

@ -818,7 +818,7 @@ static void state_smart(unsigned n, tommy_list* low)
/* |<##################################################################72>|####80>| */ /* |<##################################################################72>|####80>| */
printf("These values are the probabilities that in the next year you'll have a\n"); printf("These values are the probabilities that in the next year you'll have a\n");
printf("sequence of failures that the parity WONT be able to recover, assuming\n"); printf("sequence of failures that the parity WON'T be able to recover, assuming\n");
printf("that you regularly scrub, and in case repair, the array in the specified\n"); printf("that you regularly scrub, and in case repair, the array in the specified\n");
printf("time.\n"); printf("time.\n");

8
cmdline/elem.h
View File

@ -349,7 +349,7 @@ struct snapraid_disk {
block_off_t first_free_block; block_off_t first_free_block;
int has_volatile_inodes; /**< If the underline file-system has not persistent inodes. */ int has_volatile_inodes; /**< If the underline file-system has not persistent inodes. */
int has_volatile_hardlinks; /**< If the underline file-system has not syncronized metadata for hardlink (NTFS). */ int has_volatile_hardlinks; /**< If the underline file-system has not synchronized metadata for hardlink (NTFS). */
int has_unreliable_physical; /**< If the physical offset of files has duplicates. */ int has_unreliable_physical; /**< If the physical offset of files has duplicates. */
int has_different_uuid; /**< If the disk has a different UUID, meaning that it is not the same file-system. */ int has_different_uuid; /**< If the disk has a different UUID, meaning that it is not the same file-system. */
int has_unsupported_uuid; /**< If the disk doesn't report UUID, meaning it's not supported. */ int has_unsupported_uuid; /**< If the disk doesn't report UUID, meaning it's not supported. */
@ -1000,7 +1000,7 @@ int fs_check(struct snapraid_disk* disk);
* After this call you can use the par2file/par2block operations * After this call you can use the par2file/par2block operations
* to query the relation. * to query the relation.
* *
* \note This function is NOT thread-safe as it uses the the disk cache. + * \note This function is NOT thread-safe as it uses the disk cache. +
*/ */
void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos); void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos);
@ -1010,7 +1010,7 @@ void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snap
* After this call the par2file/par2block operations * After this call the par2file/par2block operations
* won't find anymore the parity association. * won't find anymore the parity association.
* *
* \note This function is NOT thread-safe as it uses the the disk cache. * \note This function is NOT thread-safe as it uses the disk cache.
*/ */
void fs_deallocate(struct snapraid_disk* disk, block_off_t pos); void fs_deallocate(struct snapraid_disk* disk, block_off_t pos);
@ -1129,7 +1129,7 @@ static inline snapraid_info info_make(time_t last_access, int error, int rehash,
/** /**
* Extract the time information. * Extract the time information.
* This is the last time when the block was know to be correct. * This is the last time when the block was know to be correct.
* The "scrubbed" info tells if the time is referreing at the latest sync or scrub. * The "scrubbed" info tells if the time is referring at the latest sync or scrub.
*/ */
static inline time_t info_get_time(snapraid_info info) static inline time_t info_get_time(snapraid_info info)
{ {

13
cmdline/io.c
View File

@ -19,6 +19,19 @@
#include "io.h" #include "io.h"
void (*io_start)(struct snapraid_io* io,
block_off_t blockstart, block_off_t blockmax,
int (*block_is_enabled)(void* arg, block_off_t), void* blockarg) = 0;
void (*io_stop)(struct snapraid_io* io) = 0;
block_off_t (*io_read_next)(struct snapraid_io* io, void*** buffer) = 0;
struct snapraid_task* (*io_data_read)(struct snapraid_io* io, unsigned* diskcur, unsigned* waiting_map, unsigned* waiting_mac) = 0;
struct snapraid_task* (*io_parity_read)(struct snapraid_io* io, unsigned* levcur, unsigned* waiting_map, unsigned* waiting_mac) = 0;
void (*io_parity_write)(struct snapraid_io* io, unsigned* levcur, unsigned* waiting_map, unsigned* waiting_mac) = 0;
void (*io_write_preset)(struct snapraid_io* io, block_off_t blockcur, int skip) = 0;
void (*io_write_next)(struct snapraid_io* io, block_off_t blockcur, int skip, int* writer_error) = 0;
void (*io_refresh)(struct snapraid_io* io) = 0;
/** /**
* Get the next block position to process. * Get the next block position to process.
*/ */

20
cmdline/io.h
View File

@ -306,21 +306,21 @@ void io_init(struct snapraid_io* io, struct snapraid_state* state,
struct snapraid_parity_handle* parity_handle_map, unsigned parity_handle_max); struct snapraid_parity_handle* parity_handle_map, unsigned parity_handle_max);
/** /**
* Deinitialize te InputOutput workers. * Deinitialize the InputOutput workers.
*/ */
void io_done(struct snapraid_io* io); void io_done(struct snapraid_io* io);
/** /**
* Start all the worker threads. * Start all the worker threads.
*/ */
void (*io_start)(struct snapraid_io* io, extern void (*io_start)(struct snapraid_io* io,
block_off_t blockstart, block_off_t blockmax, block_off_t blockstart, block_off_t blockmax,
int (*block_is_enabled)(void* arg, block_off_t), void* blockarg); int (*block_is_enabled)(void* arg, block_off_t), void* blockarg);
/** /**
* Stop all the worker threads. * Stop all the worker threads.
*/ */
void (*io_stop)(struct snapraid_io* io); extern void (*io_stop)(struct snapraid_io* io);
/** /**
* Next read position. * Next read position.
@ -332,7 +332,7 @@ void (*io_stop)(struct snapraid_io* io);
* \param buffer The data buffers to use for this position. * \param buffer The data buffers to use for this position.
* \return The parity position. * \return The parity position.
*/ */
block_off_t (*io_read_next)(struct snapraid_io* io, void*** buffer); extern block_off_t (*io_read_next)(struct snapraid_io* io, void*** buffer);
/** /**
* Read a data block. * Read a data block.
@ -343,7 +343,7 @@ block_off_t (*io_read_next)(struct snapraid_io* io, void*** buffer);
* \param diskcur The position of the data block in the ::handle_map vector. * \param diskcur The position of the data block in the ::handle_map vector.
* \return The completed task. * \return The completed task.
*/ */
struct snapraid_task* (*io_data_read)(struct snapraid_io* io, unsigned* diskcur, unsigned* waiting_map, unsigned* waiting_mac); extern struct snapraid_task* (*io_data_read)(struct snapraid_io* io, unsigned* diskcur, unsigned* waiting_map, unsigned* waiting_mac);
/** /**
* Read a parity block. * Read a parity block.
@ -354,7 +354,7 @@ struct snapraid_task* (*io_data_read)(struct snapraid_io* io, unsigned* diskcur,
* \param levcur The position of the parity block in the ::parity_handle_map vector. * \param levcur The position of the parity block in the ::parity_handle_map vector.
* \return The completed task. * \return The completed task.
*/ */
struct snapraid_task* (*io_parity_read)(struct snapraid_io* io, unsigned* levcur, unsigned* waiting_map, unsigned* waiting_mac); extern struct snapraid_task* (*io_parity_read)(struct snapraid_io* io, unsigned* levcur, unsigned* waiting_map, unsigned* waiting_mac);
/** /**
* Write of a parity block. * Write of a parity block.
@ -364,7 +364,7 @@ struct snapraid_task* (*io_parity_read)(struct snapraid_io* io, unsigned* levcur
* \param io InputOutput context. * \param io InputOutput context.
* \param levcur The position of the parity block in the ::parity_handle_map vector. * \param levcur The position of the parity block in the ::parity_handle_map vector.
*/ */
void (*io_parity_write)(struct snapraid_io* io, unsigned* levcur, unsigned* waiting_map, unsigned* waiting_mac); extern void (*io_parity_write)(struct snapraid_io* io, unsigned* levcur, unsigned* waiting_map, unsigned* waiting_mac);
/** /**
* Preset the write position. * Preset the write position.
@ -376,7 +376,7 @@ void (*io_parity_write)(struct snapraid_io* io, unsigned* levcur, unsigned* wait
* \param blockcur The parity position to write. * \param blockcur The parity position to write.
* \param skip Skip the writes, in case parity doesn't need to be updated. * \param skip Skip the writes, in case parity doesn't need to be updated.
*/ */
void (*io_write_preset)(struct snapraid_io* io, block_off_t blockcur, int skip); extern void (*io_write_preset)(struct snapraid_io* io, block_off_t blockcur, int skip);
/** /**
* Next write position. * Next write position.
@ -389,12 +389,12 @@ void (*io_write_preset)(struct snapraid_io* io, block_off_t blockcur, int skip);
* \param skip Skip the writes, in case parity doesn't need to be updated. * \param skip Skip the writes, in case parity doesn't need to be updated.
* \param writer_error Return the number of errors. Vector of IO_WRITER_ERROR_MAX elements. * \param writer_error Return the number of errors. Vector of IO_WRITER_ERROR_MAX elements.
*/ */
void (*io_write_next)(struct snapraid_io* io, block_off_t blockcur, int skip, int* writer_error); extern void (*io_write_next)(struct snapraid_io* io, block_off_t blockcur, int skip, int* writer_error);
/** /**
* Refresh the number of cached blocks for all data and parity disks. * Refresh the number of cached blocks for all data and parity disks.
*/ */
void (*io_refresh)(struct snapraid_io* io); extern void (*io_refresh)(struct snapraid_io* io);
#endif #endif

108
cmdline/metro.c Normal file
View File

@ -0,0 +1,108 @@
/*
* Copyright (C) 2019 Andrea Mazzoleni
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/*
* Derivative work from metrohash128.cpp
*
* metrohash128.cpp
*
* Copyright 2015-2018 J. Andrew Rogers
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
static const uint64_t k0 = 0xC83A91E1;
static const uint64_t k1 = 0x8648DBDB;
static const uint64_t k2 = 0x7BDEC03B;
static const uint64_t k3 = 0x2F5870A5;
void MetroHash128(const void* data, size_t size, const uint8_t* seed, uint8_t* digest)
{
const uint8_t* ptr = data;
uint64_t v[4];
v[0] = (util_read64(seed) - k0) * k3;
v[1] = (util_read64(seed + 8) + k1) * k2;
if (size >= 32) {
v[2] = (util_read64(seed) + k0) * k2;
v[3] = (util_read64(seed + 8) - k1) * k3;
do {
v[0] += util_read64(ptr) * k0; ptr += 8; v[0] = util_rotr64(v[0], 29) + v[2];
v[1] += util_read64(ptr) * k1; ptr += 8; v[1] = util_rotr64(v[1], 29) + v[3];
v[2] += util_read64(ptr) * k2; ptr += 8; v[2] = util_rotr64(v[2], 29) + v[0];
v[3] += util_read64(ptr) * k3; ptr += 8; v[3] = util_rotr64(v[3], 29) + v[1];
size -= 32;
} while (size >= 32);
v[2] ^= util_rotr64(((v[0] + v[3]) * k0) + v[1], 21) * k1;
v[3] ^= util_rotr64(((v[1] + v[2]) * k1) + v[0], 21) * k0;
v[0] ^= util_rotr64(((v[0] + v[2]) * k0) + v[3], 21) * k1;
v[1] ^= util_rotr64(((v[1] + v[3]) * k1) + v[2], 21) * k0;
}
if (size >= 16) {
v[0] += util_read64(ptr) * k2; ptr += 8; v[0] = util_rotr64(v[0], 33) * k3;
v[1] += util_read64(ptr) * k2; ptr += 8; v[1] = util_rotr64(v[1], 33) * k3;
v[0] ^= util_rotr64((v[0] * k2) + v[1], 45) * k1;
v[1] ^= util_rotr64((v[1] * k3) + v[0], 45) * k0;
size -= 16;
}
if (size >= 8) {
v[0] += util_read64(ptr) * k2; ptr += 8; v[0] = util_rotr64(v[0], 33) * k3;
v[0] ^= util_rotr64((v[0] * k2) + v[1], 27) * k1;
size -= 8;
}
if (size >= 4) {
v[1] += util_read32(ptr) * k2; ptr += 4; v[1] = util_rotr64(v[1], 33) * k3;
v[1] ^= util_rotr64((v[1] * k3) + v[0], 46) * k0;
size -= 4;
}
if (size >= 2) {
v[0] += util_read16(ptr) * k2; ptr += 2; v[0] = util_rotr64(v[0], 33) * k3;
v[0] ^= util_rotr64((v[0] * k2) + v[1], 22) * k1;
size -= 2;
}
if (size >= 1) {
v[1] += util_read8(ptr) * k2; v[1] = util_rotr64(v[1], 33) * k3;
v[1] ^= util_rotr64((v[1] * k3) + v[0], 58) * k0;
}
v[0] += util_rotr64((v[0] * k0) + v[1], 13);
v[1] += util_rotr64((v[1] * k1) + v[0], 37);
v[0] += util_rotr64((v[0] * k2) + v[1], 13);
v[1] += util_rotr64((v[1] * k3) + v[0], 37);
util_write64(digest, v[0]);
util_write64(digest + 8, v[0]);
}

4
cmdline/mingw.c
View File

@ -1556,7 +1556,7 @@ int windows_link(const char* existing, const char* file)
} }
/** /**
* In Windows 10 allow creationg of symblink by not priviliged user. * In Windows 10 allow creation of symblink by not privileged user.
* *
* See: Symlinks in Windows 10! * See: Symlinks in Windows 10!
* https://blogs.windows.com/buildingapps/2016/12/02/symlinks-windows-10/#cQG7cx48oGH86lkI.97 * https://blogs.windows.com/buildingapps/2016/12/02/symlinks-windows-10/#cQG7cx48oGH86lkI.97
@ -1724,7 +1724,7 @@ int fsinfo(const char* path, int* has_persistent_inode, int* has_syncronized_har
if (has_persistent_inode) if (has_persistent_inode)
*has_persistent_inode = 1; *has_persistent_inode = 1;
/* NTFS doesn't syncronize hardlinks metadata */ /* NTFS doesn't synchronize hardlinks metadata */
if (has_syncronized_hardlinks) if (has_syncronized_hardlinks)
*has_syncronized_hardlinks = 0; *has_syncronized_hardlinks = 0;

48
cmdline/parity.c
View File

@ -217,6 +217,9 @@ int parity_create(struct snapraid_parity_handle* handle, const struct snapraid_p
/* LCOV_EXCL_STOP */ /* LCOV_EXCL_STOP */
} }
/* the initial valid size is the size on disk */
split->valid_size = split->st.st_size;
/** /**
* If the parity size is not yet set, set it now. * If the parity size is not yet set, set it now.
* This happens when expanding the number of parities, * This happens when expanding the number of parities,
@ -528,6 +531,10 @@ static int parity_handle_chsize(struct snapraid_split_handle* split, data_off_t
/* LCOV_EXCL_STOP */ /* LCOV_EXCL_STOP */
} }
/* if we shrink, update the valid size, but don't update when growing */
if (split->valid_size > split->st.st_size)
split->valid_size = split->st.st_size;
return 0; return 0;
} }
@ -697,6 +704,9 @@ int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_par
/* LCOV_EXCL_STOP */ /* LCOV_EXCL_STOP */
} }
/* the initial valid size is the size on disk */
split->valid_size = split->st.st_size;
/** /**
* If the parity size is not yet set, set it now. * If the parity size is not yet set, set it now.
* This happens when expanding the number of parities, * This happens when expanding the number of parities,
@ -751,7 +761,7 @@ int parity_sync(struct snapraid_parity_handle* handle)
ret = fsync(split->f); ret = fsync(split->f);
if (ret != 0) { if (ret != 0) {
/* LCOV_EXCL_START */ /* LCOV_EXCL_START */
log_fatal("Error synching parity file '%s'. %s.\n", split->path, strerror(errno)); log_fatal("Error syncing parity file '%s'. %s.\n", split->path, strerror(errno));
return -1; return -1;
/* LCOV_EXCL_STOP */ /* LCOV_EXCL_STOP */
} }
@ -761,6 +771,30 @@ int parity_sync(struct snapraid_parity_handle* handle)
return 0; return 0;
} }
int parity_truncate(struct snapraid_parity_handle* handle)
{
unsigned s;
int f_ret = 0;
for (s = 0; s < handle->split_mac; ++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
int ret;
/* truncate any data that we know it's not valid */
ret = ftruncate(split->f, split->valid_size);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error truncating the parity file '%s' to size %" PRIu64 ". %s.\n", split->path, split->valid_size, strerror(errno));
f_ret = -1;
/* LCOV_EXCL_STOP */
/* continue to truncate the others */
}
}
return f_ret;
}
int parity_close(struct snapraid_parity_handle* handle) int parity_close(struct snapraid_parity_handle* handle)
{ {
unsigned s; unsigned s;
@ -826,6 +860,10 @@ int parity_write(struct snapraid_parity_handle* handle, block_off_t pos, unsigne
/* LCOV_EXCL_STOP */ /* LCOV_EXCL_STOP */
} }
/* update the valid range */
if (split->valid_size < offset + block_size)
split->valid_size = offset + block_size;
write_ret = pwrite(split->f, block_buffer, block_size, offset); write_ret = pwrite(split->f, block_buffer, block_size, offset);
if (write_ret != (ssize_t)block_size) { /* conversion is safe because block_size is always small */ if (write_ret != (ssize_t)block_size) { /* conversion is safe because block_size is always small */
/* LCOV_EXCL_START */ /* LCOV_EXCL_START */
@ -867,6 +905,14 @@ int parity_read(struct snapraid_parity_handle* handle, block_off_t pos, unsigned
/* LCOV_EXCL_STOP */ /* LCOV_EXCL_STOP */
} }
/* if read is completely out of the valid range */
if (offset >= split->valid_size) {
/* LCOV_EXCL_START */
out("Missing data reading file '%s' at offset %" PRIu64 " for size %u.\n", split->path, offset, block_size);
return -1;
/* LCOV_EXCL_STOP */
}
count = 0; count = 0;
do { do {
read_ret = pread(split->f, block_buffer + count, block_size - count, offset + count); read_ret = pread(split->f, block_buffer + count, block_size - count, offset + count);

15
cmdline/parity.h
View File

@ -36,6 +36,16 @@ struct snapraid_split_handle {
*/ */
data_off_t size; data_off_t size;
/**
* Valid size of the parity split.
* This is the size effectively written, and not the result of a chsize operation.
* It's used to make read operations failing if read over that size.
*
* Parity is also truncated to that size when fixing it, in case of a Break (Ctrl+C)
* of the program.
*/
data_off_t valid_size;
/** /**
* Artificial size limit for testing. * Artificial size limit for testing.
* 0 means unlimited. * 0 means unlimited.
@ -113,6 +123,11 @@ int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_par
*/ */
int parity_sync(struct snapraid_parity_handle* handle); int parity_sync(struct snapraid_parity_handle* handle);
/**
* Truncate the parity file to the valid size.
*/
int parity_truncate(struct snapraid_parity_handle* handle);
/** /**
* Close the parity file. * Close the parity file.
*/ */

2
cmdline/pool.c
View File

@ -369,7 +369,7 @@ static void make_link(tommy_hashdyn* poolset, const char* pool_dir, const char*
#ifdef _WIN32 #ifdef _WIN32
} else if (errno == EPERM) { } else if (errno == EPERM) {
/* LCOV_EXCL_START */ /* LCOV_EXCL_START */
log_fatal("You must run as Adminstrator to be able to create symlinks.\n"); log_fatal("You must run as Administrator to be able to create symlinks.\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */ /* LCOV_EXCL_STOP */
#endif #endif

16
cmdline/portable.h
View File

@ -35,7 +35,7 @@
#define __USE_MINGW_ANSI_STDIO 1 #define __USE_MINGW_ANSI_STDIO 1
/** /**
* Define the MSVCRT version targetting Windows Vista. * Define the MSVCRT version targeting Windows Vista.
*/ */
#define __MSVCRT_VERSION__ 0x0600 #define __MSVCRT_VERSION__ 0x0600
@ -198,9 +198,11 @@
#include <sys/types.h> #include <sys/types.h>
#endif #endif
#if MAJOR_IN_MKDEV #if HAVE_SYS_MKDEV
#include <sys/mkdev.h> #include <sys/mkdev.h>
#elif MAJOR_IN_SYSMACROS #endif
#if HAVE_SYS_SYSMACROS_H
#include <sys/sysmacros.h> #include <sys/sysmacros.h>
#endif #endif
@ -387,14 +389,14 @@ void os_clear(void);
* *
* If no log file is selected, it's 0. * If no log file is selected, it's 0.
*/ */
FILE* stdlog; extern FILE* stdlog;
/** /**
* Exit codes for testing. * Exit codes for testing.
*/ */
int exit_success; extern int exit_success;
int exit_failure; extern int exit_failure;
int exit_sync_needed; extern int exit_sync_needed;
#undef EXIT_SUCCESS #undef EXIT_SUCCESS
#undef EXIT_FAILURE #undef EXIT_FAILURE
#define EXIT_SUCCESS exit_success #define EXIT_SUCCESS exit_success

2
cmdline/scan.c
View File

@ -501,7 +501,7 @@ static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struc
* because the metadata in the directory is updated only when the file * because the metadata in the directory is updated only when the file
* is closed. * is closed.
* *
* The same happens for hardlinks that duplicate metatada. * The same happens for hardlinks that duplicate metadata.
* The link metadata is updated only when the link is opened. * The link metadata is updated only when the link is opened.
* This extends also to st_size and st_nlink. * This extends also to st_size and st_nlink.
* *

3
cmdline/scrub.c
View File

@ -665,7 +665,8 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
log_fatal("DANGER! Unexpected data errors! The failing blocks are now marked as bad!\n"); log_fatal("DANGER! Unexpected data errors! The failing blocks are now marked as bad!\n");
if (io_error || silent_error) { if (io_error || silent_error) {
log_fatal("Use 'snapraid status' to list the bad blocks.\n"); log_fatal("Use 'snapraid status' to list the bad blocks.\n");
log_fatal("Use 'snapraid -e fix' to recover.\n"); log_fatal("Use 'snapraid -e fix' to recover them.\n");
log_fatal("Use 'snapraid -p bad scrub' to recheck after fixing.\n");
} }
log_tag("summary:error_file:%u\n", error); log_tag("summary:error_file:%u\n", error);

2
cmdline/selftest.c
View File

@ -632,7 +632,7 @@ void selftest(void)
} }
if (raid_test_par(RAID_MODE_CAUCHY, 1, 256) != 0) { if (raid_test_par(RAID_MODE_CAUCHY, 1, 256) != 0) {
/* LCOV_EXCL_START */ /* LCOV_EXCL_START */
log_fatal("Failed GEN Cauchy test sigle data disk\n"); log_fatal("Failed GEN Cauchy test single data disk\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */ /* LCOV_EXCL_STOP */
} }

4
cmdline/snapraid.c
View File

@ -1001,6 +1001,10 @@ int main(int argc, char* argv[])
switch (operation) { switch (operation) {
case OPERATION_FIX : case OPERATION_FIX :
case OPERATION_CHECK : case OPERATION_CHECK :
case OPERATION_SMART :
case OPERATION_DEVICES :
case OPERATION_SPINUP :
case OPERATION_SPINDOWN :
break; break;
default : default :
if (opt.force_device) { if (opt.force_device) {

9
cmdline/speed.c
View File

@ -216,6 +216,7 @@ void speed(int period)
printf("%8s", "best"); printf("%8s", "best");
printf("%8s", "murmur3"); printf("%8s", "murmur3");
printf("%8s", "spooky2"); printf("%8s", "spooky2");
printf("%8s", "metro");
printf("\n"); printf("\n");
printf("%8s", "hash"); printf("%8s", "hash");
@ -245,6 +246,14 @@ void speed(int period)
memhash(HASH_SPOOKY2, seed, digest, v[j], size); memhash(HASH_SPOOKY2, seed, digest, v[j], size);
} SPEED_STOP } SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
SPEED_START {
for (j = 0; j < nd; ++j)
memhash(HASH_METRO, seed, digest, v[j], size);
} SPEED_STOP
printf("%8" PRIu64, ds / dt); printf("%8" PRIu64, ds / dt);
printf("\n"); printf("\n");
printf("\n"); printf("\n");

50
cmdline/state.c
View File

@ -279,7 +279,7 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
struct snapraid_disk* other = j->data; struct snapraid_disk* other = j->data;
if (disk->device == other->device) { if (disk->device == other->device) {
if (state->opt.force_device) { if (state->opt.force_device) {
/* note tha we just ignore the issue */ /* note that we just ignore the issue */
/* and we DON'T mark the disk to be skipped */ /* and we DON'T mark the disk to be skipped */
/* because we want to use these disks */ /* because we want to use these disks */
if (!state->opt.no_warnings) if (!state->opt.no_warnings)
@ -310,6 +310,13 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
for (l = 0; l < state->level; ++l) { for (l = 0; l < state->level; ++l) {
for (s = 0; s < state->parity[l].split_mac; ++s) { for (s = 0; s < state->parity[l].split_mac; ++s) {
if (disk->device == state->parity[l].split_map[s].device) { if (disk->device == state->parity[l].split_map[s].device) {
if (state->opt.force_device) {
/* note that we just ignore the issue */
/* and we DON'T mark the disk to be skipped */
/* because we want to use these disks */
if (!state->opt.no_warnings)
log_fatal("DANGER! Ignoring that disks '%s' and %s '%s' are on the same device\n", disk->dir, lev_name(l), state->parity[l].split_map[s].path);
} else {
/* LCOV_EXCL_START */ /* LCOV_EXCL_START */
log_fatal("Disk '%s' and %s '%s' are on the same device.\n", disk->dir, lev_name(l), state->parity[l].split_map[s].path); log_fatal("Disk '%s' and %s '%s' are on the same device.\n", disk->dir, lev_name(l), state->parity[l].split_map[s].path);
#ifdef _WIN32 #ifdef _WIN32
@ -323,6 +330,7 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
} }
} }
} }
}
if (state->pool[0] != 0 && disk->device == state->pool_device) { if (state->pool[0] != 0 && disk->device == state->pool_device) {
/* LCOV_EXCL_START */ /* LCOV_EXCL_START */
@ -364,7 +372,7 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
for (t = 0; t < state->parity[j].split_mac; ++t) { for (t = 0; t < state->parity[j].split_mac; ++t) {
if (state->parity[l].split_map[s].device == state->parity[j].split_map[t].device) { if (state->parity[l].split_map[s].device == state->parity[j].split_map[t].device) {
if (state->opt.force_device) { if (state->opt.force_device) {
/* note tha we just ignore the issue */ /* note that we just ignore the issue */
/* and we DON'T mark the disk to be skipped */ /* and we DON'T mark the disk to be skipped */
/* because we want to use these disks */ /* because we want to use these disks */
if (!state->opt.no_warnings) if (!state->opt.no_warnings)
@ -623,7 +631,7 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
} }
state->block_size *= KIBI; state->block_size *= KIBI;
} else if (strcmp(tag, "hashsize") == 0 } else if (strcmp(tag, "hashsize") == 0
|| strcmp(tag, "hash_size") == 0 /* v11.0 used incorretly this one, kept now for backward compatibility */ || strcmp(tag, "hash_size") == 0 /* v11.0 used incorrectly this one, kept now for backward compatibility */
) { ) {
uint32_t hash_size; uint32_t hash_size;
@ -1618,7 +1626,7 @@ static void state_content_check(struct snapraid_state* state, const char* path)
/** /**
* Check if the position is REQUIRED, or we can completely clear it from the state. * Check if the position is REQUIRED, or we can completely clear it from the state.
* *
* Note that position with only DELETED blocks are discarged. * Note that position with only DELETED blocks are discharged.
*/ */
static int fs_position_is_required(struct snapraid_state* state, block_off_t pos) static int fs_position_is_required(struct snapraid_state* state, block_off_t pos)
{ {
@ -1844,7 +1852,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
if (state->block_size == 0) { if (state->block_size == 0) {
/* LCOV_EXCL_START */ /* LCOV_EXCL_START */
decoding_error(path, f); decoding_error(path, f);
log_fatal("Internal incosistency due zero blocksize!\n"); log_fatal("Internal inconsistency due zero blocksize!\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */ /* LCOV_EXCL_STOP */
} }
@ -2398,6 +2406,9 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
case 'k' : case 'k' :
state->hash = HASH_SPOOKY2; state->hash = HASH_SPOOKY2;
break; break;
case 'm' :
state->hash = HASH_METRO;
break;
default : default :
/* LCOV_EXCL_START */ /* LCOV_EXCL_START */
decoding_error(path, f); decoding_error(path, f);
@ -2425,6 +2436,9 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
case 'k' : case 'k' :
state->prevhash = HASH_SPOOKY2; state->prevhash = HASH_SPOOKY2;
break; break;
case 'm' :
state->prevhash = HASH_METRO;
break;
default : default :
/* LCOV_EXCL_START */ /* LCOV_EXCL_START */
decoding_error(path, f); decoding_error(path, f);
@ -2947,6 +2961,8 @@ static void* state_write_thread(void* arg)
sputc('u', f); sputc('u', f);
} else if (state->hash == HASH_SPOOKY2) { } else if (state->hash == HASH_SPOOKY2) {
sputc('k', f); sputc('k', f);
} else if (state->hash == HASH_METRO) {
sputc('m', f);
} else { } else {
/* LCOV_EXCL_START */ /* LCOV_EXCL_START */
log_fatal("Unexpected hash when writing the content file '%s'.\n", serrorfile(f)); log_fatal("Unexpected hash when writing the content file '%s'.\n", serrorfile(f));
@ -2970,6 +2986,8 @@ static void* state_write_thread(void* arg)
sputc('u', f); sputc('u', f);
} else if (state->prevhash == HASH_SPOOKY2) { } else if (state->prevhash == HASH_SPOOKY2) {
sputc('k', f); sputc('k', f);
} else if (state->prevhash == HASH_METRO) {
sputc('m', f);
} else { } else {
/* LCOV_EXCL_START */ /* LCOV_EXCL_START */
log_fatal("Unexpected prevhash when writing the content file '%s'.\n", serrorfile(f)); log_fatal("Unexpected prevhash when writing the content file '%s'.\n", serrorfile(f));
@ -3837,10 +3855,14 @@ struct state_verify_thread_context {
static void* state_verify_thread(void* arg) static void* state_verify_thread(void* arg)
{ {
struct state_verify_thread_context* context = arg; struct state_verify_thread_context* context = arg;
struct snapraid_content* content = context->content;
STREAM* f = context->f; STREAM* f = context->f;
unsigned char buf[4]; unsigned char buf[4];
uint32_t crc_stored; uint32_t crc_stored;
uint32_t crc_computed; uint32_t crc_computed;
uint64_t start;
start = tick_ms();
if (sdeplete(f, buf) != 0) { if (sdeplete(f, buf) != 0) {
/* LCOV_EXCL_START */ /* LCOV_EXCL_START */
@ -3872,6 +3894,8 @@ static void* state_verify_thread(void* arg)
/* LCOV_EXCL_STOP */ /* LCOV_EXCL_STOP */
} }
msg_progress("Verified %s in %" PRIu64 " seconds\n", content->content, (tick_ms() - start) / 1000);
return 0; return 0;
} }
@ -4155,7 +4179,7 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tom
} }
} }
/* if we are filtering by disk, exclude any parity not explicitely included */ /* if we are filtering by disk, exclude any parity not explicitly included */
if (!tommy_list_empty(filterlist_disk)) { if (!tommy_list_empty(filterlist_disk)) {
/* for each parity disk */ /* for each parity disk */
for (l = 0; l < state->level; ++l) { for (l = 0; l < state->level; ++l) {
@ -4463,7 +4487,8 @@ int state_progress(struct snapraid_state* state, struct snapraid_io* io, block_o
) { ) {
time_t elapsed; time_t elapsed;
unsigned out_perc = 0; unsigned out_perc = 0;
unsigned out_speed = 0; unsigned out_size_speed = 0;
unsigned out_block_speed = 0;
unsigned out_cpu = 0; unsigned out_cpu = 0;
unsigned out_eta = 0; unsigned out_eta = 0;
int out_computed = 0; int out_computed = 0;
@ -4536,7 +4561,11 @@ int state_progress(struct snapraid_state* state, struct snapraid_io* io, block_o
/* estimate the speed in MB/s */ /* estimate the speed in MB/s */
if (delta_time != 0) if (delta_time != 0)
out_speed = (unsigned)(delta_size / MEGA / delta_time); out_size_speed = (unsigned)(delta_size / MEGA / delta_time);
/* estimate the speed in block/s */
if (delta_pos != 0)
out_block_speed = (unsigned)(delta_pos / delta_time);
/* estimate the cpu usage percentage */ /* estimate the cpu usage percentage */
if (delta_tick_total != 0) if (delta_tick_total != 0)
@ -4556,12 +4585,13 @@ int state_progress(struct snapraid_state* state, struct snapraid_io* io, block_o
} }
if (state->opt.gui) { if (state->opt.gui) {
log_tag("run:pos:%u:%u:%" PRIu64 ":%u:%u:%u:%u:%" PRIu64 "\n", blockpos, countpos, countsize, out_perc, out_eta, out_speed, out_cpu, (uint64_t)elapsed); log_tag("run:pos:%u:%u:%" PRIu64 ":%u:%u:%u:%u:%" PRIu64 "\n", blockpos, countpos, countsize, out_perc, out_eta, out_size_speed, out_cpu, (uint64_t)elapsed);
log_flush(); log_flush();
} else { } else {
msg_bar("%u%%, %u MB", out_perc, (unsigned)(countsize / MEGA)); msg_bar("%u%%, %u MB", out_perc, (unsigned)(countsize / MEGA));
if (out_computed) { if (out_computed) {
msg_bar(", %u MB/s", out_speed); msg_bar(", %u MB/s", out_size_speed);
msg_bar(", %u block/s", out_block_speed);
msg_bar(", CPU %u%%", out_cpu); msg_bar(", CPU %u%%", out_cpu);
msg_bar(", %u:%02u ETA", out_eta / 60, out_eta % 60); msg_bar(", %u:%02u ETA", out_eta / 60, out_eta % 60);
} }

2
cmdline/state.h
View File

@ -261,7 +261,7 @@ void state_rehash(struct snapraid_state* state);
*/ */
#define SCRUB_AUTO -1 /**< Automatic selection. */ #define SCRUB_AUTO -1 /**< Automatic selection. */
#define SCRUB_BAD -2 /**< Scrub only the bad blocks. */ #define SCRUB_BAD -2 /**< Scrub only the bad blocks. */
#define SCRUB_NEW -3 /**< Scub the new blocks. */ #define SCRUB_NEW -3 /**< Scrub the new blocks. */
#define SCRUB_FULL -4 /**< Scrub everything. */ #define SCRUB_FULL -4 /**< Scrub everything. */
#define SCRUB_EVEN -5 /**< Even blocks. */ #define SCRUB_EVEN -5 /**< Even blocks. */

8
cmdline/stream.c
View File

@ -112,7 +112,7 @@ int sopen_multi_file(STREAM* s, unsigned i, const char* file)
pathcpy(s->handle[i].path, sizeof(s->handle[i].path), file); pathcpy(s->handle[i].path, sizeof(s->handle[i].path), file);
/* O_EXCL to be resilent ensure to always create a new file and not use a stale link to the original file */ /* O_EXCL to be resilient ensure to always create a new file and not use a stale link to the original file */
f = open(file, O_WRONLY | O_CREAT | O_EXCL | O_BINARY | O_SEQUENTIAL, 0600); f = open(file, O_WRONLY | O_CREAT | O_EXCL | O_BINARY | O_SEQUENTIAL, 0600);
if (f == -1) { if (f == -1) {
/* LCOV_EXCL_START */ /* LCOV_EXCL_START */
@ -363,7 +363,7 @@ int sgettok(STREAM* f, char* str, int size)
break; break;
} }
if (c == '\n') { if (c == '\n') {
/* remove ending carrige return to support the Windows CR+LF format */ /* remove ending carriage return to support the Windows CR+LF format */
if (i != str && i[-1] == '\r') if (i != str && i[-1] == '\r')
--i; --i;
sungetc(c, f); sungetc(c, f);
@ -429,7 +429,7 @@ int sgetline(STREAM* f, char* str, int size)
while (1) { while (1) {
c = *pos++; c = *pos++;
if (c == '\n') { if (c == '\n') {
/* remove ending carrige return to support the Windows CR+LF format */ /* remove ending carriage return to support the Windows CR+LF format */
if (i != str && i[-1] == '\r') if (i != str && i[-1] == '\r')
--i; --i;
--pos; --pos;
@ -455,7 +455,7 @@ int sgetline(STREAM* f, char* str, int size)
/* LCOV_EXCL_STOP */ /* LCOV_EXCL_STOP */
} }
if (c == '\n') { if (c == '\n') {
/* remove ending carrige return to support the Windows CR+LF format */ /* remove ending carriage return to support the Windows CR+LF format */
if (i != str && i[-1] == '\r') if (i != str && i[-1] == '\r')
--i; --i;
sungetc(c, f); sungetc(c, f);

2
cmdline/stream.h
View File

@ -28,7 +28,7 @@
* *
* It's not a constant for testing purpose. * It's not a constant for testing purpose.
*/ */
unsigned STREAM_SIZE; extern unsigned STREAM_SIZE;
#define STREAM_STATE_READ 0 /**< The stream is in a normal state of read. */ #define STREAM_STATE_READ 0 /**< The stream is in a normal state of read. */
#define STREAM_STATE_WRITE 1 /**< The stream is in a normal state of write. */ #define STREAM_STATE_WRITE 1 /**< The stream is in a normal state of write. */

15
cmdline/support.c
View File

@ -79,6 +79,7 @@ void lock_done(void)
/* print */ /* print */
int msg_level = 0; int msg_level = 0;
FILE* stdlog = 0;
/* /*
* Note that in the following functions we always flush both * Note that in the following functions we always flush both
@ -811,7 +812,7 @@ int mkancestor(const char* file)
} }
#ifdef _WIN32 #ifdef _WIN32
/* if it's a drive specificaion like "C:" */ /* if it's a drive specification like "C:" */
if (isalpha(dir[0]) && dir[1] == ':' && dir[2] == 0) { if (isalpha(dir[0]) && dir[1] == ':' && dir[2] == 0) {
/* nothing more to do */ /* nothing more to do */
return 0; return 0;
@ -1236,7 +1237,7 @@ int advise_read(struct advise_struct* advise, int f, data_off_t offset, data_off
* non-blocking and do this work in a workqueue (or via some kind of * non-blocking and do this work in a workqueue (or via some kind of
* callback/continuation scheme). My worry is just doing this if a user * callback/continuation scheme). My worry is just doing this if a user
* application does something crazy, like request gigabytes and gigabytes * application does something crazy, like request gigabytes and gigabytes
* of readahead, and then repents of their craziness, there should be a * of readahead, and then repented of their craziness, there should be a
* way of cancelling the readahead request. Today, the user can just * way of cancelling the readahead request. Today, the user can just
* kill the application. But if we simply shove the work to a kernel * kill the application. But if we simply shove the work to a kernel
* thread, it becomes a lot harder to cancel the readahead request. We'd * thread, it becomes a lot harder to cancel the readahead request. We'd
@ -1632,16 +1633,16 @@ void thread_cond_wait(pthread_cond_t* cond, pthread_mutex_t* mutex)
/** /**
* Implementation note about conditional variables. * Implementation note about conditional variables.
* *
* The conditional variables can be signaled inside or ouside the mutex, * The conditional variables can be signaled inside or outside the mutex,
* what is better it's debatable but in general doing that ouside the mutex, * what is better it's debatable but in general doing that outside the mutex,
* reduces the number of context switches. * reduces the number of context switches.
* *
* But when when testing with helgrind and drd, this disallows such tools to * But when testing with helgrind and drd, this disallows such tools to
* to see the dependency between the signal and the wait. * to see the dependency between the signal and the wait.
* *
* To avoid it we signal everything inside the mutex. And we do this in both * To avoid it we signal everything inside the mutex. And we do this in both
* test mode (with CHERCKER defined) and release mode (CHECKER not defined), * test mode (with CHECKER defined) and release mode (CHECKER not defined),
* to be on the safe side and avoid any difference in beaviour between test and * to be on the safe side and avoid any difference in behaviour between test and
* release. * release.
* *
* Here some interesting discussion: * Here some interesting discussion:

4
cmdline/support.h
View File

@ -109,7 +109,7 @@ void log_flush(void);
/** /**
* Pointer to log function. * Pointer to log function.
*/ */
typedef void fptr(const char* format, ...); typedef void fptr(const char* format, ...) __attribute__((format(attribute_printf, 1, 2)));
/****************************************************************************/ /****************************************************************************/
/* message */ /* message */
@ -415,7 +415,7 @@ int smartctl_flush(FILE* f, const char* file, const char* name);
* *
* Ensure to change that before starting any thread. * Ensure to change that before starting any thread.
*/ */
int thread_cond_signal_outside; extern int thread_cond_signal_outside;
/** /**
* Thread wrappers to handle error conditions. * Thread wrappers to handle error conditions.

4
cmdline/sync.c
View File

@ -1016,7 +1016,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
} }
/* if we have only silent errors we can try to fix them on-the-fly */ /* if we have only silent errors we can try to fix them on-the-fly */
/* note the the fix is not written to disk, but used only to */ /* note the fix is not written to disk, but used only to */
/* compute the new parity */ /* compute the new parity */
if (!error_on_this_block && !io_error_on_this_block && silent_error_on_this_block) { if (!error_on_this_block && !io_error_on_this_block && silent_error_on_this_block) {
unsigned failed_mac; unsigned failed_mac;
@ -1536,7 +1536,7 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
data_off_t out_size; data_off_t out_size;
parity_size(&parity_handle[l], &out_size); parity_size(&parity_handle[l], &out_size);
parity_overflow(state, out_size); parity_overflow(state, out_size);
log_fatal("WARNING! Without an usable %s file, it isn't possible to sync.\n", lev_name(l)); log_fatal("WARNING! Without a usable %s file, it isn't possible to sync.\n", lev_name(l));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */ /* LCOV_EXCL_STOP */
} }

2
cmdline/unix.c
View File

@ -632,7 +632,7 @@ int filephy(const char* path, uint64_t size, uint64_t* physical)
/* In this way we keep them in the directory traversal order */ /* In this way we keep them in the directory traversal order */
/* that at least keeps files in the same directory together. */ /* that at least keeps files in the same directory together. */
/* Note also that in newer file-system with snapshot, like ZFS, */ /* Note also that in newer file-system with snapshot, like ZFS, */
/* the inode doesn't represent evenmore the disk position, because files */ /* the inode doesn't represent even more the disk position, because files */
/* are not overwritten in place, but rewritten in another location */ /* are not overwritten in place, but rewritten in another location */
/* of the disk. */ /* of the disk. */
*physical = FILEPHY_UNREPORTED_OFFSET; *physical = FILEPHY_UNREPORTED_OFFSET;

2
cmdline/unix.h
View File

@ -59,7 +59,7 @@ int dirent_hidden(struct dirent* dd);
const char* stat_desc(struct stat* st); const char* stat_desc(struct stat* st);
/** /**
* Return the aligment requirement for direct IO. * Return the alignment requirement for direct IO.
*/ */
size_t direct_size(void); size_t direct_size(void);

35
cmdline/util.c
View File

@ -470,8 +470,24 @@ static inline uint64_t util_rotl64(uint64_t x, int8_t r)
return (x << r) | (x >> (64 - r)); return (x << r) | (x >> (64 - r));
} }
/*
* Rotate right.
* In x86/x64 they are optimized with a single assembler instruction.
*/
#if 0 /* unused */
static inline uint32_t util_rotr32(uint32_t x, int8_t r)
{
return (x >> r) | (x << (32 - r));
}
#endif
static inline uint64_t util_rotr64(uint64_t x, int8_t r)
{
return (x >> r) | (x << (64 - r));
}
/** /**
* Swap endianess. * Swap endianness.
* They are needed only if BigEndian. * They are needed only if BigEndian.
*/ */
#if defined(__GNUC__) #if defined(__GNUC__)
@ -502,6 +518,18 @@ static inline uint64_t util_swap64(uint64_t v)
} }
#endif #endif
static inline uint8_t util_read8(const void* void_ptr)
{
const uint8_t* ptr = void_ptr;
return ptr[0];
}
static inline uint16_t util_read16(const void* void_ptr)
{
const uint8_t* ptr = void_ptr;
return ptr[0] + (ptr[1] << 8);
}
static inline uint32_t util_read32(const void* ptr) static inline uint32_t util_read32(const void* ptr)
{ {
uint32_t v; uint32_t v;
@ -543,6 +571,7 @@ static inline void util_write64(void* ptr, uint64_t v)
#include "murmur3.c" #include "murmur3.c"
#include "spooky2.c" #include "spooky2.c"
#include "metro.c"
void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* src, size_t size) void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* src, size_t size)
{ {
@ -553,6 +582,9 @@ void memhash(unsigned kind, const unsigned char* seed, void* digest, const void*
case HASH_SPOOKY2 : case HASH_SPOOKY2 :
SpookyHash128(src, size, seed, digest); SpookyHash128(src, size, seed, digest);
break; break;
case HASH_METRO :
MetroHash128(src, size, seed, digest);
break;
default : default :
/* LCOV_EXCL_START */ /* LCOV_EXCL_START */
log_fatal("Internal inconsistency in hash function %u\n", kind); log_fatal("Internal inconsistency in hash function %u\n", kind);
@ -568,6 +600,7 @@ const char* hash_config_name(unsigned kind)
case HASH_UNDEFINED : return "undefined"; case HASH_UNDEFINED : return "undefined";
case HASH_MURMUR3 : return "murmur3"; case HASH_MURMUR3 : return "murmur3";
case HASH_SPOOKY2 : return "spooky2"; case HASH_SPOOKY2 : return "spooky2";
case HASH_METRO : return "metro";
default : default :
/* LCOV_EXCL_START */ /* LCOV_EXCL_START */
return "unknown"; return "unknown";

3
cmdline/util.h
View File

@ -161,7 +161,7 @@ static inline uint32_t crc32c_plain(uint32_t crc, const unsigned char* ptr, unsi
/** /**
* Compute the CRC-32 (Castagnoli) * Compute the CRC-32 (Castagnoli)
*/ */
uint32_t (*crc32c)(uint32_t crc, const unsigned char* ptr, unsigned size); extern uint32_t (*crc32c)(uint32_t crc, const unsigned char* ptr, unsigned size);
/** /**
* Internal entry points for testing. * Internal entry points for testing.
@ -188,6 +188,7 @@ void crc32c_init(void);
#define HASH_UNDEFINED 0 #define HASH_UNDEFINED 0
#define HASH_MURMUR3 1 #define HASH_MURMUR3 1
#define HASH_SPOOKY2 2 #define HASH_SPOOKY2 2
#define HASH_METRO 3
/** /**
* Compute the HASH of a memory block. * Compute the HASH of a memory block.

14
config.h.in
View File

@ -224,6 +224,9 @@
/* Define to 1 if you have the <sys/ioctl.h> header file. */ /* Define to 1 if you have the <sys/ioctl.h> header file. */
#undef HAVE_SYS_IOCTL_H #undef HAVE_SYS_IOCTL_H
/* Define to 1 if you have the <sys/mkdev.h> header file. */
#undef HAVE_SYS_MKDEV_H
/* Define to 1 if you have the <sys/mount.h> header file. */ /* Define to 1 if you have the <sys/mount.h> header file. */
#undef HAVE_SYS_MOUNT_H #undef HAVE_SYS_MOUNT_H
@ -240,6 +243,9 @@
/* Define to 1 if you have the <sys/stat.h> header file. */ /* Define to 1 if you have the <sys/stat.h> header file. */
#undef HAVE_SYS_STAT_H #undef HAVE_SYS_STAT_H
/* Define to 1 if you have the <sys/sysmacros.h> header file. */
#undef HAVE_SYS_SYSMACROS_H
/* Define to 1 if you have the <sys/types.h> header file. */ /* Define to 1 if you have the <sys/types.h> header file. */
#undef HAVE_SYS_TYPES_H #undef HAVE_SYS_TYPES_H
@ -258,14 +264,6 @@
/* Define to 1 if you have the `vsnprintf' function. */ /* Define to 1 if you have the `vsnprintf' function. */
#undef HAVE_VSNPRINTF #undef HAVE_VSNPRINTF
/* Define to 1 if `major', `minor', and `makedev' are declared in <mkdev.h>.
*/
#undef MAJOR_IN_MKDEV
/* Define to 1 if `major', `minor', and `makedev' are declared in
<sysmacros.h>. */
#undef MAJOR_IN_SYSMACROS
/* Define to 1 if assertions should be disabled. */ /* Define to 1 if assertions should be disabled. */
#undef NDEBUG #undef NDEBUG

88
configure vendored
View File

@ -1,6 +1,6 @@
#! /bin/sh #! /bin/sh
# Guess values for system-dependent variables and create Makefiles. # Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.69 for snapraid 11.3. # Generated by GNU Autoconf 2.69 for snapraid 11.5.
# #
# #
# Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@ -577,8 +577,8 @@ MAKEFLAGS=
# Identity of this package. # Identity of this package.
PACKAGE_NAME='snapraid' PACKAGE_NAME='snapraid'
PACKAGE_TARNAME='snapraid' PACKAGE_TARNAME='snapraid'
PACKAGE_VERSION='11.3' PACKAGE_VERSION='11.5'
PACKAGE_STRING='snapraid 11.3' PACKAGE_STRING='snapraid 11.5'
PACKAGE_BUGREPORT='' PACKAGE_BUGREPORT=''
PACKAGE_URL='http://www.snapraid.it' PACKAGE_URL='http://www.snapraid.it'
@ -1304,7 +1304,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing. # Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh. # This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF cat <<_ACEOF
\`configure' configures snapraid 11.3 to adapt to many kinds of systems. \`configure' configures snapraid 11.5 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]... Usage: $0 [OPTION]... [VAR=VALUE]...
@ -1374,7 +1374,7 @@ fi
if test -n "$ac_init_help"; then if test -n "$ac_init_help"; then
case $ac_init_help in case $ac_init_help in
short | recursive ) echo "Configuration of snapraid 11.3:";; short | recursive ) echo "Configuration of snapraid 11.5:";;
esac esac
cat <<\_ACEOF cat <<\_ACEOF
@ -1496,7 +1496,7 @@ fi
test -n "$ac_init_help" && exit $ac_status test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then if $ac_init_version; then
cat <<\_ACEOF cat <<\_ACEOF
snapraid configure 11.3 snapraid configure 11.5
generated by GNU Autoconf 2.69 generated by GNU Autoconf 2.69
Copyright (C) 2012 Free Software Foundation, Inc. Copyright (C) 2012 Free Software Foundation, Inc.
@ -2102,7 +2102,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake. running configure, to aid debugging if configure makes a mistake.
It was created by snapraid $as_me 11.3, which was It was created by snapraid $as_me 11.5, which was
generated by GNU Autoconf 2.69. Invocation command line was generated by GNU Autoconf 2.69. Invocation command line was
$ $0 $@ $ $0 $@
@ -2965,7 +2965,7 @@ fi
# Define the identity of the package. # Define the identity of the package.
PACKAGE='snapraid' PACKAGE='snapraid'
VERSION='11.3' VERSION='11.5'
cat >>confdefs.h <<_ACEOF cat >>confdefs.h <<_ACEOF
@ -4657,6 +4657,22 @@ $as_echo "no" >&6; }
fi fi
rm -f conftest* rm -f conftest*
# This the new default for gcc 10
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ${CC-cc} accepts -fno-common" >&5
$as_echo_n "checking whether ${CC-cc} accepts -fno-common... " >&6; }
echo 'void f(){}' > conftest.c
if test -z "`${CC-cc} -c -fno-common conftest.c 2>&1`"; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
$as_echo "yes" >&6; }
CFLAGS="$CFLAGS -fno-common"
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi
rm -f conftest*
# Check whether --enable-largefile was given. # Check whether --enable-largefile was given.
if test "${enable_largefile+set}" = set; then : if test "${enable_largefile+set}" = set; then :
@ -5115,56 +5131,6 @@ $as_echo "#define HAVE_SYS_WAIT_H 1" >>confdefs.h
fi fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether sys/types.h defines makedev" >&5
$as_echo_n "checking whether sys/types.h defines makedev... " >&6; }
if ${ac_cv_header_sys_types_h_makedev+:} false; then :
$as_echo_n "(cached) " >&6
else
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
#include <sys/types.h>
int
main ()
{
return makedev(0, 0);
;
return 0;
}
_ACEOF
if ac_fn_c_try_link "$LINENO"; then :
ac_cv_header_sys_types_h_makedev=yes
else
ac_cv_header_sys_types_h_makedev=no
fi
rm -f core conftest.err conftest.$ac_objext \
conftest$ac_exeext conftest.$ac_ext
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_sys_types_h_makedev" >&5
$as_echo "$ac_cv_header_sys_types_h_makedev" >&6; }
if test $ac_cv_header_sys_types_h_makedev = no; then
ac_fn_c_check_header_mongrel "$LINENO" "sys/mkdev.h" "ac_cv_header_sys_mkdev_h" "$ac_includes_default"
if test "x$ac_cv_header_sys_mkdev_h" = xyes; then :
$as_echo "#define MAJOR_IN_MKDEV 1" >>confdefs.h
fi
if test $ac_cv_header_sys_mkdev_h = no; then
ac_fn_c_check_header_mongrel "$LINENO" "sys/sysmacros.h" "ac_cv_header_sys_sysmacros_h" "$ac_includes_default"
if test "x$ac_cv_header_sys_sysmacros_h" = xyes; then :
$as_echo "#define MAJOR_IN_SYSMACROS 1" >>confdefs.h
fi
fi
fi
for ac_header in fcntl.h stddef.h stdint.h stdlib.h string.h limits.h for ac_header in fcntl.h stddef.h stdint.h stdlib.h string.h limits.h
do : do :
as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh`
@ -5204,7 +5170,7 @@ fi
done done
for ac_header in sys/file.h sys/ioctl.h sys/vfs.h sys/statfs.h sys/param.h sys/mount.h for ac_header in sys/file.h sys/ioctl.h sys/vfs.h sys/statfs.h sys/param.h sys/mount.h sys/sysmacros.h sys/mkdev.h
do : do :
as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh`
ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default"
@ -7373,7 +7339,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their # report actual input values of CONFIG_FILES etc. instead of their
# values after options handling. # values after options handling.
ac_log=" ac_log="
This file was extended by snapraid $as_me 11.3, which was This file was extended by snapraid $as_me 11.5, which was
generated by GNU Autoconf 2.69. Invocation command line was generated by GNU Autoconf 2.69. Invocation command line was
CONFIG_FILES = $CONFIG_FILES CONFIG_FILES = $CONFIG_FILES
@ -7436,7 +7402,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\ ac_cs_version="\\
snapraid config.status 11.3 snapraid config.status 11.5
configured by $0, generated by GNU Autoconf 2.69, configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\" with options \\"\$ac_cs_config\\"

7
configure.ac
View File

@ -17,11 +17,13 @@ AC_CHECK_PROG([SDE],[sde],[sde],[])
AC_CHECK_PROG([ADVD2],[advd2],[advd2],[]) AC_CHECK_PROG([ADVD2],[advd2],[advd2],[])
AM_CONDITIONAL(HAVE_ADVD2, [test x"$ADVD2" != x]) AM_CONDITIONAL(HAVE_ADVD2, [test x"$ADVD2" != x])
dnl Compiler option to improve stacktrace dnl Options to improve stacktrace
AC_CHECK_CC_OPT([-fno-omit-frame-pointer], CFLAGS="$CFLAGS -fno-omit-frame-pointer", []) AC_CHECK_CC_OPT([-fno-omit-frame-pointer], CFLAGS="$CFLAGS -fno-omit-frame-pointer", [])
AC_CHECK_CC_OPT([-fno-inline-functions-called-once], CFLAGS="$CFLAGS -fno-inline-functions-called-once", []) AC_CHECK_CC_OPT([-fno-inline-functions-called-once], CFLAGS="$CFLAGS -fno-inline-functions-called-once", [])
AC_CHECK_CC_OPT([-fno-inline-small-functions], CFLAGS="$CFLAGS -fno-inline-small-functions", []) AC_CHECK_CC_OPT([-fno-inline-small-functions], CFLAGS="$CFLAGS -fno-inline-small-functions", [])
AC_CHECK_CC_OPT([-rdynamic], CFLAGS="$CFLAGS -rdynamic", []) AC_CHECK_CC_OPT([-rdynamic], CFLAGS="$CFLAGS -rdynamic", [])
# This the new default for gcc 10
AC_CHECK_CC_OPT([-fno-common], CFLAGS="$CFLAGS -fno-common", [])
dnl Checks for system. dnl Checks for system.
AC_SYS_LARGEFILE AC_SYS_LARGEFILE
@ -31,11 +33,10 @@ AC_HEADER_ASSERT
AC_HEADER_DIRENT AC_HEADER_DIRENT
AC_HEADER_TIME AC_HEADER_TIME
AC_HEADER_SYS_WAIT AC_HEADER_SYS_WAIT
AC_HEADER_MAJOR
AC_CHECK_HEADERS([fcntl.h stddef.h stdint.h stdlib.h string.h limits.h]) AC_CHECK_HEADERS([fcntl.h stddef.h stdint.h stdlib.h string.h limits.h])
AC_CHECK_HEADERS([unistd.h getopt.h fnmatch.h io.h inttypes.h byteswap.h]) AC_CHECK_HEADERS([unistd.h getopt.h fnmatch.h io.h inttypes.h byteswap.h])
AC_CHECK_HEADERS([pthread.h math.h]) AC_CHECK_HEADERS([pthread.h math.h])
AC_CHECK_HEADERS([sys/file.h sys/ioctl.h sys/vfs.h sys/statfs.h sys/param.h sys/mount.h]) AC_CHECK_HEADERS([sys/file.h sys/ioctl.h sys/vfs.h sys/statfs.h sys/param.h sys/mount.h sys/sysmacros.h sys/mkdev.h])
AC_CHECK_HEADERS([linux/fiemap.h linux/fs.h mach/mach_time.h execinfo.h]) AC_CHECK_HEADERS([linux/fiemap.h linux/fs.h mach/mach_time.h execinfo.h])
dnl Checks for typedefs, structures, and compiler characteristics. dnl Checks for typedefs, structures, and compiler characteristics.

2
raid/check.c
View File

@ -36,7 +36,7 @@
* @ip[] Vector of @nv indexes of the valid parity blocks. * @ip[] Vector of @nv indexes of the valid parity blocks.
* The indexes start from 0. They must be in order. * The indexes start from 0. They must be in order.
* @nd Number of data blocks. * @nd Number of data blocks.
* @size Size of the blocks pointed by @v. It must be a multipler of 64. * @size Size of the blocks pointed by @v. It must be a multiplier of 64.
* @v Vector of pointers to the blocks of data and parity. * @v Vector of pointers to the blocks of data and parity.
* It has (@nd + @ip[@nv - 1] + 1) elements. The starting elements are the * It has (@nd + @ip[@nv - 1] + 1) elements. The starting elements are the
* blocks for data, following with the parity blocks. * blocks for data, following with the parity blocks.

2
raid/helper.h
View File

@ -23,7 +23,7 @@
* *
* @n Number of integers currently in the vector. * @n Number of integers currently in the vector.
* @v Vector of integers already sorted. * @v Vector of integers already sorted.
* It must have extra space for the new elemet at the end. * It must have extra space for the new element at the end.
* @i Value to insert. * @i Value to insert.
*/ */
void raid_insert(int n, int *v, int i); void raid_insert(int n, int *v, int i);

4
raid/internal.h
View File

@ -174,7 +174,7 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
/* /*
* Internal naming. * Internal naming.
* *
* These are intented to provide access for testing. * These are intended to provide access for testing.
*/ */
const char *raid_gen1_tag(void); const char *raid_gen1_tag(void);
const char *raid_gen2_tag(void); const char *raid_gen2_tag(void);
@ -263,7 +263,7 @@ static __always_inline void raid_avx_end(void)
raid_sse_end(); raid_sse_end();
/* reset the upper part of the ymm registers */ /* reset the upper part of the ymm registers */
/* to avoid the 70 clocks penality on the next */ /* to avoid the 70 clocks penalty on the next */
/* xmm register use */ /* xmm register use */
asm volatile ("vzeroupper" : : : "memory"); asm volatile ("vzeroupper" : : : "memory");
} }

2
raid/memory.h
View File

@ -45,7 +45,7 @@
* gen2 6814 [MB/s] * gen2 6814 [MB/s]
* genz 3033 [MB/s] * genz 3033 [MB/s]
* *
* These are the results with displacement resulting in improvments * These are the results with displacement resulting in improvements
* in the order of 20% or more: * in the order of 20% or more:
* *
* sse2 * sse2

16
raid/raid.c
View File

@ -63,11 +63,11 @@
* computation of triple parity using power coefficients. * computation of triple parity using power coefficients.
* *
* Another important property of the Cauchy matrix is that we can setup * Another important property of the Cauchy matrix is that we can setup
* the first two rows with coeffients equal at the RAID5 and RAID6 approach * the first two rows with coefficients equal at the RAID5 and RAID6 approach
* decribed, resulting in a compatible extension, and requiring SSSE3 * described, resulting in a compatible extension, and requiring SSSE3
* or AVX2 instructions only if triple parity or beyond is used. * or AVX2 instructions only if triple parity or beyond is used.
* *
* The matrix is also adjusted, multipling each row by a constant factor * The matrix is also adjusted, multiplying each row by a constant factor
* to make the first column of all 1, to optimize the computation for * to make the first column of all 1, to optimize the computation for
* the first disk. * the first disk.
* *
@ -147,7 +147,7 @@
* "raid/test/speedtest.c" program. * "raid/test/speedtest.c" program.
* *
* For comparison, the triple parity computation using the power * For comparison, the triple parity computation using the power
* coeffients "1,2,2^-1" is only a little faster than the one based on * coefficients "1,2,2^-1" is only a little faster than the one based on
* the Cauchy matrix if SSSE3 or AVX2 is present. * the Cauchy matrix if SSSE3 or AVX2 is present.
* *
* int8 int32 int64 sse2 ssse3 avx2 * int8 int32 int64 sse2 ssse3 avx2
@ -205,12 +205,12 @@ void raid_zero(void *zero)
* All these functions give the guarantee that parities are written * All these functions give the guarantee that parities are written
* in order. First parity P, then parity Q, and so on. * in order. First parity P, then parity Q, and so on.
* This allows to specify the same memory buffer for multiple parities * This allows to specify the same memory buffer for multiple parities
* knowning that you'll get the latest written one. * knowing that you'll get the latest written one.
* This characteristic is used by the raid_delta_gen() function to * This characteristic is used by the raid_delta_gen() function to
* avoid to damage unused parities in recovering. * avoid to damage unused parities in recovering.
* *
* @nd Number of data blocks * @nd Number of data blocks
* @size Size of the blocks pointed by @v. It must be a multipler of 64. * @size Size of the blocks pointed by @v. It must be a multiplier of 64.
* @v Vector of pointers to the blocks of data and parity. * @v Vector of pointers to the blocks of data and parity.
* It has (@nd + #parities) elements. The starting elements are the blocks * It has (@nd + #parities) elements. The starting elements are the blocks
* for data, following with the parity blocks. * for data, following with the parity blocks.
@ -335,7 +335,7 @@ void raid_delta_gen(int nr, int *id, int *ip, int nd, size_t size, void **v)
} else { } else {
/* /*
* Unused parities are going to be rewritten with * Unused parities are going to be rewritten with
* not significative data, becase we don't have * not significative data, because we don't have
* functions able to compute only a subset of * functions able to compute only a subset of
* parities. * parities.
* *
@ -483,7 +483,7 @@ void raid_rec2of2_int8(int *id, int *ip, int nd, size_t size, void **vv)
* The indexes start from 0. They must be in order. * The indexes start from 0. They must be in order.
* @nd Number of data blocks. * @nd Number of data blocks.
* @np Number of parity blocks. * @np Number of parity blocks.
* @size Size of the blocks pointed by @v. It must be a multipler of 64. * @size Size of the blocks pointed by @v. It must be a multiplier of 64.
* @v Vector of pointers to the blocks of data and parity. * @v Vector of pointers to the blocks of data and parity.
* It has (@nd + @np) elements. The starting elements are the blocks * It has (@nd + @np) elements. The starting elements are the blocks
* for data, following with the parity blocks. * for data, following with the parity blocks.

4
raid/tables.c
View File

@ -13662,8 +13662,8 @@ const uint8_t __aligned(256) raid_gfcauchypshufb[251][4][2][16] =
/** /**
* PSHUFB tables for generic multiplication. * PSHUFB tables for generic multiplication.
* *
* Indexes are [MULTIPLER][LH]. * Indexes are [MULTIPLIER][LH].
* Where MULTIPLER is from 0 to 255, LH from 0 to 1. * Where MULTIPLIER is from 0 to 255, LH from 0 to 1.
*/ */
const uint8_t __aligned(256) raid_gfmulpshufb[256][2][16] = const uint8_t __aligned(256) raid_gfmulpshufb[256][2][16] =
{ {

2
raid/test/Makefile
View File

@ -3,7 +3,7 @@
# #
# selftest - Runs the same selftest and speedtest executed at the module startup. # selftest - Runs the same selftest and speedtest executed at the module startup.
# fulltest - Runs a more extensive test that checks all the built-in functions. # fulltest - Runs a more extensive test that checks all the built-in functions.
# speetest - Runs a more complete speed test. # speedtest - Runs a more complete speed test.
# invtest - Runs an extensive matrix inversion test of all the 377.342.351.231 # invtest - Runs an extensive matrix inversion test of all the 377.342.351.231
# possible square submatrices of the Cauchy matrix used. # possible square submatrices of the Cauchy matrix used.
# covtest - Runs a coverage test. # covtest - Runs a coverage test.

11
snapraid.1
View File

@ -79,6 +79,9 @@ Beside the ability to recover from disk failures, other
features of SnapRAID are: features of SnapRAID are:
.PD 0 .PD 0
.IP \(bu .IP \(bu
You can use disk already filled with files, without the need to
reformat them. You will access them like now.
.IP \(bu
All your data is hashed to ensure data integrity and to avoid All your data is hashed to ensure data integrity and to avoid
silent corruption. silent corruption.
.IP \(bu .IP \(bu
@ -89,8 +92,6 @@ All the data in the other disks is safe.
If you accidentally delete some files in a disk, you can If you accidentally delete some files in a disk, you can
recover them. recover them.
.IP \(bu .IP \(bu
You can start with already filled disks.
.IP \(bu
The disks can have different sizes. The disks can have different sizes.
.IP \(bu .IP \(bu
You can add disks at any time. You can add disks at any time.
@ -98,7 +99,7 @@ You can add disks at any time.
It doesn\'t lock\-in your data. You can stop using SnapRAID at any It doesn\'t lock\-in your data. You can stop using SnapRAID at any
time without the need to reformat or move data. time without the need to reformat or move data.
.IP \(bu .IP \(bu
To access a file, a single disk needs to spin, saving power and To access a file, only a single disk needs to spin, saving power and
producing less noise. producing less noise.
.PD .PD
.PP .PP
@ -686,7 +687,7 @@ run \[dq]sync\[dq]. Later modifications are not taken into account.
If bad blocks were detected, their block numbers are listed. If bad blocks were detected, their block numbers are listed.
To fix them, you can use the \[dq]fix \-e\[dq] command. To fix them, you can use the \[dq]fix \-e\[dq] command.
.PP .PP
It also shows a graph representing the the last time each block It also shows a graph representing the last time each block
was scrubbed or synced. Scrubbed blocks are shown with \'*\', was scrubbed or synced. Scrubbed blocks are shown with \'*\',
blocks synced but not yet scrubbed with \'o\'. blocks synced but not yet scrubbed with \'o\'.
.PP .PP
@ -889,7 +890,7 @@ Use the filter options to select a subset of files or disks to operate on.
To only fix the blocks marked bad during \[dq]sync\[dq] and \[dq]scrub\[dq], To only fix the blocks marked bad during \[dq]sync\[dq] and \[dq]scrub\[dq],
use the \-e, \-\-filter\-error option. use the \-e, \-\-filter\-error option.
As difference from other filter options, with this one the fixes are As difference from other filter options, with this one the fixes are
applied only to files that are not modified from the the latest \[dq]sync\[dq]. applied only to files that are not modified from the latest \[dq]sync\[dq].
.PP .PP
All the files that cannot be fixed are renamed adding the All the files that cannot be fixed are renamed adding the
\[dq].unrecoverable\[dq] extension. \[dq].unrecoverable\[dq] extension.

9
snapraid.d
View File

@ -31,6 +31,8 @@ Description
Beside the ability to recover from disk failures, other Beside the ability to recover from disk failures, other
features of SnapRAID are: features of SnapRAID are:
* You can use disk already filled with files, without the need to
reformat them. You will access them like now.
* All your data is hashed to ensure data integrity and to avoid * All your data is hashed to ensure data integrity and to avoid
silent corruption. silent corruption.
* If the failed disks are too many to allow a recovery, * If the failed disks are too many to allow a recovery,
@ -38,12 +40,11 @@ Description
All the data in the other disks is safe. All the data in the other disks is safe.
* If you accidentally delete some files in a disk, you can * If you accidentally delete some files in a disk, you can
recover them. recover them.
* You can start with already filled disks.
* The disks can have different sizes. * The disks can have different sizes.
* You can add disks at any time. * You can add disks at any time.
* It doesn't lock-in your data. You can stop using SnapRAID at any * It doesn't lock-in your data. You can stop using SnapRAID at any
time without the need to reformat or move data. time without the need to reformat or move data.
* To access a file, a single disk needs to spin, saving power and * To access a file, only a single disk needs to spin, saving power and
producing less noise. producing less noise.
The official site of SnapRAID is: The official site of SnapRAID is:
@ -400,7 +401,7 @@ Commands
If bad blocks were detected, their block numbers are listed. If bad blocks were detected, their block numbers are listed.
To fix them, you can use the "fix -e" command. To fix them, you can use the "fix -e" command.
It also shows a graph representing the the last time each block It also shows a graph representing the last time each block
was scrubbed or synced. Scrubbed blocks are shown with '*', was scrubbed or synced. Scrubbed blocks are shown with '*',
blocks synced but not yet scrubbed with 'o'. blocks synced but not yet scrubbed with 'o'.
@ -577,7 +578,7 @@ Commands
To only fix the blocks marked bad during "sync" and "scrub", To only fix the blocks marked bad during "sync" and "scrub",
use the -e, --filter-error option. use the -e, --filter-error option.
As difference from other filter options, with this one the fixes are As difference from other filter options, with this one the fixes are
applied only to files that are not modified from the the latest "sync". applied only to files that are not modified from the latest "sync".
All the files that cannot be fixed are renamed adding the All the files that cannot be fixed are renamed adding the
".unrecoverable" extension. ".unrecoverable" extension.

9
snapraid.txt
View File

@ -38,6 +38,8 @@ big files that rarely change.
Beside the ability to recover from disk failures, other Beside the ability to recover from disk failures, other
features of SnapRAID are: features of SnapRAID are:
* You can use disk already filled with files, without the need to
reformat them. You will access them like now.
* All your data is hashed to ensure data integrity and to avoid * All your data is hashed to ensure data integrity and to avoid
silent corruption. silent corruption.
* If the failed disks are too many to allow a recovery, * If the failed disks are too many to allow a recovery,
@ -45,12 +47,11 @@ features of SnapRAID are:
All the data in the other disks is safe. All the data in the other disks is safe.
* If you accidentally delete some files in a disk, you can * If you accidentally delete some files in a disk, you can
recover them. recover them.
* You can start with already filled disks.
* The disks can have different sizes. * The disks can have different sizes.
* You can add disks at any time. * You can add disks at any time.
* It doesn't lock-in your data. You can stop using SnapRAID at any * It doesn't lock-in your data. You can stop using SnapRAID at any
time without the need to reformat or move data. time without the need to reformat or move data.
* To access a file, a single disk needs to spin, saving power and * To access a file, only a single disk needs to spin, saving power and
producing less noise. producing less noise.
The official site of SnapRAID is: The official site of SnapRAID is:
@ -426,7 +427,7 @@ run "sync". Later modifications are not taken into account.
If bad blocks were detected, their block numbers are listed. If bad blocks were detected, their block numbers are listed.
To fix them, you can use the "fix -e" command. To fix them, you can use the "fix -e" command.
It also shows a graph representing the the last time each block It also shows a graph representing the last time each block
was scrubbed or synced. Scrubbed blocks are shown with '*', was scrubbed or synced. Scrubbed blocks are shown with '*',
blocks synced but not yet scrubbed with 'o'. blocks synced but not yet scrubbed with 'o'.
@ -617,7 +618,7 @@ Use the filter options to select a subset of files or disks to operate on.
To only fix the blocks marked bad during "sync" and "scrub", To only fix the blocks marked bad during "sync" and "scrub",
use the -e, --filter-error option. use the -e, --filter-error option.
As difference from other filter options, with this one the fixes are As difference from other filter options, with this one the fixes are
applied only to files that are not modified from the the latest "sync". applied only to files that are not modified from the latest "sync".
All the files that cannot be fixed are renamed adding the All the files that cannot be fixed are renamed adding the
".unrecoverable" extension. ".unrecoverable" extension.

4
tommyds/tommychain.h
View File

@ -114,7 +114,7 @@ tommy_inline void tommy_chain_merge(tommy_chain* first, tommy_chain* second, tom
/** /**
* Merges two chains managing special degenerated cases. * Merges two chains managing special degenerated cases.
* It's funtionally equivalent at tommy_chain_merge() but faster with already ordered chains. * It's functionally equivalent at tommy_chain_merge() but faster with already ordered chains.
*/ */
tommy_inline void tommy_chain_merge_degenerated(tommy_chain* first, tommy_chain* second, tommy_compare_func* cmp) tommy_inline void tommy_chain_merge_degenerated(tommy_chain* first, tommy_chain* second, tommy_compare_func* cmp)
{ {
@ -153,7 +153,7 @@ tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func*
/* /*
* Bit buckets of chains. * Bit buckets of chains.
* Each bucket contains 2^i nodes or it's empty. * Each bucket contains 2^i nodes or it's empty.
* The chain at address TOMMY_BIT_MAX is an independet variable operating as "carry". * The chain at address TOMMY_BIT_MAX is an independent variable operating as "carry".
* We keep it in the same "bit" vector to avoid reports from the valgrind tool sgcheck. * We keep it in the same "bit" vector to avoid reports from the valgrind tool sgcheck.
*/ */
tommy_chain bit[TOMMY_SIZE_BIT + 1]; tommy_chain bit[TOMMY_SIZE_BIT + 1];

2
tommyds/tommyhash.c
View File

@ -231,7 +231,7 @@ tommy_uint32_t tommy_strhash_u32(tommy_uint64_t init_val, const void* void_key)
key += 12; key += 12;
} }
/* for lengths that are multiplers of 12 we already have called mix */ /* for lengths that are multipliers of 12 we already have called mix */
/* this is different than the original lookup3 and the result won't match */ /* this is different than the original lookup3 and the result won't match */
tommy_final(a, b, c); tommy_final(a, b, c);

8
tommyds/tommyhash.h
View File

@ -52,7 +52,7 @@
* \param void_key Pointer to the data to hash. * \param void_key Pointer to the data to hash.
* \param key_len Size of the data to hash. * \param key_len Size of the data to hash.
* \note * \note
* This function is endianess independent. * This function is endianness independent.
* \return The hash value of 32 bits. * \return The hash value of 32 bits.
*/ */
tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tommy_size_t key_len); tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tommy_size_t key_len);
@ -72,14 +72,14 @@ tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tom
* \param void_key Pointer to the data to hash. * \param void_key Pointer to the data to hash.
* \param key_len Size of the data to hash. * \param key_len Size of the data to hash.
* \note * \note
* This function is endianess independent. * This function is endianness independent.
* \return The hash value of 64 bits. * \return The hash value of 64 bits.
*/ */
tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tommy_size_t key_len); tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tommy_size_t key_len);
/** /**
* String hash function with a 32 bits result. * String hash function with a 32 bits result.
* Implementation is based on the the Robert Jenkins "lookup3" hash 32 bits version, * Implementation is based on Robert Jenkins "lookup3" hash 32 bits version,
* from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle(). * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle().
* *
* This hash is designed to handle strings with an unknown length. If you * This hash is designed to handle strings with an unknown length. If you
@ -90,7 +90,7 @@ tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tom
* Use 0 if not relevant. * Use 0 if not relevant.
* \param void_key Pointer to the string to hash. It has to be 0 terminated. * \param void_key Pointer to the string to hash. It has to be 0 terminated.
* \note * \note
* This function is endianess independent. * This function is endianness independent.
* \return The hash value of 32 bits. * \return The hash value of 32 bits.
*/ */
tommy_uint32_t tommy_strhash_u32(tommy_uint64_t init_val, const void* void_key); tommy_uint32_t tommy_strhash_u32(tommy_uint64_t init_val, const void* void_key);

2
tommyds/tommyhashdyn.h
View File

@ -92,7 +92,7 @@
* *
* To iterate over all the elements in the hashtable with the same key, you have to * To iterate over all the elements in the hashtable with the same key, you have to
* use tommy_hashdyn_bucket() and follow the tommy_node::next pointer until NULL. * use tommy_hashdyn_bucket() and follow the tommy_node::next pointer until NULL.
* You have also to check explicitely for the key, as the bucket may contains * You have also to check explicitly for the key, as the bucket may contains
* different keys. * different keys.
* *
* \code * \code

4
tommyds/tommylist.h
View File

@ -28,7 +28,7 @@
/** \file /** \file
* Double linked list for collisions into hashtables. * Double linked list for collisions into hashtables.
* *
* This list is a double linked list mainly targetted for handling collisions * This list is a double linked list mainly targeted for handling collisions
* into an hashtables, but useable also as a generic list. * into an hashtables, but useable also as a generic list.
* *
* The main feature of this list is to require only one pointer to represent the * The main feature of this list is to require only one pointer to represent the
@ -294,7 +294,7 @@ tommy_inline void tommy_list_concat(tommy_list* first, tommy_list* second)
* It's a stable merge sort with O(N*log(N)) worst complexity. * It's a stable merge sort with O(N*log(N)) worst complexity.
* It's faster on degenerated cases like partially ordered lists. * It's faster on degenerated cases like partially ordered lists.
* \param cmp Compare function called with two elements. * \param cmp Compare function called with two elements.
* The function should return <0 if the first element is less than the second, ==0 if equal, and >0 if greather. * The function should return <0 if the first element is less than the second, ==0 if equal, and >0 if greater.
*/ */
void tommy_list_sort(tommy_list* list, tommy_compare_func* cmp); void tommy_list_sort(tommy_list* list, tommy_compare_func* cmp);

4
tommyds/tommytree.h
View File

@ -33,7 +33,7 @@
* *
* As difference than other tommy containers, duplicate elements cannot be inserted. * As difference than other tommy containers, duplicate elements cannot be inserted.
* *
* To initialize a tree you have to call tommy_tree_init() specifing a comparison * To initialize a tree you have to call tommy_tree_init() specifying a comparison
* function that will define the order in the tree. * function that will define the order in the tree.
* *
* \code * \code
@ -123,7 +123,7 @@ typedef struct tommy_tree_struct {
/** /**
* Initializes the tree. * Initializes the tree.
* \param cmp The comparison function that defines the orderin the tree. * \param cmp The comparison function that defines the order in the tree.
*/ */
void tommy_tree_init(tommy_tree* tree, tommy_compare_func* cmp); void tommy_tree_init(tommy_tree* tree, tommy_compare_func* cmp);

2
tommyds/tommytypes.h
View File

@ -227,7 +227,7 @@ typedef struct tommy_node_struct {
* Compare function for elements. * Compare function for elements.
* \param obj_a Pointer to the first object to compare. * \param obj_a Pointer to the first object to compare.
* \param obj_b Pointer to the second object to compare. * \param obj_b Pointer to the second object to compare.
* \return <0 if the first element is less than the second, ==0 equal, >0 if greather. * \return <0 if the first element is less than the second, ==0 equal, >0 if greater.
* *
* This function is like the C strcmp(). * This function is like the C strcmp().
* *