Merge tag 'upstream/12.2'

Upstream version 12.2

.version

@@ -1 +1 @@
-11.2
+12.2

HISTORY

@@ -1,6 +1,55 @@
 SnapRAID HISTORY
 ================
 
+12.2 2022/08
+============
+ * Fix build issue with GLIBC 2.36
+
+12.1 2022/01
+============
+ * Reduce stack usage to work in environments with limited stack size, like MUSL.
+ * Increase the default disk cache from 8 MiB to 16 MiB.
+
+12.0 2021/12
+============
+ * Parallel disk scanning. It's always enabled but it doesn't cover the -m option
+   that still process disks sequentially.
+
+11.6 2021/10
+============
+ * The 'fix' and 'check' command with the -e option now process the whole
+   files that have bad blocks, and not only the block marked bad.
+   This allows to restore the timestamp and to print the paths of
+   processed files and the final state of the files like 'recovered' or
+   'unrecovered'. The previous behaviour is available with the -b,
+   --filter-block-error option.
+ * Improved the speed of the filtering in 'fix' and 'check'. This phase
+   happens after the "Selecting..." message.  [UhClem]
+
+11.5 2020/05
+============
+ * Removed the default -march=native to allow to deploy in any machine.
+ * Fixed typos [Andrea Gelmini]
+
+11.4 2020/05
+============
+ * Fix build errors due new gcc 10 default for -fno-common.
+ * In fixing, if a parity is filtered out, don't attempt to recover its size,
+   and proceed without it if missing.
+ * Avoid unnecessary parity read when fixing the parity itself.
+   This improves the 'fix' speed when a parity file is completely missing.
+ * Removed a build warning about major/minor defined now in sys/sysmacros.h.
+
+11.3 2018/11
+============
+ * Fixed handing of Linux devices that have multiple slaves. This affects
+   the smart/list/devices/down commands [Valentin Hilbig].
+ * The 'list' command in verbose mode prints the full nanosecond
+   timestamp precision.
+ * After writing content files also sync their directory.
+ * Fix a invalid time computation that could result in future scrub dates.
+   Such dates are fixed automatically at the next scrub or sync.
+
 11.2 2017/12
 ============
  * Fixed recognition of NTFS hardlinks. They behave differently than
@@ -129,7 +178,7 @@ SnapRAID HISTORY
    A workaround was to just run 'sync' one time with the -N,
    --force-nocopy option to disable the copy detection.
  * Restored the -O2 optimization option for Windows binaries, as -Og has
-   a too big performance penality.
+   a too big performance penalty.
 
 9.2 2016/01
 ===========
@@ -402,7 +451,7 @@ SnapRAID HISTORY
    and Opteron.
  * Faster RAID5 and RAID6 implementation for ARM 64 bit CPUs.
  * If a silent error is found during a "sync" command, directly marks
-   the block as bad like in "scrub", without stopping the the "sync"
+   the block as bad like in "scrub", without stopping the "sync"
    process.
  * Sort files by inode when listing the directory. This improves
    the scanning performance.

Makefile.am

@@ -83,6 +83,7 @@ noinst_HEADERS = \
 	cmdline/murmur3test.c \
 	cmdline/spooky2.c \
 	cmdline/spooky2test.c \
+	cmdline/metro.c \
 	cmdline/fnmatch.h \
 	cmdline/import.h \
 	cmdline/search.h \
@@ -158,7 +159,7 @@ maintainer-clean-local:
 CHECKFLAGS_COMMON = --test-skip-device --test-skip-self --test-force-progress --no-warnings --test-parity-limit=3333333
 
 # --test-force-order-alpha
-# Ensures to process files always in the same order despites
+# Ensures to process files always in the same order despite
 # the inode, physical location, and dir order assigned by the OS.
 CHECKFLAGS_ALPHA = $(CHECKFLAGS_COMMON) --test-force-order-alpha
 
@@ -499,7 +500,7 @@ endif
 	$(FAILENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-run "rm bench/disk1/RUN-RM" --test-expect-failure sync
 	echo RUN > bench/disk1/RUN-CHMOD
 if HAVE_POSIX
-# Doesn't run this test as root because the root user overrride permissions
+# Doesn't run this test as root because the root user override permissions
 	if [[ $$EUID -ne 0 ]]; then \
 		$(FAILENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-run "chmod a-r bench/disk1/RUN-CHMOD" --test-expect-failure sync; \
 	fi

Makefile.in

@@ -1,7 +1,7 @@
-# Makefile.in generated by automake 1.15 from Makefile.am.
+# Makefile.in generated by automake 1.16.2 from Makefile.am.
 # @configure_input@
 
-# Copyright (C) 1994-2014 Free Software Foundation, Inc.
+# Copyright (C) 1994-2020 Free Software Foundation, Inc.
 
 # This Makefile.in is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -153,7 +153,7 @@ am__v_at_0 = @
 am__v_at_1 = 
 DEFAULT_INCLUDES = -I.@am__isrc@
 depcomp =
-am__depfiles_maybe =
+am__maybe_remake_depfiles =
 COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
 	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
 AM_V_CC = $(am__v_CC_@AM_V@)
@@ -205,8 +205,8 @@ man1dir = $(mandir)/man1
 NROFF = nroff
 MANS = $(man_MANS)
 HEADERS = $(noinst_HEADERS)
-am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) \
-	$(LISP)config.h.in
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) \
+	config.h.in
 # Read a list of newline-separated strings from the standard input,
 # and print each of them once, without duplicates.  Input order is
 # *not* preserved.
@@ -429,6 +429,7 @@ noinst_HEADERS = \
 	cmdline/murmur3test.c \
 	cmdline/spooky2.c \
 	cmdline/spooky2test.c \
+	cmdline/metro.c \
 	cmdline/fnmatch.h \
 	cmdline/import.h \
 	cmdline/search.h \
@@ -487,7 +488,7 @@ man_MANS = snapraid.1
 CHECKFLAGS_COMMON = --test-skip-device --test-skip-self --test-force-progress --no-warnings --test-parity-limit=3333333
 
 # --test-force-order-alpha
-# Ensures to process files always in the same order despites
+# Ensures to process files always in the same order despite
 # the inode, physical location, and dir order assigned by the OS.
 CHECKFLAGS_ALPHA = $(CHECKFLAGS_COMMON) --test-force-order-alpha
 
@@ -554,8 +555,8 @@ Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
 	    echo ' $(SHELL) ./config.status'; \
 	    $(SHELL) ./config.status;; \
 	  *) \
-	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $@ $(am__depfiles_maybe)'; \
-	    cd $(top_builddir) && $(SHELL) ./config.status $@ $(am__depfiles_maybe);; \
+	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $@ $(am__maybe_remake_depfiles)'; \
+	    cd $(top_builddir) && $(SHELL) ./config.status $@ $(am__maybe_remake_depfiles);; \
 	esac;
 
 $(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
@@ -808,7 +809,10 @@ distclean-tags:
 	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
 	-rm -f cscope.out cscope.in.out cscope.po.out cscope.files
 
-distdir: $(DISTFILES)
+distdir: $(BUILT_SOURCES)
+	$(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
 	$(am__remove_distdir)
 	test -d "$(distdir)" || mkdir "$(distdir)"
 	@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
@@ -851,7 +855,7 @@ distdir: $(DISTFILES)
 	  ! -type d ! -perm -444 -exec $(install_sh) -c -m a+r {} {} \; \
 	|| chmod -R a+r "$(distdir)"
 dist-gzip: distdir
-	tardir=$(distdir) && $(am__tar) | GZIP=$(GZIP_ENV) gzip -c >$(distdir).tar.gz
+	tardir=$(distdir) && $(am__tar) | eval GZIP= gzip $(GZIP_ENV) -c >$(distdir).tar.gz
 	$(am__post_remove_distdir)
 
 dist-bzip2: distdir
@@ -866,6 +870,10 @@ dist-xz: distdir
 	tardir=$(distdir) && $(am__tar) | XZ_OPT=$${XZ_OPT--e} xz -c >$(distdir).tar.xz
 	$(am__post_remove_distdir)
 
+dist-zstd: distdir
+	tardir=$(distdir) && $(am__tar) | zstd -c $${ZSTD_CLEVEL-$${ZSTD_OPT--19}} >$(distdir).tar.zst
+	$(am__post_remove_distdir)
+
 dist-tarZ: distdir
 	@echo WARNING: "Support for distribution archives compressed with" \
 		       "legacy program 'compress' is deprecated." >&2
@@ -877,7 +885,7 @@ dist-shar: distdir
 	@echo WARNING: "Support for shar distribution archives is" \
 	               "deprecated." >&2
 	@echo WARNING: "It will be removed altogether in Automake 2.0" >&2
-	shar $(distdir) | GZIP=$(GZIP_ENV) gzip -c >$(distdir).shar.gz
+	shar $(distdir) | eval GZIP= gzip $(GZIP_ENV) -c >$(distdir).shar.gz
 	$(am__post_remove_distdir)
 
 dist-zip: distdir
@@ -895,7 +903,7 @@ dist dist-all:
 distcheck: dist
 	case '$(DIST_ARCHIVES)' in \
 	*.tar.gz*) \
-	  GZIP=$(GZIP_ENV) gzip -dc $(distdir).tar.gz | $(am__untar) ;;\
+	  eval GZIP= gzip $(GZIP_ENV) -dc $(distdir).tar.gz | $(am__untar) ;;\
 	*.tar.bz2*) \
 	  bzip2 -dc $(distdir).tar.bz2 | $(am__untar) ;;\
 	*.tar.lz*) \
@@ -905,9 +913,11 @@ distcheck: dist
 	*.tar.Z*) \
 	  uncompress -c $(distdir).tar.Z | $(am__untar) ;;\
 	*.shar.gz*) \
-	  GZIP=$(GZIP_ENV) gzip -dc $(distdir).shar.gz | unshar ;;\
+	  eval GZIP= gzip $(GZIP_ENV) -dc $(distdir).shar.gz | unshar ;;\
 	*.zip*) \
 	  unzip $(distdir).zip ;;\
+	*.tar.zst*) \
+	  zstd -dc $(distdir).tar.zst | $(am__untar) ;;\
 	esac
 	chmod -R a-w $(distdir)
 	chmod u+w $(distdir)
@@ -1096,20 +1106,20 @@ uninstall-man: uninstall-man1
 	check-local clean clean-binPROGRAMS clean-checkPROGRAMS \
 	clean-cscope clean-generic clean-local cscope cscopelist-am \
 	ctags ctags-am dist dist-all dist-bzip2 dist-gzip dist-hook \
-	dist-lzip dist-shar dist-tarZ dist-xz dist-zip distcheck \
-	distclean distclean-compile distclean-generic distclean-hdr \
-	distclean-tags distcleancheck distdir distuninstallcheck dvi \
-	dvi-am html html-am info info-am install install-am \
-	install-binPROGRAMS install-data install-data-am install-dvi \
-	install-dvi-am install-exec install-exec-am install-html \
-	install-html-am install-info install-info-am install-man \
-	install-man1 install-pdf install-pdf-am install-ps \
-	install-ps-am install-strip installcheck installcheck-am \
-	installdirs maintainer-clean maintainer-clean-generic \
-	maintainer-clean-local mostlyclean mostlyclean-compile \
-	mostlyclean-generic pdf pdf-am ps ps-am tags tags-am uninstall \
-	uninstall-am uninstall-binPROGRAMS uninstall-man \
-	uninstall-man1
+	dist-lzip dist-shar dist-tarZ dist-xz dist-zip dist-zstd \
+	distcheck distclean distclean-compile distclean-generic \
+	distclean-hdr distclean-tags distcleancheck distdir \
+	distuninstallcheck dvi dvi-am html html-am info info-am \
+	install install-am install-binPROGRAMS install-data \
+	install-data-am install-dvi install-dvi-am install-exec \
+	install-exec-am install-html install-html-am install-info \
+	install-info-am install-man install-man1 install-pdf \
+	install-pdf-am install-ps install-ps-am install-strip \
+	installcheck installcheck-am installdirs maintainer-clean \
+	maintainer-clean-generic maintainer-clean-local mostlyclean \
+	mostlyclean-compile mostlyclean-generic pdf pdf-am ps ps-am \
+	tags tags-am uninstall uninstall-am uninstall-binPROGRAMS \
+	uninstall-man uninstall-man1
 
 .PRECIOUS: Makefile
 
@@ -1409,7 +1419,7 @@ check-local:
 @HAVE_THREAD_CHECKER_FALSE@	echo RUN > bench/disk1/RUN-RM
 @HAVE_THREAD_CHECKER_FALSE@	$(FAILENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-run "rm bench/disk1/RUN-RM" --test-expect-failure sync
 @HAVE_THREAD_CHECKER_FALSE@	echo RUN > bench/disk1/RUN-CHMOD
-# Doesn't run this test as root because the root user overrride permissions
+# Doesn't run this test as root because the root user override permissions
 @HAVE_POSIX_TRUE@@HAVE_THREAD_CHECKER_FALSE@	if [[ $$EUID -ne 0 ]]; then \
 @HAVE_POSIX_TRUE@@HAVE_THREAD_CHECKER_FALSE@		$(FAILENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-run "chmod a-r bench/disk1/RUN-CHMOD" --test-expect-failure sync; \
 @HAVE_POSIX_TRUE@@HAVE_THREAD_CHECKER_FALSE@	fi

README

@@ -11,6 +11,8 @@ have a lot of big files that rarely change.
 Beside the ability to recover from disk failures, the other
 features of SnapRAID are:
 
+* You can use disk already filled with files, without the need to
+  reformat them. You will access them like now.
 * All your data is hashed to ensure data integrity and to avoid
   silent corruption.
 * If the failed disks are too many to allow a recovery,
@@ -18,12 +20,11 @@ features of SnapRAID are:
   All the data in the other disks is safe.
 * If you accidentally delete some files in a disk, you can
   recover them.
-* You can start with already filled disks.
 * The disks can have different sizes.
 * You can add disks at any time.
 * It doesn't lock-in your data. You can stop using SnapRAID at any
   time without the need to reformat or move data.
-* To access a file, a single disk needs to spin, saving power and
+* To access a file, only a single disk needs to spin, saving power and
   producing less noise.
 
 The official site of SnapRAID is:

TODO

@@ -17,6 +17,18 @@ Not checked disks should be allowed to be missing.
 * Add an option to ignore subsecond timestamp.
 Like when you copy data to a filesystem with less timestamp precision.
 
+* Use threads to scan all the disks at the same time.
+- After 7.0 Windows changes it seems fast enough even
+with a mono thread implementation with 100.0000 files.
++ But if you have millions of files, it could take minutes.
+
+* Support more parity levels
+It can be done with a generic computation function, using
+intrinsic for SSSE3 and AVX instructions.
+It would be interesting to compare performance with the hand-written
+assembler functions. Eventually we can convert them to use intrinsic also.
+https://sourceforge.net/p/snapraid/discussion/1677233/thread/9dbd7581/
+
 * Extend haspdeep to support the SnapRAID hash :
 https://github.com/jessek/hashdeep/
 https://sourceforge.net/p/snapraid/discussion/1677233/thread/90b0e9b2/?limit=25
@@ -139,7 +151,7 @@ See: https://sourceforge.net/p/snapraid/discussion/1677233/thread/cdea773f/
 * Allocate parity minimizing concurrent use of it
 Each parity allocation should check for a parity
 range with less utilization by other disks.
-We need to take care do disable this meachnism when the parity space
+We need to take care do disable this mechanism when the parity space
 is near to fillup the parity partition.
 See: https://sourceforge.net/p/snapraid/discussion/1677233/thread/1797bf7d/
 + This increase the possibility of recovering with multiple failures with not
@@ -170,7 +182,7 @@ allowing the user to choose where to put it.
 - It won't work with disks of different size.
 Suppose to have all disks of size N, with only one of size M>N.
 To fully use the M space, you can allocate a full N parity in such disk,
-but the remaning space will also need additional parity in the other disks,
+but the remaining space will also need additional parity in the other disks,
 in fact requiring a total of M parity for the array.
 In the end, we cannot avoid that the first biggest disk added is fully
 dedicated to parity, even if it means to leave some space unused.
@@ -187,7 +199,7 @@ But it should be only few bits for each file. So, it should be manageable.
 A lot of discussions about this feature :)
 https://sourceforge.net/p/snapraid/discussion/1677233/thread/b2cd9385/
 - The only benefit is to distribute better the data. This could help the recovery process,
-in case of multiple failures. But no real usability or funtionality benefit in the normal
+in case of multiple failures. But no real usability or functionality benefit in the normal
 case.
 
 * https://sourceforge.net/p/snapraid/discussion/1677233/thread/2cb97e8a/
@@ -217,15 +229,11 @@ are automatically deleted.
 
 * Checks if splitting hash/parity computation in 4K pages
 can improve speed in sync. That should increase cache locality,
-because we read the data two times for hash and and parity,
+because we read the data two times for hash and parity,
 and if we manage to keep it in the cache, we should save time.
 - We now hash first the faster disks, and this could
 reduce performance as we'll have to wait for all disks.
 
-* Use threads to scan all the disks at the same time.
-- After 7.0 Windows changes it seems fast enough even
-with a mono thread implementation.
-
 * Enable storing of creation time NTFS, crtime/birth time EXT4.
 But see: http://unix.stackexchange.com/questions/50177/birth-is-empty-on-ext4
 coreutils stat has an example, but it doesn't work in Linux (see lib/stat-time.h)

acinclude.m4

@@ -1,7 +1,7 @@
 dnl @synopsis AC_CHECK_CC_OPT(flag, ifyes, ifno)
 dnl 
-dnl Shows a message as like "checking wether gcc accepts flag ... no"
-dnl and executess ifyes or ifno.
+dnl Shows a message as like "checking whether gcc accepts flag ... no"
+dnl and executes ifyes or ifno.
 
 AC_DEFUN([AC_CHECK_CC_OPT],
 [

aclocal.m4

@@ -1,6 +1,6 @@
-# generated automatically by aclocal 1.15 -*- Autoconf -*-
+# generated automatically by aclocal 1.16.2 -*- Autoconf -*-
 
-# Copyright (C) 1996-2014 Free Software Foundation, Inc.
+# Copyright (C) 1996-2020 Free Software Foundation, Inc.
 
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -20,7 +20,7 @@ You have another version of autoconf.  It may work, but is not guaranteed to.
 If you have problems, you may need to regenerate the build system entirely.
 To do so, use the procedure documented by the package, typically 'autoreconf'.])])
 
-# Copyright (C) 2002-2014 Free Software Foundation, Inc.
+# Copyright (C) 2002-2020 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -32,10 +32,10 @@ To do so, use the procedure documented by the package, typically 'autoreconf'.])
 # generated from the m4 files accompanying Automake X.Y.
 # (This private macro should not be called outside this file.)
 AC_DEFUN([AM_AUTOMAKE_VERSION],
-[am__api_version='1.15'
+[am__api_version='1.16'
 dnl Some users find AM_AUTOMAKE_VERSION and mistake it for a way to
 dnl require some minimum version.  Point them to the right macro.
-m4_if([$1], [1.15], [],
+m4_if([$1], [1.16.2], [],
       [AC_FATAL([Do not call $0, use AM_INIT_AUTOMAKE([$1]).])])dnl
 ])
 
@@ -51,14 +51,14 @@ m4_define([_AM_AUTOCONF_VERSION], [])
 # Call AM_AUTOMAKE_VERSION and AM_AUTOMAKE_VERSION so they can be traced.
 # This function is AC_REQUIREd by AM_INIT_AUTOMAKE.
 AC_DEFUN([AM_SET_CURRENT_AUTOMAKE_VERSION],
-[AM_AUTOMAKE_VERSION([1.15])dnl
+[AM_AUTOMAKE_VERSION([1.16.2])dnl
 m4_ifndef([AC_AUTOCONF_VERSION],
   [m4_copy([m4_PACKAGE_VERSION], [AC_AUTOCONF_VERSION])])dnl
 _AM_AUTOCONF_VERSION(m4_defn([AC_AUTOCONF_VERSION]))])
 
 # AM_AUX_DIR_EXPAND                                         -*- Autoconf -*-
 
-# Copyright (C) 2001-2014 Free Software Foundation, Inc.
+# Copyright (C) 2001-2020 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -110,7 +110,7 @@ am_aux_dir=`cd "$ac_aux_dir" && pwd`
 
 # AM_CONDITIONAL                                            -*- Autoconf -*-
 
-# Copyright (C) 1997-2014 Free Software Foundation, Inc.
+# Copyright (C) 1997-2020 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -143,7 +143,7 @@ fi])])
 
 # Do all the work for Automake.                             -*- Autoconf -*-
 
-# Copyright (C) 1996-2014 Free Software Foundation, Inc.
+# Copyright (C) 1996-2020 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -230,8 +230,8 @@ AC_REQUIRE([AM_PROG_INSTALL_STRIP])dnl
 AC_REQUIRE([AC_PROG_MKDIR_P])dnl
 # For better backward compatibility.  To be removed once Automake 1.9.x
 # dies out for good.  For more background, see:
-# <http://lists.gnu.org/archive/html/automake/2012-07/msg00001.html>
-# <http://lists.gnu.org/archive/html/automake/2012-07/msg00014.html>
+# <https://lists.gnu.org/archive/html/automake/2012-07/msg00001.html>
+# <https://lists.gnu.org/archive/html/automake/2012-07/msg00014.html>
 AC_SUBST([mkdir_p], ['$(MKDIR_P)'])
 # We need awk for the "check" target (and possibly the TAP driver).  The
 # system "awk" is bad on some platforms.
@@ -298,7 +298,7 @@ END
 Aborting the configuration process, to ensure you take notice of the issue.
 
 You can download and install GNU coreutils to get an 'rm' implementation
-that behaves properly: <http://www.gnu.org/software/coreutils/>.
+that behaves properly: <https://www.gnu.org/software/coreutils/>.
 
 If you want to complete the configuration process using your problematic
 'rm' anyway, export the environment variable ACCEPT_INFERIOR_RM_PROGRAM
@@ -340,7 +340,7 @@ for _am_header in $config_headers :; do
 done
 echo "timestamp for $_am_arg" >`AS_DIRNAME(["$_am_arg"])`/stamp-h[]$_am_stamp_count])
 
-# Copyright (C) 2001-2014 Free Software Foundation, Inc.
+# Copyright (C) 2001-2020 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -361,7 +361,7 @@ if test x"${install_sh+set}" != xset; then
 fi
 AC_SUBST([install_sh])])
 
-# Copyright (C) 2003-2014 Free Software Foundation, Inc.
+# Copyright (C) 2003-2020 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -382,7 +382,7 @@ AC_SUBST([am__leading_dot])])
 
 # Fake the existence of programs that GNU maintainers use.  -*- Autoconf -*-
 
-# Copyright (C) 1997-2014 Free Software Foundation, Inc.
+# Copyright (C) 1997-2020 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -421,7 +421,7 @@ fi
 
 # Helper functions for option handling.                     -*- Autoconf -*-
 
-# Copyright (C) 2001-2014 Free Software Foundation, Inc.
+# Copyright (C) 2001-2020 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -450,7 +450,7 @@ AC_DEFUN([_AM_SET_OPTIONS],
 AC_DEFUN([_AM_IF_OPTION],
 [m4_ifset(_AM_MANGLE_OPTION([$1]), [$2], [$3])])
 
-# Copyright (C) 1999-2014 Free Software Foundation, Inc.
+# Copyright (C) 1999-2020 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -497,7 +497,7 @@ AC_LANG_POP([C])])
 # For backward compatibility.
 AC_DEFUN_ONCE([AM_PROG_CC_C_O], [AC_REQUIRE([AC_PROG_CC])])
 
-# Copyright (C) 2001-2014 Free Software Foundation, Inc.
+# Copyright (C) 2001-2020 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -516,7 +516,7 @@ AC_DEFUN([AM_RUN_LOG],
 
 # Check to make sure that the build environment is sane.    -*- Autoconf -*-
 
-# Copyright (C) 1996-2014 Free Software Foundation, Inc.
+# Copyright (C) 1996-2020 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -597,7 +597,7 @@ AC_CONFIG_COMMANDS_PRE(
 rm -f conftest.file
 ])
 
-# Copyright (C) 2009-2014 Free Software Foundation, Inc.
+# Copyright (C) 2009-2020 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -657,7 +657,7 @@ AC_SUBST([AM_BACKSLASH])dnl
 _AM_SUBST_NOTMAKE([AM_BACKSLASH])dnl
 ])
 
-# Copyright (C) 2001-2014 Free Software Foundation, Inc.
+# Copyright (C) 2001-2020 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -685,7 +685,7 @@ fi
 INSTALL_STRIP_PROGRAM="\$(install_sh) -c -s"
 AC_SUBST([INSTALL_STRIP_PROGRAM])])
 
-# Copyright (C) 2006-2014 Free Software Foundation, Inc.
+# Copyright (C) 2006-2020 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,
@@ -704,7 +704,7 @@ AC_DEFUN([AM_SUBST_NOTMAKE], [_AM_SUBST_NOTMAKE($@)])
 
 # Check how to create a tarball.                            -*- Autoconf -*-
 
-# Copyright (C) 2004-2014 Free Software Foundation, Inc.
+# Copyright (C) 2004-2020 Free Software Foundation, Inc.
 #
 # This file is free software; the Free Software Foundation
 # gives unlimited permission to copy and/or distribute it,

cmdline/check.c

@@ -574,7 +574,7 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
 	} else {
 		log_tag("recover_unsync:%u:%u: Skipped for%s%s\n", pos, n,
 			!something_to_recover ? " nothing to recover" : "",
-			!something_unsynced ? " nothing unsynched" : ""
+			!something_unsynced ? " nothing unsynced" : ""
 		);
 	}
 
@@ -590,9 +590,11 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
  * For each file, if we are at the last block, closes it,
  * adjust the timestamp, and print the result.
  *
- * This works with the assumption to always process the whole files to
- * fix. This assumption is not always correct, and in such case we have to
- * skip the whole postprocessing. And example, is when fixing only bad blocks.
+ * This works only if the whole file is processed, including its last block.
+ * This doesn't always happen, like with an explicit end block.
+ *
+ * In such case, the check/fix command won't report any information of the
+ * files partially checked.
  */
 static int file_post(struct snapraid_state* state, int fix, unsigned i, struct snapraid_handle* handle, unsigned diskmax)
 {
@@ -601,11 +603,6 @@ static int file_post(struct snapraid_state* state, int fix, unsigned i, struct s
 	char esc_buffer[ESC_MAX];
 	char esc_buffer_alt[ESC_MAX];
 
-	/* if we are processing only bad blocks, we don't have to do any post-processing */
-	/* as we don't have any guarantee to process the last block of the fixed files */
-	if (state->opt.badonly)
-		return 0;
-
 	/* for all the files print the final status, and does the final time fix */
 	/* we also ensure to close files after processing the last block */
 	for (j = 0; j < diskmax; ++j) {
@@ -614,7 +611,6 @@ static int file_post(struct snapraid_state* state, int fix, unsigned i, struct s
 		struct snapraid_file* collide_file;
 		struct snapraid_file* file;
 		block_off_t file_pos;
-		char path[PATH_MAX];
 		uint64_t inode;
 
 		disk = handle[j].disk;
@@ -630,7 +626,6 @@ static int file_post(struct snapraid_state* state, int fix, unsigned i, struct s
 		}
 
 		file = fs_par2file_get(disk, i, &file_pos);
-		pathprint(path, sizeof(path), "%s%s", disk->dir, file->sub);
 
 		/* if it isn't the last block in the file */
 		if (!file_block_is_last(file, file_pos)) {
@@ -654,8 +649,10 @@ static int file_post(struct snapraid_state* state, int fix, unsigned i, struct s
 			/* if the file is damaged, meaning that a fix failed */
 			if (file_flag_has(file, FILE_IS_DAMAGED)) {
 				/* rename it to .unrecoverable */
+				char path[PATH_MAX];
 				char path_to[PATH_MAX];
 
+				pathprint(path, sizeof(path), "%s%s", disk->dir, file->sub);
 				pathprint(path_to, sizeof(path_to), "%s%s.unrecoverable", disk->dir, file->sub);
 
 				/* ensure to close the file before renaming */
@@ -801,30 +798,52 @@ close_and_continue:
  */
 static int block_is_enabled(struct snapraid_state* state, block_off_t i, struct snapraid_handle* handle, unsigned diskmax)
 {
-	snapraid_info info;
 	unsigned j;
 	unsigned l;
 
-	/* get block specific info */
-	info = info_get(&state->infoarr, i);
+	/* filter for bad blocks */
+	if (state->opt.badblockonly) {
+		snapraid_info info;
 
-	/* if we filter for only bad blocks */
-	if (state->opt.badonly) {
-		/* skip if this is not bad */
-		if (!info_get_bad(info))
-			return 0;
+		/* get block specific info */
+		info = info_get(&state->infoarr, i);
+
+		/*
+		 * Filter specifically only for bad blocks
+		 */
+		return info_get_bad(info);
 	}
 
-	/* now apply the filters */
+	/* filter for the parity */
+	if (state->opt.badfileonly) {
+		snapraid_info info;
 
-	/* if a parity is not excluded, include all blocks, even unused ones */
-	for (l = 0; l < state->level; ++l) {
-		if (!state->parity[l].is_excluded_by_filter) {
+		/* get block specific info */
+		info = info_get(&state->infoarr, i);
+
+		/*
+		 * If the block is bad, it has to be processed
+		 *
+		 * This is not necessary in normal cases because if a block is bad,
+		 * it necessary needs to have a file related to it, and files with
+		 * bad blocks are fully included.
+		 *
+		 * But some files may be excluded by additional filter options,
+		 * so it's not always true, and this ensures to always check all
+		 * the bad blocks.
+		 */
+		if (info_get_bad(info))
 			return 1;
+	} else {
+		/* if a parity is not excluded, include all blocks, even unused ones */
+		for (l = 0; l < state->level; ++l) {
+			if (!state->parity[l].is_excluded_by_filter) {
+				return 1;
+			}
 		}
 	}
 
-	/* otherwise include only used blocks */
+	/* filter for the files */
 	for (j = 0; j < diskmax; ++j) {
 		struct snapraid_block* block;
 
@@ -868,6 +887,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
 	unsigned l;
 	char esc_buffer[ESC_MAX];
 	char esc_buffer_alt[ESC_MAX];
+	bit_vect_t* block_enabled;
 
 	handle = handle_mapping(state, &diskmax);
 
@@ -889,14 +909,25 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
 	unrecoverable_error = 0;
 	recovered_error = 0;
 
+	msg_progress("Selecting...\n");
+
 	/* first count the number of blocks to process */
 	countmax = 0;
+	block_enabled = calloc_nofail(1, bit_vect_size(blockmax)); /* preinitialize to 0 */
 	for (i = blockstart; i < blockmax; ++i) {
 		if (!block_is_enabled(state, i, handle, diskmax))
 			continue;
+		bit_vect_set(block_enabled, i);
 		++countmax;
 	}
 
+	if (fix)
+		msg_progress("Fixing...\n");
+	else if (!state->opt.auditonly)
+		msg_progress("Checking...\n");
+	else
+		msg_progress("Hashing...\n");
+
 	/* check all the blocks in files */
 	countsize = 0;
 	countpos = 0;
@@ -908,18 +939,8 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
 		snapraid_info info;
 		int rehash;
 
-		if (!block_is_enabled(state, i, handle, diskmax)) {
-			/* post process the files */
-			ret = file_post(state, fix, i, handle, diskmax);
-			if (ret == -1) {
-				/* LCOV_EXCL_START */
-				log_fatal("Stopping at block %u\n", i);
-				++unrecoverable_error;
-				goto bail;
-				/* LCOV_EXCL_STOP */
-			}
-
-			/* and now continue with the next block */
+		if (!bit_vect_test(block_enabled, i)) {
+			/* continue with the next block */
 			continue;
 		}
 
@@ -1441,7 +1462,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
 			char path[PATH_MAX];
 			struct stat st;
 			struct snapraid_file* file;
-			int unsuccesful = 0;
+			int unsuccessful = 0;
 
 			file = node->data;
 			node = node->next; /* next node */
@@ -1460,24 +1481,24 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
 			pathprint(path, sizeof(path), "%s%s", disk->dir, file->sub);
 			ret = stat(path, &st);
 			if (ret == -1) {
-				unsuccesful = 1;
+				unsuccessful = 1;
 
 				log_error("Error stating empty file '%s'. %s.\n", path, strerror(errno));
 				log_tag("error:%s:%s: Empty file stat error\n", disk->name, esc_tag(file->sub, esc_buffer));
 				++error;
 			} else if (!S_ISREG(st.st_mode)) {
-				unsuccesful = 1;
+				unsuccessful = 1;
 
 				log_tag("error:%s:%s: Empty file error for not regular file\n", disk->name, esc_tag(file->sub, esc_buffer));
 				++error;
 			} else if (st.st_size != 0) {
-				unsuccesful = 1;
+				unsuccessful = 1;
 
 				log_tag("error:%s:%s: Empty file error for size '%" PRIu64 "'\n", disk->name, esc_tag(file->sub, esc_buffer), (uint64_t)st.st_size);
 				++error;
 			}
 
-			if (fix && unsuccesful) {
+			if (fix && unsuccessful) {
 				int f;
 
 				/* create the ancestor directories */
@@ -1552,7 +1573,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
 			struct stat st;
 			struct stat stto;
 			struct snapraid_link* slink;
-			int unsuccesful = 0;
+			int unsuccessful = 0;
 			int unrecoverable = 0;
 
 			slink = node->data;
@@ -1568,13 +1589,13 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
 				pathprint(path, sizeof(path), "%s%s", disk->dir, slink->sub);
 				ret = stat(path, &st);
 				if (ret == -1) {
-					unsuccesful = 1;
+					unsuccessful = 1;
 
 					log_error("Error stating hardlink '%s'. %s.\n", path, strerror(errno));
 					log_tag("hardlink_error:%s:%s:%s: Hardlink stat error\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt));
 					++error;
 				} else if (!S_ISREG(st.st_mode)) {
-					unsuccesful = 1;
+					unsuccessful = 1;
 
 					log_tag("hardlink_error:%s:%s:%s: Hardlink error for not regular file\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt));
 					++error;
@@ -1584,7 +1605,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
 				pathprint(pathto, sizeof(pathto), "%s%s", disk->dir, slink->linkto);
 				ret = stat(pathto, &stto);
 				if (ret == -1) {
-					unsuccesful = 1;
+					unsuccessful = 1;
 
 					if (errno == ENOENT) {
 						unrecoverable = 1;
@@ -1604,12 +1625,12 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
 					log_tag("hardlink_error:%s:%s:%s: Hardlink to stat error\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt));
 					++error;
 				} else if (!S_ISREG(stto.st_mode)) {
-					unsuccesful = 1;
+					unsuccessful = 1;
 
 					log_tag("hardlink_error:%s:%s:%s: Hardlink-to error for not regular file\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt));
 					++error;
-				} else if (!unsuccesful && st.st_ino != stto.st_ino) {
-					unsuccesful = 1;
+				} else if (!unsuccessful && st.st_ino != stto.st_ino) {
+					unsuccessful = 1;
 
 					log_error("Mismatch hardlink '%s' and '%s'. Different inode.\n", path, pathto);
 					log_tag("hardlink_error:%s:%s:%s: Hardlink mismatch for different inode\n", disk->name, esc_tag(slink->sub, esc_buffer), esc_tag(slink->linkto, esc_buffer_alt));
@@ -1620,13 +1641,13 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
 				pathprint(path, sizeof(path), "%s%s", disk->dir, slink->sub);
 				ret = readlink(path, linkto, sizeof(linkto));
 				if (ret < 0) {
-					unsuccesful = 1;
+					unsuccessful = 1;
 
 					log_error("Error reading symlink '%s'. %s.\n", path, strerror(errno));
 					log_tag("symlink_error:%s:%s: Symlink read error\n", disk->name, esc_tag(slink->sub, esc_buffer));
 					++error;
 				} else if (ret >= PATH_MAX) {
-					unsuccesful = 1;
+					unsuccessful = 1;
 
 					log_error("Error reading symlink '%s'. Symlink too long.\n", path);
 					log_tag("symlink_error:%s:%s: Symlink read error\n", disk->name, esc_tag(slink->sub, esc_buffer));
@@ -1635,7 +1656,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
 					linkto[ret] = 0;
 
 					if (strcmp(linkto, slink->linkto) != 0) {
-						unsuccesful = 1;
+						unsuccessful = 1;
 
 						log_tag("symlink_error:%s:%s: Symlink data error '%s' instead of '%s'\n", disk->name, esc_tag(slink->sub, esc_buffer), linkto, slink->linkto);
 						++error;
@@ -1643,7 +1664,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
 				}
 			}
 
-			if (fix && unsuccesful && !unrecoverable) {
+			if (fix && unsuccessful && !unrecoverable) {
 				/* create the ancestor directories */
 				ret = mkancestor(path);
 				if (ret != 0) {
@@ -1720,7 +1741,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
 			char path[PATH_MAX];
 			struct stat st;
 			struct snapraid_dir* dir;
-			int unsuccesful = 0;
+			int unsuccessful = 0;
 
 			dir = node->data;
 			node = node->next; /* next node */
@@ -1734,19 +1755,19 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
 			pathprint(path, sizeof(path), "%s%s", disk->dir, dir->sub);
 			ret = stat(path, &st);
 			if (ret == -1) {
-				unsuccesful = 1;
+				unsuccessful = 1;
 
 				log_error("Error stating dir '%s'. %s.\n", path, strerror(errno));
 				log_tag("dir_error:%s:%s: Dir stat error\n", disk->name, esc_tag(dir->sub, esc_buffer));
 				++error;
 			} else if (!S_ISDIR(st.st_mode)) {
-				unsuccesful = 1;
+				unsuccessful = 1;
 
 				log_tag("dir_error:%s:%s: Dir error for not directory\n", disk->name, esc_tag(dir->sub, esc_buffer));
 				++error;
 			}
 
-			if (fix && unsuccesful) {
+			if (fix && unsuccessful) {
 				/* create the ancestor directories */
 				ret = mkancestor(path);
 				if (ret != 0) {
@@ -1906,6 +1927,7 @@ bail:
 
 	free(failed);
 	free(failed_map);
+	free(block_enabled);
 	free(handle);
 	free(buffer_alloc);
 	free(buffer);
@@ -1973,20 +1995,32 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b
 			}
 
 			parity_ptr[l] = &parity[l];
-			ret = parity_create(parity_ptr[l], &state->parity[l], l, state->file_mode, state->block_size, state->opt.parity_limit_size);
-			if (ret == -1) {
-				/* LCOV_EXCL_START */
-				log_fatal("WARNING! Without an accessible %s file, it isn't possible to fix any error.\n", lev_name(l));
-				exit(EXIT_FAILURE);
-				/* LCOV_EXCL_STOP */
-			}
 
-			ret = parity_chsize(parity_ptr[l], &state->parity[l], 0, size, state->block_size, state->opt.skip_fallocate, state->opt.skip_space_holder);
-			if (ret == -1) {
-				/* LCOV_EXCL_START */
-				log_fatal("WARNING! Without an accessible %s file, it isn't possible to sync.\n", lev_name(l));
-				exit(EXIT_FAILURE);
-				/* LCOV_EXCL_STOP */
+			/* if the parity is excluded */
+			if (state->parity[l].is_excluded_by_filter) {
+				/* open for reading, and ignore error */
+				ret = parity_open(parity_ptr[l], &state->parity[l], l, state->file_mode, state->block_size, state->opt.parity_limit_size);
+				if (ret == -1) {
+					/* continue anyway */
+					parity_ptr[l] = 0;
+				}
+			} else {
+				/* open for writing */
+				ret = parity_create(parity_ptr[l], &state->parity[l], l, state->file_mode, state->block_size, state->opt.parity_limit_size);
+				if (ret == -1) {
+					/* LCOV_EXCL_START */
+					log_fatal("WARNING! Without an accessible %s file, it isn't possible to fix any error.\n", lev_name(l));
+					exit(EXIT_FAILURE);
+					/* LCOV_EXCL_STOP */
+				}
+
+				ret = parity_chsize(parity_ptr[l], &state->parity[l], 0, size, state->block_size, state->opt.skip_fallocate, state->opt.skip_space_holder);
+				if (ret == -1) {
+					/* LCOV_EXCL_START */
+					log_fatal("WARNING! Without an accessible %s file, it isn't possible to sync.\n", lev_name(l));
+					exit(EXIT_FAILURE);
+					/* LCOV_EXCL_STOP */
+				}
 			}
 		}
 	} else if (!state->opt.auditonly) {
@@ -2007,13 +2041,6 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b
 			parity_ptr[l] = 0;
 	}
 
-	if (fix)
-		msg_progress("Fixing...\n");
-	else if (!state->opt.auditonly)
-		msg_progress("Checking...\n");
-	else
-		msg_progress("Hashing...\n");
-
 	error = 0;
 
 	/* skip degenerated cases of empty parity, or skipping all */
@@ -2030,14 +2057,26 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b
 	/* try to close only if opened */
 	for (l = 0; l < state->level; ++l) {
 		if (parity_ptr[l]) {
+			/* if fixing and not excluded, truncate parity not valid */
+			if (fix && !state->parity[l].is_excluded_by_filter) {
+				ret = parity_truncate(parity_ptr[l]);
+				if (ret == -1) {
+					/* LCOV_EXCL_START */
+					log_fatal("DANGER! Unexpected truncate error in %s disk.\n", lev_name(l));
+					++error;
+					/* continue, as we are already exiting */
+					/* LCOV_EXCL_STOP */
+				}
+			}
+
 			ret = parity_close(parity_ptr[l]);
-			/* LCOV_EXCL_START */
 			if (ret == -1) {
+				/* LCOV_EXCL_START */
 				log_fatal("DANGER! Unexpected close error in %s disk.\n", lev_name(l));
 				++error;
 				/* continue, as we are already exiting */
+				/* LCOV_EXCL_STOP */
 			}
-			/* LCOV_EXCL_STOP */
 		}
 	}
 

cmdline/device.c

@@ -818,7 +818,7 @@ static void state_smart(unsigned n, tommy_list* low)
 
 	/*      |<##################################################################72>|####80>| */
 	printf("These values are the probabilities that in the next year you'll have a\n");
-	printf("sequence of failures that the parity WONT be able to recover, assuming\n");
+	printf("sequence of failures that the parity WON'T be able to recover, assuming\n");
 	printf("that you regularly scrub, and in case repair, the array in the specified\n");
 	printf("time.\n");
 

cmdline/dry.c

@@ -28,17 +28,6 @@
 /****************************************************************************/
 /* dry */
 
-/**
- * Check if we have to process the specified block index ::i.
- */
-static int block_is_enabled(void* void_plan, block_off_t i)
-{
-	(void)void_plan;
-	(void)i;
-
-	return 1;
-}
-
 static void dry_data_reader(struct snapraid_worker* worker, struct snapraid_task* task)
 {
 	struct snapraid_io* io = worker->io;
@@ -208,7 +197,7 @@ static int state_dry_process(struct snapraid_state* state, struct snapraid_parit
 	countpos = 0;
 
 	/* start all the worker threads */
-	io_start(&io, blockstart, blockmax, &block_is_enabled, 0);
+	io_start(&io, blockstart, blockmax, 0);
 
 	state_progress_begin(state, blockstart, blockmax, countmax);
 	while (1) {

cmdline/dup.c

@@ -41,18 +41,19 @@ struct snapraid_hash* hash_alloc(struct snapraid_state* state, struct snapraid_d
 	struct snapraid_hash* hash;
 	block_off_t i;
 	unsigned char* buf;
+	size_t hash_size = BLOCK_HASH_SIZE;
 
 	hash = malloc_nofail(sizeof(struct snapraid_hash));
 	hash->disk = disk;
 	hash->file = file;
 
-	buf = malloc_nofail(file->blockmax * BLOCK_HASH_SIZE);
+	buf = malloc_nofail(file->blockmax * hash_size);
 
 	/* set the back pointer */
 	for (i = 0; i < file->blockmax; ++i) {
 		struct snapraid_block* block = fs_file2block_get(file, i);
 
-		memcpy(buf + i * BLOCK_HASH_SIZE, block->hash, BLOCK_HASH_SIZE);
+		memcpy(buf + i * hash_size, block->hash, hash_size);
 
 		if (!block_has_updated_hash(block)) {
 			free(buf);
@@ -61,7 +62,7 @@ struct snapraid_hash* hash_alloc(struct snapraid_state* state, struct snapraid_d
 		}
 	}
 
-	memhash(state->besthash, state->hashseed, hash->hash, buf, file->blockmax * BLOCK_HASH_SIZE);
+	memhash(state->besthash, state->hashseed, hash->hash, buf, file->blockmax * hash_size);
 
 	free(buf);
 

cmdline/elem.c

@@ -736,7 +736,7 @@ int dir_name_compare(const void* void_arg, const void* void_data)
 	return strcmp(arg, dir->sub);
 }
 
-struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev, const char* uuid, int skip)
+struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev, const char* uuid, int skip_access)
 {
 	struct snapraid_disk* disk;
 
@@ -748,8 +748,9 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev
 	/* ensure that the dir terminate with "/" if it isn't empty */
 	pathslash(disk->dir, sizeof(disk->dir));
 
-#if HAVE_PTHREAD
-	thread_mutex_init(&disk->fs_mutex, 0);
+#if HAVE_THREAD
+	thread_mutex_init(&disk->fs_mutex);
+	disk->fs_mutex_enabled = 0; /* lock will be enabled at threads start */
 #endif
 
 	disk->smartctl[0] = 0;
@@ -767,7 +768,7 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev
 	disk->has_unsupported_uuid = *uuid == 0; /* empty UUID means unsupported */
 	disk->had_empty_uuid = 0;
 	disk->mapping_idx = -1;
-	disk->skip_access = skip;
+	disk->skip_access = skip_access;
 	tommy_list_init(&disk->filelist);
 	tommy_list_init(&disk->deletedlist);
 	tommy_hashdyn_init(&disk->inodeset);
@@ -797,17 +798,27 @@ void disk_free(struct snapraid_disk* disk)
 	tommy_list_foreach(&disk->dirlist, (tommy_foreach_func*)dir_free);
 	tommy_hashdyn_done(&disk->dirset);
 
-#if HAVE_PTHREAD
+#if HAVE_THREAD
 	thread_mutex_destroy(&disk->fs_mutex);
 #endif
 
 	free(disk);
 }
 
+void disk_start_thread(struct snapraid_disk* disk)
+{
+#if HAVE_THREAD
+	disk->fs_mutex_enabled = 1;
+#else
+	(void)disk;
+#endif
+}
+
 static inline void fs_lock(struct snapraid_disk* disk)
 {
-#if HAVE_PTHREAD
-	thread_mutex_lock(&disk->fs_mutex);
+#if HAVE_THREAD
+	if (disk->fs_mutex_enabled)
+		thread_mutex_lock(&disk->fs_mutex);
 #else
 	(void)disk;
 #endif
@@ -815,8 +826,9 @@ static inline void fs_lock(struct snapraid_disk* disk)
 
 static inline void fs_unlock(struct snapraid_disk* disk)
 {
-#if HAVE_PTHREAD
-	thread_mutex_unlock(&disk->fs_mutex);
+#if HAVE_THREAD
+	if (disk->fs_mutex_enabled)
+		thread_mutex_unlock(&disk->fs_mutex);
 #else
 	(void)disk;
 #endif

cmdline/elem.h

@@ -349,7 +349,7 @@ struct snapraid_disk {
 	block_off_t first_free_block;
 
 	int has_volatile_inodes; /**< If the underline file-system has not persistent inodes. */
-	int has_volatile_hardlinks; /**< If the underline file-system has not syncronized metadata for hardlink (NTFS). */
+	int has_volatile_hardlinks; /**< If the underline file-system has not synchronized metadata for hardlink (NTFS). */
 	int has_unreliable_physical; /**< If the physical offset of files has duplicates. */
 	int has_different_uuid; /**< If the disk has a different UUID, meaning that it is not the same file-system. */
 	int has_unsupported_uuid; /**< If the disk doesn't report UUID, meaning it's not supported. */
@@ -357,7 +357,7 @@ struct snapraid_disk {
 	int mapping_idx; /**< Index in the mapping vector. Used only as buffer when writing the content file. */
 	int skip_access; /**< If the disk is inaccessible and it should be skipped. */
 
-#if HAVE_PTHREAD
+#if HAVE_THREAD
 	/**
 	 * Mutex for protecting the filesystem structure.
 	 *
@@ -367,7 +367,15 @@ struct snapraid_disk {
 	 * Files, links and dirs are not protected as they are not expected to
 	 * change during multithread processing.
 	 */
-	pthread_mutex_t fs_mutex;
+	thread_mutex_t fs_mutex;
+	int fs_mutex_enabled; /*< If the lock has to be used. */
+
+	/**
+	 * Mutex for protecting the scan process.
+	 *
+	 * It's used during the scan process to protect the stampset to identity copy of files
+	 */
+	thread_mutex_t stamp_mutex;
 #endif
 
 	/**
@@ -968,13 +976,18 @@ static inline tommy_uint32_t dir_name_hash(const char* name)
 /**
  * Allocate a disk.
  */
-struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev, const char* uuid, int skip);
+struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev, const char* uuid, int skip_access);
 
 /**
  * Deallocate a disk.
  */
 void disk_free(struct snapraid_disk* disk);
 
+/**
+ * Enable multithread support for the disk.
+ */
+void disk_start_thread(struct snapraid_disk* disk);
+
 /**
  * Get the size of the disk in blocks.
  */
@@ -1000,7 +1013,7 @@ int fs_check(struct snapraid_disk* disk);
  * After this call you can use the par2file/par2block operations
  * to query the relation.
  *
- * \note This function is NOT thread-safe as it uses the the disk cache. +
+ * \note This function is NOT thread-safe as it uses the disk cache. +
  */
 void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos);
 
@@ -1010,7 +1023,7 @@ void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snap
  * After this call the par2file/par2block operations
  * won't find anymore the parity association.
  *
- * \note This function is NOT thread-safe as it uses the the disk cache.
+ * \note This function is NOT thread-safe as it uses the disk cache.
  */
 void fs_deallocate(struct snapraid_disk* disk, block_off_t pos);
 
@@ -1129,7 +1142,7 @@ static inline snapraid_info info_make(time_t last_access, int error, int rehash,
 /**
  * Extract the time information.
  * This is the last time when the block was know to be correct.
- * The "scrubbed" info tells if the time is referreing at the latest sync or scrub.
+ * The "scrubbed" info tells if the time is referring at the latest sync or scrub.
  */
 static inline time_t info_get_time(snapraid_info info)
 {

cmdline/io.c

@@ -19,6 +19,19 @@
 
 #include "io.h"
 
+void (*io_start)(struct snapraid_io* io,
+	block_off_t blockstart, block_off_t blockmax,
+	bit_vect_t* block_enabled) = 0;
+void (*io_stop)(struct snapraid_io* io) = 0;
+block_off_t (*io_read_next)(struct snapraid_io* io, void*** buffer) = 0;
+struct snapraid_task* (*io_data_read)(struct snapraid_io* io, unsigned* diskcur, unsigned* waiting_map, unsigned* waiting_mac) = 0;
+struct snapraid_task* (*io_parity_read)(struct snapraid_io* io, unsigned* levcur, unsigned* waiting_map, unsigned* waiting_mac) = 0;
+void (*io_parity_write)(struct snapraid_io* io, unsigned* levcur, unsigned* waiting_map, unsigned* waiting_mac) = 0;
+void (*io_write_preset)(struct snapraid_io* io, block_off_t blockcur, int skip) = 0;
+void (*io_write_next)(struct snapraid_io* io, block_off_t blockcur, int skip, int* writer_error) = 0;
+void (*io_refresh)(struct snapraid_io* io) = 0;
+
+
 /**
  * Get the next block position to process.
  */
@@ -27,8 +40,10 @@ static block_off_t io_position_next(struct snapraid_io* io)
 	block_off_t blockcur;
 
 	/* get the next position */
-	while (io->block_next < io->block_max && !io->block_is_enabled(io->block_arg, io->block_next))
-		++io->block_next;
+	if (io->block_enabled) {
+		while (io->block_next < io->block_max && !bit_vect_test(io->block_enabled, io->block_next))
+			++io->block_next;
+	}
 
 	blockcur = io->block_next;
 
@@ -244,12 +259,11 @@ static void io_parity_write_mono(struct snapraid_io* io, unsigned* pos, unsigned
 
 static void io_start_mono(struct snapraid_io* io,
 	block_off_t blockstart, block_off_t blockmax,
-	int (*block_is_enabled)(void* arg, block_off_t), void* blockarg)
+	bit_vect_t* block_enabled)
 {
 	io->block_start = blockstart;
 	io->block_max = blockmax;
-	io->block_is_enabled = block_is_enabled;
-	io->block_arg = blockarg;
+	io->block_enabled = block_enabled;
 	io->block_next = blockstart;
 }
 
@@ -262,7 +276,7 @@ static void io_stop_mono(struct snapraid_io* io)
 /* multi thread */
 
 /* disable multithread if pthread is not present */
-#if HAVE_PTHREAD
+#if HAVE_THREAD
 
 /**
  * Get the next task to work on for a reader.
@@ -754,14 +768,20 @@ static void* io_writer_thread(void* arg)
 
 static void io_start_thread(struct snapraid_io* io,
 	block_off_t blockstart, block_off_t blockmax,
-	int (*block_is_enabled)(void* arg, block_off_t), void* blockarg)
+	bit_vect_t* block_enabled)
 {
 	unsigned i;
+	tommy_node* j;
+
+	/* enable the filesystem mutex in all disks */
+	for (j = io->state->disklist; j != 0; j = j->next) {
+		struct snapraid_disk* disk = j->data;
+		disk_start_thread(disk);
+	}
 
 	io->block_start = blockstart;
 	io->block_max = blockmax;
-	io->block_is_enabled = block_is_enabled;
-	io->block_arg = blockarg;
+	io->block_enabled = block_enabled;
 	io->block_next = blockstart;
 
 	io->done = 0;
@@ -791,7 +811,7 @@ static void io_start_thread(struct snapraid_io* io,
 
 		worker->index = 0;
 
-		thread_create(&worker->thread, 0, io_reader_thread, worker);
+		thread_create(&worker->thread, io_reader_thread, worker);
 	}
 
 	/* start the writer threads */
@@ -800,7 +820,7 @@ static void io_start_thread(struct snapraid_io* io,
 
 		worker->index = io->io_max - 1;
 
-		thread_create(&worker->thread, 0, io_writer_thread, worker);
+		thread_create(&worker->thread, io_writer_thread, worker);
 	}
 }
 
@@ -852,15 +872,16 @@ void io_init(struct snapraid_io* io, struct snapraid_state* state,
 	struct snapraid_parity_handle* parity_handle_map, unsigned parity_handle_max)
 {
 	unsigned i;
-	size_t allocated;
+	size_t allocated_size;
+	size_t block_size = state->block_size;
 
 	io->state = state;
 
-#if HAVE_PTHREAD
+#if HAVE_THREAD
 	if (io_cache == 0) {
-		/* default is 8 MiB of cache */
+		/* default is 16 MiB of cache */
 		/* this seems to be a good tradeoff between speed and memory usage */
-		io->io_max = 8 * 1024 * 1024 / state->block_size;
+		io->io_max = 16 * 1024 * 1024 / state->block_size;
 		if (io->io_max < IO_MIN)
 			io->io_max = IO_MIN;
 		if (io->io_max > IO_MAX)
@@ -878,18 +899,18 @@ void io_init(struct snapraid_io* io, struct snapraid_state* state,
 	assert(io->io_max == 1 || (io->io_max >= IO_MIN && io->io_max <= IO_MAX));
 
 	io->buffer_max = buffer_max;
-	allocated = 0;
+	allocated_size = 0;
 	for (i = 0; i < io->io_max; ++i) {
 		if (state->file_mode != ADVISE_DIRECT)
-			io->buffer_map[i] = malloc_nofail_vector_align(handle_max, buffer_max, state->block_size, &io->buffer_alloc_map[i]);
+			io->buffer_map[i] = malloc_nofail_vector_align(handle_max, buffer_max, block_size, &io->buffer_alloc_map[i]);
 		else
-			io->buffer_map[i] = malloc_nofail_vector_direct(handle_max, buffer_max, state->block_size, &io->buffer_alloc_map[i]);
+			io->buffer_map[i] = malloc_nofail_vector_direct(handle_max, buffer_max, block_size, &io->buffer_alloc_map[i]);
 		if (!state->opt.skip_self)
 			mtest_vector(io->buffer_max, state->block_size, io->buffer_map[i]);
-		allocated += state->block_size * buffer_max;
+		allocated_size += block_size * buffer_max;
 	}
 
-	msg_progress("Using %u MiB of memory for %u blocks of IO cache.\n", (unsigned)(allocated / MEBI), io->io_max);
+	msg_progress("Using %u MiB of memory for %u cached blocks.\n", (unsigned)(allocated_size / MEBI), io->io_max);
 
 	if (parity_writer) {
 		io->reader_max = handle_max;
@@ -947,7 +968,7 @@ void io_init(struct snapraid_io* io, struct snapraid_state* state,
 		worker->buffer_skew = handle_max;
 	}
 
-#if HAVE_PTHREAD
+#if HAVE_THREAD
 	if (io->io_max > 1) {
 		io_read_next = io_read_next_thread;
 		io_write_preset = io_write_preset_thread;
@@ -959,11 +980,11 @@ void io_init(struct snapraid_io* io, struct snapraid_state* state,
 		io_start = io_start_thread;
 		io_stop = io_stop_thread;
 
-		thread_mutex_init(&io->io_mutex, 0);
-		thread_cond_init(&io->read_done, 0);
-		thread_cond_init(&io->read_sched, 0);
-		thread_cond_init(&io->write_done, 0);
-		thread_cond_init(&io->write_sched, 0);
+		thread_mutex_init(&io->io_mutex);
+		thread_cond_init(&io->read_done);
+		thread_cond_init(&io->read_sched);
+		thread_cond_init(&io->write_done);
+		thread_cond_init(&io->write_sched);
 	} else
 #endif
 	{
@@ -993,7 +1014,7 @@ void io_done(struct snapraid_io* io)
 	free(io->writer_map);
 	free(io->writer_list);
 
-#if HAVE_PTHREAD
+#if HAVE_THREAD
 	if (io->io_max > 1) {
 		thread_mutex_destroy(&io->io_mutex);
 		thread_cond_destroy(&io->read_done);

cmdline/io.h

@@ -35,8 +35,8 @@
  *  4  - 452 MB/s, CPU 54%, speed 118%
  *  8  - 487 MB/s, CPU 60%, speed 128%
  * 16  - 505 MB/s, CPU 63%, speed 132%
- * 32  - 520 MB/s, CPU 64%, speed 136%
- * 64  - 524 MB/s, CPU 65%, speed 137%
+ * 32  - 520 MB/s, CPU 64%, speed 136% [SnapRAID <= 12.0]
+ * 64  - 524 MB/s, CPU 65%, speed 137% [SnapRAID > 12.0]
  * 128 - 525 MB/s, CPU 66%, speed 138%
  */
 #define IO_MIN 3 /* required by writers, readers can work also with 2 */
@@ -87,8 +87,8 @@ struct snapraid_task {
  * from a specific disk.
  */
 struct snapraid_worker {
-#if HAVE_PTHREAD
-	pthread_t thread; /**< Thread context for the worker. */
+#if HAVE_THREAD
+	thread_id_t thread; /**< Thread context for the worker. */
 #endif
 
 	struct snapraid_io* io; /**< Parent pointer. */
@@ -147,12 +147,12 @@ struct snapraid_io {
 	 */
 	unsigned io_max;
 
-#if HAVE_PTHREAD
+#if HAVE_THREAD
 	/**
 	 * Mutex used to protect the synchronization
 	 * between the io and the workers.
 	 */
-	pthread_mutex_t io_mutex;
+	thread_mutex_t io_mutex;
 
 	/**
 	 * Condition for a new read is completed.
@@ -161,7 +161,7 @@ struct snapraid_io {
 	 * The IO waits on this condition when it's waiting for
 	 * a new read to be completed.
 	 */
-	pthread_cond_t read_done;
+	thread_cond_t read_done;
 
 	/**
 	 * Condition for a new read scheduled.
@@ -170,7 +170,7 @@ struct snapraid_io {
 	 * read to process.
 	 * The IO signals this condition when new reads are scheduled.
 	 */
-	pthread_cond_t read_sched;
+	thread_cond_t read_sched;
 
 	/**
 	 * Condition for a new write is completed.
@@ -179,7 +179,7 @@ struct snapraid_io {
 	 * The IO waits on this condition when it's waiting for
 	 * a new write to be completed.
 	 */
-	pthread_cond_t write_done;
+	thread_cond_t write_done;
 
 	/**
 	 * Condition for a new write scheduled.
@@ -188,7 +188,7 @@ struct snapraid_io {
 	 * write to process.
 	 * The IO signals this condition when new writes are scheduled.
 	 */
-	pthread_cond_t write_sched;
+	thread_cond_t write_sched;
 #endif
 
 	/**
@@ -217,8 +217,7 @@ struct snapraid_io {
 	block_off_t block_start;
 	block_off_t block_max;
 	block_off_t block_next;
-	int (*block_is_enabled)(void* arg, block_off_t);
-	void* block_arg;
+	bit_vect_t* block_enabled;
 
 	/**
 	 * Buffers for data.
@@ -306,21 +305,21 @@ void io_init(struct snapraid_io* io, struct snapraid_state* state,
 	struct snapraid_parity_handle* parity_handle_map, unsigned parity_handle_max);
 
 /**
- * Deinitialize te InputOutput workers.
+ * Deinitialize the InputOutput workers.
  */
 void io_done(struct snapraid_io* io);
 
 /**
  * Start all the worker threads.
  */
-void (*io_start)(struct snapraid_io* io,
+extern void (*io_start)(struct snapraid_io* io,
 	block_off_t blockstart, block_off_t blockmax,
-	int (*block_is_enabled)(void* arg, block_off_t), void* blockarg);
+	bit_vect_t* block_enabled);
 
 /**
  * Stop all the worker threads.
  */
-void (*io_stop)(struct snapraid_io* io);
+extern void (*io_stop)(struct snapraid_io* io);
 
 /**
  * Next read position.
@@ -332,7 +331,7 @@ void (*io_stop)(struct snapraid_io* io);
  * \param buffer The data buffers to use for this position.
  * \return The parity position.
  */
-block_off_t (*io_read_next)(struct snapraid_io* io, void*** buffer);
+extern block_off_t (*io_read_next)(struct snapraid_io* io, void*** buffer);
 
 /**
  * Read a data block.
@@ -343,7 +342,7 @@ block_off_t (*io_read_next)(struct snapraid_io* io, void*** buffer);
  * \param diskcur The position of the data block in the ::handle_map vector.
  * \return The completed task.
  */
-struct snapraid_task* (*io_data_read)(struct snapraid_io* io, unsigned* diskcur, unsigned* waiting_map, unsigned* waiting_mac);
+extern struct snapraid_task* (*io_data_read)(struct snapraid_io* io, unsigned* diskcur, unsigned* waiting_map, unsigned* waiting_mac);
 
 /**
  * Read a parity block.
@@ -354,7 +353,7 @@ struct snapraid_task* (*io_data_read)(struct snapraid_io* io, unsigned* diskcur,
  * \param levcur The position of the parity block in the ::parity_handle_map vector.
  * \return The completed task.
  */
-struct snapraid_task* (*io_parity_read)(struct snapraid_io* io, unsigned* levcur, unsigned* waiting_map, unsigned* waiting_mac);
+extern struct snapraid_task* (*io_parity_read)(struct snapraid_io* io, unsigned* levcur, unsigned* waiting_map, unsigned* waiting_mac);
 
 /**
  * Write of a parity block.
@@ -364,7 +363,7 @@ struct snapraid_task* (*io_parity_read)(struct snapraid_io* io, unsigned* levcur
  * \param io InputOutput context.
  * \param levcur The position of the parity block in the ::parity_handle_map vector.
  */
-void (*io_parity_write)(struct snapraid_io* io, unsigned* levcur, unsigned* waiting_map, unsigned* waiting_mac);
+extern void (*io_parity_write)(struct snapraid_io* io, unsigned* levcur, unsigned* waiting_map, unsigned* waiting_mac);
 
 /**
  * Preset the write position.
@@ -376,7 +375,7 @@ void (*io_parity_write)(struct snapraid_io* io, unsigned* levcur, unsigned* wait
  * \param blockcur The parity position to write.
  * \param skip Skip the writes, in case parity doesn't need to be updated.
  */
-void (*io_write_preset)(struct snapraid_io* io, block_off_t blockcur, int skip);
+extern void (*io_write_preset)(struct snapraid_io* io, block_off_t blockcur, int skip);
 
 /**
  * Next write position.
@@ -389,12 +388,12 @@ void (*io_write_preset)(struct snapraid_io* io, block_off_t blockcur, int skip);
  * \param skip Skip the writes, in case parity doesn't need to be updated.
  * \param writer_error Return the number of errors. Vector of IO_WRITER_ERROR_MAX elements.
  */
-void (*io_write_next)(struct snapraid_io* io, block_off_t blockcur, int skip, int* writer_error);
+extern void (*io_write_next)(struct snapraid_io* io, block_off_t blockcur, int skip, int* writer_error);
 
 /**
  * Refresh the number of cached blocks for all data and parity disks.
  */
-void (*io_refresh)(struct snapraid_io* io);
+extern void (*io_refresh)(struct snapraid_io* io);
 
 #endif
 

cmdline/list.c

@@ -74,7 +74,7 @@ void state_list(struct snapraid_state* state)
 			if (tm) {
 				printf("%04u/%02u/%02u %02u:%02u", tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min);
 				if (msg_level >= MSG_VERBOSE)
-					printf(":%02u.%03u", tm->tm_sec, file->mtime_nsec / 1000000);
+					printf(":%02u.%09u", tm->tm_sec, file->mtime_nsec);
 				printf(" ");
 			}
 			printf("%s\n", fmt_term(disk, file->sub, esc_buffer));
@@ -105,7 +105,7 @@ void state_list(struct snapraid_state* state)
 			printf("%12s ", type);
 			printf("                 ");
 			if (msg_level >= MSG_VERBOSE)
-				printf("       ");
+				printf("             ");
 			printf("%s -> %s\n", fmt_term(disk, slink->sub, esc_buffer), fmt_term(disk, slink->linkto, esc_buffer_alt));
 		}
 	}

cmdline/metro.c

@@ -0,0 +1,108 @@
+/*
+ * Copyright (C) 2019 Andrea Mazzoleni
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * Derivative work from metrohash128.cpp
+ *
+ * metrohash128.cpp
+ *
+ * Copyright 2015-2018 J. Andrew Rogers
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+static const uint64_t k0 = 0xC83A91E1;
+static const uint64_t k1 = 0x8648DBDB;
+static const uint64_t k2 = 0x7BDEC03B;
+static const uint64_t k3 = 0x2F5870A5;
+
+void MetroHash128(const void* data, size_t size, const uint8_t* seed, uint8_t* digest)
+{
+	const uint8_t* ptr = data;
+	uint64_t v[4];
+
+	v[0] = (util_read64(seed) - k0) * k3;
+	v[1] = (util_read64(seed + 8) + k1) * k2;
+
+	if (size >= 32) {
+		v[2] = (util_read64(seed) + k0) * k2;
+		v[3] = (util_read64(seed + 8) - k1) * k3;
+
+		do {
+			v[0] += util_read64(ptr) * k0; ptr += 8; v[0] = util_rotr64(v[0], 29) + v[2];
+			v[1] += util_read64(ptr) * k1; ptr += 8; v[1] = util_rotr64(v[1], 29) + v[3];
+			v[2] += util_read64(ptr) * k2; ptr += 8; v[2] = util_rotr64(v[2], 29) + v[0];
+			v[3] += util_read64(ptr) * k3; ptr += 8; v[3] = util_rotr64(v[3], 29) + v[1];
+			size -= 32;
+		} while (size >= 32);
+
+		v[2] ^= util_rotr64(((v[0] + v[3]) * k0) + v[1], 21) * k1;
+		v[3] ^= util_rotr64(((v[1] + v[2]) * k1) + v[0], 21) * k0;
+		v[0] ^= util_rotr64(((v[0] + v[2]) * k0) + v[3], 21) * k1;
+		v[1] ^= util_rotr64(((v[1] + v[3]) * k1) + v[2], 21) * k0;
+	}
+
+	if (size >= 16) {
+		v[0] += util_read64(ptr) * k2; ptr += 8; v[0] = util_rotr64(v[0], 33) * k3;
+		v[1] += util_read64(ptr) * k2; ptr += 8; v[1] = util_rotr64(v[1], 33) * k3;
+		v[0] ^= util_rotr64((v[0] * k2) + v[1], 45) * k1;
+		v[1] ^= util_rotr64((v[1] * k3) + v[0], 45) * k0;
+		size -= 16;
+	}
+
+	if (size >= 8) {
+		v[0] += util_read64(ptr) * k2; ptr += 8; v[0] = util_rotr64(v[0], 33) * k3;
+		v[0] ^= util_rotr64((v[0] * k2) + v[1], 27) * k1;
+		size -= 8;
+	}
+
+	if (size >= 4) {
+		v[1] += util_read32(ptr) * k2; ptr += 4; v[1] = util_rotr64(v[1], 33) * k3;
+		v[1] ^= util_rotr64((v[1] * k3) + v[0], 46) * k0;
+		size -= 4;
+	}
+
+	if (size >= 2) {
+		v[0] += util_read16(ptr) * k2; ptr += 2; v[0] = util_rotr64(v[0], 33) * k3;
+		v[0] ^= util_rotr64((v[0] * k2) + v[1], 22) * k1;
+		size -= 2;
+	}
+
+	if (size >= 1) {
+		v[1] += util_read8(ptr) * k2; v[1] = util_rotr64(v[1], 33) * k3;
+		v[1] ^= util_rotr64((v[1] * k3) + v[0], 58) * k0;
+	}
+
+	v[0] += util_rotr64((v[0] * k0) + v[1], 13);
+	v[1] += util_rotr64((v[1] * k1) + v[0], 37);
+	v[0] += util_rotr64((v[0] * k2) + v[1], 13);
+	v[1] += util_rotr64((v[1] * k3) + v[0], 37);
+
+	util_write64(digest, v[0]);
+	util_write64(digest + 8, v[0]);
+}
+

cmdline/mingw.c

@@ -31,14 +31,26 @@ int exit_sync_needed = 2;
 /* Add missing Windows declaration */
 
 /* For SetThreadExecutionState */
+#ifndef WIN32_ES_SYSTEM_REQUIRED
 #define WIN32_ES_SYSTEM_REQUIRED      0x00000001L
+#endif
+#ifndef WIN32_ES_DISPLAY_REQUIRED
 #define WIN32_ES_DISPLAY_REQUIRED     0x00000002L
+#endif
+#ifndef WIN32_ES_USER_PRESENT
 #define WIN32_ES_USER_PRESENT         0x00000004L
+#endif
+#ifndef WIN32_ES_AWAYMODE_REQUIRED
 #define WIN32_ES_AWAYMODE_REQUIRED    0x00000040L
+#endif
+#ifndef WIN32_ES_CONTINUOUS
 #define WIN32_ES_CONTINUOUS           0x80000000L
+#endif
 
 /* File Index */
+#ifndef FILE_INVALID_FILE_ID
 #define FILE_INVALID_FILE_ID          ((ULONGLONG)-1LL)
+#endif
 
 /**
  * Direct access to RtlGenRandom().
@@ -56,12 +68,12 @@ static ULONGLONG (WINAPI* ptr_GetTickCount64)(void);
  * Description of the last error.
  * It's stored in the thread local storage.
  */
-static pthread_key_t last_error;
+static windows_key_t last_error;
 
 /**
  * Monotone tick counter
  */
-static pthread_mutex_t tick_lock;
+static windows_mutex_t tick_lock;
 static uint64_t tick_last;
 
 /**
@@ -119,14 +131,14 @@ void os_init(int opt)
 	is_scan_winfind = opt != 0;
 
 	/* initialize the thread local storage for strerror(), using free() as destructor */
-	if (pthread_key_create(&last_error, free) != 0) {
-		log_fatal("Error calling pthread_key_create().\n");
+	if (windows_key_create(&last_error, free) != 0) {
+		log_fatal("Error calling windows_key_create().\n");
 		exit(EXIT_FAILURE);
 	}
 
 	tick_last = 0;
-	if (pthread_mutex_init(&tick_lock, 0) != 0) {
-		log_fatal("Error calling pthread_mutex_init().\n");
+	if (windows_mutex_init(&tick_lock, 0) != 0) {
+		log_fatal("Error calling windows_mutex_init().\n");
 		exit(EXIT_FAILURE);
 	}
 
@@ -173,9 +185,9 @@ void os_init(int opt)
 void os_done(void)
 {
 	/* delete the thread local storage for strerror() */
-	pthread_key_delete(last_error);
+	windows_key_delete(last_error);
 
-	pthread_mutex_destroy(&tick_lock);
+	windows_mutex_destroy(&tick_lock);
 
 	/* restore the normal execution level */
 	SetThreadExecutionState(WIN32_ES_CONTINUOUS);
@@ -475,7 +487,7 @@ static void windows_attr2stat(DWORD FileAttributes, DWORD ReparseTag, struct win
  */
 static int windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE_ATTRIBUTE_TAG_INFO* tag, struct windows_stat* st)
 {
-	uint64_t mtime;
+	int64_t mtime;
 
 	windows_attr2stat(info->dwFileAttributes, tag->ReparseTag, st);
 
@@ -518,7 +530,7 @@ static int windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE_
 	 * "that includes 128-bit file identifiers.  If GetFileInformationByHandle returns"
 	 * "FILE_INVALID_FILE_ID, the identifier may only be described in 128 bit form."
 	 */
-	if (st->st_ino == FILE_INVALID_FILE_ID) {
+	if (st->st_ino == (uint64_t)FILE_INVALID_FILE_ID) {
 		log_fatal("Invalid inode number! Is this ReFS?\n");
 		errno = EINVAL;
 		return -1;
@@ -532,7 +544,7 @@ static int windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE_
  */
 static int windows_stream2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE_ID_BOTH_DIR_INFO* stream, struct windows_stat* st)
 {
-	uint64_t mtime;
+	int64_t mtime;
 
 	/* The FILE_ID_BOTH_DIR_INFO doesn't have the ReparseTag information */
 	/* we could use instead FILE_ID_EXTD_DIR_INFO, but it's available only */
@@ -563,7 +575,7 @@ static int windows_stream2stat(const BY_HANDLE_FILE_INFORMATION* info, const FIL
 	st->st_sync = 0;
 
 	/* in ReFS the IDs are 128 bit, and the 64 bit interface may fail */
-	if (st->st_ino == FILE_INVALID_FILE_ID) {
+	if (st->st_ino == (uint64_t)FILE_INVALID_FILE_ID) {
 		log_fatal("Invalid inode number! Is this ReFS?\n");
 		errno = EINVAL;
 		return -1;
@@ -577,7 +589,7 @@ static int windows_stream2stat(const BY_HANDLE_FILE_INFORMATION* info, const FIL
  */
 static void windows_finddata2stat(const WIN32_FIND_DATAW* info, struct windows_stat* st)
 {
-	uint64_t mtime;
+	int64_t mtime;
 
 	windows_attr2stat(info->dwFileAttributes, info->dwReserved0, st);
 
@@ -694,6 +706,7 @@ static void windows_errno(DWORD error)
 		errno = EPERM;
 		break;
 	case ERROR_IO_DEVICE : /* in ReadFile() and WriteFile() */
+	case ERROR_CRC : /* in ReadFile() */
 		errno = EIO;
 		break;
 	default :
@@ -788,7 +801,7 @@ static BOOL GetFilePhysicalOffset(HANDLE h, uint64_t* physical)
 	BOOL ret;
 	DWORD n;
 
-	/* in Wine FSCTL_GET_RETRIVIAL_POINTERS is not supported */
+	/* in Wine FSCTL_GET_RETRIEVAL_POINTERS is not supported */
 	if (is_wine) {
 		*physical = FILEPHY_UNREPORTED_OFFSET;
 		return TRUE;
@@ -1038,7 +1051,7 @@ int windows_futimens(int fd, struct windows_timespec tv[2])
 {
 	HANDLE h;
 	FILETIME ft;
-	uint64_t mtime;
+	int64_t mtime;
 
 	if (fd == -1) {
 		errno = EBADF;
@@ -1060,7 +1073,7 @@ int windows_futimens(int fd, struct windows_timespec tv[2])
 	mtime = tv[0].tv_sec;
 	mtime *= 10000000;
 	mtime += tv[0].tv_nsec / 100;
-	mtime += 116444736000000000;
+	mtime += 116444736000000000LL;
 
 	ft.dwHighDateTime = mtime >> 32;
 	ft.dwLowDateTime = mtime;
@@ -1078,7 +1091,7 @@ int windows_utimensat(int fd, const char* file, struct windows_timespec tv[2], i
 	wchar_t conv_buf[CONV_MAX];
 	HANDLE h;
 	FILETIME ft;
-	uint64_t mtime;
+	int64_t mtime;
 	DWORD wflags;
 
 	/*
@@ -1118,7 +1131,7 @@ int windows_utimensat(int fd, const char* file, struct windows_timespec tv[2], i
 	mtime = tv[0].tv_sec;
 	mtime *= 10000000;
 	mtime += tv[0].tv_nsec / 100;
-	mtime += 116444736000000000;
+	mtime += 116444736000000000LL;
 
 	ft.dwHighDateTime = mtime >> 32;
 	ft.dwLowDateTime = mtime;
@@ -1555,7 +1568,7 @@ int windows_link(const char* existing, const char* file)
 }
 
 /**
- * In Windows 10 allow creationg of symblink by not priviliged user.
+ * In Windows 10 allow creation of symlink by not privileged user.
  *
  * See: Symlinks in Windows 10!
  * https://blogs.windows.com/buildingapps/2016/12/02/symlinks-windows-10/#cQG7cx48oGH86lkI.97
@@ -1723,7 +1736,7 @@ int fsinfo(const char* path, int* has_persistent_inode, int* has_syncronized_har
 	if (has_persistent_inode)
 		*has_persistent_inode = 1;
 
-	/* NTFS doesn't syncronize hardlinks metadata */
+	/* NTFS doesn't synchronize hardlinks metadata */
 	if (has_syncronized_hardlinks)
 		*has_syncronized_hardlinks = 0;
 
@@ -1810,10 +1823,10 @@ const char* windows_strerror(int err)
 	snprintf(error, len, "%s [%d/%u]", str, err, (unsigned)GetLastError());
 
 	/* get previous one, if any */
-	previous = pthread_getspecific(last_error);
+	previous = windows_getspecific(last_error);
 
 	/* store in the thread local storage */
-	if (pthread_setspecific(last_error, error) != 0) {
+	if (windows_setspecific(last_error, error) != 0) {
 		free(error);
 		return str;
 	}
@@ -2023,7 +2036,7 @@ uint64_t tick(void)
 	 * We had reports of invalid stats due faulty High Precision Event Timer.
 	 * See: https://sourceforge.net/p/snapraid/discussion/1677233/thread/a2122fd6/
 	 */
-	pthread_mutex_lock(&tick_lock);
+	windows_mutex_lock(&tick_lock);
 
 	/*
 	 * MSDN 'QueryPerformanceCounter'
@@ -2038,7 +2051,7 @@ uint64_t tick(void)
 		r = tick_last;
 	tick_last = r;
 
-	pthread_mutex_unlock(&tick_lock);
+	windows_mutex_unlock(&tick_lock);
 
 	return r;
 }
@@ -2599,7 +2612,7 @@ static int device_thread(tommy_list* list, void* (*func)(void* arg))
 	for (i = tommy_list_head(list); i != 0; i = i->next) {
 		devinfo_t* devinfo = i->data;
 
-		thread_create(&devinfo->thread, 0, func, devinfo);
+		thread_create(&devinfo->thread, func, devinfo);
 	}
 
 	/* joins all threads */
@@ -2689,109 +2702,221 @@ int devquery(tommy_list* high, tommy_list* low, int operation, int others)
 }
 
 /****************************************************************************/
-/* thread */
+/* pthread like interface */
 
 int windows_mutex_init(windows_mutex_t* mutex, void* attr)
 {
-	CRITICAL_SECTION* cs;
-
 	(void)attr;
 
-	cs = malloc(sizeof(CRITICAL_SECTION));
-	if (!cs)
-		return -1;
-
-	InitializeCriticalSection(cs);
-
-	*mutex = cs;
+	InitializeCriticalSection(mutex);
 
 	return 0;
 }
 
 int windows_mutex_destroy(windows_mutex_t* mutex)
 {
-	CRITICAL_SECTION* cs = *mutex;
-
-	DeleteCriticalSection(cs);
-
-	free(cs);
+	DeleteCriticalSection(mutex);
 
 	return 0;
 }
 
 int windows_mutex_lock(windows_mutex_t* mutex)
 {
-	CRITICAL_SECTION* cs = *mutex;
-
-	EnterCriticalSection(cs);
+	EnterCriticalSection(mutex);
 
 	return 0;
 }
 
 int windows_mutex_unlock(windows_mutex_t* mutex)
 {
-	CRITICAL_SECTION* cs = *mutex;
-
-	LeaveCriticalSection(cs);
+	LeaveCriticalSection(mutex);
 
 	return 0;
 }
 
 int windows_cond_init(windows_cond_t* cond, void* attr)
 {
-	CONDITION_VARIABLE* cv;
-
 	(void)attr;
 
-	cv = malloc(sizeof(CONDITION_VARIABLE));
-	if (!cv)
-		return -1;
-
-	InitializeConditionVariable(cv);
-
-	*cond = cv;
+	InitializeConditionVariable(cond);
 
 	return 0;
 }
 
 int windows_cond_destroy(windows_cond_t* cond)
 {
-	CONDITION_VARIABLE* cv = *cond;
-
 	/* note that in Windows there is no DeleteConditionVariable() to call */
-	free(cv);
+	(void)cond;
 
 	return 0;
 }
 
 int windows_cond_signal(windows_cond_t* cond)
 {
-	CONDITION_VARIABLE* cv = *cond;
-
-	WakeConditionVariable(cv);
+	WakeConditionVariable(cond);
 
 	return 0;
 }
 
 int windows_cond_broadcast(windows_cond_t* cond)
 {
-	CONDITION_VARIABLE* cv = *cond;
-
-	WakeAllConditionVariable(cv);
+	WakeAllConditionVariable(cond);
 
 	return 0;
 }
 
 int windows_cond_wait(windows_cond_t* cond, windows_mutex_t* mutex)
 {
-	CONDITION_VARIABLE* cv = *cond;
-	CRITICAL_SECTION* cs = *mutex;
-
-	if (!SleepConditionVariableCS(cv, cs, INFINITE))
+	if (!SleepConditionVariableCS(cond, mutex, INFINITE))
 		return -1;
 
 	return 0;
 }
 
+struct windows_key_context {
+	void (* func)(void *);
+	DWORD key;
+	tommy_node node;
+};
+
+/* list of all keys with destructor */
+static tommy_list windows_key_list = { 0 };
+
+int windows_key_create(windows_key_t* key, void(* destructor)(void*))
+{
+	struct windows_key_context* context;
+
+	context = malloc(sizeof(struct windows_key_context));
+	if (!context)
+		return -1;
+
+	context->func = destructor;
+	context->key = TlsAlloc();
+	if (context->key == 0xFFFFFFFF) {
+		windows_errno(GetLastError());
+		free(context);
+		return -1;
+	}
+
+	/* insert in the list of destructors */
+	if (context->func)
+		tommy_list_insert_tail(&windows_key_list, &context->node, context);
+
+	*key = context;
+
+	return 0;
+}
+
+int windows_key_delete(windows_key_t key)
+{
+	struct windows_key_context* context = key;
+
+	/* remove from the list of destructors */
+	if (context->func)
+		tommy_list_remove_existing(&windows_key_list, &context->node);
+
+	TlsFree(context->key);
+
+	free(context);
+
+	return 0;
+}
+
+void* windows_getspecific(windows_key_t key)
+{
+	struct windows_key_context* context = key;
+
+	return TlsGetValue(context->key);
+}
+
+int windows_setspecific(windows_key_t key, void* value)
+{
+	struct windows_key_context* context = key;
+
+	if (!TlsSetValue(context->key, value)) {
+		windows_errno(GetLastError());
+		return -1;
+	}
+
+	return 0;
+}
+
+struct windows_thread_context {
+	HANDLE h;
+	unsigned id;
+	void* (* func)(void *);
+	void* arg;
+	void* ret;
+};
+
+/* forwarder to change the function declaration */
+static unsigned __stdcall windows_thread_func(void* arg)
+{
+	struct windows_thread_context* context = arg;
+	tommy_node* i;
+
+	context->ret = context->func(context->arg);
+
+	/* call the destructor of all the keys */
+	i = tommy_list_head(&windows_key_list);
+	while (i) {
+		struct windows_key_context* key = i->data;
+		if (key->func) {
+			void* value = windows_getspecific(key);
+			if (value)
+				key->func(value);
+		}
+		i = i->next;
+	}
+
+	return 0;
+}
+
+int windows_create(thread_id_t* thread, void* attr, void* (* func)(void *), void* arg)
+{
+	struct windows_thread_context* context;
+
+	(void)attr;
+
+	context = malloc(sizeof(struct windows_thread_context));
+	if (!context)
+		return -1;
+
+	context->func = func;
+	context->arg = arg;
+	context->ret = 0;
+	context->h = (void*)_beginthreadex(0, 0, windows_thread_func, context, 0, &context->id);
+
+	if (context->h == 0) {
+		free(context);
+		return -1;
+	}
+
+	*thread = context;
+
+	return 0;
+}
+
+int windows_join(thread_id_t thread, void** retval)
+{
+	struct windows_thread_context* context = thread;
+
+	if (WaitForSingleObject(context->h, INFINITE) != WAIT_OBJECT_0) {
+		windows_errno(GetLastError());
+		return -1;
+	}
+
+	if (!CloseHandle(context->h)) {
+		windows_errno(GetLastError());
+		return -1;
+	}
+
+	*retval = context->ret;
+
+	free(context);
+
+	return 0;
+}
+
 #endif
 

cmdline/mingw.h

@@ -385,23 +385,8 @@ size_t windows_direct_size(void);
 /****************************************************************************/
 /* thread */
 
-#define pthread_mutex_t windows_mutex_t
-#define pthread_cond_t windows_cond_t
-#define pthread_mutex_init windows_mutex_init
-#define pthread_mutex_destroy windows_mutex_destroy
-#define pthread_mutex_lock windows_mutex_lock
-#define pthread_mutex_unlock windows_mutex_unlock
-#define pthread_cond_init windows_cond_init
-#define pthread_cond_destroy windows_cond_destroy
-#define pthread_cond_signal windows_cond_signal
-#define pthread_cond_broadcast windows_cond_broadcast
-#define pthread_cond_wait windows_cond_wait
-
-typedef void* windows_mutex_t;
-typedef void* windows_cond_t;
-
 /**
- * Like pthread_* equivalent.
+ * Like the pthread_* equivalent.
  */
 int windows_mutex_init(windows_mutex_t* mutex, void* attr);
 int windows_mutex_destroy(windows_mutex_t* mutex);
@@ -412,6 +397,12 @@ int windows_cond_destroy(windows_cond_t* cond);
 int windows_cond_signal(windows_cond_t* cond);
 int windows_cond_broadcast(windows_cond_t* cond);
 int windows_cond_wait(windows_cond_t* cond, windows_mutex_t* mutex);
+int windows_key_create(windows_key_t* key, void(* destructor)(void*));
+int windows_key_delete(windows_key_t key);
+void* windows_getspecific(windows_key_t key);
+int windows_setspecific(windows_key_t key, void* value);
+int windows_create(thread_id_t* thread, void* attr, void* (* func)(void *), void *arg);
+int windows_join(thread_id_t thread, void** retval);
 
 #endif
 #endif

cmdline/mkstream.c

@@ -41,6 +41,7 @@ void test(void)
 	s = sopen_multi_write(STREAM_MAX);
 	for (i = 0; i < STREAM_MAX; ++i) {
 		snprintf(file, sizeof(file), "stream%u.bin", i);
+		remove(file);
 		if (sopen_multi_file(s, i, file) != 0) {
 			/* LCOV_EXCL_START */
 			exit(EXIT_FAILURE);

cmdline/murmur3.c

@@ -136,25 +136,29 @@ void MurmurHash3_x86_128(const void* data, size_t size, const uint8_t* seed, voi
 		uint32_t k4 = 0;
 
 		switch (size_remainder) {
-		case 15 : k4 ^= (uint32_t)tail[14] << 16;
-		case 14 : k4 ^= (uint32_t)tail[13] << 8;
-		case 13 : k4 ^= (uint32_t)tail[12] << 0;
+		case 15 : k4 ^= (uint32_t)tail[14] << 16; /* fallthrough */
+		case 14 : k4 ^= (uint32_t)tail[13] << 8; /* fallthrough */
+		case 13 : k4 ^= (uint32_t)tail[12] << 0; /* fallthrough */
 			k4 *= c4; k4 = util_rotl32(k4, 18); k4 *= c1; h4 ^= k4;
-		case 12 : k3 ^= (uint32_t)tail[11] << 24;
-		case 11 : k3 ^= (uint32_t)tail[10] << 16;
-		case 10 : k3 ^= (uint32_t)tail[ 9] << 8;
-		case 9 : k3 ^= (uint32_t)tail[ 8] << 0;
+			/* fallthrough */
+		case 12 : k3 ^= (uint32_t)tail[11] << 24; /* fallthrough */
+		case 11 : k3 ^= (uint32_t)tail[10] << 16; /* fallthrough */
+		case 10 : k3 ^= (uint32_t)tail[ 9] << 8; /* fallthrough */
+		case 9 : k3 ^= (uint32_t)tail[ 8] << 0; /* fallthrough */
 			k3 *= c3; k3 = util_rotl32(k3, 17); k3 *= c4; h3 ^= k3;
-		case 8 : k2 ^= (uint32_t)tail[ 7] << 24;
-		case 7 : k2 ^= (uint32_t)tail[ 6] << 16;
-		case 6 : k2 ^= (uint32_t)tail[ 5] << 8;
-		case 5 : k2 ^= (uint32_t)tail[ 4] << 0;
+			/* fallthrough */
+		case 8 : k2 ^= (uint32_t)tail[ 7] << 24; /* fallthrough */
+		case 7 : k2 ^= (uint32_t)tail[ 6] << 16; /* fallthrough */
+		case 6 : k2 ^= (uint32_t)tail[ 5] << 8; /* fallthrough */
+		case 5 : k2 ^= (uint32_t)tail[ 4] << 0; /* fallthrough */
 			k2 *= c2; k2 = util_rotl32(k2, 16); k2 *= c3; h2 ^= k2;
-		case 4 : k1 ^= (uint32_t)tail[ 3] << 24;
-		case 3 : k1 ^= (uint32_t)tail[ 2] << 16;
-		case 2 : k1 ^= (uint32_t)tail[ 1] << 8;
-		case 1 : k1 ^= (uint32_t)tail[ 0] << 0;
+			/* fallthrough */
+		case 4 : k1 ^= (uint32_t)tail[ 3] << 24; /* fallthrough */
+		case 3 : k1 ^= (uint32_t)tail[ 2] << 16; /* fallthrough */
+		case 2 : k1 ^= (uint32_t)tail[ 1] << 8; /* fallthrough */
+		case 1 : k1 ^= (uint32_t)tail[ 0] << 0; /* fallthrough */
 			k1 *= c1; k1 = util_rotl32(k1, 15); k1 *= c2; h1 ^= k1;
+			/* fallthrough */
 		}
 	}
 

cmdline/parity.c

@@ -217,6 +217,9 @@ int parity_create(struct snapraid_parity_handle* handle, const struct snapraid_p
 			/* LCOV_EXCL_STOP */
 		}
 
+		/* the initial valid size is the size on disk */
+		split->valid_size = split->st.st_size;
+
 		/**
 		 * If the parity size is not yet set, set it now.
 		 * This happens when expanding the number of parities,
@@ -528,6 +531,10 @@ static int parity_handle_chsize(struct snapraid_split_handle* split, data_off_t
 		/* LCOV_EXCL_STOP */
 	}
 
+	/* if we shrink, update the valid size, but don't update when growing */
+	if (split->valid_size > split->st.st_size)
+		split->valid_size = split->st.st_size;
+
 	return 0;
 }
 
@@ -697,6 +704,9 @@ int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_par
 			/* LCOV_EXCL_STOP */
 		}
 
+		/* the initial valid size is the size on disk */
+		split->valid_size = split->st.st_size;
+
 		/**
 		 * If the parity size is not yet set, set it now.
 		 * This happens when expanding the number of parities,
@@ -751,7 +761,7 @@ int parity_sync(struct snapraid_parity_handle* handle)
 		ret = fsync(split->f);
 		if (ret != 0) {
 			/* LCOV_EXCL_START */
-			log_fatal("Error synching parity file '%s'. %s.\n", split->path, strerror(errno));
+			log_fatal("Error syncing parity file '%s'. %s.\n", split->path, strerror(errno));
 			return -1;
 			/* LCOV_EXCL_STOP */
 		}
@@ -761,6 +771,30 @@ int parity_sync(struct snapraid_parity_handle* handle)
 	return 0;
 }
 
+int parity_truncate(struct snapraid_parity_handle* handle)
+{
+	unsigned s;
+	int f_ret = 0;
+
+	for (s = 0; s < handle->split_mac; ++s) {
+		struct snapraid_split_handle* split = &handle->split_map[s];
+		int ret;
+
+		/* truncate any data that we know it's not valid */
+		ret = ftruncate(split->f, split->valid_size);
+		if (ret != 0) {
+			/* LCOV_EXCL_START */
+			log_fatal("Error truncating the parity file '%s' to size %" PRIu64 ". %s.\n", split->path, split->valid_size, strerror(errno));
+			f_ret = -1;
+			/* LCOV_EXCL_STOP */
+
+			/* continue to truncate the others */
+		}
+	}
+
+	return f_ret;
+}
+
 int parity_close(struct snapraid_parity_handle* handle)
 {
 	unsigned s;
@@ -826,6 +860,10 @@ int parity_write(struct snapraid_parity_handle* handle, block_off_t pos, unsigne
 		/* LCOV_EXCL_STOP */
 	}
 
+	/* update the valid range */
+	if (split->valid_size < offset + block_size)
+		split->valid_size = offset + block_size;
+
 	write_ret = pwrite(split->f, block_buffer, block_size, offset);
 	if (write_ret != (ssize_t)block_size) { /* conversion is safe because block_size is always small */
 		/* LCOV_EXCL_START */
@@ -867,6 +905,14 @@ int parity_read(struct snapraid_parity_handle* handle, block_off_t pos, unsigned
 		/* LCOV_EXCL_STOP */
 	}
 
+	/* if read is completely out of the valid range */
+	if (offset >= split->valid_size) {
+		/* LCOV_EXCL_START */
+		out("Missing data reading file '%s' at offset %" PRIu64 " for size %u.\n", split->path, offset, block_size);
+		return -1;
+		/* LCOV_EXCL_STOP */
+	}
+
 	count = 0;
 	do {
 		read_ret = pread(split->f, block_buffer + count, block_size - count, offset + count);

cmdline/parity.h

@@ -36,6 +36,16 @@ struct snapraid_split_handle {
 	 */
 	data_off_t size;
 
+	/**
+	 * Valid size of the parity split.
+	 * This is the size effectively written, and not the result of a chsize operation.
+	 * It's used to make read operations failing if read over that size.
+	 *
+	 * Parity is also truncated to that size when fixing it, in case of a Break (Ctrl+C)
+	 * of the program.
+	 */
+	data_off_t valid_size;
+
 	/**
 	 * Artificial size limit for testing.
 	 * 0 means unlimited.
@@ -113,6 +123,11 @@ int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_par
  */
 int parity_sync(struct snapraid_parity_handle* handle);
 
+/**
+ * Truncate the parity file to the valid size.
+ */
+int parity_truncate(struct snapraid_parity_handle* handle);
+
 /**
  * Close the parity file.
  */

cmdline/pool.c

@@ -369,7 +369,7 @@ static void make_link(tommy_hashdyn* poolset, const char* pool_dir, const char*
 #ifdef _WIN32
 		} else if (errno == EPERM) {
 			/* LCOV_EXCL_START */
-			log_fatal("You must run as Adminstrator to be able to create symlinks.\n");
+			log_fatal("You must run as Administrator to be able to create symlinks.\n");
 			exit(EXIT_FAILURE);
 			/* LCOV_EXCL_STOP */
 #endif

cmdline/portable.h

@@ -35,7 +35,7 @@
 #define __USE_MINGW_ANSI_STDIO 1
 
 /**
- * Define the MSVCRT version targetting Windows Vista.
+ * Define the MSVCRT version targeting Windows Vista.
  */
 #define __MSVCRT_VERSION__ 0x0600
 
@@ -198,9 +198,11 @@
 #include <sys/types.h>
 #endif
 
-#if MAJOR_IN_MKDEV
+#if HAVE_SYS_MKDEV
 #include <sys/mkdev.h>
-#elif MAJOR_IN_SYSMACROS
+#endif
+
+#if HAVE_SYS_SYSMACROS_H
 #include <sys/sysmacros.h>
 #endif
 
@@ -228,10 +230,6 @@
 #include "fnmatch.h"
 #endif
 
-#if HAVE_PTHREAD_H
-#include <pthread.h>
-#endif
-
 #if HAVE_MATH_H
 #include <math.h>
 #endif
@@ -243,8 +241,37 @@
 /**
  * Enable thread use.
  */
+#ifdef _WIN32
+#define HAVE_THREAD 1
+typedef void* windows_thread_t;
+typedef CRITICAL_SECTION windows_mutex_t;
+typedef CONDITION_VARIABLE windows_cond_t;
+typedef void* windows_key_t;
+/* remap to pthread */
+#define thread_id_t windows_thread_t
+#define thread_mutex_t windows_mutex_t
+#define thread_cond_t windows_cond_t
+#define pthread_mutex_init windows_mutex_init
+#define pthread_mutex_destroy windows_mutex_destroy
+#define pthread_mutex_lock windows_mutex_lock
+#define pthread_mutex_unlock windows_mutex_unlock
+#define pthread_cond_init windows_cond_init
+#define pthread_cond_destroy windows_cond_destroy
+#define pthread_cond_signal windows_cond_signal
+#define pthread_cond_broadcast windows_cond_broadcast
+#define pthread_cond_wait windows_cond_wait
+#define pthread_create windows_create
+#define pthread_join windows_join
+#else
+#if HAVE_PTHREAD_H
+#include <pthread.h>
+#endif
 #if HAVE_PTHREAD_CREATE
-#define HAVE_PTHREAD 1
+#define HAVE_THREAD 1
+typedef pthread_t thread_id_t;
+typedef pthread_mutex_t thread_mutex_t;
+typedef pthread_cond_t thread_cond_t;
+#endif
 #endif
 
 /**
@@ -335,7 +362,7 @@ int devuuid(uint64_t device, char* uuid, size_t size);
 #define FILEPHY_REAL_OFFSET 3
 
 /**
- * Get the physcal address of the specified file.
+ * Get the physical address of the specified file.
  * This is expected to be just a hint and not necessarily correct or unique.
  * Return 0 on success.
  */
@@ -387,14 +414,14 @@ void os_clear(void);
  *
  * If no log file is selected, it's 0.
  */
-FILE* stdlog;
+extern FILE* stdlog;
 
 /**
  * Exit codes for testing.
  */
-int exit_success;
-int exit_failure;
-int exit_sync_needed;
+extern int exit_success;
+extern int exit_failure;
+extern int exit_sync_needed;
 #undef EXIT_SUCCESS
 #undef EXIT_FAILURE
 #define EXIT_SUCCESS exit_success
@@ -468,8 +495,8 @@ struct devinfo_struct {
 	char smart_serial[SMART_MAX]; /**< SMART serial number. */
 	char smart_vendor[SMART_MAX]; /**< SMART vendor. */
 	char smart_model[SMART_MAX]; /**< SMART model. */
-#if HAVE_PTHREAD
-	pthread_t thread;
+#if HAVE_THREAD
+	thread_id_t thread;
 #endif
 	tommy_node node;
 };

cmdline/scan.c

@@ -25,6 +25,17 @@
 struct snapraid_scan {
 	struct snapraid_state* state; /**< State used. */
 	struct snapraid_disk* disk; /**< Disk used. */
+	thread_id_t thread; /**< Thread used for scanning the disk */
+
+	int is_diff; /**< If it's a diff command or a scanning */
+	int need_write; /**< If a state write is required */
+
+#if HAVE_THREAD
+	/**
+	 * Mutex for protecting the disk stampset table
+	 */
+	thread_mutex_t mutex;
+#endif
 
 	/**
 	 * Counters of changes.
@@ -45,16 +56,69 @@ struct snapraid_scan {
 	tommy_node node;
 };
 
+static struct snapraid_scan* scan_alloc(struct snapraid_state* state, struct snapraid_disk* disk, int is_diff)
+{
+	struct snapraid_scan* scan;
+
+	scan = malloc_nofail(sizeof(struct snapraid_scan));
+	scan->state = state;
+	scan->disk = disk;
+	scan->count_equal = 0;
+	scan->count_move = 0;
+	scan->count_copy = 0;
+	scan->count_restore = 0;
+	scan->count_change = 0;
+	scan->count_remove = 0;
+	scan->count_insert = 0;
+	tommy_list_init(&scan->file_insert_list);
+	tommy_list_init(&scan->link_insert_list);
+	tommy_list_init(&scan->dir_insert_list);
+	scan->is_diff = is_diff;
+	scan->need_write = 0;
+
+#if HAVE_THREAD
+	thread_mutex_init(&disk->stamp_mutex);
+#endif
+
+	return scan;
+}
+
+static void scan_free(struct snapraid_scan* scan)
+{
+#if HAVE_THREAD
+	thread_mutex_destroy(&scan->disk->stamp_mutex);
+#endif
+
+	free(scan);
+}
+
+static void stamp_lock(struct snapraid_disk* disk)
+{
+#if HAVE_THREAD
+	thread_mutex_lock(&disk->stamp_mutex);
+#else
+	(void)disk;
+#endif
+}
+
+static void stamp_unlock(struct snapraid_disk* disk)
+{
+#if HAVE_THREAD
+	thread_mutex_unlock(&disk->stamp_mutex);
+#else
+	(void)disk;
+#endif
+}
+
 /**
  * Remove the specified link from the data set.
  */
 static void scan_link_remove(struct snapraid_scan* scan, struct snapraid_link* slink)
 {
-	struct snapraid_state* state = scan->state;
 	struct snapraid_disk* disk = scan->disk;
 
 	/* state changed */
-	state->need_write = 1;
+	scan->need_write = 1;
 
 	/* remove the file from the link containers */
 	tommy_hashdyn_remove_existing(&disk->linkset, &slink->nodeset);
@@ -69,11 +133,10 @@ static void scan_link_remove(struct snapraid_scan* scan, struct snapraid_link* s
  */
 static void scan_link_insert(struct snapraid_scan* scan, struct snapraid_link* slink)
 {
-	struct snapraid_state* state = scan->state;
 	struct snapraid_disk* disk = scan->disk;
 
 	/* state changed */
-	state->need_write = 1;
+	scan->need_write = 1;
 
 	/* insert the link in the link containers */
 	tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
@@ -116,13 +179,13 @@ static void scan_link(struct snapraid_scan* scan, int is_diff, const char* sub,
 			/* it's an update */
 
 			/* we have to save the linkto/type */
-			state->need_write = 1;
+			scan->need_write = 1;
 
 			++scan->count_change;
 
 			log_tag("scan:update:%s:%s\n", disk->name, esc_tag(slink->sub, esc_buffer));
 			if (is_diff) {
-				printf("update %s\n", fmt_term(disk, slink->sub, esc_buffer));
+				msg_info("update %s\n", fmt_term(disk, slink->sub, esc_buffer));
 			}
 
 			/* update it */
@@ -139,7 +202,7 @@ static void scan_link(struct snapraid_scan* scan, int is_diff, const char* sub,
 
 		log_tag("scan:add:%s:%s\n", disk->name, esc_tag(sub, esc_buffer));
 		if (is_diff) {
-			printf("add %s\n", fmt_term(disk, sub, esc_buffer));
+			msg_info("add %s\n", fmt_term(disk, sub, esc_buffer));
 		}
 
 		/* and continue to insert it */
@@ -166,7 +229,7 @@ static void scan_file_allocate(struct snapraid_scan* scan, struct snapraid_file*
 	block_off_t parity_pos;
 
 	/* state changed */
-	state->need_write = 1;
+	scan->need_write = 1;
 
 	/* allocate the blocks of the file */
 	parity_pos = disk->first_free_block;
@@ -267,7 +330,7 @@ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_fil
 	tommy_list_remove_existing(&disk->filelist, &file->nodelist);
 
 	/* state changed */
-	state->need_write = 1;
+	scan->need_write = 1;
 
 	/* here we are supposed to adjust the ::first_free_block position */
 	/* with the parity position we are deleting */
@@ -501,7 +564,7 @@ static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struc
 			 * because the metadata in the directory is updated only when the file
 			 * is closed.
 			 *
-			 * The same happens for hardlinks that duplicate metatada.
+			 * The same happens for hardlinks that duplicate metadata.
 			 * The link metadata is updated only when the link is opened.
 			 * This extends also to st_size and st_nlink.
 			 *
@@ -563,8 +626,11 @@ static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* f
 	/* insert the file in the containers */
 	if (!file_flag_has(file, FILE_IS_WITHOUT_INODE))
 		tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
+
+	stamp_lock(disk);
 	tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
 	tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
+	stamp_unlock(disk);
 
 	/* delayed allocation of the parity */
 	scan_file_delayed_allocate(scan, file);
@@ -583,7 +649,10 @@ static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* f
 	if (!file_flag_has(file, FILE_IS_WITHOUT_INODE))
 		tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
 	tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
+
+	stamp_lock(disk);
 	tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset);
+	stamp_unlock(disk);
 
 	/* deallocate the file from the parity */
 	scan_file_deallocate(scan, file);
@@ -706,7 +775,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
 				file->mtime_nsec = STAT_NSEC(st);
 
 				/* we have to save the new mtime */
-				state->need_write = 1;
+				scan->need_write = 1;
 			}
 
 			if (strcmp(file->sub, sub) != 0) {
@@ -715,7 +784,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
 
 				log_tag("scan:move:%s:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer), esc_tag(sub, esc_buffer_alt));
 				if (is_diff) {
-					printf("move %s -> %s\n", fmt_term(disk, file->sub, esc_buffer), fmt_term(disk, sub, esc_buffer_alt));
+					msg_info("move %s -> %s\n", fmt_term(disk, file->sub, esc_buffer), fmt_term(disk, sub, esc_buffer_alt));
 				}
 
 				/* remove from the name set */
@@ -728,7 +797,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
 				tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
 
 				/* we have to save the new name */
-				state->need_write = 1;
+				scan->need_write = 1;
 			} else {
 				/* otherwise it's equal */
 				++scan->count_equal;
@@ -864,7 +933,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
 				file->mtime_nsec = STAT_NSEC(st);
 
 				/* we have to save the new mtime */
-				state->need_write = 1;
+				scan->need_write = 1;
 			}
 
 			/* if when processing the disk we used the past inodes values */
@@ -878,7 +947,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
 
 				log_tag("scan:restore:%s:%s\n", disk->name, esc_tag(sub, esc_buffer));
 				if (is_diff) {
-					printf("restore %s\n", fmt_term(disk, sub, esc_buffer));
+					msg_info("restore %s\n", fmt_term(disk, sub, esc_buffer));
 				}
 
 				/* remove from the inode set */
@@ -891,7 +960,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
 				tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
 
 				/* we have to save the new inode */
-				state->need_write = 1;
+				scan->need_write = 1;
 			} else {
 				/* otherwise it's the case of not persistent inode, where doesn't */
 				/* matter if the inode is different or equal, because they have no */
@@ -976,12 +1045,14 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
 			struct snapraid_disk* other_disk = i->data;
 			struct snapraid_file* other_file;
 
+			stamp_lock(other_disk);
 			/* if the nanosecond part of the time stamp is valid, search */
 			/* for name and stamp, otherwise for path and stamp */
 			if (file->mtime_nsec != 0 && file->mtime_nsec != STAT_NSEC_INVALID)
 				other_file = tommy_hashdyn_search(&other_disk->stampset, file_namestamp_compare, file, hash);
 			else
 				other_file = tommy_hashdyn_search(&other_disk->stampset, file_pathstamp_compare, file, hash);
+			stamp_unlock(other_disk);
 
 			/* if found, and it's a fully hashed file */
 			if (other_file && file_is_full_hashed_and_stable(scan->state, other_disk, other_file)) {
@@ -993,7 +1064,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
 
 				log_tag("scan:copy:%s:%s:%s:%s\n", other_disk->name, esc_tag(other_file->sub, esc_buffer), disk->name, esc_tag(file->sub, esc_buffer_alt));
 				if (is_diff) {
-					printf("copy %s -> %s\n", fmt_term(other_disk, other_file->sub, esc_buffer), fmt_term(disk, file->sub, esc_buffer_alt));
+					msg_info("copy %s -> %s\n", fmt_term(other_disk, other_file->sub, esc_buffer), fmt_term(disk, file->sub, esc_buffer_alt));
 				}
 
 				/* mark it as reported */
@@ -1017,14 +1088,14 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
 			);
 
 			if (is_diff) {
-				printf("update %s\n", fmt_term(disk, sub, esc_buffer));
+				msg_info("update %s\n", fmt_term(disk, sub, esc_buffer));
 			}
 		} else {
 			++scan->count_insert;
 
 			log_tag("scan:add:%s:%s\n", disk->name, esc_tag(sub, esc_buffer));
 			if (is_diff) {
-				printf("add %s\n", fmt_term(disk, sub, esc_buffer));
+				msg_info("add %s\n", fmt_term(disk, sub, esc_buffer));
 			}
 		}
 	}
@@ -1038,11 +1109,10 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
  */
 static void scan_emptydir_remove(struct snapraid_scan* scan, struct snapraid_dir* dir)
 {
-	struct snapraid_state* state = scan->state;
 	struct snapraid_disk* disk = scan->disk;
 
 	/* state changed */
-	state->need_write = 1;
+	scan->need_write = 1;
 
 	/* remove the file from the dir containers */
 	tommy_hashdyn_remove_existing(&disk->dirset, &dir->nodeset);
@@ -1057,11 +1127,10 @@ static void scan_emptydir_remove(struct snapraid_scan* scan, struct snapraid_dir
  */
 static void scan_emptydir_insert(struct snapraid_scan* scan, struct snapraid_dir* dir)
 {
-	struct snapraid_state* state = scan->state;
 	struct snapraid_disk* disk = scan->disk;
 
 	/* state changed */
-	state->need_write = 1;
+	scan->need_write = 1;
 
 	/* insert the dir in the dir containers */
 	tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash(dir->sub));
@@ -1172,7 +1241,7 @@ struct stat* dstat(const char* file, struct stat* st)
  * Process a directory.
  * Return != 0 if at least one file or link is processed.
  */
-static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const char* dir, const char* sub)
+static int scan_sub(struct snapraid_scan* scan, int level, int is_diff, char* path_next, char* sub_next, char* tmp)
 {
 	struct snapraid_state* state = scan->state;
 	struct snapraid_disk* disk = scan->disk;
@@ -1180,25 +1249,28 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
 	DIR* d;
 	tommy_list list;
 	tommy_node* node;
+	size_t path_len;
+	size_t sub_len;
+
+	path_len = strlen(path_next);
+	sub_len = strlen(sub_next);
 
 	tommy_list_init(&list);
 
-	d = opendir(dir);
+	d = opendir(path_next);
 	if (!d) {
 		/* LCOV_EXCL_START */
-		log_fatal("Error opening directory '%s'. %s.\n", dir, strerror(errno));
+		log_fatal("Error opening directory '%s'. %s.\n", path_next, strerror(errno));
 		if (level == 0)
 			log_fatal("If this is the disk mount point, remember to create it manually\n");
 		else
-			log_fatal("If it's a permission problem, you can exclude it in the config file with:\n\texclude /%s\n", sub);
+			log_fatal("If it's a permission problem, you can exclude it in the config file with:\n\texclude /%s\n", sub_next);
 		exit(EXIT_FAILURE);
 		/* LCOV_EXCL_STOP */
 	}
 
 	/* read the full directory */
 	while (1) {
-		char path_next[PATH_MAX];
-		char sub_next[PATH_MAX];
 		struct dirent_sorted* entry;
 		const char* name;
 		struct dirent* dd;
@@ -1215,8 +1287,11 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
 		dd = readdir(d);
 		if (dd == 0 && errno != 0) {
 			/* LCOV_EXCL_START */
-			log_fatal("Error reading directory '%s'. %s.\n", dir, strerror(errno));
-			log_fatal("You can exclude it in the config file with:\n\texclude /%s\n", sub);
+			/* restore removing additions */
+			path_next[path_len] = 0;
+			sub_next[sub_len] = 0;
+			log_fatal("Error reading directory '%s'. %s.\n", path_next, strerror(errno));
+			log_fatal("You can exclude it in the config file with:\n\texclude /%s\n", sub_next);
 			exit(EXIT_FAILURE);
 			/* LCOV_EXCL_STOP */
 		}
@@ -1229,8 +1304,7 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
 		if (name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0)))
 			continue;
 
-		pathprint(path_next, sizeof(path_next), "%s%s", dir, name);
-		pathprint(sub_next, sizeof(sub_next), "%s%s", sub, name);
+		pathcatl(path_next, path_len, PATH_MAX, name);
 
 		/* check for not supported file names */
 		if (name[0] == 0) {
@@ -1276,7 +1350,9 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
 
 	if (closedir(d) != 0) {
 		/* LCOV_EXCL_START */
-		log_fatal("Error closing directory '%s'. %s.\n", dir, strerror(errno));
+		/* restore removing additions */
+		path_next[path_len] = 0;
+		log_fatal("Error closing directory '%s'. %s.\n", path_next, strerror(errno));
 		exit(EXIT_FAILURE);
 		/* LCOV_EXCL_STOP */
 	}
@@ -1299,9 +1375,6 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
 	/* process the sorted dir entries */
 	node = list;
 	while (node != 0) {
-		char path_next[PATH_MAX];
-		char sub_next[PATH_MAX];
-		char out[PATH_MAX];
 		struct snapraid_filter* reason = 0;
 		struct dirent_sorted* dd = node->data;
 		const char* name = dd->d_name;
@@ -1311,8 +1384,8 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
 		struct stat st_buf;
 #endif
 
-		pathprint(path_next, sizeof(path_next), "%s%s", dir, name);
-		pathprint(sub_next, sizeof(sub_next), "%s%s", sub, name);
+		pathcatl(path_next, path_len, PATH_MAX, name);
+		pathcatl(sub_next, sub_len, PATH_MAX, name);
 
 		/* start with an unknown type */
 		type = -1;
@@ -1383,14 +1456,13 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
 				scan_file(scan, is_diff, sub_next, st, FILEPHY_UNREAD_OFFSET);
 				processed = 1;
 			} else {
-				msg_verbose("Excluding file '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
+				msg_verbose("Excluding file '%s' for rule '%s'\n", path_next, filter_type(reason, tmp, PATH_MAX));
 			}
 		} else if (type == 1) { /* LNK */
 			if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0) {
-				char subnew[PATH_MAX];
 				int ret;
 
-				ret = readlink(path_next, subnew, sizeof(subnew));
+				ret = readlink(path_next, tmp, PATH_MAX);
 				if (ret >= PATH_MAX) {
 					/* LCOV_EXCL_START */
 					log_fatal("Error in readlink file '%s'. Symlink too long.\n", path_next);
@@ -1407,13 +1479,13 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
 					log_fatal("WARNING! Empty symbolic link '%s'.\n", path_next);
 
 				/* readlink doesn't put the final 0 */
-				subnew[ret] = 0;
+				tmp[ret] = 0;
 
 				/* process as a symbolic link */
-				scan_link(scan, is_diff, sub_next, subnew, FILE_IS_SYMLINK);
+				scan_link(scan, is_diff, sub_next, tmp, FILE_IS_SYMLINK);
 				processed = 1;
 			} else {
-				msg_verbose("Excluding link '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
+				msg_verbose("Excluding link '%s' for rule '%s'\n", path_next, filter_type(reason, tmp, PATH_MAX));
 			}
 		} else if (type == 2) { /* DIR */
 			if (filter_subdir(&state->filterlist, &reason, disk->name, sub_next) == 0) {
@@ -1429,13 +1501,12 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
 				} else
 #endif
 				{
-					char sub_dir[PATH_MAX];
-
 					/* recurse */
-					pathslash(path_next, sizeof(path_next));
-					pathcpy(sub_dir, sizeof(sub_dir), sub_next);
-					pathslash(sub_dir, sizeof(sub_dir));
-					if (scan_dir(scan, level + 1, is_diff, path_next, sub_dir) == 0) {
+					pathslash(path_next, PATH_MAX);
+					pathslash(sub_next, PATH_MAX);
+					if (scan_sub(scan, level + 1, is_diff, path_next, sub_next, tmp) == 0) {
+						/* restore removing additions */
+						pathcatl(sub_next, sub_len, PATH_MAX, name);
 						/* scan the directory as empty dir */
 						scan_emptydir(scan, sub_next);
 					}
@@ -1443,7 +1514,7 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
 					processed = 1;
 				}
 			} else {
-				msg_verbose("Excluding directory '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
+				msg_verbose("Excluding directory '%s' for rule '%s'\n", path_next, filter_type(reason, tmp, PATH_MAX));
 			}
 		} else {
 			if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0) {
@@ -1453,7 +1524,7 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
 
 				log_fatal("WARNING! Ignoring special '%s' file '%s'\n", stat_desc(st), path_next);
 			} else {
-				msg_verbose("Excluding special file '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
+				msg_verbose("Excluding special file '%s' for rule '%s'\n", path_next, filter_type(reason, tmp, PATH_MAX));
 			}
 		}
 
@@ -1467,6 +1538,80 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
 	return processed;
 }
 
+/**
+ * Process a directory.
+ * Return != 0 if at least one file or link is processed.
+ */
+static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const char* dir, const char* sub)
+{
+	/* working buffers used by scan_sub() */
+	char path_next[PATH_MAX];
+	char sub_next[PATH_MAX];
+	char tmp[PATH_MAX];
+
+	pathcpy(path_next, sizeof(path_next), dir);
+	pathcpy(sub_next, sizeof(sub_next), sub);
+
+	return scan_sub(scan, level, is_diff, path_next, sub_next, tmp);
+}
+
+static void* scan_disk(void* arg)
+{
+	struct snapraid_scan* scan = arg;
+	struct snapraid_disk* disk = scan->disk;
+	int ret;
+	int has_persistent_inodes;
+	int has_syncronized_hardlinks;
+	uint64_t start;
+
+	/* check if the disk supports persistent inodes */
+	ret = fsinfo(disk->dir, &has_persistent_inodes, &has_syncronized_hardlinks, 0, 0);
+	if (ret < 0) {
+		/* LCOV_EXCL_START */
+		log_fatal("Error accessing disk '%s' to get file-system info. %s.\n", disk->dir, strerror(errno));
+		exit(EXIT_FAILURE);
+		/* LCOV_EXCL_STOP */
+	}
+	if (!has_persistent_inodes) {
+		disk->has_volatile_inodes = 1;
+	}
+	if (!has_syncronized_hardlinks) {
+		disk->has_volatile_hardlinks = 1;
+	}
+
+	/* if inodes or UUID are not persistent/changed/unsupported */
+	if (disk->has_volatile_inodes || disk->has_different_uuid || disk->has_unsupported_uuid) {
+		/* remove all the inodes from the inode collection */
+		/* if they are not persistent, all of them could be changed now */
+		/* and we don't want to find false matching ones */
+		/* see scan_file() for more details */
+		tommy_node* node = disk->filelist;
+		while (node) {
+			struct snapraid_file* file = node->data;
+
+			node = node->next;
+
+			/* remove from the inode set */
+			tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
+
+			/* clear the inode */
+			file->inode = 0;
+
+			/* mark as missing inode */
+			file_flag_set(file, FILE_IS_WITHOUT_INODE);
+		}
+	}
+
+	start = tick_ms();
+
+	scan_dir(scan, 0, scan->is_diff, disk->dir, "");
+
+	if (!scan->is_diff)
+		msg_progress("Scanned %s in %" PRIu64 " seconds\n", disk->name, (tick_ms() - start) / 1000);
+
+	return 0;
+}
+
 static int state_diffscan(struct snapraid_state* state, int is_diff)
 {
 	tommy_node* i;
@@ -1482,76 +1627,44 @@ static int state_diffscan(struct snapraid_state* state, int is_diff)
 
 	if (is_diff)
 		msg_progress("Comparing...\n");
+	else
+		msg_progress("Scanning...\n");
 
-	/* first scan all the directory and find new and deleted files */
+	/* allocate all the scan data */
 	for (i = state->disklist; i != 0; i = i->next) {
 		struct snapraid_disk* disk = i->data;
 		struct snapraid_scan* scan;
-		tommy_node* node;
-		int ret;
-		int has_persistent_inodes;
-		int has_syncronized_hardlinks;
 
-		scan = malloc_nofail(sizeof(struct snapraid_scan));
-		scan->state = state;
-		scan->disk = disk;
-		scan->count_equal = 0;
-		scan->count_move = 0;
-		scan->count_copy = 0;
-		scan->count_restore = 0;
-		scan->count_change = 0;
-		scan->count_remove = 0;
-		scan->count_insert = 0;
-		tommy_list_init(&scan->file_insert_list);
-		tommy_list_init(&scan->link_insert_list);
-		tommy_list_init(&scan->dir_insert_list);
+		scan = scan_alloc(state, disk, is_diff);
 
 		tommy_list_insert_tail(&scanlist, &scan->node, scan);
-
-		if (!is_diff)
-			msg_progress("Scanning disk %s...\n", disk->name);
-
-		/* check if the disk supports persistent inodes */
-		ret = fsinfo(disk->dir, &has_persistent_inodes, &has_syncronized_hardlinks, 0, 0);
-		if (ret < 0) {
-			/* LCOV_EXCL_START */
-			log_fatal("Error accessing disk '%s' to get file-system info. %s.\n", disk->dir, strerror(errno));
-			exit(EXIT_FAILURE);
-			/* LCOV_EXCL_STOP */
-		}
-		if (!has_persistent_inodes) {
-			disk->has_volatile_inodes = 1;
-		}
-		if (!has_syncronized_hardlinks) {
-			disk->has_volatile_hardlinks = 1;
-		}
-
-		/* if inodes or UUID are not persistent/changed/unsupported */
-		if (disk->has_volatile_inodes || disk->has_different_uuid || disk->has_unsupported_uuid) {
-			/* remove all the inodes from the inode collection */
-			/* if they are not persistent, all of them could be changed now */
-			/* and we don't want to find false matching ones */
-			/* see scan_file() for more details */
-			node = disk->filelist;
-			while (node) {
-				struct snapraid_file* file = node->data;
-
-				node = node->next;
-
-				/* remove from the inode set */
-				tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
-
-				/* clear the inode */
-				file->inode = 0;
-
-				/* mark as missing inode */
-				file_flag_set(file, FILE_IS_WITHOUT_INODE);
-			}
-		}
-
-		scan_dir(scan, 0, is_diff, disk->dir, "");
 	}
 
+	/* first scan all the directory and find new and deleted files */
+	for (i = scanlist; i != 0; i = i->next) {
+		struct snapraid_scan* scan = i->data;
+#if HAVE_THREAD
+		if (state->opt.skip_multi_scan)
+			scan_disk(scan);
+		else
+			thread_create(&scan->thread, scan_disk, scan);
+#else
+		scan_disk(scan);
+#endif
+	}
+
+#if HAVE_THREAD
+	/* wait for all threads to terminate */
+	for (i = scanlist; i != 0; i = i->next) {
+		struct snapraid_scan* scan = i->data;
+		void* retval;
+
+		/* wait for thread termination */
+		if (!state->opt.skip_multi_scan)
+			thread_join(scan->thread, &retval);
+	}
+#endif
+
 	/* we split the search in two phases because to detect files */
 	/* moved from one disk to another we have to start deletion */
 	/* only when all disks have all the new files found */
@@ -1580,7 +1693,7 @@ static int state_diffscan(struct snapraid_state* state, int is_diff)
 
 				log_tag("scan:remove:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
 				if (is_diff) {
-					printf("remove %s\n", fmt_term(disk, file->sub, esc_buffer));
+					msg_info("remove %s\n", fmt_term(disk, file->sub, esc_buffer));
 				}
 
 				scan_file_remove(scan, file);
@@ -1601,7 +1714,7 @@ static int state_diffscan(struct snapraid_state* state, int is_diff)
 
 				log_tag("scan:remove:%s:%s\n", disk->name, esc_tag(slink->sub, esc_buffer));
 				if (is_diff) {
-					printf("remove %s\n", fmt_term(disk, slink->sub, esc_buffer));
+					msg_info("remove %s\n", fmt_term(disk, slink->sub, esc_buffer));
 				}
 
 				scan_link_remove(scan, slink);
@@ -1707,6 +1820,14 @@ static int state_diffscan(struct snapraid_state* state, int is_diff)
 		}
 	}
 
+	/* propagate the state change (after all the scan operations are called) */
+	for (i = scanlist; i != 0; i = i->next) {
+		struct snapraid_scan* scan = i->data;
+		if (scan->need_write) {
+			state->need_write = 1;
+		}
+	}
+
 	/* check for disks where all the previously existing files where removed */
 	if (!state->opt.force_empty) {
 		int all_missing = 0;
@@ -1896,7 +2017,7 @@ static int state_diffscan(struct snapraid_state* state, int is_diff)
 	}
 	log_flush();
 
-	tommy_list_foreach(&scanlist, (tommy_foreach_func*)free);
+	tommy_list_foreach(&scanlist, (tommy_foreach_func*)scan_free);
 
 	/* check the file-system on all disks */
 	state_fscheck(state, "after scan");

cmdline/scrub.c

@@ -50,9 +50,8 @@ struct snapraid_plan {
 /**
  * Check if we have to process the specified block index ::i.
  */
-static int block_is_enabled(void* void_plan, block_off_t i)
+static int block_is_enabled(struct snapraid_plan* plan, block_off_t i)
 {
-	struct snapraid_plan* plan = void_plan;
 	time_t blocktime;
 	snapraid_info info;
 
@@ -268,6 +267,7 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
 	unsigned* waiting_map;
 	unsigned waiting_mac;
 	char esc_buffer[ESC_MAX];
+	bit_vect_t* block_enabled;
 
 	/* maps the disks to handles */
 	handle = handle_mapping(state, &diskmax);
@@ -289,12 +289,16 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
 	silent_error = 0;
 	io_error = 0;
 
+	msg_progress("Selecting...\n");
+
 	/* first count the number of blocks to process */
 	countmax = 0;
 	plan->countlast = 0;
+	block_enabled = calloc_nofail(1, bit_vect_size(blockmax)); /* preinitialize to 0 */
 	for (blockcur = blockstart; blockcur < blockmax; ++blockcur) {
 		if (!block_is_enabled(plan, blockcur))
 			continue;
+		bit_vect_set(block_enabled, blockcur);
 		++countmax;
 	}
 
@@ -310,10 +314,11 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
 
 	countsize = 0;
 	countpos = 0;
-	plan->countlast = 0;
+
+	msg_progress("Scrubbing...\n");
 
 	/* start all the worker threads */
-	io_start(&io, blockstart, blockmax, &block_is_enabled, plan);
+	io_start(&io, blockstart, blockmax, block_enabled);
 
 	state_progress_begin(state, blockstart, blockmax, countmax);
 	while (1) {
@@ -665,7 +670,8 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
 		log_fatal("DANGER! Unexpected data errors! The failing blocks are now marked as bad!\n");
 	if (io_error || silent_error) {
 		log_fatal("Use 'snapraid status' to list the bad blocks.\n");
-		log_fatal("Use 'snapraid -e fix' to recover.\n");
+		log_fatal("Use 'snapraid -e fix' to recover them.\n");
+		log_fatal("Use 'snapraid -p bad scrub' to recheck after fixing.\n");
 	}
 
 	log_tag("summary:error_file:%u\n", error);
@@ -699,6 +705,7 @@ bail:
 	free(rehandle_alloc);
 	free(waiting_map);
 	io_done(&io);
+	free(block_enabled);
 
 	if (state->opt.expect_recoverable) {
 		if (error + silent_error + io_error == 0)
@@ -873,8 +880,6 @@ int state_scrub(struct snapraid_state* state, int plan, int olderthan)
 		}
 	}
 
-	msg_progress("Scrubbing...\n");
-
 	error = 0;
 
 	ret = state_scrub_process(state, parity_handle, 0, blockmax, &ps, now);

cmdline/selftest.c

@@ -632,7 +632,7 @@ void selftest(void)
 	}
 	if (raid_test_par(RAID_MODE_CAUCHY, 1, 256) != 0) {
 		/* LCOV_EXCL_START */
-		log_fatal("Failed GEN Cauchy test sigle data disk\n");
+		log_fatal("Failed GEN Cauchy test single data disk\n");
 		exit(EXIT_FAILURE);
 		/* LCOV_EXCL_STOP */
 	}

cmdline/snapraid.c

@@ -87,7 +87,7 @@ void memory(void)
 	log_tag("memory:link:%" PRIu64 "\n", (uint64_t)(sizeof(struct snapraid_link)));
 	log_tag("memory:dir:%" PRIu64 "\n", (uint64_t)(sizeof(struct snapraid_dir)));
 
-	msg_progress("Using %u MiB of memory for the FileSystem.\n", (unsigned)(malloc_counter_get() / MEBI));
+	msg_progress("Using %u MiB of memory for the file-system.\n", (unsigned)(malloc_counter_get() / MEBI));
 }
 
 void test(int argc, char* argv[])
@@ -131,13 +131,20 @@ void log_open(const char* file)
 	char text_D[32];
 	time_t t;
 	struct tm* tm;
+#if HAVE_LOCALTIME_R
+	struct tm tm_res;
+#endif
 
 	/* leave stdlog at 0 if not specified */
 	if (file == 0)
 		return;
 
 	t = time(0);
+#if HAVE_LOCALTIME_R
+	tm = localtime_r(&t, &tm_res);
+#else
 	tm = localtime(&t);
+#endif
 	if (tm) {
 		strftime(text_T, sizeof(text_T), "%H%M%S", tm);
 		strftime(text_D, sizeof(text_T), "%Y%m%d", tm);
@@ -300,6 +307,7 @@ void config(char* conf, size_t conf_size, const char* argv0)
 #define OPT_TEST_SKIP_CONTENT_WRITE 302
 #define OPT_TEST_SKIP_SPACE_HOLDER 303
 #define OPT_TEST_FORMAT 304
+#define OPT_TEST_SKIP_MULTI_SCAN 305
 
 #if HAVE_GETOPT_LONG
 struct option long_options[] = {
@@ -308,6 +316,7 @@ struct option long_options[] = {
 	{ "filter-disk", 1, 0, 'd' },
 	{ "filter-missing", 0, 0, 'm' },
 	{ "filter-error", 0, 0, 'e' },
+	{ "filter-block-error", 0, 0, 'b' },
 	{ "percentage", 1, 0, 'p' }, /* legacy name for --plan */
 	{ "plan", 1, 0, 'p' },
 	{ "older-than", 1, 0, 'o' },
@@ -467,11 +476,14 @@ struct option long_options[] = {
 	/* Set the output format */
 	{ "test-fmt", 1, 0, OPT_TEST_FORMAT },
 
+	/* Skip thread in disk scan */
+	{ "test-skip-multi-scan", 0, 0, OPT_TEST_SKIP_MULTI_SCAN },
+
 	{ 0, 0, 0, 0 }
 };
 #endif
 
-#define OPTIONS "c:f:d:mep:o:S:B:L:i:l:ZEUDNFRahTC:vqHVG"
+#define OPTIONS "c:f:d:mebp:o:S:B:L:i:l:ZEUDNFRahTC:vqHVG"
 
 volatile int global_interrupt = 0;
 
@@ -552,6 +564,9 @@ int main(int argc, char* argv[])
 	int period;
 	time_t t;
 	struct tm* tm;
+#if HAVE_LOCALTIME_R
+	struct tm tm_res;
+#endif
 	int i;
 
 	test(argc, argv);
@@ -617,10 +632,18 @@ int main(int argc, char* argv[])
 			opt.expected_missing = 1;
 			break;
 		case 'e' :
-			/* when processing only error, we filter both files and blocks */
+			/* when processing only error, we filter files */
 			/* and we apply fixes only to synced ones */
 			filter_error = 1;
-			opt.badonly = 1;
+			opt.badfileonly = 1;
+			opt.syncedonly = 1;
+			break;
+		case 'b' :
+			/* when processing only block with error, we filter both files and blocks */
+			/* and we apply fixes only to synced ones */
+			filter_error = 1;
+			opt.badfileonly = 1;
+			opt.badblockonly = 1;
 			opt.syncedonly = 1;
 			break;
 		case 'p' :
@@ -862,7 +885,7 @@ int main(int argc, char* argv[])
 			opt.force_stats = 1;
 			break;
 		case OPT_TEST_COND_SIGNAL_OUTSIDE :
-#if HAVE_PTHREAD
+#if HAVE_THREAD
 			thread_cond_signal_outside = 1;
 #endif
 			break;
@@ -910,6 +933,9 @@ int main(int argc, char* argv[])
 				/* LCOV_EXCL_STOP */
 			}
 			break;
+		case OPT_TEST_SKIP_MULTI_SCAN :
+			opt.skip_multi_scan = 1;
+			break;
 		default :
 			/* LCOV_EXCL_START */
 			log_fatal("Unknown option '%c'\n", (char)c);
@@ -1001,6 +1027,10 @@ int main(int argc, char* argv[])
 	switch (operation) {
 	case OPERATION_FIX :
 	case OPERATION_CHECK :
+	case OPERATION_SMART :
+	case OPERATION_DEVICES :
+	case OPERATION_SPINUP :
+	case OPERATION_SPINDOWN :
 		break;
 	default :
 		if (opt.force_device) {
@@ -1091,7 +1121,7 @@ int main(int argc, char* argv[])
 			exit(EXIT_FAILURE);
 			/* LCOV_EXCL_STOP */
 		}
-		/* follow */
+		/* fallthrough */
 	case OPERATION_SPINUP :
 	case OPERATION_SPINDOWN :
 		if (!tommy_list_empty(&filterlist_file)) {
@@ -1221,7 +1251,11 @@ int main(int argc, char* argv[])
 
 	/* print generic info into the log */
 	t = time(0);
+#if HAVE_LOCALTIME_R
+	tm = localtime_r(&t, &tm_res);
+#else
 	tm = localtime(&t);
+#endif
 	log_tag("version:%s\n", PACKAGE_VERSION);
 	log_tag("unixtime:%" PRIi64 "\n", (int64_t)t);
 	if (tm) {

cmdline/speed.c

@@ -216,6 +216,7 @@ void speed(int period)
 	printf("%8s", "best");
 	printf("%8s", "murmur3");
 	printf("%8s", "spooky2");
+	printf("%8s", "metro");
 	printf("\n");
 
 	printf("%8s", "hash");
@@ -245,6 +246,14 @@ void speed(int period)
 			memhash(HASH_SPOOKY2, seed, digest, v[j], size);
 	} SPEED_STOP
 
+	printf("%8" PRIu64, ds / dt);
+	fflush(stdout);
+
+	SPEED_START {
+		for (j = 0; j < nd; ++j)
+			memhash(HASH_METRO, seed, digest, v[j], size);
+	} SPEED_STOP
+
 	printf("%8" PRIu64, ds / dt);
 	printf("\n");
 	printf("\n");

cmdline/state.c

@@ -37,14 +37,10 @@
  * writing to disk, but you'll need to access multiple times the same data,
  * being potentially slower.
  *
- * For upcoming SnapRAID version it's planned to add a mutex protection
- * at the file-system structure, slowing down multiple data access,
- * so we disable it.
- *
  * Multi thread for verify is instead always generally faster,
  * so we enable it if possible.
  */
-#if HAVE_PTHREAD
+#if HAVE_THREAD
 /* #define HAVE_MT_WRITE 1 */
 #define HAVE_MT_VERIFY 1
 #endif
@@ -279,7 +275,7 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
 				struct snapraid_disk* other = j->data;
 				if (disk->device == other->device) {
 					if (state->opt.force_device) {
-						/* note tha we just ignore the issue */
+						/* note that we just ignore the issue */
 						/* and we DON'T mark the disk to be skipped */
 						/* because we want to use these disks */
 						if (!state->opt.no_warnings)
@@ -310,15 +306,23 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
 				for (l = 0; l < state->level; ++l) {
 					for (s = 0; s < state->parity[l].split_mac; ++s) {
 						if (disk->device == state->parity[l].split_map[s].device) {
-							/* LCOV_EXCL_START */
-							log_fatal("Disk '%s' and %s '%s' are on the same device.\n", disk->dir, lev_name(l), state->parity[l].split_map[s].path);
+							if (state->opt.force_device) {
+								/* note that we just ignore the issue */
+								/* and we DON'T mark the disk to be skipped */
+								/* because we want to use these disks */
+								if (!state->opt.no_warnings)
+									log_fatal("DANGER! Ignoring that disks '%s' and %s '%s' are on the same device\n", disk->dir, lev_name(l), state->parity[l].split_map[s].path);
+							} else {
+								/* LCOV_EXCL_START */
+								log_fatal("Disk '%s' and %s '%s' are on the same device.\n", disk->dir, lev_name(l), state->parity[l].split_map[s].path);
 #ifdef _WIN32
-							log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
-							log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
-							log_fatal("to change one of the disk serial.\n");
+								log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
+								log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
+								log_fatal("to change one of the disk serial.\n");
 #endif
-							exit(EXIT_FAILURE);
-							/* LCOV_EXCL_STOP */
+								exit(EXIT_FAILURE);
+								/* LCOV_EXCL_STOP */
+							}
 						}
 					}
 				}
@@ -364,7 +368,7 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
 					for (t = 0; t < state->parity[j].split_mac; ++t) {
 						if (state->parity[l].split_map[s].device == state->parity[j].split_map[t].device) {
 							if (state->opt.force_device) {
-								/* note tha we just ignore the issue */
+								/* note that we just ignore the issue */
 								/* and we DON'T mark the disk to be skipped */
 								/* because we want to use these disks */
 								if (!state->opt.no_warnings)
@@ -623,7 +627,7 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
 			}
 			state->block_size *= KIBI;
 		} else if (strcmp(tag, "hashsize") == 0
-			|| strcmp(tag, "hash_size") == 0 /* v11.0 used incorretly this one, kept now for backward compatibility */
+			|| strcmp(tag, "hash_size") == 0 /* v11.0 used incorrectly this one, kept now for backward compatibility */
 		) {
 			uint32_t hash_size;
 
@@ -1618,7 +1622,7 @@ static void state_content_check(struct snapraid_state* state, const char* path)
 /**
  * Check if the position is REQUIRED, or we can completely clear it from the state.
  *
- * Note that position with only DELETED blocks are discarged.
+ * Note that position with only DELETED blocks are discharged.
  */
 static int fs_position_is_required(struct snapraid_state* state, block_off_t pos)
 {
@@ -1844,7 +1848,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
 			if (state->block_size == 0) {
 				/* LCOV_EXCL_START */
 				decoding_error(path, f);
-				log_fatal("Internal incosistency due zero blocksize!\n");
+				log_fatal("Internal inconsistency due zero blocksize!\n");
 				exit(EXIT_FAILURE);
 				/* LCOV_EXCL_STOP */
 			}
@@ -2398,6 +2402,9 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
 			case 'k' :
 				state->hash = HASH_SPOOKY2;
 				break;
+			case 'm' :
+				state->hash = HASH_METRO;
+				break;
 			default :
 				/* LCOV_EXCL_START */
 				decoding_error(path, f);
@@ -2425,6 +2432,9 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
 			case 'k' :
 				state->prevhash = HASH_SPOOKY2;
 				break;
+			case 'm' :
+				state->prevhash = HASH_METRO;
+				break;
 			default :
 				/* LCOV_EXCL_START */
 				decoding_error(path, f);
@@ -2867,11 +2877,12 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
 struct state_write_thread_context {
 	struct snapraid_state* state;
 #if HAVE_MT_WRITE
-	pthread_t thread;
+	thread_id_t thread;
 #endif
 	/* input */
 	block_off_t blockmax;
 	time_t info_oldest;
+	time_t info_now;
 	int info_has_rehash;
 	STREAM* f;
 	/* output */
@@ -2888,6 +2899,7 @@ static void* state_write_thread(void* arg)
 	struct snapraid_state* state = context->state;
 	block_off_t blockmax = context->blockmax;
 	time_t info_oldest = context->info_oldest;
+	time_t info_now = context->info_now;
 	int info_has_rehash = context->info_has_rehash;
 	STREAM* f = context->f;
 	uint32_t crc;
@@ -2945,6 +2957,8 @@ static void* state_write_thread(void* arg)
 		sputc('u', f);
 	} else if (state->hash == HASH_SPOOKY2) {
 		sputc('k', f);
+	} else if (state->hash == HASH_METRO) {
+		sputc('m', f);
 	} else {
 		/* LCOV_EXCL_START */
 		log_fatal("Unexpected hash when writing the content file '%s'.\n", serrorfile(f));
@@ -2968,6 +2982,8 @@ static void* state_write_thread(void* arg)
 				sputc('u', f);
 			} else if (state->prevhash == HASH_SPOOKY2) {
 				sputc('k', f);
+			} else if (state->prevhash == HASH_METRO) {
+				sputc('m', f);
 			} else {
 				/* LCOV_EXCL_START */
 				log_fatal("Unexpected prevhash when writing the content file '%s'.\n", serrorfile(f));
@@ -3278,7 +3294,18 @@ static void* state_write_thread(void* arg)
 				flag |= 8;
 			sputb32(flag, f);
 
-			t = info_get_time(info) - info_oldest;
+			t = info_get_time(info);
+
+			/* truncate any time that is in the future */
+			if (t > info_now)
+				t = info_now;
+
+			/* the oldest info is computed only on required blocks, so it may not be the absolute oldest */
+			if (t < info_oldest)
+				t = 0;
+			else
+				t -= info_oldest;
+
 			sputb32(t, f);
 		} else {
 			/* write a special 0 flag to mark missing info */
@@ -3353,6 +3380,7 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
 	tommy_node* i;
 	block_off_t blockmax;
 	time_t info_oldest;
+	time_t info_now;
 	int info_has_rehash;
 	int mapping_idx;
 	block_off_t idx;
@@ -3371,6 +3399,7 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
 	/* clear the info for unused blocks */
 	/* and get some other info */
 	info_oldest = 0; /* oldest time in info */
+	info_now = time(0); /* get the present time */
 	info_has_rehash = 0; /* if there is a rehash info */
 	for (idx = 0; idx < blockmax; ++idx) {
 		/* if the position is used */
@@ -3434,10 +3463,21 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
 		msg_progress("Saving state to %s...\n", content->content);
 
 		pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
+
+		/* ensure to delete a previous stale file */
+		if (remove(tmp) != 0) {
+			if (errno != ENOENT) {
+				/* LCOV_EXCL_START */
+				log_fatal("Error removing the stale content file '%s'. %s.\n", tmp, strerror(errno));
+				exit(EXIT_FAILURE);
+				/* LCOV_EXCL_STOP */
+			}
+		}
+
 		f = sopen_write(tmp);
 		if (f == 0) {
 			/* LCOV_EXCL_START */
-			log_fatal("Error opening the content file '%s'. %s.\n", tmp, strerror(errno));
+			log_fatal("Error opening the temporary content file '%s'. %s.\n", tmp, strerror(errno));
 			exit(EXIT_FAILURE);
 			/* LCOV_EXCL_STOP */
 		}
@@ -3450,10 +3490,11 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
 		context->state = state;
 		context->blockmax = blockmax;
 		context->info_oldest = info_oldest;
+		context->info_now = info_now;
 		context->info_has_rehash = info_has_rehash;
 		context->f = f;
 
-		thread_create(&context->thread, 0, state_write_thread, context);
+		thread_create(&context->thread, state_write_thread, context);
 
 		i = i->next;
 	}
@@ -3562,12 +3603,24 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
 		struct snapraid_content* content = i->data;
 		char tmp[PATH_MAX];
 		pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
+
+		/* ensure to delete a previous stale file */
+		if (remove(tmp) != 0) {
+			if (errno != ENOENT) {
+				/* LCOV_EXCL_START */
+				log_fatal("Error removing the stale content file '%s'. %s.\n", tmp, strerror(errno));
+				exit(EXIT_FAILURE);
+				/* LCOV_EXCL_STOP */
+			}
+		}
+
 		if (sopen_multi_file(f, k, tmp) != 0) {
 			/* LCOV_EXCL_START */
-			log_fatal("Error opening the content file '%s'. %s.\n", tmp, strerror(errno));
+			log_fatal("Error opening the temporary content file '%s'. %s.\n", tmp, strerror(errno));
 			exit(EXIT_FAILURE);
 			/* LCOV_EXCL_STOP */
 		}
+
 		++k;
 		i = i->next;
 	}
@@ -3579,6 +3632,7 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
 	context->state = state;
 	context->blockmax = blockmax;
 	context->info_oldest = info_oldest;
+	context->info_now = info_now;
 	context->info_has_rehash = info_has_rehash;
 	context->f = f;
 
@@ -3785,7 +3839,7 @@ struct state_verify_thread_context {
 	struct snapraid_state* state;
 	struct snapraid_content* content;
 #if HAVE_MT_VERIFY
-	pthread_t thread;
+	thread_id_t thread;
 #else
 	void* retval;
 #endif
@@ -3797,10 +3851,14 @@ struct state_verify_thread_context {
 static void* state_verify_thread(void* arg)
 {
 	struct state_verify_thread_context* context = arg;
+	struct snapraid_content* content = context->content;
 	STREAM* f = context->f;
 	unsigned char buf[4];
 	uint32_t crc_stored;
 	uint32_t crc_computed;
+	uint64_t start;
+
+	start = tick_ms();
 
 	if (sdeplete(f, buf) != 0) {
 		/* LCOV_EXCL_START */
@@ -3832,6 +3890,8 @@ static void* state_verify_thread(void* arg)
 		/* LCOV_EXCL_STOP */
 	}
 
+	msg_progress("Verified %s in %" PRIu64 " seconds\n", content->content, (tick_ms() - start) / 1000);
+
 	return 0;
 }
 
@@ -3840,6 +3900,8 @@ static void state_verify_content(struct snapraid_state* state, uint32_t crc)
 	tommy_node* i;
 	int fail;
 
+	msg_progress("Verifying...\n");
+
 	/* start all reading threads */
 	i = tommy_list_head(&state->contentlist);
 	while (i) {
@@ -3848,14 +3910,11 @@ static void state_verify_content(struct snapraid_state* state, uint32_t crc)
 		char tmp[PATH_MAX];
 		STREAM* f;
 
-		msg_progress("Verifying %s...\n", content->content);
-
 		pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
-
 		f = sopen_read(tmp);
 		if (f == 0) {
 			/* LCOV_EXCL_START */
-			log_fatal("Error reopening the content file '%s'. %s.\n", tmp, strerror(errno));
+			log_fatal("Error reopening the temporary content file '%s'. %s.\n", tmp, strerror(errno));
 			exit(EXIT_FAILURE);
 			/* LCOV_EXCL_STOP */
 		}
@@ -3871,7 +3930,7 @@ static void state_verify_content(struct snapraid_state* state, uint32_t crc)
 		context->f = f;
 
 #if HAVE_MT_VERIFY
-		thread_create(&context->thread, 0, state_verify_thread, context);
+		thread_create(&context->thread, state_verify_thread, context);
 #else
 		context->retval = state_verify_thread(context);
 #endif
@@ -3924,6 +3983,59 @@ static void state_rename_content(struct snapraid_state* state)
 {
 	tommy_node* i;
 
+#if defined(_linux) /* this sequence is linux specific */
+	i = tommy_list_head(&state->contentlist);
+	while (i) {
+		struct snapraid_content* content = i->data;
+		char tmp[PATH_MAX];
+		char dir[PATH_MAX];
+		char* slash;
+		int handle;
+
+		pathcpy(dir, sizeof(dir), content->content);
+
+		slash = strrchr(tmp, '/');
+		if (slash)
+			*slash = 0;
+		else
+			pathcpy(dir, sizeof(dir), ".");
+
+		/* open the directory to get the handle */
+		handle = open(dir, O_RDONLY | O_DIRECTORY);
+		if (handle < 0) {
+			/* LCOV_EXCL_START */
+			log_fatal("Error opening the directory '%s'. %s.\n", dir, strerror(errno));
+			exit(EXIT_FAILURE);
+			/* LCOV_EXCL_STOP */
+		}
+
+		/* now rename the just written copy with the correct name */
+		pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
+		if (rename(tmp, content->content) != 0) {
+			/* LCOV_EXCL_START */
+			log_fatal("Error renaming the content file '%s' to '%s'. %s.\n", tmp, content->content, strerror(errno));
+			exit(EXIT_FAILURE);
+			/* LCOV_EXCL_STOP */
+		}
+
+		/* sync the directory */
+		if (fsync(handle) != 0) {
+			/* LCOV_EXCL_START */
+			log_fatal("Error syncing the directory '%s'. %s.\n", dir, strerror(errno));
+			exit(EXIT_FAILURE);
+			/* LCOV_EXCL_STOP */
+		}
+
+		if (close(handle) != 0) {
+			/* LCOV_EXCL_START */
+			log_fatal("Error closing the directory '%s'. %s.\n", dir, strerror(errno));
+			exit(EXIT_FAILURE);
+			/* LCOV_EXCL_STOP */
+		}
+
+		i = i->next;
+	}
+#else
 	i = tommy_list_head(&state->contentlist);
 	while (i) {
 		struct snapraid_content* content = i->data;
@@ -3940,6 +4052,7 @@ static void state_rename_content(struct snapraid_state* state)
 
 		i = i->next;
 	}
+#endif
 }
 
 void state_write(struct snapraid_state* state)
@@ -4000,7 +4113,7 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tom
 	if (!filter_missing && !filter_error && tommy_list_empty(filterlist_file) && tommy_list_empty(filterlist_disk))
 		return;
 
-	msg_progress("Filtering...\n");
+	msg_progress("Selecting...\n");
 
 	for (i = tommy_list_head(filterlist_disk); i != 0; i = i->next) {
 		struct snapraid_filter* filter = i->data;
@@ -4062,7 +4175,7 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tom
 		}
 	}
 
-	/* if we are filtering by disk, exclude any parity not explicitely included */
+	/* if we are filtering by disk, exclude any parity not explicitly included */
 	if (!tommy_list_empty(filterlist_disk)) {
 		/* for each parity disk */
 		for (l = 0; l < state->level; ++l) {
@@ -4370,7 +4483,8 @@ int state_progress(struct snapraid_state* state, struct snapraid_io* io, block_o
 	) {
 		time_t elapsed;
 		unsigned out_perc = 0;
-		unsigned out_speed = 0;
+		unsigned out_size_speed = 0;
+		unsigned out_block_speed = 0;
 		unsigned out_cpu = 0;
 		unsigned out_eta = 0;
 		int out_computed = 0;
@@ -4443,7 +4557,11 @@ int state_progress(struct snapraid_state* state, struct snapraid_io* io, block_o
 
 			/* estimate the speed in MB/s */
 			if (delta_time != 0)
-				out_speed = (unsigned)(delta_size / MEGA / delta_time);
+				out_size_speed = (unsigned)(delta_size / MEGA / delta_time);
+
+			/* estimate the speed in block/s */
+			if (delta_time != 0)
+				out_block_speed = (unsigned)(delta_pos / delta_time);
 
 			/* estimate the cpu usage percentage */
 			if (delta_tick_total != 0)
@@ -4463,12 +4581,13 @@ int state_progress(struct snapraid_state* state, struct snapraid_io* io, block_o
 		}
 
 		if (state->opt.gui) {
-			log_tag("run:pos:%u:%u:%" PRIu64 ":%u:%u:%u:%u:%" PRIu64 "\n", blockpos, countpos, countsize, out_perc, out_eta, out_speed, out_cpu, (uint64_t)elapsed);
+			log_tag("run:pos:%u:%u:%" PRIu64 ":%u:%u:%u:%u:%" PRIu64 "\n", blockpos, countpos, countsize, out_perc, out_eta, out_size_speed, out_cpu, (uint64_t)elapsed);
 			log_flush();
 		} else {
 			msg_bar("%u%%, %u MB", out_perc, (unsigned)(countsize / MEGA));
 			if (out_computed) {
-				msg_bar(", %u MB/s", out_speed);
+				msg_bar(", %u MB/s", out_size_speed);
+				msg_bar(", %u stripe/s", out_block_speed);
 				msg_bar(", CPU %u%%", out_cpu);
 				msg_bar(", %u:%02u ETA", out_eta / 60, out_eta % 60);
 			}

cmdline/state.h

@@ -73,7 +73,8 @@ extern volatile int global_interrupt;
 struct snapraid_option {
 	int gui; /**< Gui output. */
 	int auditonly; /**< In check, checks only the hash and not the parity. */
-	int badonly; /**< In fix, fixes only the blocks marked as bad. */
+	int badfileonly; /**< In fix, fixes only files marked as bad. */
+	int badblockonly; /**< In fix, fixes only the blocks marked as bad. */
 	int syncedonly; /**< In fix, fixes only files that are synced. */
 	int prehash; /**< Enables the prehash mode for sync. */
 	unsigned io_error_limit; /**< Max number of input/output errors before aborting. */
@@ -118,6 +119,7 @@ struct snapraid_option {
 	int auto_conf; /**< Allow to run without configuration file. */
 	int force_stats; /**< Force stats print during process. */
 	uint64_t parity_limit_size; /**< Test limit for parity files. */
+	int skip_multi_scan; /**< Don't use threads in scan. */
 };
 
 struct snapraid_state {
@@ -261,7 +263,7 @@ void state_rehash(struct snapraid_state* state);
  */
 #define SCRUB_AUTO -1 /**< Automatic selection. */
 #define SCRUB_BAD -2 /**< Scrub only the bad blocks. */
-#define SCRUB_NEW -3 /**< Scub the new blocks. */
+#define SCRUB_NEW -3 /**< Scrub the new blocks. */
 #define SCRUB_FULL -4 /**< Scrub everything. */
 #define SCRUB_EVEN -5 /**< Even blocks. */
 

cmdline/status.c

@@ -29,6 +29,10 @@
 
 unsigned day_ago(time_t ref, time_t now)
 {
+	/* in case some dates is in the future */
+	if (now < ref)
+		return 0;
+
 	return (now - ref) / (24 * 3600);
 }
 
@@ -458,6 +462,10 @@ int state_status(struct snapraid_state* state)
 
 	printf("\n");
 
+	if (newest > now) {
+		printf("WARNING! You have scrub dates in the future! The next sync/scrub will truncate them!\n");
+	}
+
 	if (unsynced_blocks) {
 		printf("WARNING! The array is NOT fully synced.\n");
 		printf("You have a sync in progress at %u%%.\n", (blockmax - unsynced_blocks) * 100 / blockmax);

cmdline/stream.c

@@ -112,7 +112,8 @@ int sopen_multi_file(STREAM* s, unsigned i, const char* file)
 
 	pathcpy(s->handle[i].path, sizeof(s->handle[i].path), file);
 
-	f = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY | O_SEQUENTIAL, 0600);
+	/* O_EXCL to be resilient ensure to always create a new file and not use a stale link to the original file */
+	f = open(file, O_WRONLY | O_CREAT | O_EXCL | O_BINARY | O_SEQUENTIAL, 0600);
 	if (f == -1) {
 		/* LCOV_EXCL_START */
 		return -1;
@@ -362,7 +363,7 @@ int sgettok(STREAM* f, char* str, int size)
 			break;
 		}
 		if (c == '\n') {
-			/* remove ending carrige return to support the Windows CR+LF format */
+			/* remove ending carriage return to support the Windows CR+LF format */
 			if (i != str && i[-1] == '\r')
 				--i;
 			sungetc(c, f);
@@ -428,7 +429,7 @@ int sgetline(STREAM* f, char* str, int size)
 		while (1) {
 			c = *pos++;
 			if (c == '\n') {
-				/* remove ending carrige return to support the Windows CR+LF format */
+				/* remove ending carriage return to support the Windows CR+LF format */
 				if (i != str && i[-1] == '\r')
 					--i;
 				--pos;
@@ -454,7 +455,7 @@ int sgetline(STREAM* f, char* str, int size)
 				/* LCOV_EXCL_STOP */
 			}
 			if (c == '\n') {
-				/* remove ending carrige return to support the Windows CR+LF format */
+				/* remove ending carriage return to support the Windows CR+LF format */
 				if (i != str && i[-1] == '\r')
 					--i;
 				sungetc(c, f);
@@ -500,15 +501,34 @@ int sgetu32(STREAM* f, uint32_t* value)
 	int c;
 
 	c = sgetc(f);
-	if (c >= '0' && c <= '9') {
+	if (c == '0') {
+		*value = 0;
+		return 0;
+	} else if (c >= '1' && c <= '9') {
 		uint32_t v;
 
 		v = c - '0';
 
 		c = sgetc(f);
 		while (c >= '0' && c <= '9') {
+			uint32_t digit;
+			if (v > 0xFFFFFFFFU / 10) {
+				/* LCOV_EXCL_START */
+				/* overflow */
+				return -1;
+				/* LCOV_EXCL_STOP */
+			}
 			v *= 10;
-			v += c - '0';
+
+			digit = c - '0';
+			if (v > 0xFFFFFFFFU - digit) {
+				/* LCOV_EXCL_START */
+				/* overflow */
+				return -1;
+				/* LCOV_EXCL_STOP */
+			}
+			v += digit;
+
 			c = sgetc(f);
 		}
 

cmdline/stream.h

@@ -28,7 +28,7 @@
  *
  * It's not a constant for testing purpose.
  */
-unsigned STREAM_SIZE;
+extern unsigned STREAM_SIZE;
 
 #define STREAM_STATE_READ 0 /**< The stream is in a normal state of read. */
 #define STREAM_STATE_WRITE 1 /**< The stream is in a normal state of write. */

cmdline/support.c

@@ -25,51 +25,51 @@
 /**
  * Locks used externally.
  */
-#if HAVE_PTHREAD
-static pthread_mutex_t msg_lock;
-static pthread_mutex_t memory_lock;
+#if HAVE_THREAD
+static thread_mutex_t msg_lock;
+static thread_mutex_t memory_lock;
 #endif
 
 void lock_msg(void)
 {
-#if HAVE_PTHREAD
+#if HAVE_THREAD
 	thread_mutex_lock(&msg_lock);
 #endif
 }
 
 void unlock_msg(void)
 {
-#if HAVE_PTHREAD
+#if HAVE_THREAD
 	thread_mutex_unlock(&msg_lock);
 #endif
 }
 
 void lock_memory(void)
 {
-#if HAVE_PTHREAD
+#if HAVE_THREAD
 	thread_mutex_lock(&memory_lock);
 #endif
 }
 
 void unlock_memory(void)
 {
-#if HAVE_PTHREAD
+#if HAVE_THREAD
 	thread_mutex_unlock(&memory_lock);
 #endif
 }
 
 void lock_init(void)
 {
-#if HAVE_PTHREAD
+#if HAVE_THREAD
 	/* initialize the locks as first operation as log_fatal depends on them */
-	thread_mutex_init(&msg_lock, 0);
-	thread_mutex_init(&memory_lock, 0);
+	thread_mutex_init(&msg_lock);
+	thread_mutex_init(&memory_lock);
 #endif
 }
 
 void lock_done(void)
 {
-#if HAVE_PTHREAD
+#if HAVE_THREAD
 	thread_mutex_destroy(&msg_lock);
 	thread_mutex_destroy(&memory_lock);
 #endif
@@ -79,6 +79,7 @@ void lock_done(void)
 /* print */
 
 int msg_level = 0;
+FILE* stdlog = 0;
 
 /*
  * Note that in the following functions we always flush both
@@ -662,6 +663,20 @@ void pathcat(char* dst, size_t size, const char* src)
 	memcpy(dst + dst_len, src, src_len + 1);
 }
 
+void pathcatl(char* dst, size_t dst_len, size_t size, const char* src)
+{
+	size_t src_len = strlen(src);
+
+	if (dst_len + src_len + 1 > size) {
+		/* LCOV_EXCL_START */
+		log_fatal("Path too long '%s%s'\n", dst, src);
+		os_abort();
+		/* LCOV_EXCL_STOP */
+	}
+
+	memcpy(dst + dst_len, src, src_len + 1);
+}
+
 void pathcatc(char* dst, size_t size, char c)
 {
 	size_t dst_len = strlen(dst);
@@ -811,7 +826,7 @@ int mkancestor(const char* file)
 	}
 
 #ifdef _WIN32
-	/* if it's a drive specificaion like "C:" */
+	/* if it's a drive specification like "C:" */
 	if (isalpha(dir[0]) && dir[1] == ':' && dir[2] == 0) {
 		/* nothing more to do */
 		return 0;
@@ -1236,7 +1251,7 @@ int advise_read(struct advise_struct* advise, int f, data_off_t offset, data_off
 	 * non-blocking and do this work in a workqueue (or via some kind of
 	 * callback/continuation scheme).  My worry is just doing this if a user
 	 * application does something crazy, like request gigabytes and gigabytes
-	 * of readahead, and then repents of their craziness, there should be a
+	 * of readahead, and then repented of their craziness, there should be a
 	 * way of cancelling the readahead request.  Today, the user can just
 	 * kill the application.  But if we simply shove the work to a kernel
 	 * thread, it becomes a lot harder to cancel the readahead request.  We'd
@@ -1538,10 +1553,10 @@ int smartctl_flush(FILE* f, const char* file, const char* name)
 /****************************************************************************/
 /* thread */
 
-#if HAVE_PTHREAD
-void thread_mutex_init(pthread_mutex_t* mutex, pthread_mutexattr_t* attr)
+#if HAVE_THREAD
+void thread_mutex_init(thread_mutex_t* mutex)
 {
-	if (pthread_mutex_init(mutex, attr) != 0) {
+	if (pthread_mutex_init(mutex, 0) != 0) {
 		/* LCOV_EXCL_START */
 		log_fatal("Failed call to pthread_mutex_init().\n");
 		os_abort();
@@ -1549,7 +1564,7 @@ void thread_mutex_init(pthread_mutex_t* mutex, pthread_mutexattr_t* attr)
 	}
 }
 
-void thread_mutex_destroy(pthread_mutex_t* mutex)
+void thread_mutex_destroy(thread_mutex_t* mutex)
 {
 	if (pthread_mutex_destroy(mutex) != 0) {
 		/* LCOV_EXCL_START */
@@ -1559,7 +1574,7 @@ void thread_mutex_destroy(pthread_mutex_t* mutex)
 	}
 }
 
-void thread_mutex_lock(pthread_mutex_t* mutex)
+void thread_mutex_lock(thread_mutex_t* mutex)
 {
 	if (pthread_mutex_lock(mutex) != 0) {
 		/* LCOV_EXCL_START */
@@ -1569,7 +1584,7 @@ void thread_mutex_lock(pthread_mutex_t* mutex)
 	}
 }
 
-void thread_mutex_unlock(pthread_mutex_t* mutex)
+void thread_mutex_unlock(thread_mutex_t* mutex)
 {
 	if (pthread_mutex_unlock(mutex) != 0) {
 		/* LCOV_EXCL_START */
@@ -1579,9 +1594,9 @@ void thread_mutex_unlock(pthread_mutex_t* mutex)
 	}
 }
 
-void thread_cond_init(pthread_cond_t* cond, pthread_condattr_t* attr)
+void thread_cond_init(thread_cond_t* cond)
 {
-	if (pthread_cond_init(cond, attr) != 0) {
+	if (pthread_cond_init(cond, 0) != 0) {
 		/* LCOV_EXCL_START */
 		log_fatal("Failed call to pthread_cond_init().\n");
 		os_abort();
@@ -1589,7 +1604,7 @@ void thread_cond_init(pthread_cond_t* cond, pthread_condattr_t* attr)
 	}
 }
 
-void thread_cond_destroy(pthread_cond_t* cond)
+void thread_cond_destroy(thread_cond_t* cond)
 {
 	if (pthread_cond_destroy(cond) != 0) {
 		/* LCOV_EXCL_START */
@@ -1599,7 +1614,7 @@ void thread_cond_destroy(pthread_cond_t* cond)
 	}
 }
 
-void thread_cond_signal(pthread_cond_t* cond)
+void thread_cond_signal(thread_cond_t* cond)
 {
 	if (pthread_cond_signal(cond) != 0) {
 		/* LCOV_EXCL_START */
@@ -1609,7 +1624,7 @@ void thread_cond_signal(pthread_cond_t* cond)
 	}
 }
 
-void thread_cond_broadcast(pthread_cond_t* cond)
+void thread_cond_broadcast(thread_cond_t* cond)
 {
 	if (pthread_cond_broadcast(cond) != 0) {
 		/* LCOV_EXCL_START */
@@ -1619,7 +1634,7 @@ void thread_cond_broadcast(pthread_cond_t* cond)
 	}
 }
 
-void thread_cond_wait(pthread_cond_t* cond, pthread_mutex_t* mutex)
+void thread_cond_wait(thread_cond_t* cond, thread_mutex_t* mutex)
 {
 	if (pthread_cond_wait(cond, mutex) != 0) {
 		/* LCOV_EXCL_START */
@@ -1632,16 +1647,16 @@ void thread_cond_wait(pthread_cond_t* cond, pthread_mutex_t* mutex)
 /**
  * Implementation note about conditional variables.
  *
- * The conditional variables can be signaled inside or ouside the mutex,
- * what is better it's debatable but in general doing that ouside the mutex,
+ * The conditional variables can be signaled inside or outside the mutex,
+ * what is better it's debatable but in general doing that outside the mutex,
  * reduces the number of context switches.
  *
- * But when when testing with helgrind and drd, this disallows such tools to
+ * But when testing with helgrind and drd, this disallows such tools to
  * to see the dependency between the signal and the wait.
  *
  * To avoid it we signal everything inside the mutex. And we do this in both
- * test mode (with CHERCKER defined) and release mode (CHECKER not defined),
- * to be on the safe side and avoid any difference in beaviour between test and
+ * test mode (with CHECKER defined) and release mode (CHECKER not defined),
+ * to be on the safe side and avoid any difference in behaviour between test and
  * release.
  *
  * Here some interesting discussion:
@@ -1658,7 +1673,7 @@ void thread_cond_wait(pthread_cond_t* cond, pthread_mutex_t* mutex)
  */
 int thread_cond_signal_outside = 0;
 
-void thread_cond_signal_and_unlock(pthread_cond_t* cond, pthread_mutex_t* mutex)
+void thread_cond_signal_and_unlock(thread_cond_t* cond, thread_mutex_t* mutex)
 {
 	if (thread_cond_signal_outside) {
 		/* without the thread checker unlock before signaling, */
@@ -1675,7 +1690,7 @@ void thread_cond_signal_and_unlock(pthread_cond_t* cond, pthread_mutex_t* mutex)
 	}
 }
 
-void thread_cond_broadcast_and_unlock(pthread_cond_t* cond, pthread_mutex_t* mutex)
+void thread_cond_broadcast_and_unlock(thread_cond_t* cond, thread_mutex_t* mutex)
 {
 	if (thread_cond_signal_outside) {
 		/* without the thread checker unlock before signaling, */
@@ -1692,9 +1707,9 @@ void thread_cond_broadcast_and_unlock(pthread_cond_t* cond, pthread_mutex_t* mut
 	}
 }
 
-void thread_create(pthread_t* thread, pthread_attr_t* attr, void *(* func)(void *), void *arg)
+void thread_create(thread_id_t* thread, void* (* func)(void *), void *arg)
 {
-	if (pthread_create(thread, attr, func, arg) != 0) {
+	if (pthread_create(thread, 0, func, arg) != 0) {
 		/* LCOV_EXCL_START */
 		log_fatal("Failed call to pthread_create().\n");
 		os_abort();
@@ -1702,7 +1717,7 @@ void thread_create(pthread_t* thread, pthread_attr_t* attr, void *(* func)(void
 	}
 }
 
-void thread_join(pthread_t thread, void** retval)
+void thread_join(thread_id_t thread, void** retval)
 {
 	if (pthread_join(thread, retval) != 0) {
 		/* LCOV_EXCL_START */

cmdline/support.h

@@ -109,7 +109,7 @@ void log_flush(void);
 /**
  * Pointer to log function.
  */
-typedef void fptr(const char* format, ...);
+typedef void fptr(const char* format, ...) __attribute__((format(attribute_printf, 1, 2)));
 
 /****************************************************************************/
 /* message */
@@ -265,6 +265,12 @@ void pathcpy(char* dst, size_t size, const char* src);
  */
 void pathcat(char* dst, size_t size, const char* src);
 
+/**
+ * Concatenate a path limiting the size knowing the length.
+ * Abort if too long.
+ */
+void pathcatl(char* dst, size_t dst_len, size_t size, const char* src);
+
 /**
  * Concatenate a path limiting the size.
  * Abort if too long.
@@ -299,6 +305,8 @@ void pathslash(char* dst, size_t size);
 
 /**
  * Cut everything after the latest slash.
+ *
+ * If the string doesn't contain any slash, it returns the empty string.
  */
 void pathcut(char* dst);
 
@@ -405,7 +413,7 @@ int smartctl_flush(FILE* f, const char* file, const char* name);
 /****************************************************************************/
 /* thread */
 
-#if HAVE_PTHREAD
+#if HAVE_THREAD
 /**
  * Control when to signal the condition variables.
  *
@@ -413,24 +421,24 @@ int smartctl_flush(FILE* f, const char* file, const char* name);
  *
  * Ensure to change that before starting any thread.
  */
-int thread_cond_signal_outside;
+extern int thread_cond_signal_outside;
 
 /**
  * Thread wrappers to handle error conditions.
  */
-void thread_mutex_init(pthread_mutex_t* mutex, pthread_mutexattr_t* attr);
-void thread_mutex_destroy(pthread_mutex_t* mutex);
-void thread_mutex_lock(pthread_mutex_t* mutex);
-void thread_mutex_unlock(pthread_mutex_t* mutex);
-void thread_cond_init(pthread_cond_t* cond, pthread_condattr_t* attr);
-void thread_cond_destroy(pthread_cond_t* cond);
-void thread_cond_signal(pthread_cond_t* cond);
-void thread_cond_broadcast(pthread_cond_t* cond);
-void thread_cond_wait(pthread_cond_t* cond, pthread_mutex_t* mutex);
-void thread_cond_signal_and_unlock(pthread_cond_t* cond, pthread_mutex_t* mutex);
-void thread_cond_broadcast_and_unlock(pthread_cond_t* cond, pthread_mutex_t* mutex);
-void thread_create(pthread_t* thread, pthread_attr_t* attr, void *(* func)(void *), void *arg);
-void thread_join(pthread_t thread, void** retval);
+void thread_mutex_init(thread_mutex_t* mutex);
+void thread_mutex_destroy(thread_mutex_t* mutex);
+void thread_mutex_lock(thread_mutex_t* mutex);
+void thread_mutex_unlock(thread_mutex_t* mutex);
+void thread_cond_init(thread_cond_t* cond);
+void thread_cond_destroy(thread_cond_t* cond);
+void thread_cond_signal(thread_cond_t* cond);
+void thread_cond_broadcast(thread_cond_t* cond);
+void thread_cond_wait(thread_cond_t* cond, thread_mutex_t* mutex);
+void thread_cond_signal_and_unlock(thread_cond_t* cond, thread_mutex_t* mutex);
+void thread_cond_broadcast_and_unlock(thread_cond_t* cond, thread_mutex_t* mutex);
+void thread_create(thread_id_t* thread, void* (* func)(void *), void *arg);
+void thread_join(thread_id_t thread, void** retval);
 #endif
 
 #endif

cmdline/sync.c

@@ -440,9 +440,8 @@ struct snapraid_rehash {
 /**
  * Check if we have to process the specified block index ::i.
  */
-static int block_is_enabled(void* void_plan, block_off_t i)
+static int block_is_enabled(struct snapraid_plan* plan, block_off_t i)
 {
-	struct snapraid_plan* plan = void_plan;
 	unsigned j;
 	int one_invalid;
 	int one_valid;
@@ -693,6 +692,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
 	unsigned* waiting_map;
 	unsigned waiting_mac;
 	char esc_buffer[ESC_MAX];
+	bit_vect_t* block_enabled;
 
 	/* the sync process assumes that all the hashes are correct */
 	/* including the ones from CHG and DELETED blocks */
@@ -732,14 +732,18 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
 	silent_error = 0;
 	io_error = 0;
 
+	msg_progress("Selecting...\n");
+
 	/* first count the number of blocks to process */
 	countmax = 0;
 	plan.handle_max = diskmax;
 	plan.handle_map = handle;
 	plan.force_full = state->opt.force_full;
+	block_enabled = calloc_nofail(1, bit_vect_size(blockmax)); /* preinitialize to 0 */
 	for (blockcur = blockstart; blockcur < blockmax; ++blockcur) {
 		if (!block_is_enabled(&plan, blockcur))
 			continue;
+		bit_vect_set(block_enabled, blockcur);
 		++countmax;
 	}
 
@@ -756,8 +760,10 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
 	countsize = 0;
 	countpos = 0;
 
+	msg_progress("Syncing...\n");
+
 	/* start all the worker threads */
-	io_start(&io, blockstart, blockmax, &block_is_enabled, &plan);
+	io_start(&io, blockstart, blockmax, block_enabled);
 
 	if (!state_progress_begin(state, blockstart, blockmax, countmax))
 		goto end;
@@ -1016,7 +1022,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
 		}
 
 		/* if we have only silent errors we can try to fix them on-the-fly */
-		/* note the the fix is not written to disk, but used only to */
+		/* note the fix is not written to disk, but used only to */
 		/* compute the new parity */
 		if (!error_on_this_block && !io_error_on_this_block && silent_error_on_this_block) {
 			unsigned failed_mac;
@@ -1413,6 +1419,7 @@ bail:
 	free(failed_map);
 	free(waiting_map);
 	io_done(&io);
+	free(block_enabled);
 
 	if (state->opt.expect_recoverable) {
 		if (error + silent_error + io_error == 0)
@@ -1536,7 +1543,7 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
 				data_off_t out_size;
 				parity_size(&parity_handle[l], &out_size);
 				parity_overflow(state, out_size);
-				log_fatal("WARNING! Without an unsable %s file, it isn't possible to sync.\n", lev_name(l));
+				log_fatal("WARNING! Without a usable %s file, it isn't possible to sync.\n", lev_name(l));
 				exit(EXIT_FAILURE);
 				/* LCOV_EXCL_STOP */
 			}
@@ -1575,8 +1582,6 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
 			log_fatal("WARNING! Skipped state write for --test-skip-content-write option.\n");
 		}
 
-		msg_progress("Syncing...\n");
-
 		/* skip degenerated cases of empty parity, or skipping all */
 		if (blockstart < blockmax) {
 			ret = state_sync_process(state, parity_handle, blockstart, blockmax);

cmdline/unix.c

@@ -474,7 +474,7 @@ static int devuuid_dev(uint64_t device, char* uuid, size_t uuid_size)
 #endif
 
 /**
- * Get the UUID using liblkid.
+ * Get the UUID using libblkid.
  * It uses a cache to work without root permission, resulting in UUID
  * not necessarily recent.
  * We could call blkid_probe_all() to refresh the UUID, but it would
@@ -632,7 +632,7 @@ int filephy(const char* path, uint64_t size, uint64_t* physical)
 	/* In this way we keep them in the directory traversal order */
 	/* that at least keeps files in the same directory together. */
 	/* Note also that in newer file-system with snapshot, like ZFS, */
-	/* the inode doesn't represent evenmore the disk position, because files */
+	/* the inode doesn't represent even more the disk position, because files */
 	/* are not overwritten in place, but rewritten in another location */
 	/* of the disk. */
 	*physical = FILEPHY_UNREPORTED_OFFSET;
@@ -917,18 +917,20 @@ static int devtree(const char* name, const char* custom, dev_t device, devinfo_t
 
 		while ((dd = readdir(d)) != 0) {
 			if (dd->d_name[0] != '.') {
+				dev_t subdev;
+
 				/* for each slave, expand the full potential tree */
 				pathprint(path, sizeof(path), "/sys/dev/block/%u:%u/slaves/%s/dev", major(device), minor(device), dd->d_name);
 
-				device = devread(path);
-				if (!device) {
+				subdev = devread(path);
+				if (!subdev) {
 					/* LCOV_EXCL_START */
 					closedir(d);
 					return -1;
 					/* LCOV_EXCL_STOP */
 				}
 
-				if (devtree(name, custom, device, parent, list) != 0) {
+				if (devtree(name, custom, subdev, parent, list) != 0) {
 					/* LCOV_EXCL_START */
 					closedir(d);
 					return -1;
@@ -1072,8 +1074,8 @@ static int devscan(tommy_list* list)
 #if HAVE_LINUX_DEVICE
 static int devsmart(dev_t device, const char* name, const char* custom, uint64_t* smart, char* serial, char* vendor, char* model)
 {
-	char cmd[128];
-	char file[128];
+	char cmd[PATH_MAX + 64];
+	char file[PATH_MAX];
 	FILE* f;
 	int ret;
 
@@ -1086,7 +1088,7 @@ static int devsmart(dev_t device, const char* name, const char* custom, uint64_t
 
 	/* if there is a custom command */
 	if (custom[0]) {
-		char option[128];
+		char option[PATH_MAX];
 		snprintf(option, sizeof(option), custom, file);
 		snprintf(cmd, sizeof(cmd), "smartctl -a %s", option);
 	} else {
@@ -1140,8 +1142,8 @@ static int devsmart(dev_t device, const char* name, const char* custom, uint64_t
 #if HAVE_LINUX_DEVICE
 static int devdown(dev_t device, const char* name, const char* custom)
 {
-	char cmd[128];
-	char file[128];
+	char cmd[PATH_MAX + 64];
+	char file[PATH_MAX];
 	FILE* f;
 	int ret;
 
@@ -1154,7 +1156,7 @@ static int devdown(dev_t device, const char* name, const char* custom)
 
 	/* if there is a custom command */
 	if (custom[0]) {
-		char option[128];
+		char option[PATH_MAX];
 		snprintf(option, sizeof(option), custom, file);
 		snprintf(cmd, sizeof(cmd), "smartctl -s standby,now %s", option);
 	} else {
@@ -1323,12 +1325,12 @@ static int device_thread(tommy_list* list, void* (*func)(void* arg))
 	int fail = 0;
 	tommy_node* i;
 
-#if HAVE_PTHREAD
+#if HAVE_THREAD
 	/* start all threads */
 	for (i = tommy_list_head(list); i != 0; i = i->next) {
 		devinfo_t* devinfo = i->data;
 
-		thread_create(&devinfo->thread, 0, func, devinfo);
+		thread_create(&devinfo->thread, func, devinfo);
 	}
 
 	/* join all threads */

cmdline/unix.h

@@ -59,7 +59,7 @@ int dirent_hidden(struct dirent* dd);
 const char* stat_desc(struct stat* st);
 
 /**
- * Return the aligment requirement for direct IO.
+ * Return the alignment requirement for direct IO.
  */
 size_t direct_size(void);
 

cmdline/util.c

@@ -470,8 +470,24 @@ static inline uint64_t util_rotl64(uint64_t x, int8_t r)
 	return (x << r) | (x >> (64 - r));
 }
 
+/*
+ * Rotate right.
+ * In x86/x64 they are optimized with a single assembler instruction.
+ */
+#if 0 /* unused */
+static inline uint32_t util_rotr32(uint32_t x, int8_t r)
+{
+	return (x >> r) | (x << (32 - r));
+}
+#endif
+static inline uint64_t util_rotr64(uint64_t x, int8_t r)
+{
+	return (x >> r) | (x << (64 - r));
+}
+
+
 /**
- * Swap endianess.
+ * Swap endianness.
  * They are needed only if BigEndian.
  */
 #if defined(__GNUC__)
@@ -502,6 +518,18 @@ static inline uint64_t util_swap64(uint64_t v)
 }
 #endif
 
+static inline uint8_t util_read8(const void* void_ptr)
+{
+	const uint8_t* ptr = void_ptr;
+	return ptr[0];
+}
+
+static inline uint16_t util_read16(const void* void_ptr)
+{
+	const uint8_t* ptr = void_ptr;
+	return ptr[0] + (ptr[1] << 8);
+}
+
 static inline uint32_t util_read32(const void* ptr)
 {
 	uint32_t v;
@@ -543,6 +571,7 @@ static inline void util_write64(void* ptr, uint64_t v)
 
 #include "murmur3.c"
 #include "spooky2.c"
+#include "metro.c"
 
 void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* src, size_t size)
 {
@@ -553,6 +582,9 @@ void memhash(unsigned kind, const unsigned char* seed, void* digest, const void*
 	case HASH_SPOOKY2 :
 		SpookyHash128(src, size, seed, digest);
 		break;
+	case HASH_METRO :
+		MetroHash128(src, size, seed, digest);
+		break;
 	default :
 		/* LCOV_EXCL_START */
 		log_fatal("Internal inconsistency in hash function %u\n", kind);
@@ -568,6 +600,7 @@ const char* hash_config_name(unsigned kind)
 	case HASH_UNDEFINED : return "undefined";
 	case HASH_MURMUR3 : return "murmur3";
 	case HASH_SPOOKY2 : return "spooky2";
+	case HASH_METRO : return "metro";
 	default :
 		/* LCOV_EXCL_START */
 		return "unknown";

cmdline/util.h

@@ -18,6 +18,7 @@
 #ifndef __UTIL_H
 #define __UTIL_H
 
+
 /****************************************************************************/
 /* memory */
 
@@ -161,7 +162,7 @@ static inline uint32_t crc32c_plain(uint32_t crc, const unsigned char* ptr, unsi
 /**
  * Compute the CRC-32 (Castagnoli)
  */
-uint32_t (*crc32c)(uint32_t crc, const unsigned char* ptr, unsigned size);
+extern uint32_t (*crc32c)(uint32_t crc, const unsigned char* ptr, unsigned size);
 
 /**
  * Internal entry points for testing.
@@ -188,6 +189,7 @@ void crc32c_init(void);
 #define HASH_UNDEFINED 0
 #define HASH_MURMUR3 1
 #define HASH_SPOOKY2 2
+#define HASH_METRO 3
 
 /**
  * Compute the HASH of a memory block.
@@ -220,5 +222,34 @@ int lock_lock(const char* file);
  */
 int lock_unlock(int f);
 
+/****************************************************************************/
+/* bitvect */
+
+typedef unsigned char bit_vect_t;
+#define BIT_VECT_SIZE (sizeof(bit_vect_t) * 8)
+
+static inline size_t bit_vect_size(size_t max)
+{
+	return (max + BIT_VECT_SIZE - 1) / BIT_VECT_SIZE;
+}
+
+static inline void bit_vect_set(bit_vect_t* bit_vect, size_t off)
+{
+	bit_vect_t mask = 1 << (off % BIT_VECT_SIZE);
+	bit_vect[off / BIT_VECT_SIZE] |= mask;
+}
+
+static inline void bit_vect_clear(bit_vect_t* bit_vect, size_t off)
+{
+	bit_vect_t mask = 1 << (off % BIT_VECT_SIZE);
+	bit_vect[off / BIT_VECT_SIZE] &= ~mask;
+}
+
+static inline int bit_vect_test(bit_vect_t* bit_vect, size_t off)
+{
+	bit_vect_t mask = 1 << (off % BIT_VECT_SIZE);
+	return (bit_vect[off / BIT_VECT_SIZE] & mask) != 0;
+}
+
 #endif
 

config.h.in

@@ -33,6 +33,10 @@
 /* Define to 1 if you have the `clock_gettime' function. */
 #undef HAVE_CLOCK_GETTIME
 
+/* Define to 1 if you have the declaration of `statfs', and to 0 if you don't.
+   */
+#undef HAVE_DECL_STATFS
+
 /* Define to 1 if you have the <dirent.h> header file, and it defines `DIR'.
    */
 #undef HAVE_DIRENT_H
@@ -224,6 +228,9 @@
 /* Define to 1 if you have the <sys/ioctl.h> header file. */
 #undef HAVE_SYS_IOCTL_H
 
+/* Define to 1 if you have the <sys/mkdev.h> header file. */
+#undef HAVE_SYS_MKDEV_H
+
 /* Define to 1 if you have the <sys/mount.h> header file. */
 #undef HAVE_SYS_MOUNT_H
 
@@ -240,6 +247,9 @@
 /* Define to 1 if you have the <sys/stat.h> header file. */
 #undef HAVE_SYS_STAT_H
 
+/* Define to 1 if you have the <sys/sysmacros.h> header file. */
+#undef HAVE_SYS_SYSMACROS_H
+
 /* Define to 1 if you have the <sys/types.h> header file. */
 #undef HAVE_SYS_TYPES_H
 
@@ -258,14 +268,6 @@
 /* Define to 1 if you have the `vsnprintf' function. */
 #undef HAVE_VSNPRINTF
 
-/* Define to 1 if `major', `minor', and `makedev' are declared in <mkdev.h>.
-   */
-#undef MAJOR_IN_MKDEV
-
-/* Define to 1 if `major', `minor', and `makedev' are declared in
-   <sysmacros.h>. */
-#undef MAJOR_IN_SYSMACROS
-
 /* Define to 1 if assertions should be disabled. */
 #undef NDEBUG
 

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for snapraid 11.2.
+# Generated by GNU Autoconf 2.69 for snapraid 12.2.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -577,8 +577,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='snapraid'
 PACKAGE_TARNAME='snapraid'
-PACKAGE_VERSION='11.2'
-PACKAGE_STRING='snapraid 11.2'
+PACKAGE_VERSION='12.2'
+PACKAGE_STRING='snapraid 12.2'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL='http://www.snapraid.it'
 
@@ -1304,7 +1304,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures snapraid 11.2 to adapt to many kinds of systems.
+\`configure' configures snapraid 12.2 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1374,7 +1374,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of snapraid 11.2:";;
+     short | recursive ) echo "Configuration of snapraid 12.2:";;
    esac
   cat <<\_ACEOF
 
@@ -1496,7 +1496,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-snapraid configure 11.2
+snapraid configure 12.2
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2098,11 +2098,57 @@ $as_echo "$ac_res" >&6; }
   eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
 
 } # ac_fn_c_check_func
+
+# ac_fn_c_check_decl LINENO SYMBOL VAR INCLUDES
+# ---------------------------------------------
+# Tests whether SYMBOL is declared in INCLUDES, setting cache variable VAR
+# accordingly.
+ac_fn_c_check_decl ()
+{
+  as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack
+  as_decl_name=`echo $2|sed 's/ *(.*//'`
+  as_decl_use=`echo $2|sed -e 's/(/((/' -e 's/)/) 0&/' -e 's/,/) 0& (/g'`
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $as_decl_name is declared" >&5
+$as_echo_n "checking whether $as_decl_name is declared... " >&6; }
+if eval \${$3+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+$4
+int
+main ()
+{
+#ifndef $as_decl_name
+#ifdef __cplusplus
+  (void) $as_decl_use;
+#else
+  (void) $as_decl_name;
+#endif
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+  eval "$3=yes"
+else
+  eval "$3=no"
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+eval ac_res=\$$3
+	       { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5
+$as_echo "$ac_res" >&6; }
+  eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno
+
+} # ac_fn_c_check_decl
 cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by snapraid $as_me 11.2, which was
+It was created by snapraid $as_me 12.2, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -2450,7 +2496,7 @@ ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $
 ac_compiler_gnu=$ac_cv_c_compiler_gnu
 
 
-am__api_version='1.15'
+am__api_version='1.16'
 
 ac_aux_dir=
 for ac_dir in "$srcdir" "$srcdir/.." "$srcdir/../.."; do
@@ -2965,7 +3011,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='snapraid'
- VERSION='11.2'
+ VERSION='12.2'
 
 
 cat >>confdefs.h <<_ACEOF
@@ -2995,8 +3041,8 @@ MAKEINFO=${MAKEINFO-"${am_missing_run}makeinfo"}
 
 # For better backward compatibility.  To be removed once Automake 1.9.x
 # dies out for good.  For more background, see:
-# <http://lists.gnu.org/archive/html/automake/2012-07/msg00001.html>
-# <http://lists.gnu.org/archive/html/automake/2012-07/msg00014.html>
+# <https://lists.gnu.org/archive/html/automake/2012-07/msg00001.html>
+# <https://lists.gnu.org/archive/html/automake/2012-07/msg00014.html>
 mkdir_p='$(MKDIR_P)'
 
 # We need awk for the "check" target (and possibly the TAP driver).  The
@@ -3047,7 +3093,7 @@ END
 Aborting the configuration process, to ensure you take notice of the issue.
 
 You can download and install GNU coreutils to get an 'rm' implementation
-that behaves properly: <http://www.gnu.org/software/coreutils/>.
+that behaves properly: <https://www.gnu.org/software/coreutils/>.
 
 If you want to complete the configuration process using your problematic
 'rm' anyway, export the environment variable ACCEPT_INFERIOR_RM_PROGRAM
@@ -4657,6 +4703,22 @@ $as_echo "no" >&6; }
 fi
 rm -f conftest*
 
+# This the new default for gcc 10
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ${CC-cc} accepts -fno-common" >&5
+$as_echo_n "checking whether ${CC-cc} accepts -fno-common... " >&6; }
+echo 'void f(){}' > conftest.c
+if test -z "`${CC-cc} -c -fno-common conftest.c 2>&1`"; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+  CFLAGS="$CFLAGS -fno-common"
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+fi
+rm -f conftest*
+
 
 # Check whether --enable-largefile was given.
 if test "${enable_largefile+set}" = set; then :
@@ -5115,56 +5177,6 @@ $as_echo "#define HAVE_SYS_WAIT_H 1" >>confdefs.h
 
 fi
 
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether sys/types.h defines makedev" >&5
-$as_echo_n "checking whether sys/types.h defines makedev... " >&6; }
-if ${ac_cv_header_sys_types_h_makedev+:} false; then :
-  $as_echo_n "(cached) " >&6
-else
-  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h.  */
-#include <sys/types.h>
-int
-main ()
-{
-return makedev(0, 0);
-  ;
-  return 0;
-}
-_ACEOF
-if ac_fn_c_try_link "$LINENO"; then :
-  ac_cv_header_sys_types_h_makedev=yes
-else
-  ac_cv_header_sys_types_h_makedev=no
-fi
-rm -f core conftest.err conftest.$ac_objext \
-    conftest$ac_exeext conftest.$ac_ext
-
-fi
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_sys_types_h_makedev" >&5
-$as_echo "$ac_cv_header_sys_types_h_makedev" >&6; }
-
-if test $ac_cv_header_sys_types_h_makedev = no; then
-ac_fn_c_check_header_mongrel "$LINENO" "sys/mkdev.h" "ac_cv_header_sys_mkdev_h" "$ac_includes_default"
-if test "x$ac_cv_header_sys_mkdev_h" = xyes; then :
-
-$as_echo "#define MAJOR_IN_MKDEV 1" >>confdefs.h
-
-fi
-
-
-
-  if test $ac_cv_header_sys_mkdev_h = no; then
-    ac_fn_c_check_header_mongrel "$LINENO" "sys/sysmacros.h" "ac_cv_header_sys_sysmacros_h" "$ac_includes_default"
-if test "x$ac_cv_header_sys_sysmacros_h" = xyes; then :
-
-$as_echo "#define MAJOR_IN_SYSMACROS 1" >>confdefs.h
-
-fi
-
-
-  fi
-fi
-
 for ac_header in fcntl.h stddef.h stdint.h stdlib.h string.h limits.h
 do :
   as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh`
@@ -5204,7 +5216,7 @@ fi
 
 done
 
-for ac_header in sys/file.h sys/ioctl.h sys/vfs.h sys/statfs.h sys/param.h sys/mount.h
+for ac_header in sys/file.h sys/ioctl.h sys/sysmacros.h sys/mkdev.h
 do :
   as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh`
 ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default"
@@ -5670,54 +5682,6 @@ cat >>confdefs.h <<_ACEOF
 _ACEOF
 
 
-fi
-
-ac_fn_c_check_member "$LINENO" "struct statfs" "f_type" "ac_cv_member_struct_statfs_f_type" "
-#if HAVE_SYS_PARAM_H
-#include <sys/param.h>
-#endif
-#if HAVE_SYS_MOUNT_H
-#include <sys/mount.h>
-#endif
-#if HAVE_SYS_VFS_H
-#include <sys/vfs.h>
-#endif
-#if HAVE_SYS_STATFS_H
-#include <sys/statfs.h>
-#endif
-
-"
-if test "x$ac_cv_member_struct_statfs_f_type" = xyes; then :
-
-cat >>confdefs.h <<_ACEOF
-#define HAVE_STRUCT_STATFS_F_TYPE 1
-_ACEOF
-
-
-fi
-
-ac_fn_c_check_member "$LINENO" "struct statfs" "f_fstypename" "ac_cv_member_struct_statfs_f_fstypename" "
-#if HAVE_SYS_PARAM_H
-#include <sys/param.h>
-#endif
-#if HAVE_SYS_MOUNT_H
-#include <sys/mount.h>
-#endif
-#if HAVE_SYS_VFS_H
-#include <sys/vfs.h>
-#endif
-#if HAVE_SYS_STATFS_H
-#include <sys/statfs.h>
-#endif
-
-"
-if test "x$ac_cv_member_struct_statfs_f_fstypename" = xyes; then :
-
-cat >>confdefs.h <<_ACEOF
-#define HAVE_STRUCT_STATFS_F_FSTYPENAME 1
-_ACEOF
-
-
 fi
 
 
@@ -5793,7 +5757,7 @@ _ACEOF
 fi
 done
 
-for ac_func in fstatat flock statfs
+for ac_func in fstatat flock
 do :
   as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
 ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var"
@@ -5978,6 +5942,119 @@ if test "$ac_res" != no; then :
 fi
 
 
+for ac_header in sys/vfs.h sys/statfs.h
+do :
+  as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh`
+ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default"
+if eval test \"x\$"$as_ac_Header"\" = x"yes"; then :
+  cat >>confdefs.h <<_ACEOF
+#define `$as_echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+
+done
+
+ac_fn_c_check_decl "$LINENO" "statfs" "ac_cv_have_decl_statfs" "
+#if HAVE_SYS_VFS_H
+#include <sys/vfs.h>
+#endif
+#if HAVE_SYS_STATFS_H
+#include <sys/statfs.h>
+#endif
+
+"
+if test "x$ac_cv_have_decl_statfs" = xyes; then :
+  ac_have_decl=1
+else
+  ac_have_decl=0
+fi
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_DECL_STATFS $ac_have_decl
+_ACEOF
+if test $ac_have_decl = 1; then :
+
+else
+
+	for ac_header in sys/param.h sys/mount.h
+do :
+  as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh`
+ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default"
+if eval test \"x\$"$as_ac_Header"\" = x"yes"; then :
+  cat >>confdefs.h <<_ACEOF
+#define `$as_echo "HAVE_$ac_header" | $as_tr_cpp` 1
+_ACEOF
+
+fi
+
+done
+
+
+fi
+
+for ac_func in statfs
+do :
+  ac_fn_c_check_func "$LINENO" "statfs" "ac_cv_func_statfs"
+if test "x$ac_cv_func_statfs" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_STATFS 1
+_ACEOF
+
+fi
+done
+
+
+ac_fn_c_check_member "$LINENO" "struct statfs" "f_type" "ac_cv_member_struct_statfs_f_type" "
+#if HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
+#if HAVE_SYS_MOUNT_H
+#include <sys/mount.h>
+#endif
+#if HAVE_SYS_VFS_H
+#include <sys/vfs.h>
+#endif
+#if HAVE_SYS_STATFS_H
+#include <sys/statfs.h>
+#endif
+
+"
+if test "x$ac_cv_member_struct_statfs_f_type" = xyes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_STRUCT_STATFS_F_TYPE 1
+_ACEOF
+
+
+fi
+
+
+ac_fn_c_check_member "$LINENO" "struct statfs" "f_fstypename" "ac_cv_member_struct_statfs_f_fstypename" "
+#if HAVE_SYS_PARAM_H
+#include <sys/param.h>
+#endif
+#if HAVE_SYS_MOUNT_H
+#include <sys/mount.h>
+#endif
+#if HAVE_SYS_VFS_H
+#include <sys/vfs.h>
+#endif
+#if HAVE_SYS_STATFS_H
+#include <sys/statfs.h>
+#endif
+
+"
+if test "x$ac_cv_member_struct_statfs_f_fstypename" = xyes; then :
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_STRUCT_STATFS_F_FSTYPENAME 1
+_ACEOF
+
+
+fi
+
+
 
 # Check whether --with-blkid was given.
 if test "${with_blkid+set}" = set; then :
@@ -6367,6 +6444,22 @@ fi
 rm -f conftest*
 
 
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ${CC-cc} accepts -Wno-zero-length-bounds" >&5
+$as_echo_n "checking whether ${CC-cc} accepts -Wno-zero-length-bounds... " >&6; }
+echo 'void f(){}' > conftest.c
+if test -z "`${CC-cc} -c -Wno-zero-length-bounds conftest.c 2>&1`"; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+  CFLAGS="$CFLAGS -Wno-zero-length-bounds"
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+fi
+rm -f conftest*
+
+
 # Check whether --enable-asm was given.
 if test "${enable_asm+set}" = set; then :
   enableval=$enable_asm;
@@ -7373,7 +7466,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by snapraid $as_me 11.2, which was
+This file was extended by snapraid $as_me 12.2, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -7436,7 +7529,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-snapraid config.status 11.2
+snapraid config.status 12.2
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -17,11 +17,13 @@ AC_CHECK_PROG([SDE],[sde],[sde],[])
 AC_CHECK_PROG([ADVD2],[advd2],[advd2],[])
 AM_CONDITIONAL(HAVE_ADVD2, [test x"$ADVD2" != x])
 
-dnl Compiler option to improve stacktrace
+dnl Options to improve stacktrace
 AC_CHECK_CC_OPT([-fno-omit-frame-pointer], CFLAGS="$CFLAGS -fno-omit-frame-pointer", [])
 AC_CHECK_CC_OPT([-fno-inline-functions-called-once], CFLAGS="$CFLAGS -fno-inline-functions-called-once", [])
 AC_CHECK_CC_OPT([-fno-inline-small-functions], CFLAGS="$CFLAGS -fno-inline-small-functions", [])
 AC_CHECK_CC_OPT([-rdynamic], CFLAGS="$CFLAGS -rdynamic", [])
+# This the new default for gcc 10
+AC_CHECK_CC_OPT([-fno-common], CFLAGS="$CFLAGS -fno-common", [])
 
 dnl Checks for system.
 AC_SYS_LARGEFILE
@@ -31,11 +33,10 @@ AC_HEADER_ASSERT
 AC_HEADER_DIRENT
 AC_HEADER_TIME
 AC_HEADER_SYS_WAIT
-AC_HEADER_MAJOR
 AC_CHECK_HEADERS([fcntl.h stddef.h stdint.h stdlib.h string.h limits.h])
 AC_CHECK_HEADERS([unistd.h getopt.h fnmatch.h io.h inttypes.h byteswap.h])
 AC_CHECK_HEADERS([pthread.h math.h])
-AC_CHECK_HEADERS([sys/file.h sys/ioctl.h sys/vfs.h sys/statfs.h sys/param.h sys/mount.h])
+AC_CHECK_HEADERS([sys/file.h sys/ioctl.h sys/sysmacros.h sys/mkdev.h])
 AC_CHECK_HEADERS([linux/fiemap.h linux/fs.h mach/mach_time.h execinfo.h])
 
 dnl Checks for typedefs, structures, and compiler characteristics.
@@ -64,6 +65,38 @@ AC_CHECK_MEMBERS([struct stat.st_nlink, struct stat.st_mtim.tv_nsec, struct stat
 #include <unistd.h>
 #endif
 ]])
+
+dnl Checks for library functions.
+AC_CHECK_FUNCS([memset strchr strerror strrchr mkdir gettimeofday strtoul])
+AC_CHECK_FUNCS([getopt getopt_long snprintf vsnprintf sigaction])
+AC_CHECK_FUNCS([ftruncate fallocate access])
+AC_CHECK_FUNCS([fsync posix_fadvise sync_file_range])
+AC_CHECK_FUNCS([getc_unlocked ferror_unlocked fnmatch])
+AC_CHECK_FUNCS([futimes futimens futimesat localtime_r lutimes utimensat])
+AC_CHECK_FUNCS([fstatat flock])
+AC_CHECK_FUNCS([mach_absolute_time])
+AC_CHECK_FUNCS([backtrace backtrace_symbols])
+AC_SEARCH_LIBS([clock_gettime], [rt])
+AC_CHECK_FUNCS([clock_gettime])
+AC_CHECK_CC_OPT([-pthread], CFLAGS="$CFLAGS -pthread", CFLAGS="$CFLAGS -D_REENTRANT")
+AC_CHECK_FUNCS([pthread_create])
+AC_SEARCH_LIBS([exp], [m])
+
+dnl Checks for statfs for linux avoiding to include sys/mount.h that is required only in darwin
+dnl In glibc since 7eae6a91e9b1670330c9f15730082c91c0b1d570, milestone 2.36, sys/mount.h defines fsconfig_command which conflicts with linux/mount.h
+AC_CHECK_HEADERS([sys/vfs.h sys/statfs.h])
+AC_CHECK_DECLS([statfs], [], [
+	AC_CHECK_HEADERS([sys/param.h sys/mount.h])
+	], [[
+#if HAVE_SYS_VFS_H
+#include <sys/vfs.h>
+#endif
+#if HAVE_SYS_STATFS_H
+#include <sys/statfs.h>
+#endif
+]])
+AC_CHECK_FUNCS([statfs])
+
 AC_CHECK_MEMBERS([struct statfs.f_type], [], [], [[
 #if HAVE_SYS_PARAM_H
 #include <sys/param.h>
@@ -78,6 +111,7 @@ AC_CHECK_MEMBERS([struct statfs.f_type], [], [], [[
 #include <sys/statfs.h>
 #endif
 ]])
+
 AC_CHECK_MEMBERS([struct statfs.f_fstypename], [], [], [[
 #if HAVE_SYS_PARAM_H
 #include <sys/param.h>
@@ -93,22 +127,6 @@ AC_CHECK_MEMBERS([struct statfs.f_fstypename], [], [], [[
 #endif
 ]])
 
-dnl Checks for library functions.
-AC_CHECK_FUNCS([memset strchr strerror strrchr mkdir gettimeofday strtoul])
-AC_CHECK_FUNCS([getopt getopt_long snprintf vsnprintf sigaction])
-AC_CHECK_FUNCS([ftruncate fallocate access])
-AC_CHECK_FUNCS([fsync posix_fadvise sync_file_range])
-AC_CHECK_FUNCS([getc_unlocked ferror_unlocked fnmatch])
-AC_CHECK_FUNCS([futimes futimens futimesat localtime_r lutimes utimensat])
-AC_CHECK_FUNCS([fstatat flock statfs])
-AC_CHECK_FUNCS([mach_absolute_time])
-AC_CHECK_FUNCS([backtrace backtrace_symbols])
-AC_SEARCH_LIBS([clock_gettime], [rt])
-AC_CHECK_FUNCS([clock_gettime])
-AC_CHECK_CC_OPT([-pthread], CFLAGS="$CFLAGS -pthread", CFLAGS="$CFLAGS -D_REENTRANT")
-AC_CHECK_FUNCS([pthread_create])
-AC_SEARCH_LIBS([exp], [m])
-
 dnl Checks for libblkid
 AC_ARG_WITH([blkid],
     AS_HELP_STRING([--without-blkid], [Ignore presence of blkid and disable it]))
@@ -134,6 +152,9 @@ AC_CHECK_CC_OPT([-Wextra], CFLAGS="$CFLAGS -Wextra", [])
 AC_CHECK_CC_OPT([-Wuninitialized], CFLAGS="$CFLAGS -Wuninitialized", [])
 AC_CHECK_CC_OPT([-Wshadow], CFLAGS="$CFLAGS -Wshadow", [])
 
+dnl Disable warning about zero-length-bounds raised by gcc 10 on linux kernel header on the fm_extents field
+AC_CHECK_CC_OPT([-Wno-zero-length-bounds], CFLAGS="$CFLAGS -Wno-zero-length-bounds", [])
+
 dnl Checks for asm
 AC_ARG_ENABLE([asm],
 	AS_HELP_STRING([--disable-asm], [Disable inline assembly]))
@@ -178,7 +199,7 @@ dnl Checks for AS supporting the SSE4.2 instructions.
 AC_MSG_CHECKING([for sse42])
 asmsse42=no
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
-#if defined(__i386__) || defined(__x86_64__) 
+#if defined(__i386__) || defined(__x86_64__)
 	unsigned f(unsigned crc, unsigned char b)
 	{
 		asm volatile("crc32b %1, %0" : "+r" (crc) : "rm" (b));

debian/changelog

@@ -0,0 +1,693 @@
+snapraid (11.6-0) UNRELEASED; urgency=medium
+
+  * Bump
+  * Bump
+
+ -- Mario Fetka <mario.fetka@gmail.com>  Fri, 11 Sep 2020 13:42:38 +0200
+
+snapraid (11.2-0tikhonov1) xenial; urgency=medium
+
+  [11.2 2017/12]
+  * Fixed recognition of NTFS hardlinks. They behave differently than
+    standard Unix hardlinks and this could result in SnapRAID reporting
+    internal inconsistency errors for detecting links to the same file
+    with different metadata attributes.
+  * More efficient 'pool' command that updates only the links
+    that need to be updated. This ensures that no change is
+    done, avoiding to trigger a directory rescan of other programs.
+  * In Linux use by default the advise "discard" mode instead of "flush".
+    This avoids to swap-out the other process memory, leaving the system
+    more responsive.
+  * Changed the fallocate() use to work better with Btrfs with parity disks.
+  * Changed the --test-io-stats screen to print the file name in process
+    for each disk.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Wed, 18 Jul 2018 19:44:46 +0400
+
+snapraid (11.1-0tikhonov1~trusty) trusty; urgency=medium
+
+  [11.1 2017/05]
+  * Fixed the check command to correctly ignore errors on unused parity.
+    This was broken in version 9.0.
+  * Allow increasing the number of parity splits of existing parity.
+  * Fixed quoting when printing in Linux. This fixes the UTF-8 screen
+    output. Windows version was not affected.
+  * Fixed recognition of 'hashsize' in the configuration file.
+    The previous incorrect 'hash_size' is still supported for backward
+    compatibility.
+  * Fixed building in platforms that don't provide major/minor definitions
+    in sys/types.h.
+  * When creating 'pool' symbolic links, set their time as the linked files.
+  * Added support for the Windows 10 symbolic link unprivileged creation,
+    using SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE.
+  * Windows binaries built with gcc 4.9.4 using the MXE cross compiler at
+    commit ae56efa2b23a793b0146508bfef33027cdb09fd2 with targets
+    i686-w64-mingw32 and x86_64-w64-mingw32 and optimization -O2.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Mon, 28 Aug 2017 12:41:29 +0400
+
+snapraid (11.0-0tikhonov1~trusty) trusty; urgency=medium
+
+  [11.0 2016/11]
+  * Added support for splitting the parity in multiple partitions. You
+    can now specify multiple files for a single parity. As soon a file
+    cannot grow anymore, the next one starts growing.
+    In the configuration file, just put more files in the same 'parity'
+    line, separated by , (comma).
+    Note that if this feature is used, the saved content file won't be
+    read by older SnapRAID versions.
+    In Windows, 256 MB are left free in each disk to avoid the warning
+    about full disks.
+  * Added a new 'hashsize' configuration option. It could be useful in
+    systems with low memory, to reduce the memory usage.
+    Note that if this feature is used, the saved content file won't be
+    read by older SnapRAID versions.
+  * In Linux added the missing support for Btrfs file-systems. Note that
+    to have full support you need also the 'libblkid' library, otherwise
+    you won't get the UUIDs.
+  * In screen messages don't print the disk directory in file path. You
+    can control the format with the test option:
+    --test-fmt file|disk|path.
+  * In Windows allows to use the escape char '^' to handle file patterns
+    containing real characters matching the globbing '*?[]' ones. In Unix
+    it was already possible to do the same escaping with '\'.
+  * Added a new -R, --force-realloc option to reallocate all the
+    parity information keeping the precomputed hash.
+    This is the previous -F, --force-full that instead now maintains the
+    same parity organization and just recomputes it.
+  * Added test options for selecting the file advise mode to use:
+    --test-io-advise-none for standard mode
+    --test-io-advise-sequential advise sequential access (Linux/Windows)
+    --test-io-advise-flush flush cache after every operation (Linux)
+    --test-io-advise-flush-window flush cache every 8 MB (Linux)
+    --test-io-advise-discard discard cache after every operation (Linux)
+    --test-io-advise-discard-window discard cache every 8 MB (Linux)
+    --test-io-advise-direct use direct/unbuffered mode (Linux/Windows)
+    The new default mode is 'flush' in Linux (before it was 'sequential'),
+    and 'sequential' in Windows (like before).
+  * For Seagate SMR (Shingled Magnetic Recording) ignore the SMART
+    attribute Command_Timeout 188 as not reliable.
+  * Fixed running in Windows platforms that miss the RtlGenRandom()
+    function.
+  * Added the --test-io-cache=1 option to disable the multi-thread IO
+    mode.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Thu, 01 Dec 2016 16:25:21 +0400
+
+snapraid (10.0-0tikhonov1~trusty) trusty; urgency=medium
+
+  [ 10.0 2016/02 ]
+  * Boosts the speed of the 'sync' and 'scrub' commands with a new
+    multi-thread implementation. It uses one thread for each disk,
+    dedicated exclusively to read-ahead data and parity and to
+    write-behind parity. This maximizes the data throughput keeping
+    disks always busy.
+    You can control the number of blocks to cache with the option
+    --test-io-cache=NUMBER, where the number is between 3 and 128.
+    The default is 8 MiB of blocks.
+    You can show run-time stats during the process with the
+    --test-io-stats option. You will see a graph with the number of
+    cached blocks, and a graph with the wait time percentage for all the
+    disks and computations.
+  * The -h, --pre-hash command, saves the content file only after having
+    verified all the hashes. This allows recovering of moved files in
+    case a silent error is found during the hash verification check.
+  * Allows to use the -d, --filter-disk option in the 'up' and 'down'
+    commands.
+  * Allows to run the 'smart' command without a configuration file.
+    In such case it operates on all the disks of the machine.
+  * In the configuration file 'data' is now a synonymous of 'disk'.
+  * Adds the 'touch' command intended to arbitrarily set all the zero
+    sub-second timestamps. This improves the SnapRAID capabilities to
+    identify files. The 'status' command recommends to run 'touch' if
+    required.
+  * Restores the functionality of the -D, --force-device option when used
+    to workaround the use of the same disk for two logical data drives
+    when running the 'fix' command.
+  * Uses a correct shell quoting in the example commands that involve
+    files.
+  * The minimum Windows version supported is now Windows Vista. This is
+    required to use the native Windows thread support for the new
+    multi-thread implementation. If you need to run on Windows XP, you
+    have to stick on SnapRAID 9.x.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Sat, 05 Mar 2016 17:44:53 +0400
+
+snapraid (9.3-0tikhonov1~trusty) trusty; urgency=medium
+
+  [ 9.3 2016/01 ]
+  * Fixes an invalid assumption in the copy detection mechanism that
+    could result in an internal inconsistency, and with the impossibility
+    to run the 'sync' and 'diff' commands.
+    This was triggered by a very specific pattern of identical files.
+    At least three of them, with one already in the parity, and at a
+    higher disk number than the others that should be instead new ones.
+    This had no bad effect, if not preventing the 'sync' command to run.
+    A workaround was to just run 'sync' one time with the -N,
+    --force-nocopy option to disable the copy detection.
+  * Restored the -O2 optimization option for Windows binaries, as -Og has
+    a too big performance penality.
+
+  [ 9.2 2016/01 ]
+  * Fixes support for symlinks pointing to an empty target. Before they
+    were only partially supported, and their presence could result in a
+    content file not readable.
+    This also disables multi-thread content write, as this was the issue
+    we tried to detect with this feature, and it doesn't provide a
+    performance advantage. Content verification is instead still multi
+    thread.
+  * Autorename disks using the matching UUID. To rename a disk you can
+    now change directly the name in the configuration file, and run a
+    'sync' command.
+  * Improves the physical offset ordering for the Btrfs file-system,
+    correctly detecting files that have not a physical offset, for
+    whatever reason.
+  * Adds UUID support to Btrfs file-systems. It's present only if the
+    'libblkid' development library is available on the system.
+    Usually this requires to install the libblkid-dev or libblkid-devel
+    package.
+  * Added a new --no-warnings option to disable some repetitive warnings
+    that could be annoying to power users.
+  * Improves the error reporting, printing a complete stack trace, that
+    can be used to track down bugs more easily.
+    For this reason the Windows binaries are now built with optimization
+    option -Og, instead than -O2.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Tue, 02 Feb 2016 12:59:13 +0400
+
+snapraid (9.1-0tikhonov1~trusty) trusty; urgency=medium
+
+  [ 9.1 2015/11 ]
+  * Fixes a bug when reading a content file with a deleted entry bigger
+    than 4 GB. This was a regression introduced in version 9.0 that
+    could result in the impossibility to read a valid content file,
+    after a deletion of a file bigger than 4 GB in the array.
+    If this happened to you, just upgrading to 9.1 fixes the issue, and
+    it allows you to continue to work.
+    Note that this bug only prevented to run 9.0, but your data was still
+    protected and could have been recovered using the versions 8.1 or 9.1.
+  * In Windows disables the file zero check requiring the --force-zero
+    option. This check is intended for possible case using ext3/4 in Linux,
+    and there is no evidence that in Windows it's possible at all.
+  * Windows binaries built with gcc 4.9.3 using the MXE cross compiler at
+    commit 62bcdbee56e87c81f1faa105b8777a5879d4e2e with targets
+    i686-w64-mingw32 and x86_64-w64-mingw32 and optimization -O2.
+
+  [ 9.0 2015/11 ]
+  * Fixes an invalid assumption that could happen when using the
+    -e, --filter-error option with "fix" or "check".
+    This was triggered by a very specific pattern of fragmented files
+    and bad blocks combination, not so easy to reproduce.
+    This had no bad effect, if not preventing the command to run.
+  * Drastically reduces the memory usage. For each block, it now
+    uses 17 bytes of memory, instead of the previous 28 bytes
+    (for 32 bit) or 36 bytes (for 64 bit).
+    This could result is a memory saving of up the 50%.
+  * The -p, --plan option (old --percentage) can be used to
+    define a scrub plan: "new", "bad" and "full".
+    The "new" plan scrubs all the new synced  blocks not yet scrubbed.
+    This allows to verify as early as possible that the written
+    parity during sync is really correct. You can use the "status"
+    command to show the amount blocks not yet scrubbed.
+    The "bad" plan scrubs only bad blocks.
+    The "full" plan scrubs all blocks.
+  * The graph in the "status" command now show scrubbed blocks
+    with '*', and synced, but not yet scrubbed, blocks with 'o'.
+    Note that when upgrading from a previous version, all blocks
+    are assumed scrubbed the first time.
+  * Content files are now written asyncronously from different
+    threads to avoid the unfortunate condition that a memory
+    error affects all of them in the same way.
+    After writing, they are read again to verify their CRC.
+    This is done to ensure thay they are really OK, even in the
+    case of the worst possible silent errors.
+  * Extends the -D, --force-device option to ignore more
+    erroneous conditions in the 'fix' command, like unaccessible
+    disks, or disks sharing the same physical device.
+  * Extends the -d, --filter-disk option to allow to filter also
+    by parity disk.
+  * Extends the -h, --pre-hash option to also verify moved and
+    copied files into the array before running a 'sync'.
+  * Updates 'best' RAID functions for recent Atom CPUs.
+  * Validates filters specifications rejecting relative paths.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Tue, 01 Dec 2015 23:50:52 +0400
+
+snapraid (8.1-0tikhonov1~trusty) trusty; urgency=medium
+
+  [ 8.1 2015/05 ]
+  * Fix build issues in generic Unix platforms, including Mac OS X.
+  * The "diff" command returns with error code 2 if a "sync" is
+    required, to differentiate with the generic error code 1.
+  * Reduced the effect of SMART attribute 193 on the failure
+    probability to avoid some false positive reports.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Sun, 17 May 2015 23:29:11 +0400
+
+snapraid (8.0-0tikhonov1~trusty) trusty; urgency=medium
+
+  [ 8.0 2015/04 ]
+  * Allows "sync" and "scrub" to continue after the first bunch of disk
+    errors. Blocks with errors are marked as bad, and you can fix them
+    with the "fix -e" command.
+    The fix is expected to force the disk firmware to reallocate the
+    bad sector, likely fixing the problem.
+    You can control the number of allowed errors with the new
+    -L, --error-limit option. The default is 100.
+  * The -e, --filter-error option doesn't write anymore fixes to
+    unsynced files. This helps in case you are running it on a not
+    synced array, removing the risk to revert some files to an old state.
+  * The -e, --filter-error option is now optimal and reads only the
+    minimal amount of data necessary to fix the errors.
+  * The "diff" command returns with an error code if a "sync" is
+    required.
+  * Adds new "smart" command to print a SMART report of the array.
+  * Adds new "up" and "down" commands to spin up and down the disks of
+    the array.
+  * Adds new "devices" command to print the devices associations in
+    the array.
+  * Changes the log handling. If no log file is specified, all the
+    warnings and not fatal errors messages goes to stderr. If a log file
+    is specified, only fatal error messages are printed on the screen.
+    You can control the amount of informative messages on stdout with
+    the -q, --quiet and -v, --verbose options, that can be specified
+    multiple times to be more quiet or verbose.
+  * In the "status" command the "Wasted" column now shows a negative
+    number for the amount of space that you can still waste without
+    filling up the parity.
+  * In the "status" and others commands we now use GB instead of GiB,
+    when referring to disk space.
+  * Renames the -s and -t options to -S and -B as they are intended to
+    be manual only operations.
+  * Windows binary built with gcc 4.8.1 using the MXE cross compiler 2.23,
+    with targets i686-w64-mingw32 and x86_64-w64-mingw32. Before the x86
+    target was i686-pc-mingw32.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Thu, 30 Apr 2015 23:01:55 +0400
+
+snapraid (7.1-0tikhonov1~precise) precise; urgency=medium
+
+  [ 7.1 2015/01 ]
+  * In 'scrub' and 'sync' detects and reports Input/Output errors
+    separately from generic file system errors.
+  * In 'diff' doesn't print the "add" entry if a "copy" one is already
+    printed.
+  * Fixes build with old compilers in the x64 platforms [Leigh Phillips].
+  * Fixes out-of-dir builds [Christoph Junghans].
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Wed, 11 Mar 2015 20:36:58 +0400
+
+snapraid (7.0-0tikhonov2~precise) precise; urgency=medium
+
+  [ 7.0 2014/11 ]
+  * In 'check' and 'fix' the array is scanned to find any moved files
+    that could be used to recover missing data. Files are identified by
+    timestamp, and then they are recognized also if moved to a different
+    disk. Note that even if there are false positive they are identified
+    checking the hash, so they have not effect, besides making the
+    process a little slower. To disable this new behaviour you can use
+    the -N, --force-nocopy option.
+  * The -i, --import command now identifies files by timestamp making it
+    very fast in importing directories.
+  * More detailed 'status' report with single disk stats and free space
+    available.
+  * A lot faster directory listing for Windows.
+  * Adds AVX2 support to improve parity generation speed.
+  * Prints the time spent waiting for each disk also in 'scrub'.
+  * The CPU usage, speed and ETA estimations are now based on the last 100
+    seconds rather than from the start.
+  * Keeps track of the UUID of the parity disks to check them before
+    operating.
+  * Windows binary built with gcc 4.8.1 using the MXE cross compiler 2.23.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Thu, 27 Nov 2014 00:36:59 +0400
+
+snapraid (6.2-0tikhonov1~precise) precise; urgency=low
+
+  [ 6.2 2014/5 ]
+  * Fixed the regression test when run as root.
+  * Added a new heuristic to detect file copies. Now a file is assumed
+    to be a copy if name, size and nanosecond timestamp are matching,
+    but if the nanosecond part of the timestamp is 0, it requires
+    the full path matching and not only the name.
+  * Added the -N, --force-nocopy option to disable completely
+    the copy detection. SnapRAID also suggests to use this option
+    in the error message of a data mismatch if likely caused by the
+    copy detection.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Wed, 16 Jul 2014 01:01:09 +0400
+
+snapraid (6.1-0tikhonov1~precise) precise; urgency=high
+
+  [ 6.1 2014/4 ]
+  * Fix build and regression test in Mac OS X.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Tue, 22 Apr 2014 00:22:07 +0400
+
+snapraid (6.0-0tikhonov1~precise) precise; urgency=low
+
+  [ 6.0 2014/3 ]
+  * In "sync", even if a silent error is found, continue to update
+    the parity if it's possible to correct the error.
+    Note that the block will be marked bad, and the data
+    will be fixed only at the next "fix -e" call.
+    But any new data added will be protected if you are using
+    enough parity to fix both the silent error and at least
+    another potential error.
+  * Detect copied files from one disk to another and reuse the already
+    computed hash information to validate them in "sync".
+    Files are assumed copied if they matches the name, size and timestamp.
+  * For "sync", added a new -h, --pre-hash option to run a preliminary
+    hashing step for all the new files to ensure to detect silent errors
+    caused by the heavy machine usage of the parity computation.
+  * In "fix", if a previous fixing attempt was made resulting in a
+    .unrecoverable file, uses this file as starting point for the
+    new attempt.
+  * In the log file name allows the use of the '>>', %D, %T modifiers
+    to select append mode, and to insert the date and time in the name.
+  * The options  -p, --percentage and -o, --older-than now keep their
+    default value even if the other one is specified.
+  * Moved the .lock file in the same dir of the first specified content
+    file. This avoid to spin-up the parity disks in all commands.
+  * The "diff", "list", "dup", "status" and "pool" commands don't access
+    anymore the parity disks that can now stay powered down.
+  * The default configuration file in Windows is now searched in the same
+    directory where the snapraid.exe file resides.
+  * New source code organization. The RAID engine is now
+    an external component usable also in other projects.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Fri, 11 Apr 2014 02:43:52 +0400
+
+snapraid (5.3-0tikhonov1~precise) precise; urgency=low
+
+  [ 5.3 2014/3 ]
+  * Don't warn about UUID changed if it's for an empty disk.
+  * Fixed the number of blocks that scrub has to process when
+    selecting a high percentage of the array.
+  * Removed duplicate recovery attempts in synced state.
+
+  [ 5.2 2013/12 ]
+  * If a disk changes UUID, automatically disable the inode
+    recognition, because this is likely a new filesystem with
+    all the inodes reassigned, and we don't want to risk a false
+    positive when searching for inode/timestamp/size.
+  * Allow to run a fix command with disks that doesn't need to be
+    fixed mounted as read-only.
+  * After a failed sync, always reallocates new files with a not
+    yet computed parity to ensure to minimize the parity usage,
+    if some other file is deleted in the meantime.
+  * Doesn't count empty dirs as files in the diff counters.
+  * Added a new "share" configuration option to allow to share
+    in the network the pool directory also in Windows.
+  * Fixed build problems in OpenBSD due the old assembler.
+  * Fixed build problems in platforms different than x86.
+
+  [ 5.1 2013/12 ]
+  * Fixed a potential crash if a file is deleted during a "sync/scrub".
+    This is a problem introduced in version 5.0 due new logging.
+    If happened to you to have a crash in sync, you don't need to
+    take any special action, just run "sync" again.
+  * Restored the functionality of -C, --gen-conf command.
+  * Prints the files with duplicate physical offset if the
+    -v, --verbose option is specified.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Tue, 11 Mar 2014 18:56:46 +0400
+
+snapraid (5.0-0tikhonov1~precise) precise; urgency=low
+
+  * New upstream release.
+
+  [ Upstream changelog 4.4 ]
+  * Added support for up to six levels of parity.
+  * Added a specific and faster triple parity format for CPUs that
+    don't support SSSE3 instructions like ARM and AMD Phenom, Athlon
+    and Opteron.
+  * Faster RAID5 and RAID6 implementation for ARM 64 bit CPUs.
+  * If a silent error is found during a "sync" command, directly marks
+    the block as bad like in "scrub", without stopping the the "sync"
+    process.
+  * Sort files by inode when listing the directory. This improves
+    the scanning performance.
+  * For files with changes only in some blocks, updates the parity
+    only for blocks that really are changed.
+    This improves the performance in sync for modified files.
+  * Added a new "list" command to see the stored list of files.
+  * Removed the detailed list of errors from the screen output.
+    To get it you must explicitely use the -l, --log option.
+    It's now too detailed for the screen, because it contains a lot
+    of info.
+  * Changed the output format of some commands to make it similar
+    at the new "list" one.
+  * Reduced memory usage removing some unnecessary allocations.
+  * Added a memory test on the memory buffers used in sync/scrub/check/fix
+    before using them.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Mon, 25 Nov 2013 15:58:37 +0400
+
+snapraid (4.4-0tikhonov1~precise) precise; urgency=low
+
+  * New upstream release.
+
+  [ Upstream changelog 4.4 ]
+  * Relaxed the check about small parity files, to allow to recover after a 
+    failed sync before resizing the parity files.
+
+  [ Upstream changelog 4.3 ]
+  * Fixed the scrub command with the -p0 option. Now it really scrubs only the 
+    blocks marked as bad and not the full array.
+
+  [ Upstream changelog 4.2 ]
+  * Fixed the wrong warning about physical offsets not supported caused by files 
+    not having a real offset because too small. For example, in NTFS it's possible 
+    to store such files in the MFT. It's just a cosmetic change, and not a 
+    functional one.
+  * Remove unexpected 'Restore' entries in the diff output when dealing with 
+    filesystem without persistent inodes like NTFS in Linux.
+  * Added support for filenames containing newlines. This happens in Mac OS X.
+
+  [ Upstream changelog 4.1 ]
+  * If the underline filesystem doesn't support the FIEMAP command, automatically 
+    fallback to use FIBMAP for sorting files.
+  * Fixed the import of content files from previous version of SnapRAID that are 
+    the result of an incomplete sync.
+  * Added a new -C, --gen-conf option to generate a dummy configuration file from 
+    the info in the content file. Just in case that you lose everything, except 
+    the content file.
+  * At the end of sync/scrub/check/fix prints "Everything OK" if no error was found. 
+    This should make clear that everything is really OK.
+
+  [ Upstream changelog 4.0 ]
+  * New 'scrub' command to periodically check the oldest blocks for silent 
+    errors without the need to scan the whole array. 
+  * New 'status' command to check the fragmentation, the last check time 
+    distribution, and the silent error status of the array.
+  * Added the new Spooky hash. It's faster in 64 bit architectures. To 
+    convert you can use the new 'rehash' command.
+  * Changed to a binary content file to improve speed and reduce size.
+  * Removed the --find-by-name, -N option. Now it always searches
+    by name if a file is not found searching by inode, automatically
+    reassigning inodes in restored files without needing to sync
+    again the file. This happens only if the file has the same path, size 
+    and timestamp at nanosecond precision.
+  * Added a hash seed to make harder intentional collision attacks.
+  * When inserting files for the first time, sort them by their physical
+    address to improve read performance.
+  * Optimized the cache use for the all the RAID computations.
+    This improves a lot the RAID performance.
+  * Better selection of the RAID6 implementation for different CPUs.
+  * Added RAID5/RAID6 mmx and sse2 implementations with unrolling by 4.
+    They are a little faster than the previous unroll by 2.
+  * Added a lock file to avoid multiple running instances on the same array.
+    The file is named as parity file adding the .lock extension.
+    There is also the undocumented --test-skip-lock to avoid to check it.
+  * Automatically ignores, with warning, mount points inside the array
+    directory tree.
+  * Changes the 'dup' output format to include the size of each duplicate file.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Fri, 01 Nov 2013 00:04:46 +0400
+
+snapraid (3.2-0tikhonov1~precise) precise; urgency=low
+
+  * New upstream release.
+
+  [ Upstream changelog 3.2 ]
+  * Fix a directory creation problem in Windows when the "disk" option points to
+    the root directory of a drive. Now SnapRAID won't complain about the
+    inability to create such directory.
+    If you encouter this problem when trying to recover your data, just upgrade
+    to this version, and you'll be able to complete the recovering process.
+    No need to upgrade for platforms different than Windows.
+
+  [ Upstream changelog 3.1 ]
+  * Direct use of Windows API for disk access to improve error reporting.
+  * If the 'fix' process is aborted, it removes all the new files partially
+    recovered, to allow to reuse again the '-m, --filter-missing' flag.
+  * In Windows don't exclude anymore system files. Only system directories are
+    excluded.
+  * In Windows applies filters in case insensitive way.
+  * The Windows binaries are now built with gcc 4.7.2.
+  * Reduced memory occupation for hardlinks and directories.
+  * In 'dup' don't list file with 0 size.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Fri, 09 Aug 2013 17:43:06 +0400
+
+snapraid (3.0-0tikhonov1~precise) precise; urgency=low
+
+  * New upstream release.
+
+  [ Upstream changelog ]
+  * Added pooling support with the new 'pool' command. It creates a virtual
+    view of the array using symbolic links pointing to the original files.
+  * Added a new -m, --filter-missing option that allow to undelete files,
+    without checking/fixing the others.
+  * Added a new -i, --import option to automatically import deleted files
+    when fixing.
+  * Added a new -l, --log option to save to disk the detailed log.
+  * Added support also for hardlinks and empty directories.
+  * Added support to save symlinks to files in Windows. Note that only the
+    symlink is saved and not the linked file.
+    Symlinks to dirs and junctions are still not supported in Windows.
+  * Files without read permission generate an error instead of a warning.
+    You now must explicitely exclude them in the configuration file with
+    exclusion rules.
+  * In 'check' and 'fix', if verbose is enabled, prints the result for each
+    processed file.
+  * Added an UUID check to detect when a disk is replaced, and to prevent
+    unwanted disk swaps.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Wed, 03 Apr 2013 13:01:53 +0400
+
+snapraid (2.1-0tikhonov1~precise) precise; urgency=low
+
+  * New upstream release.
+
+  [ Upstream changelog ]
+  * Checks for wrong empty fields in the configuration file.
+  * Filter rules for files are not anymore applied to directories.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Mon, 28 Jan 2013 02:10:28 +0400
+
+snapraid (2.0-0tikhonov1~precise) precise; urgency=low
+
+  * New upstream release.
+
+  [ Upstream changelog ]
+  * Added a new -a option to make the 'check' command to only check file hashes
+    without checking the parity data.
+  * Added a new -d option to filter by disk name.
+  * The file modification time is now saved using nanosecond precision.
+    This allows to restore the exact modification time in 'fix'.
+    The new 'content' files written with this version are not backward
+    compatible, but it's still possible to read the old 'content' format.
+  * Fixed hard-links automatic exclusion. All the hardlinks after the first one
+    are now correctly ignored.
+  * If it isn't possible to grow a parity file, prints the list of files
+    outside the maximum size allocated.
+  * Autosave isn't triggered if we are near the end of the 'sync' process.
+  * Before starting a 'sync', we wait for two seconds, to workaround the FAT
+    limitation of having two seconds modification time precision.
+    This a safe measure to be 100% sure to always detect file changes.
+  * Always fill the memory after allocating it to avoid the OOM (Out Of Memory)
+    killer in Linux.
+  * Fixed compilation in Solaris/OpenIndiana for lacking both futimes()
+    and futimens().
+  * Now 'sync' ensures that the parity files are not too small to contain the
+    just loaded data.
+  * Removed the '-H,--filter-nohidden' option. It doesn't make sense to
+    have it as command line option.
+    You must use the 'nohidden' option in the configuration file.
+  * When opening files in read-only mode, also specify the noatime flag,
+    to avoid to update the file access time.
+  * Exclude rules for files are now also applied to directories.
+    This allows to excludes some file/directory without the need to call
+    the stat() function on them.
+  * The -N, --find-by-name option also ignores the nanosecond part of
+    timestamps to work with copy programs not supporting nanoseconds.
+  * Fixed deduplicated files handling in Windows Server 2012.
+  * Removed MD5 support.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Thu, 06 Dec 2012 00:38:20 +0400
+
+snapraid (1.13-0tikhonov1~precise) precise; urgency=low
+
+  * New upstream release.
+
+  [ Upstream changelog ]
+  * Fixed a Segmentation Fault when checking/fixing if there are three or 
+    more errors in a specific block.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Mon, 05 Nov 2012 16:31:22 +0400
+
+snapraid (1.12-0tikhonov1~precise) precise; urgency=low
+
+  * New upstream release 1.12.
+
+  [ Upstream changelog ]
+  * Fixed file renaming in Windows during a 'fix' command. This is only a 
+    Windows issue, no reason to upgrade for other platforms.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Thu, 20 Sep 2012 13:13:40 +0400
+
+snapraid (1.11-0tikhonov1~precise) precise; urgency=high
+
+  * New upstream release 1.11.
+
+  [ Upstream changelog ]
+  * Fixed again directories inclusion. Exclusion rules for directories were ignored.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Mon, 09 Jul 2012 19:06:01 +0400
+
+snapraid (1.10-0tikhonov1~precise) precise; urgency=low
+
+  * New upstream release 1.10.
+
+  [ Upstream changelog ]
+  * Fixed directory inclusion, in case the last rule is an "include" one.
+  * Fixed very long paths in Windows. We now always use the special '\\?' prefix to remove the 260 chars limitation.
+  * If a file is excluded, it prints explicitely which attribute caused the exclusion.
+  * Automatically excludes also the temporary copy of content file, the one with the ".tmp" extension.
+  * Avoid the Windows system to go in automatic sleep mode when running.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Sun, 01 Jul 2012 21:43:16 +0400
+
+snapraid (1.9-0tikhonov1~precise) precise; urgency=low
+
+  * New upstream release 1.9.
+
+  [ Upstream changelog ]
+  * Implemented a more sophisticated recovering in case a harddisk failure happens during a 'sync' 
+    command. When using RAID6 it improves the chances of recovering data after an aborted 'sync'.
+  * Fixed the count of new files.
+  * Added a new 'autosave' configuration option to save the intermediate 'sync' state.
+  * Supported filesystems with read requests returning less data than requested.
+  * In Windows ensures that the disk serial number is not zero.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Tue, 10 Apr 2012 17:31:51 +0400
+
+snapraid (1.8-0tikhonov1~oneiric) oneiric; urgency=low
+
+  * New upstream release 1.8.
+
+  Upstream changelog:
+  * Added a new "dup" command to find all the duplicate files.
+  * Added a new option "--filter-nohidden" to exclude hidden files.
+  * Faster and parallel writing of content files.
+  * The example configuration files now put the content files in the data
+    disks instead than in the parity disks.
+  * Added a checksum at the content file to ensure its integrity.
+  * Using fallocate() instead posix_fallocate() to avoid the very slow 
+    posix_fallocate() fallback of writing the whole file.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Sat, 17 Mar 2012 19:15:07 +0400
+
+snapraid (1.7-0tikhonov2~oneiric) oneiric; urgency=low
+
+  * Fixed example snapraid.conf.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Sat, 24 Dec 2011 01:28:39 +0400
+
+snapraid (1.7-0tikhonov1~oneiric) oneiric; urgency=low
+
+  * Initial release.
+
+ -- Maxim Tikhonov <flaterichd@gmail.com>  Thu, 22 Dec 2011 19:46:17 +0400

debian/compat

@@ -0,0 +1 @@
+7

debian/control

@@ -0,0 +1,28 @@
+Source: snapraid
+Section: utils
+Priority: extra
+Maintainer: Maxim Tikhonov <flaterichd@gmail.com>
+Build-Depends: debhelper (>= 7.0.0), autotools-dev
+Standards-Version: 3.9.2
+Homepage: http://snapraid.sourceforge.net/
+
+Package: snapraid
+Architecture: any
+Depends:
+    ${shlibs:Depends}, 
+    ${misc:Depends}
+Description: SnapRAID is a backup program for disk arrays.
+ SnapRAID is a backup program for disk arrays.
+ .
+ SnapRAID stores redundancy information in the disk array, and it allows recovering from up to two disk failures.
+ .
+ SnapRAID is mainly targeted for a home media center, where you have a lot of big files that rarely change.
+ .
+ Beside the ability to recover from disk failures, the other features of SnapRAID are:
+ - You can start using SnapRAID with already filled disks.
+ - The disks of the array can have different sizes.
+ - You can add more disks at any time.
+ - If you accidentally delete some files in a disk, you can recover them.
+ - If more than two disks fail, you lose the data only on the failed disks. All the data in the other disks is safe.
+ - It doesn't lock-in your data. You can stop using SnapRAID at any time without the need to reformat or move data.
+ - All your data is hashed to ensure data integrity and avoid silent corruption.

debian/copyright

@@ -0,0 +1,40 @@
+Format: http://dep.debian.net/deps/dep5
+Name: snapraid
+Upstream-Name: snapraid
+Maintainer: Maxim Tikhonov <flaterichd@gmail.com>
+Source: http://sourceforge.net/projects/snapraid/files/
+
+Files: *
+Copyright: 2011 Andrea Mazzoleni <amadvance@users.sourceforge.net>
+License: GPL-3
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+ .
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ On Debian systems, the complete text of the GNU General
+ Public License can be found in `/usr/share/common-licenses/GPL-3'.
+
+Files: debian/*
+Copyright: 2011 Maxim Tikhonov <flaterichd@gmail.com>
+License: GPL-2+
+ This package is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+ .
+ This package is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>
+ .
+ On Debian systems, the complete text of the GNU General
+ Public License version 2 can be found in "/usr/share/common-licenses/GPL-2".

debian/docs

@@ -0,0 +1,9 @@
+AUTHORS
+CHECK
+COPYING
+HISTORY
+INSTALL
+README
+TODO
+snapraid.conf.example
+snapraid.txt

debian/install

@@ -0,0 +1,2 @@
+debian/snapraid.conf    etc
+snapraid                usr/bin

debian/manpage.1

@@ -0,0 +1 @@
+../snapraid.1

debian/rules

@@ -0,0 +1,21 @@
+#!/usr/bin/make -f
+# -*- makefile -*-
+# Sample debian/rules that uses debhelper.
+#
+# This file was originally written by Joey Hess and Craig Small.
+# As a special exception, when this file is copied by dh-make into a
+# dh-make output file, you may use that output file without restriction.
+# This special exception was added by Craig Small in version 0.37 of dh-make.
+#
+# Modified to make a template file for a multi-binary package with separated
+# build-arch and build-indep targets  by Bill Allombert 2001
+
+# Uncomment this to turn on verbose mode.
+#export DH_VERBOSE=1
+
+# This has to be exported to make some magic below work.
+export DH_OPTIONS
+
+
+%:
+	dh $@ 

debian/snapraid.conf

@@ -0,0 +1,86 @@
+# Example configuration for snapraid
+
+# Defines the file to use as parity storage
+# It must NOT be in a data disk
+# Format: "parity FILE_PATH"
+parity /mnt/diskp/snapraid.parity
+
+# Defines the files to use as additional parity storage.
+# If specified, they enable the multiple failures protection
+# from two to six level of parity.
+# To enable, uncomment one parity file for each level of extra
+# protection required. Start from 2-parity, and follow in order.
+# It must NOT be in a data disk
+# Format: "X-parity FILE_PATH"
+#2-parity /mnt/diskq/snapraid.2-parity
+#3-parity /mnt/diskr/snapraid.3-parity
+#4-parity /mnt/disks/snapraid.4-parity
+#5-parity /mnt/diskt/snapraid.5-parity
+#6-parity /mnt/disku/snapraid.6-parity
+
+# Defines the files to use as content list
+# You can use multiple specification to store more copies
+# You must have least one copy for each parity file plus one. Some more don't hurt
+# They can be in the disks used for data, parity or boot,
+# but each file must be in a different disk
+# Format: "content FILE_PATH"
+content /var/snapraid.content
+content /mnt/disk1/snapraid.content
+content /mnt/disk2/snapraid.content
+
+# Defines the data disks to use
+# The name and mount point association is relevant for parity, do not change it
+# WARNING: Adding here your /home, /var or /tmp disks is NOT a good idea!
+# SnapRAID is better suited for files that rarely changes!
+# Format: "disk DISK_NAME DISK_MOUNT_POINT"
+data d1 /mnt/disk1/
+data d2 /mnt/disk2/
+data d3 /mnt/disk3/
+
+# Excludes hidden files and directories (uncomment to enable).
+#nohidden
+
+# Defines files and directories to exclude
+# Remember that all the paths are relative at the mount points
+# Format: "exclude FILE"
+# Format: "exclude DIR/"
+# Format: "exclude /PATH/FILE"
+# Format: "exclude /PATH/DIR/"
+exclude *.unrecoverable
+exclude /tmp/
+exclude /lost+found/
+
+# Defines the block size in kibi bytes (1024 bytes) (uncomment to enable).
+# Default value is 256 -> 256 kibi bytes -> 262144 bytes
+# Format: "blocksize SIZE_IN_KiB"
+#blocksize 256
+
+# Automatically save the state when syncing after the specified amount
+# of GB processed (uncomment to enable).
+# This option is useful to avoid to restart from scratch long 'sync'
+# commands interrupted by a machine crash.
+# It also improves the recovering if a disk break during a 'sync'.
+# Default value is 0, meaning disabled.
+# Format: "autosave SIZE_IN_GB"
+#autosave 500
+
+# Defines the pooling directory where the virtual view of the disk
+# array is created using the "pool" command (uncomment to enable).
+# The files are not really copied here, but just linked using
+# symbolic links.
+# This directory must be outside the array.
+# Format: "pool DIR"
+#pool /pool
+
+# Defines a custom smartctl command to obtain the SMART attributes
+# for each disk. This may be required for RAID controllers and for
+# some USB disk that cannot be autodetected.
+# In the specified options, the "%s" string is replaced by the device name.
+# Refers at the smartmontools documentation about the possible options:
+# RAID -> https://www.smartmontools.org/wiki/Supported_RAID-Controllers
+# USB -> https://www.smartmontools.org/wiki/Supported_USB-Devices
+#smartctl d1 -d sat %s
+#smartctl d2 -d usbjmicron %s
+#smartctl parity -d areca,1/1 /dev/sg0
+#smartctl 2-parity -d areca,2/1 /dev/sg0
+

debian/source/format

@@ -0,0 +1 @@
+3.0 (quilt)

debian/watch

@@ -0,0 +1,2 @@
+version=3
+https://github.com/amadvance/snapraid/releases .*snapraid-([0-9.]*\.[0-9.]*).tar.gz

raid/check.c

@@ -36,7 +36,7 @@
  * @ip[] Vector of @nv indexes of the valid parity blocks.
  *   The indexes start from 0. They must be in order.
  * @nd Number of data blocks.
- * @size Size of the blocks pointed by @v. It must be a multipler of 64.
+ * @size Size of the blocks pointed by @v. It must be a multiplier of 64.
  * @v Vector of pointers to the blocks of data and parity.
  *   It has (@nd + @ip[@nv - 1] + 1) elements. The starting elements are the
  *   blocks for data, following with the parity blocks.

raid/helper.h

@@ -23,7 +23,7 @@
  *
  * @n Number of integers currently in the vector.
  * @v Vector of integers already sorted.
- *   It must have extra space for the new elemet at the end.
+ *   It must have extra space for the new element at the end.
  * @i Value to insert.
  */
 void raid_insert(int n, int *v, int i);

raid/internal.h

@@ -174,7 +174,7 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 /*
  * Internal naming.
  *
- * These are intented to provide access for testing.
+ * These are intended to provide access for testing.
  */
 const char *raid_gen1_tag(void);
 const char *raid_gen2_tag(void);
@@ -263,7 +263,7 @@ static __always_inline void raid_avx_end(void)
 	raid_sse_end();
 
 	/* reset the upper part of the ymm registers */
-	/* to avoid the 70 clocks penality on the next */
+	/* to avoid the 70 clocks penalty on the next */
 	/* xmm register use */
 	asm volatile ("vzeroupper" : : : "memory");
 }

raid/memory.h

@@ -45,7 +45,7 @@
  *    gen2    6814 [MB/s]
  *    genz    3033 [MB/s]
  *
- * These are the results with displacement resulting in improvments
+ * These are the results with displacement resulting in improvements
  * in the order of 20% or more:
  *
  *            sse2

raid/raid.c

@@ -63,11 +63,11 @@
  * computation of triple parity using power coefficients.
  *
  * Another important property of the Cauchy matrix is that we can setup
- * the first two rows with coeffients equal at the RAID5 and RAID6 approach
- * decribed, resulting in a compatible extension, and requiring SSSE3
+ * the first two rows with coefficients equal at the RAID5 and RAID6 approach
+ * described, resulting in a compatible extension, and requiring SSSE3
  * or AVX2 instructions only if triple parity or beyond is used.
  *
- * The matrix is also adjusted, multipling each row by a constant factor
+ * The matrix is also adjusted, multiplying each row by a constant factor
  * to make the first column of all 1, to optimize the computation for
  * the first disk.
  *
@@ -147,7 +147,7 @@
  * "raid/test/speedtest.c" program.
  *
  * For comparison, the triple parity computation using the power
- * coeffients "1,2,2^-1" is only a little faster than the one based on
+ * coefficients "1,2,2^-1" is only a little faster than the one based on
  * the Cauchy matrix if SSSE3 or AVX2 is present.
  *
  *             int8   int32   int64    sse2   ssse3    avx2
@@ -205,12 +205,12 @@ void raid_zero(void *zero)
  * All these functions give the guarantee that parities are written
  * in order. First parity P, then parity Q, and so on.
  * This allows to specify the same memory buffer for multiple parities
- * knowning that you'll get the latest written one.
+ * knowing that you'll get the latest written one.
  * This characteristic is used by the raid_delta_gen() function to
  * avoid to damage unused parities in recovering.
  *
  * @nd Number of data blocks
- * @size Size of the blocks pointed by @v. It must be a multipler of 64.
+ * @size Size of the blocks pointed by @v. It must be a multiplier of 64.
  * @v Vector of pointers to the blocks of data and parity.
  *   It has (@nd + #parities) elements. The starting elements are the blocks
  *   for data, following with the parity blocks.
@@ -335,7 +335,7 @@ void raid_delta_gen(int nr, int *id, int *ip, int nd, size_t size, void **v)
 		} else {
 			/*
 			 * Unused parities are going to be rewritten with
-			 * not significative data, becase we don't have
+			 * not significative data, because we don't have
 			 * functions able to compute only a subset of
 			 * parities.
 			 *
@@ -483,7 +483,7 @@ void raid_rec2of2_int8(int *id, int *ip, int nd, size_t size, void **vv)
  *   The indexes start from 0. They must be in order.
  * @nd Number of data blocks.
  * @np Number of parity blocks.
- * @size Size of the blocks pointed by @v. It must be a multipler of 64.
+ * @size Size of the blocks pointed by @v. It must be a multiplier of 64.
  * @v Vector of pointers to the blocks of data and parity.
  *   It has (@nd + @np) elements. The starting elements are the blocks
  *   for data, following with the parity blocks.

raid/raid.h

@@ -192,7 +192,7 @@ int raid_check(int nr, int *ir, int nd, int np, size_t size, void **v);
  * This function identifies the failed data and parity blocks using the
  * available redundancy.
  *
- * It uses a brute force method, and then the call can be expansive.
+ * It uses a brute force method, and then the call can be expensive.
  * The expected execution time is proportional at the binomial coefficient
  * @np + @nd choose @np - 1, usually written as:
  *

raid/tables.c

@@ -13662,8 +13662,8 @@ const uint8_t __aligned(256) raid_gfcauchypshufb[251][4][2][16] =
 /**
  * PSHUFB tables for generic multiplication.
  *
- * Indexes are [MULTIPLER][LH].
- * Where MULTIPLER is from 0 to 255, LH from 0 to 1.
+ * Indexes are [MULTIPLIER][LH].
+ * Where MULTIPLIER is from 0 to 255, LH from 0 to 1.
  */
 const uint8_t __aligned(256) raid_gfmulpshufb[256][2][16] =
 {

raid/test/Makefile

@@ -3,7 +3,7 @@
 #
 # selftest - Runs the same selftest and speedtest executed at the module startup.
 # fulltest - Runs a more extensive test that checks all the built-in functions.
-# speetest - Runs a more complete speed test.
+# speedtest - Runs a more complete speed test.
 # invtest - Runs an extensive matrix inversion test of all the 377.342.351.231
 #           possible square submatrices of the Cauchy matrix used.
 # covtest - Runs a coverage test.

snapraid.1

@@ -79,6 +79,9 @@ Beside the ability to recover from disk failures, other
 features of SnapRAID are:
 .PD 0
 .IP \(bu
+You can use disk already filled with files, without the need to
+reformat them. You will access them like now.
+.IP \(bu
 All your data is hashed to ensure data integrity and to avoid
 silent corruption.
 .IP \(bu
@@ -89,8 +92,6 @@ All the data in the other disks is safe.
 If you accidentally delete some files in a disk, you can
 recover them.
 .IP \(bu
-You can start with already filled disks.
-.IP \(bu
 The disks can have different sizes.
 .IP \(bu
 You can add disks at any time.
@@ -98,7 +99,7 @@ You can add disks at any time.
 It doesn\'t lock\-in your data. You can stop using SnapRAID at any
 time without the need to reformat or move data.
 .IP \(bu
-To access a file, a single disk needs to spin, saving power and
+To access a file, only a single disk needs to spin, saving power and
 producing less noise.
 .PD
 .PP
@@ -647,7 +648,7 @@ and ignore all the parity data.
 This command will take a long time, but if you are not paranoid,
 you can skip it.
 .SS STEP 4 \-> Sync 
-Run the \[dq]sync\[dq] command to resynchronize the array with the new disk.
+Run the \[dq]sync\[dq] command to re\-synchronize the array with the new disk.
 .PP
 .RS 4
 snapraid sync
@@ -686,7 +687,7 @@ run \[dq]sync\[dq]. Later modifications are not taken into account.
 If bad blocks were detected, their block numbers are listed.
 To fix them, you can use the \[dq]fix \-e\[dq] command.
 .PP
-It also shows a graph representing the the last time each block
+It also shows a graph representing the last time each block
 was scrubbed or synced. Scrubbed blocks are shown with \'*\',
 blocks synced but not yet scrubbed with \'o\'.
 .PP
@@ -745,7 +746,7 @@ Spins up all the disks of the array.
 .PP
 You can spin\-up only some specific disks using the \-d, \-\-filter\-disk option.
 .PP
-Take care that spinniup\-up all the disks at the same time needs a lot of power.
+Take care that spinning\-up all the disks at the same time needs a lot of power.
 Ensure that your power\-supply can sustain that.
 .PP
 Nothing is modified.
@@ -889,7 +890,7 @@ Use the filter options to select a subset of files or disks to operate on.
 To only fix the blocks marked bad during \[dq]sync\[dq] and \[dq]scrub\[dq],
 use the \-e, \-\-filter\-error option.
 As difference from other filter options, with this one the fixes are
-applied only to files that are not modified from the the latest \[dq]sync\[dq].
+applied only to files that are not modified from the latest \[dq]sync\[dq].
 .PP
 All the files that cannot be fixed are renamed adding the
 \[dq].unrecoverable\[dq] extension.
@@ -912,7 +913,7 @@ Verify all the files and the parity data.
 It works like \[dq]fix\[dq], but it only simulates a recovery and no change
 is written in the array.
 .PP
-This command is mostly intended for manual verifications,
+This command is mostly intended for manual verification,
 like after a recovery process or in other special conditions.
 For periodic and scheduled checks uses \[dq]scrub\[dq].
 .PP
@@ -931,7 +932,7 @@ Nothing is modified.
 .SS dup 
 Lists all the duplicate files. Two files are assumed equal if their
 hashes are matching. The file data is not read, but only the
-precomputed hashes are used.
+pre\-computed hashes are used.
 .PP
 Nothing is modified.
 .SS pool 
@@ -942,7 +943,7 @@ The files are not really copied here, but just linked using
 symbolic links.
 .PP
 When updating, all the present symbolic links and empty
-subdirectories are deleted and replaced with the new
+sub\-directories are deleted and replaced with the new
 view of the array. Any other regular file is left in place.
 .PP
 Nothing is modified outside the pool directory.
@@ -962,20 +963,20 @@ low level devices used by a single disk in the array.
 .PP
 Nothing is modified.
 .SS touch 
-Sets arbitrarely the sub\-second timestamp of all the files
+Sets arbitrarily the sub\-second time\-stamp of all the files
 that have it at zero.
 .PP
 This improves the SnapRAID capability to recognize moved
-and copied files as it makes the timestamp almost unique,
+and copied files as it makes the time\-stamp almost unique,
 removing possible duplicates.
 .PP
-More specifically, if the sub\-second timestamp is not zero,
+More specifically, if the sub\-second time\-stamp is not zero,
 a moved or copied file is identified as such if it matches
-the name, size and timestamp. If instead the sub\-second timestamp
+the name, size and time\-stamp. If instead the sub\-second time\-stamp
 is zero, it\'s considered a copy only if it matches the full path,
-size and timestamp.
+size and time\-stamp.
 .PP
-Note that the second precision timestamp is not modified,
+Note that the second precision time\-stamp is not modified,
 and all the dates and times of your files will be maintained.
 .SS rehash 
 Schedules a rehash of the whole array.
@@ -1040,9 +1041,9 @@ Note that it cannot be used with \[dq]sync\[dq] and \[dq]scrub\[dq], because the
 process the whole array.
 .TP
 .B \-e, \-\-filter\-error
-Filters the blocks to process in \[dq]check\[dq] and \[dq]fix\[dq].
-It processes only the blocks marked with silent or input/output
-errors during \[dq]sync\[dq] and \[dq]scrub\[dq], and listed in \[dq]status\[dq].
+Process the files with errors in \[dq]check\[dq] and \[dq]fix\[dq].
+It processes only files that have blocks marked with silent
+or input/output errors during \[dq]sync\[dq] and \[dq]scrub\[dq], and listed in \[dq]status\[dq].
 This option can be used only with \[dq]check\[dq] and \[dq]fix\[dq].
 .TP
 .B \-p, \-\-plan PERC|bad|new|full
@@ -1090,7 +1091,7 @@ Imports from the specified directory any file that you deleted
 from the array after the last \[dq]sync\[dq].
 If you still have such files, they could be used by \[dq]check\[dq]
 and \[dq]fix\[dq] to improve the recover process.
-The files are read also in subdirectories and they are
+The files are read also in sub\-directories and they are
 identified regardless of their name.
 This option can be used only with \[dq]check\[dq] and \[dq]fix\[dq].
 .TP
@@ -1146,15 +1147,15 @@ This allows to identify copied or moved files from one disk
 to another, and to reuse the already computed hash information
 to detect silent errors or to recover missing files.
 This behavior, in some rare cases, may result in false positives,
-or in a slow process due the many hash verifications, and this
+or in a slow process due the many hash verification, and this
 option allows to resolve them.
 This option can be used only with \[dq]sync\[dq], \[dq]check\[dq] and \[dq]fix\[dq].
 .TP
 .B \-F, \-\-force\-full
-In \[dq]sync\[dq] forces a full rebuild of the parity.
+In \[dq]sync\[dq] forces a full recomputation of the parity.
 This option can be used when you add a new parity level, or if
 you reverted back to an old content file using a more recent parity data.
-Instead of recomputing the parity from scratch, this allows
+Instead of recreating the parity from scratch, this allows
 to reuse the hashes present in the content file to validate data,
 and to maintain data protection during the \[dq]sync\[dq] process using
 the parity data you have.
@@ -1165,9 +1166,10 @@ In \[dq]sync\[dq] forces a full reallocation of files and rebuild of the parity.
 This option can be used to completely reallocate all the files
 removing the fragmentation, but reusing the hashes present in the content
 file to validate data.
-Compared to \-F, \-\-force\-full, this option reallocates all the parity
-not having data protection during the operation.
 This option can be used only with \[dq]sync\[dq].
+WARNING! This option is for experts only, and it\'s highly
+recommended to not use it.
+You DO NOT have data protection during the \[dq]sync\[dq] operation.
 .TP
 .B \-l, \-\-log FILE
 Write a detailed log in the specified file.
@@ -1280,7 +1282,7 @@ is enabled:
 6\-parity enables hexa (six) parity
 .PD
 .PP
-Each parity level requires the precence of all the previous parity
+Each parity level requires the presence of all the previous parity
 levels.
 .PP
 The same considerations of the \'parity\' option apply.

snapraid.d

@@ -31,6 +31,8 @@ Description
 	Beside the ability to recover from disk failures, other
 	features of SnapRAID are:
 
+	* You can use disk already filled with files, without the need to
+		reformat them. You will access them like now.
 	* All your data is hashed to ensure data integrity and to avoid
 		silent corruption.
 	* If the failed disks are too many to allow a recovery,
@@ -38,12 +40,11 @@ Description
 		All the data in the other disks is safe.
 	* If you accidentally delete some files in a disk, you can
 		recover them.
-	* You can start with already filled disks.
 	* The disks can have different sizes.
 	* You can add disks at any time.
 	* It doesn't lock-in your data. You can stop using SnapRAID at any
 		time without the need to reformat or move data.
-	* To access a file, a single disk needs to spin, saving power and
+	* To access a file, only a single disk needs to spin, saving power and
 		producing less noise.
 
 	The official site of SnapRAID is:
@@ -400,7 +401,7 @@ Commands
 	If bad blocks were detected, their block numbers are listed.
 	To fix them, you can use the "fix -e" command.
 
-	It also shows a graph representing the the last time each block
+	It also shows a graph representing the last time each block
 	was scrubbed or synced. Scrubbed blocks are shown with '*',
 	blocks synced but not yet scrubbed with 'o'.
 
@@ -577,7 +578,7 @@ Commands
 	To only fix the blocks marked bad during "sync" and "scrub",
 	use the -e, --filter-error option.
 	As difference from other filter options, with this one the fixes are
-	applied only to files that are not modified from the the latest "sync".
+	applied only to files that are not modified from the latest "sync".
 
 	All the files that cannot be fixed are renamed adding the
 	".unrecoverable" extension.
@@ -736,9 +737,9 @@ Options
 		process the whole array.
 
 	-e, --filter-error
-		Filters the blocks to process in "check" and "fix".
-		It processes only the blocks marked with silent or input/output
-		errors during "sync" and "scrub", and listed in "status".
+		Process the files with errors in "check" and "fix".
+		It processes only files that have blocks marked with silent
+		or input/output errors during "sync" and "scrub", and listed in "status".
 		This option can be used only with "check" and "fix".
 
 	-p, --plan PERC|bad|new|full
@@ -847,10 +848,10 @@ Options
 		This option can be used only with "sync", "check" and "fix".
 
 	-F, --force-full
-		In "sync" forces a full rebuild of the parity.
+		In "sync" forces a full recomputation of the parity.
 		This option can be used when you add a new parity level, or if
 		you reverted back to an old content file using a more recent parity data.
-		Instead of recomputing the parity from scratch, this allows
+		Instead of recreating the parity from scratch, this allows
 		to reuse the hashes present in the content file to validate data,
 		and to maintain data protection during the "sync" process using
 		the parity data you have.
@@ -861,9 +862,10 @@ Options
 		This option can be used to completely reallocate all the files
 		removing the fragmentation, but reusing the hashes present in the content
 		file to validate data.
-		Compared to -F, --force-full, this option reallocates all the parity
-		not having data protection during the operation.
 		This option can be used only with "sync".
+		WARNING! This option is for experts only, and it's highly
+		recommended to not use it.
+		You DO NOT have data protection during the "sync" operation.
 
 	-l, --log FILE
 		Write a detailed log in the specified file.

snapraid.txt

@@ -38,6 +38,8 @@ big files that rarely change.
 Beside the ability to recover from disk failures, other
 features of SnapRAID are:
 
+* You can use disk already filled with files, without the need to
+  reformat them. You will access them like now.
 * All your data is hashed to ensure data integrity and to avoid
   silent corruption.
 * If the failed disks are too many to allow a recovery,
@@ -45,12 +47,11 @@ features of SnapRAID are:
   All the data in the other disks is safe.
 * If you accidentally delete some files in a disk, you can
   recover them.
-* You can start with already filled disks.
 * The disks can have different sizes.
 * You can add disks at any time.
 * It doesn't lock-in your data. You can stop using SnapRAID at any
   time without the need to reformat or move data.
-* To access a file, a single disk needs to spin, saving power and
+* To access a file, only a single disk needs to spin, saving power and
   producing less noise.
 
 The official site of SnapRAID is:
@@ -391,7 +392,7 @@ This command will take a long time, but if you are not paranoid,
 you can skip it.
 
 ---- 4.4.4 STEP 4 -> Sync ----
-Run the "sync" command to resynchronize the array with the new disk.
+Run the "sync" command to re-synchronize the array with the new disk.
 
     snapraid sync
 
@@ -426,7 +427,7 @@ run "sync". Later modifications are not taken into account.
 If bad blocks were detected, their block numbers are listed.
 To fix them, you can use the "fix -e" command.
 
-It also shows a graph representing the the last time each block
+It also shows a graph representing the last time each block
 was scrubbed or synced. Scrubbed blocks are shown with '*',
 blocks synced but not yet scrubbed with 'o'.
 
@@ -476,7 +477,7 @@ Spins up all the disks of the array.
 
 You can spin-up only some specific disks using the -d, --filter-disk option.
 
-Take care that spinniup-up all the disks at the same time needs a lot of power.
+Take care that spinning-up all the disks at the same time needs a lot of power.
 Ensure that your power-supply can sustain that.
 
 Nothing is modified.
@@ -617,7 +618,7 @@ Use the filter options to select a subset of files or disks to operate on.
 To only fix the blocks marked bad during "sync" and "scrub",
 use the -e, --filter-error option.
 As difference from other filter options, with this one the fixes are
-applied only to files that are not modified from the the latest "sync".
+applied only to files that are not modified from the latest "sync".
 
 All the files that cannot be fixed are renamed adding the
 ".unrecoverable" extension.
@@ -643,7 +644,7 @@ Verify all the files and the parity data.
 It works like "fix", but it only simulates a recovery and no change
 is written in the array.
 
-This command is mostly intended for manual verifications,
+This command is mostly intended for manual verification,
 like after a recovery process or in other special conditions.
 For periodic and scheduled checks uses "scrub".
 
@@ -668,7 +669,7 @@ Nothing is modified.
 
 Lists all the duplicate files. Two files are assumed equal if their
 hashes are matching. The file data is not read, but only the
-precomputed hashes are used.
+pre-computed hashes are used.
 
 Nothing is modified.
 
@@ -682,7 +683,7 @@ The files are not really copied here, but just linked using
 symbolic links.
 
 When updating, all the present symbolic links and empty
-subdirectories are deleted and replaced with the new
+sub-directories are deleted and replaced with the new
 view of the array. Any other regular file is left in place.
 
 Nothing is modified outside the pool directory.
@@ -708,20 +709,20 @@ Nothing is modified.
 5.14 touch
 ----------
 
-Sets arbitrarely the sub-second timestamp of all the files
+Sets arbitrarily the sub-second time-stamp of all the files
 that have it at zero.
 
 This improves the SnapRAID capability to recognize moved
-and copied files as it makes the timestamp almost unique,
+and copied files as it makes the time-stamp almost unique,
 removing possible duplicates.
 
-More specifically, if the sub-second timestamp is not zero,
+More specifically, if the sub-second time-stamp is not zero,
 a moved or copied file is identified as such if it matches
-the name, size and timestamp. If instead the sub-second timestamp
+the name, size and time-stamp. If instead the sub-second time-stamp
 is zero, it's considered a copy only if it matches the full path,
-size and timestamp.
+size and time-stamp.
 
-Note that the second precision timestamp is not modified,
+Note that the second precision time-stamp is not modified,
 and all the dates and times of your files will be maintained.
 
 5.15 rehash
@@ -793,9 +794,9 @@ SnapRAID provides the following options:
         process the whole array.
 
     -e, --filter-error
-        Filters the blocks to process in "check" and "fix".
-        It processes only the blocks marked with silent or input/output
-        errors during "sync" and "scrub", and listed in "status".
+        Process the files with errors in "check" and "fix".
+        It processes only files that have blocks marked with silent
+        or input/output errors during "sync" and "scrub", and listed in "status".
         This option can be used only with "check" and "fix".
 
     -p, --plan PERC|bad|new|full
@@ -843,7 +844,7 @@ SnapRAID provides the following options:
         from the array after the last "sync".
         If you still have such files, they could be used by "check"
         and "fix" to improve the recover process.
-        The files are read also in subdirectories and they are
+        The files are read also in sub-directories and they are
         identified regardless of their name.
         This option can be used only with "check" and "fix".
 
@@ -899,15 +900,15 @@ SnapRAID provides the following options:
         to another, and to reuse the already computed hash information
         to detect silent errors or to recover missing files.
         This behavior, in some rare cases, may result in false positives,
-        or in a slow process due the many hash verifications, and this
+        or in a slow process due the many hash verification, and this
         option allows to resolve them.
         This option can be used only with "sync", "check" and "fix".
 
     -F, --force-full
-        In "sync" forces a full rebuild of the parity.
+        In "sync" forces a full recomputation of the parity.
         This option can be used when you add a new parity level, or if
         you reverted back to an old content file using a more recent parity data.
-        Instead of recomputing the parity from scratch, this allows
+        Instead of recreating the parity from scratch, this allows
         to reuse the hashes present in the content file to validate data,
         and to maintain data protection during the "sync" process using
         the parity data you have.
@@ -918,9 +919,10 @@ SnapRAID provides the following options:
         This option can be used to completely reallocate all the files
         removing the fragmentation, but reusing the hashes present in the content
         file to validate data.
-        Compared to -F, --force-full, this option reallocates all the parity
-        not having data protection during the operation.
         This option can be used only with "sync".
+        WARNING! This option is for experts only, and it's highly
+        recommended to not use it.
+        You DO NOT have data protection during the "sync" operation.
 
     -l, --log FILE
         Write a detailed log in the specified file.
@@ -1037,7 +1039,7 @@ is enabled:
 * 5-parity enables penta (five) parity
 * 6-parity enables hexa (six) parity
 
-Each parity level requires the precence of all the previous parity
+Each parity level requires the presence of all the previous parity
 levels.
 
 The same considerations of the 'parity' option apply.

tommyds/tommyarray.c

@@ -36,7 +36,7 @@ void tommy_array_init(tommy_array* array)
 
 	/* fixed initial size */
 	array->bucket_bit = TOMMY_ARRAY_BIT;
-	array->bucket_max = 1 << array->bucket_bit;
+	array->bucket_max = (tommy_size_t)1 << array->bucket_bit;
 	array->bucket[0] = tommy_cast(void**, tommy_calloc(array->bucket_max, sizeof(void*)));
 	for (i = 1; i < TOMMY_ARRAY_BIT; ++i)
 		array->bucket[i] = array->bucket[0];
@@ -51,11 +51,11 @@ void tommy_array_done(tommy_array* array)
 	tommy_free(array->bucket[0]);
 	for (i = TOMMY_ARRAY_BIT; i < array->bucket_bit; ++i) {
 		void** segment = array->bucket[i];
-		tommy_free(&segment[((tommy_ptrdiff_t)1) << i]);
+		tommy_free(&segment[(tommy_ptrdiff_t)1 << i]);
 	}
 }
 
-void tommy_array_grow(tommy_array* array, tommy_count_t count)
+void tommy_array_grow(tommy_array* array, tommy_size_t count)
 {
 	if (array->count >= count)
 		return;
@@ -72,7 +72,7 @@ void tommy_array_grow(tommy_array* array, tommy_count_t count)
 		array->bucket[array->bucket_bit] = &segment[-(tommy_ptrdiff_t)array->bucket_max];
 
 		++array->bucket_bit;
-		array->bucket_max = 1 << array->bucket_bit;
+		array->bucket_max = (tommy_size_t)1 << array->bucket_bit;
 	}
 }
 

tommyds/tommyarray.h

@@ -53,20 +53,15 @@
  */
 #define TOMMY_ARRAY_BIT 6
 
-/** \internal
- * Max number of elements as a power of 2.
- */
-#define TOMMY_ARRAY_BIT_MAX 32
-
 /**
  * Array container type.
  * \note Don't use internal fields directly, but access the container only using functions.
  */
 typedef struct tommy_array_struct {
-	void** bucket[TOMMY_ARRAY_BIT_MAX]; /**< Dynamic array of buckets. */
+	void** bucket[TOMMY_SIZE_BIT]; /**< Dynamic array of buckets. */
+	tommy_size_t bucket_max; /**< Number of buckets. */
+	tommy_size_t count; /**< Number of initialized elements in the array. */
 	tommy_uint_t bucket_bit; /**< Bits used in the bit mask. */
-	tommy_count_t bucket_max; /**< Number of buckets. */
-	tommy_count_t count; /**< Number of initialized elements in the array. */
 } tommy_array;
 
 /**
@@ -83,21 +78,21 @@ void tommy_array_done(tommy_array* array);
  * Grows the size up to the specified value.
  * All the new elements in the array are initialized with the 0 value.
  */
-void tommy_array_grow(tommy_array* array, tommy_count_t size);
+void tommy_array_grow(tommy_array* array, tommy_size_t size);
 
 /**
  * Gets a reference of the element at the specified position.
  * You must be sure that space for this position is already
  * allocated calling tommy_array_grow().
  */
-tommy_inline void** tommy_array_ref(tommy_array* array, tommy_count_t pos)
+tommy_inline void** tommy_array_ref(tommy_array* array, tommy_size_t pos)
 {
 	tommy_uint_t bsr;
 
 	assert(pos < array->count);
 
 	/* get the highest bit set, in case of all 0, return 0 */
-	bsr = tommy_ilog2_u32(pos | 1);
+	bsr = tommy_ilog2(pos | 1);
 
 	return &array->bucket[bsr][pos];
 }
@@ -107,7 +102,7 @@ tommy_inline void** tommy_array_ref(tommy_array* array, tommy_count_t pos)
  * You must be sure that space for this position is already
  * allocated calling tommy_array_grow().
  */
-tommy_inline void tommy_array_set(tommy_array* array, tommy_count_t pos, void* element)
+tommy_inline void tommy_array_set(tommy_array* array, tommy_size_t pos, void* element)
 {
 	*tommy_array_ref(array, pos) = element;
 }
@@ -117,7 +112,7 @@ tommy_inline void tommy_array_set(tommy_array* array, tommy_count_t pos, void* e
  * You must be sure that space for this position is already
  * allocated calling tommy_array_grow().
  */
-tommy_inline void* tommy_array_get(tommy_array* array, tommy_count_t pos)
+tommy_inline void* tommy_array_get(tommy_array* array, tommy_size_t pos)
 {
 	return *tommy_array_ref(array, pos);
 }
@@ -127,7 +122,7 @@ tommy_inline void* tommy_array_get(tommy_array* array, tommy_count_t pos)
  */
 tommy_inline void tommy_array_insert(tommy_array* array, void* element)
 {
-	tommy_count_t pos = array->count;
+	tommy_size_t pos = array->count;
 
 	tommy_array_grow(array, pos + 1);
 
@@ -137,7 +132,7 @@ tommy_inline void tommy_array_insert(tommy_array* array, void* element)
 /**
  * Gets the initialized size of the array.
  */
-tommy_inline tommy_count_t tommy_array_size(tommy_array* array)
+tommy_inline tommy_size_t tommy_array_size(tommy_array* array)
 {
 	return array->count;
 }

tommyds/tommyarrayblkof.c

@@ -40,7 +40,7 @@ void tommy_arrayblkof_init(tommy_arrayblkof* array, tommy_size_t element_size)
 
 void tommy_arrayblkof_done(tommy_arrayblkof* array)
 {
-	tommy_count_t i;
+	tommy_size_t i;
 
 	for (i = 0; i < tommy_array_size(&array->block); ++i)
 		tommy_free(tommy_array_get(&array->block, i));
@@ -48,10 +48,10 @@ void tommy_arrayblkof_done(tommy_arrayblkof* array)
 	tommy_array_done(&array->block);
 }
 
-void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_count_t count)
+void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_size_t count)
 {
-	tommy_count_t block_max;
-	tommy_count_t block_mac;
+	tommy_size_t block_max;
+	tommy_size_t block_mac;
 
 	if (array->count >= count)
 		return;

tommyds/tommyarrayblkof.h

@@ -61,7 +61,7 @@
 typedef struct tommy_arrayblkof_struct {
 	tommy_array block; /**< Array of blocks. */
 	tommy_size_t element_size; /**< Size of the stored element in bytes. */
-	tommy_count_t count; /**< Number of initialized elements in the array. */
+	tommy_size_t count; /**< Number of initialized elements in the array. */
 } tommy_arrayblkof;
 
 /**
@@ -79,14 +79,14 @@ void tommy_arrayblkof_done(tommy_arrayblkof* array);
  * Grows the size up to the specified value.
  * All the new elements in the array are initialized with the 0 value.
  */
-void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_count_t size);
+void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_size_t size);
 
 /**
  * Gets a reference of the element at the specified position.
  * You must be sure that space for this position is already
  * allocated calling tommy_arrayblkof_grow().
  */
-tommy_inline void* tommy_arrayblkof_ref(tommy_arrayblkof* array, tommy_count_t pos)
+tommy_inline void* tommy_arrayblkof_ref(tommy_arrayblkof* array, tommy_size_t pos)
 {
 	unsigned char* base;
 
@@ -100,7 +100,7 @@ tommy_inline void* tommy_arrayblkof_ref(tommy_arrayblkof* array, tommy_count_t p
 /**
  * Gets the initialized size of the array.
  */
-tommy_inline tommy_count_t tommy_arrayblkof_size(tommy_arrayblkof* array)
+tommy_inline tommy_size_t tommy_arrayblkof_size(tommy_arrayblkof* array)
 {
 	return array->count;
 }

tommyds/tommychain.h

@@ -114,7 +114,7 @@ tommy_inline void tommy_chain_merge(tommy_chain* first, tommy_chain* second, tom
 
 /**
  * Merges two chains managing special degenerated cases.
- * It's funtionally equivalent at tommy_chain_merge() but faster with already ordered chains.
+ * It's functionally equivalent at tommy_chain_merge() but faster with already ordered chains.
  */
 tommy_inline void tommy_chain_merge_degenerated(tommy_chain* first, tommy_chain* second, tommy_compare_func* cmp)
 {
@@ -136,11 +136,6 @@ tommy_inline void tommy_chain_merge_degenerated(tommy_chain* first, tommy_chain*
 	tommy_chain_merge(first, second, cmp);
 }
 
-/**
- * Max number of elements as a power of 2.
- */
-#define TOMMY_CHAIN_BIT_MAX 32
-
 /**
  * Sorts a chain.
  * It's a stable merge sort using power of 2 buckets, with O(N*log(N)) complexity,
@@ -158,20 +153,20 @@ tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func*
 	/*
 	 * Bit buckets of chains.
 	 * Each bucket contains 2^i nodes or it's empty.
-	 * The chain at address TOMMY_CHAIN_BIT_MAX is an independet variable operating as "carry".
+	 * The chain at address TOMMY_BIT_MAX is an independent variable operating as "carry".
 	 * We keep it in the same "bit" vector to avoid reports from the valgrind tool sgcheck.
 	 */
-	tommy_chain bit[TOMMY_CHAIN_BIT_MAX + 1];
+	tommy_chain bit[TOMMY_SIZE_BIT + 1];
 
 	/**
 	 * Value stored inside the bit bucket.
 	 * It's used to know which bucket is empty of full.
 	 */
-	tommy_count_t counter;
+	tommy_size_t counter;
 	tommy_node* node = chain->head;
 	tommy_node* tail = chain->tail;
-	tommy_count_t mask;
-	tommy_count_t i;
+	tommy_size_t mask;
+	tommy_size_t i;
 
 	counter = 0;
 	while (1) {
@@ -179,9 +174,9 @@ tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func*
 		tommy_chain* last;
 
 		/* carry bit to add */
-		last = &bit[TOMMY_CHAIN_BIT_MAX];
-		bit[TOMMY_CHAIN_BIT_MAX].head = node;
-		bit[TOMMY_CHAIN_BIT_MAX].tail = node;
+		last = &bit[TOMMY_SIZE_BIT];
+		bit[TOMMY_SIZE_BIT].head = node;
+		bit[TOMMY_SIZE_BIT].tail = node;
 		next = node->next;
 
 		/* add the bit, propagating the carry */
@@ -206,7 +201,7 @@ tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func*
 	}
 
 	/* merge the buckets */
-	i = tommy_ctz_u32(counter);
+	i = tommy_ctz(counter);
 	mask = counter >> i;
 	while (mask != 1) {
 		mask >>= 1;

tommyds/tommyhash.c

@@ -92,22 +92,22 @@ tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tom
 		b += tommy_le_uint32_read(key + 4);
 		a += tommy_le_uint32_read(key + 0);
 		break;
-	case 11 : c += ((tommy_uint32_t)key[10]) << 16;
-	case 10 : c += ((tommy_uint32_t)key[9]) << 8;
-	case 9 : c += key[8];
+	case 11 : c += ((tommy_uint32_t)key[10]) << 16; /* fallthrough */
+	case 10 : c += ((tommy_uint32_t)key[9]) << 8; /* fallthrough */
+	case 9 : c += key[8]; /* fallthrough */
 	case 8 :
 		b += tommy_le_uint32_read(key + 4);
 		a += tommy_le_uint32_read(key + 0);
 		break;
-	case 7 : b += ((tommy_uint32_t)key[6]) << 16;
-	case 6 : b += ((tommy_uint32_t)key[5]) << 8;
-	case 5 : b += key[4];
+	case 7 : b += ((tommy_uint32_t)key[6]) << 16; /* fallthrough */
+	case 6 : b += ((tommy_uint32_t)key[5]) << 8; /* fallthrough */
+	case 5 : b += key[4]; /* fallthrough */
 	case 4 :
 		a += tommy_le_uint32_read(key + 0);
 		break;
-	case 3 : a += ((tommy_uint32_t)key[2]) << 16;
-	case 2 : a += ((tommy_uint32_t)key[1]) << 8;
-	case 1 : a += key[0];
+	case 3 : a += ((tommy_uint32_t)key[2]) << 16; /* fallthrough */
+	case 2 : a += ((tommy_uint32_t)key[1]) << 8; /* fallthrough */
+	case 1 : a += key[0]; /* fallthrough */
 	}
 
 	tommy_final(a, b, c);
@@ -142,22 +142,22 @@ tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tom
 		b += tommy_le_uint32_read(key + 4);
 		a += tommy_le_uint32_read(key + 0);
 		break;
-	case 11 : c += ((tommy_uint32_t)key[10]) << 16;
-	case 10 : c += ((tommy_uint32_t)key[9]) << 8;
-	case 9 : c += key[8];
+	case 11 : c += ((tommy_uint32_t)key[10]) << 16; /* fallthrough */
+	case 10 : c += ((tommy_uint32_t)key[9]) << 8; /* fallthrough */
+	case 9 : c += key[8]; /* fallthrough */
 	case 8 :
 		b += tommy_le_uint32_read(key + 4);
 		a += tommy_le_uint32_read(key + 0);
 		break;
-	case 7 : b += ((tommy_uint32_t)key[6]) << 16;
-	case 6 : b += ((tommy_uint32_t)key[5]) << 8;
-	case 5 : b += key[4];
+	case 7 : b += ((tommy_uint32_t)key[6]) << 16; /* fallthrough */
+	case 6 : b += ((tommy_uint32_t)key[5]) << 8; /* fallthrough */
+	case 5 : b += key[4]; /* fallthrough */
 	case 4 :
 		a += tommy_le_uint32_read(key + 0);
 		break;
-	case 3 : a += ((tommy_uint32_t)key[2]) << 16;
-	case 2 : a += ((tommy_uint32_t)key[1]) << 8;
-	case 1 : a += key[0];
+	case 3 : a += ((tommy_uint32_t)key[2]) << 16; /* fallthrough */
+	case 2 : a += ((tommy_uint32_t)key[1]) << 8; /* fallthrough */
+	case 1 : a += key[0]; /* fallthrough */
 	}
 
 	tommy_final(a, b, c);
@@ -231,7 +231,7 @@ tommy_uint32_t tommy_strhash_u32(tommy_uint64_t init_val, const void* void_key)
 		key += 12;
 	}
 
-	/* for lengths that are multiplers of 12 we already have called mix */
+	/* for lengths that are multipliers of 12 we already have called mix */
 	/* this is different than the original lookup3 and the result won't match */
 
 	tommy_final(a, b, c);

tommyds/tommyhash.h

@@ -37,11 +37,6 @@
 /******************************************************************************/
 /* hash */
 
-/**
- * Hash type used in hashtables.
- */
-typedef tommy_key_t tommy_hash_t;
-
 /**
  * Hash function with a 32 bits result.
  * Implementation of the Robert Jenkins "lookup3" hash 32 bits version,
@@ -57,7 +52,7 @@ typedef tommy_key_t tommy_hash_t;
  * \param void_key Pointer to the data to hash.
  * \param key_len Size of the data to hash.
  * \note
- * This function is endianess independent.
+ * This function is endianness independent.
  * \return The hash value of 32 bits.
  */
 tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tommy_size_t key_len);
@@ -77,14 +72,14 @@ tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tom
  * \param void_key Pointer to the data to hash.
  * \param key_len Size of the data to hash.
  * \note
- * This function is endianess independent.
+ * This function is endianness independent.
  * \return The hash value of 64 bits.
  */
 tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tommy_size_t key_len);
 
 /**
  * String hash function with a 32 bits result.
- * Implementation is based on the the Robert Jenkins "lookup3" hash 32 bits version,
+ * Implementation is based on Robert Jenkins "lookup3" hash 32 bits version,
  * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle().
  *
  * This hash is designed to handle strings with an unknown length. If you
@@ -95,7 +90,7 @@ tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tom
  * Use 0 if not relevant.
  * \param void_key Pointer to the string to hash. It has to be 0 terminated.
  * \note
- * This function is endianess independent.
+ * This function is endianness independent.
  * \return The hash value of 32 bits.
  */
 tommy_uint32_t tommy_strhash_u32(tommy_uint64_t init_val, const void* void_key);

tommyds/tommyhashdyn.c

@@ -35,7 +35,7 @@ void tommy_hashdyn_init(tommy_hashdyn* hashdyn)
 {
 	/* fixed initial size */
 	hashdyn->bucket_bit = TOMMY_HASHDYN_BIT;
-	hashdyn->bucket_max = 1 << hashdyn->bucket_bit;
+	hashdyn->bucket_max = (tommy_size_t)1 << hashdyn->bucket_bit;
 	hashdyn->bucket_mask = hashdyn->bucket_max - 1;
 	hashdyn->bucket = tommy_cast(tommy_hashdyn_node**, tommy_calloc(hashdyn->bucket_max, sizeof(tommy_hashdyn_node*)));
 
@@ -50,18 +50,18 @@ void tommy_hashdyn_done(tommy_hashdyn* hashdyn)
 /**
  * Resize the bucket vector.
  */
-static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_count_t new_bucket_bit)
+static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_size_t new_bucket_bit)
 {
-	tommy_count_t bucket_bit;
-	tommy_count_t bucket_max;
-	tommy_count_t new_bucket_max;
-	tommy_count_t new_bucket_mask;
+	tommy_size_t bucket_bit;
+	tommy_size_t bucket_max;
+	tommy_size_t new_bucket_max;
+	tommy_size_t new_bucket_mask;
 	tommy_hashdyn_node** new_bucket;
 
 	bucket_bit = hashdyn->bucket_bit;
 	bucket_max = hashdyn->bucket_max;
 
-	new_bucket_max = 1 << new_bucket_bit;
+	new_bucket_max = (tommy_size_t)1 << new_bucket_bit;
 	new_bucket_mask = new_bucket_max - 1;
 
 	/* allocate the new vector using malloc() and not calloc() */
@@ -70,7 +70,7 @@ static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_count_t new_bucke
 
 	/* reinsert all the elements */
 	if (new_bucket_bit > bucket_bit) {
-		tommy_count_t i;
+		tommy_size_t i;
 
 		/* grow */
 		for (i = 0; i < bucket_max; ++i) {
@@ -84,7 +84,7 @@ static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_count_t new_bucke
 			j = hashdyn->bucket[i];
 			while (j) {
 				tommy_hashdyn_node* j_next = j->next;
-				tommy_count_t pos = j->key & new_bucket_mask;
+				tommy_size_t pos = j->index & new_bucket_mask;
 				if (new_bucket[pos])
 					tommy_list_insert_tail_not_empty(new_bucket[pos], j);
 				else
@@ -93,7 +93,7 @@ static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_count_t new_bucke
 			}
 		}
 	} else {
-		tommy_count_t i;
+		tommy_size_t i;
 
 		/* shrink */
 		for (i = 0; i < new_bucket_max; ++i) {
@@ -136,11 +136,11 @@ tommy_inline void hashdyn_shrink_step(tommy_hashdyn* hashdyn)
 
 void tommy_hashdyn_insert(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void* data, tommy_hash_t hash)
 {
-	tommy_count_t pos = hash & hashdyn->bucket_mask;
+	tommy_size_t pos = hash & hashdyn->bucket_mask;
 
 	tommy_list_insert_tail(&hashdyn->bucket[pos], node, data);
 
-	node->key = hash;
+	node->index = hash;
 
 	++hashdyn->count;
 
@@ -149,7 +149,7 @@ void tommy_hashdyn_insert(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void
 
 void* tommy_hashdyn_remove_existing(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node)
 {
-	tommy_count_t pos = node->key & hashdyn->bucket_mask;
+	tommy_size_t pos = node->index & hashdyn->bucket_mask;
 
 	tommy_list_remove_existing(&hashdyn->bucket[pos], node);
 
@@ -162,12 +162,12 @@ void* tommy_hashdyn_remove_existing(tommy_hashdyn* hashdyn, tommy_hashdyn_node*
 
 void* tommy_hashdyn_remove(tommy_hashdyn* hashdyn, tommy_search_func* cmp, const void* cmp_arg, tommy_hash_t hash)
 {
-	tommy_count_t pos = hash & hashdyn->bucket_mask;
+	tommy_size_t pos = hash & hashdyn->bucket_mask;
 	tommy_hashdyn_node* node = hashdyn->bucket[pos];
 
 	while (node) {
 		/* we first check if the hash matches, as in the same bucket we may have multiples hash values */
-		if (node->key == hash && cmp(cmp_arg, node->data) == 0) {
+		if (node->index == hash && cmp(cmp_arg, node->data) == 0) {
 			tommy_list_remove_existing(&hashdyn->bucket[pos], node);
 
 			--hashdyn->count;
@@ -184,9 +184,9 @@ void* tommy_hashdyn_remove(tommy_hashdyn* hashdyn, tommy_search_func* cmp, const
 
 void tommy_hashdyn_foreach(tommy_hashdyn* hashdyn, tommy_foreach_func* func)
 {
-	tommy_count_t bucket_max = hashdyn->bucket_max;
+	tommy_size_t bucket_max = hashdyn->bucket_max;
 	tommy_hashdyn_node** bucket = hashdyn->bucket;
-	tommy_count_t pos;
+	tommy_size_t pos;
 
 	for (pos = 0; pos < bucket_max; ++pos) {
 		tommy_hashdyn_node* node = bucket[pos];
@@ -201,9 +201,9 @@ void tommy_hashdyn_foreach(tommy_hashdyn* hashdyn, tommy_foreach_func* func)
 
 void tommy_hashdyn_foreach_arg(tommy_hashdyn* hashdyn, tommy_foreach_arg_func* func, void* arg)
 {
-	tommy_count_t bucket_max = hashdyn->bucket_max;
+	tommy_size_t bucket_max = hashdyn->bucket_max;
 	tommy_hashdyn_node** bucket = hashdyn->bucket;
-	tommy_count_t pos;
+	tommy_size_t pos;
 
 	for (pos = 0; pos < bucket_max; ++pos) {
 		tommy_hashdyn_node* node = bucket[pos];

tommyds/tommyhashdyn.h

@@ -35,7 +35,7 @@
  * All the elements are reallocated in a single resize operation done inside
  * tommy_hashdyn_insert() or tommy_hashdyn_remove().
  *
- * Note that the resize operation takes approximatively 100 [ms] with 1 million of elements,
+ * Note that the resize operation takes approximately 100 [ms] with 1 million of elements,
  * and 1 [second] with 10 millions. This could be a problem in real-time applications.
  *
  * The resize also fragment the heap, as it involves allocating a double-sized table, copy elements,
@@ -92,7 +92,7 @@
  *
  * To iterate over all the elements in the hashtable with the same key, you have to
  * use tommy_hashdyn_bucket() and follow the tommy_node::next pointer until NULL.
- * You have also to check explicitely for the key, as the bucket may contains
+ * You have also to check explicitly for the key, as the bucket may contains
  * different keys.
  *
  * \code
@@ -160,10 +160,10 @@ typedef tommy_node tommy_hashdyn_node;
  */
 typedef struct tommy_hashdyn_struct {
 	tommy_hashdyn_node** bucket; /**< Hash buckets. One list for each hash modulus. */
+	tommy_size_t bucket_max; /**< Number of buckets. */
+	tommy_size_t bucket_mask; /**< Bit mask to access the buckets. */
+	tommy_size_t count; /**< Number of elements. */
 	tommy_uint_t bucket_bit; /**< Bits used in the bit mask. */
-	tommy_count_t bucket_max; /**< Number of buckets. */
-	tommy_count_t bucket_mask; /**< Bit mask to access the buckets. */
-	tommy_count_t count; /**< Number of elements. */
 } tommy_hashdyn;
 
 /**
@@ -226,7 +226,7 @@ tommy_inline void* tommy_hashdyn_search(tommy_hashdyn* hashdyn, tommy_search_fun
 
 	while (i) {
 		/* we first check if the hash matches, as in the same bucket we may have multiples hash values */
-		if (i->key == hash && cmp(cmp_arg, i->data) == 0)
+		if (i->index == hash && cmp(cmp_arg, i->data) == 0)
 			return i->data;
 		i = i->next;
 	}
@@ -281,7 +281,7 @@ void tommy_hashdyn_foreach_arg(tommy_hashdyn* hashdyn, tommy_foreach_arg_func* f
 /**
  * Gets the number of elements.
  */
-tommy_inline tommy_count_t tommy_hashdyn_count(tommy_hashdyn* hashdyn)
+tommy_inline tommy_size_t tommy_hashdyn_count(tommy_hashdyn* hashdyn)
 {
 	return hashdyn->count;
 }

tommyds/tommylist.h

@@ -28,7 +28,7 @@
 /** \file
  * Double linked list for collisions into hashtables.
  *
- * This list is a double linked list mainly targetted for handling collisions
+ * This list is a double linked list mainly targeted for handling collisions
  * into an hashtables, but useable also as a generic list.
  *
  * The main feature of this list is to require only one pointer to represent the
@@ -226,24 +226,6 @@ tommy_inline void tommy_list_insert_tail(tommy_list* list, tommy_node* node, voi
 	node->data = data;
 }
 
-/** \internal
- * Removes an element from the head of a not empty list.
- * \param list The list. The list cannot be empty.
- * \return The node removed.
- */
-tommy_inline tommy_node* tommy_list_remove_head_not_empty(tommy_list* list)
-{
-	tommy_node* head = tommy_list_head(list);
-
-	/* remove from the "circular" prev list */
-	head->next->prev = head->prev;
-
-	/* remove from the "0 terminated" next list */
-	*list = head->next; /* the new head, in case 0 */
-
-	return head;
-}
-
 /**
  * Removes an element from the list.
  * You must already have the address of the element to remove.
@@ -312,7 +294,7 @@ tommy_inline void tommy_list_concat(tommy_list* first, tommy_list* second)
  * It's a stable merge sort with O(N*log(N)) worst complexity.
  * It's faster on degenerated cases like partially ordered lists.
  * \param cmp Compare function called with two elements.
- * The function should return <0 if the first element is less than the second, ==0 if equal, and >0 if greather.
+ * The function should return <0 if the first element is less than the second, ==0 if equal, and >0 if greater.
  */
 void tommy_list_sort(tommy_list* list, tommy_compare_func* cmp);
 
@@ -329,9 +311,9 @@ tommy_inline tommy_bool_t tommy_list_empty(tommy_list* list)
  * Gets the number of elements.
  * \note This operation is O(n).
  */
-tommy_inline tommy_count_t tommy_list_count(tommy_list* list)
+tommy_inline tommy_size_t tommy_list_count(tommy_list* list)
 {
-	tommy_count_t count = 0;
+	tommy_size_t count = 0;
 	tommy_node* i = tommy_list_head(list);
 
 	while (i) {

tommyds/tommytree.c

@@ -39,10 +39,10 @@ void tommy_tree_init(tommy_tree* tree, tommy_compare_func* cmp)
 	tree->cmp = cmp;
 }
 
-static int tommy_tree_delta(tommy_tree_node* root)
+static tommy_ssize_t tommy_tree_delta(tommy_tree_node* root)
 {
-	int left_height = root->prev ? root->prev->key : 0;
-	int right_height = root->next ? root->next->key : 0;
+	tommy_ssize_t left_height = root->prev ? root->prev->index : 0;
+	tommy_ssize_t right_height = root->next ? root->next->index : 0;
 
 	return left_height - right_height;
 }
@@ -84,7 +84,7 @@ static tommy_tree_node* tommy_tree_move_right(tommy_tree_node* root, tommy_tree_
 
 static tommy_tree_node* tommy_tree_balance(tommy_tree_node* root)
 {
-	int delta = tommy_tree_delta(root);
+	tommy_ssize_t delta = tommy_tree_delta(root);
 
 	if (delta < -1) {
 		if (tommy_tree_delta(root->next) > 0)
@@ -99,16 +99,16 @@ static tommy_tree_node* tommy_tree_balance(tommy_tree_node* root)
 	}
 
 	/* recompute key */
-	root->key = 0;
+	root->index = 0;
 
-	if (root->prev && root->prev->key > root->key)
-		root->key = root->prev->key;
+	if (root->prev && root->prev->index > root->index)
+		root->index = root->prev->index;
 
-	if (root->next && root->next->key > root->key)
-		root->key = root->next->key;
+	if (root->next && root->next->index > root->index)
+		root->index = root->next->index;
 
 	/* count itself */
-	root->key += 1;
+	root->index += 1;
 
 	return root;
 }
@@ -145,7 +145,7 @@ void* tommy_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data)
 	insert->data = data;
 	insert->prev = 0;
 	insert->next = 0;
-	insert->key = 0;
+	insert->index = 0;
 
 	tree->root = tommy_tree_insert_node(tree->cmp, tree->root, &insert);
 

tommyds/tommytree.h

@@ -33,7 +33,7 @@
  *
  * As difference than other tommy containers, duplicate elements cannot be inserted.
  *
- * To initialize a tree you have to call tommy_tree_init() specifing a comparison
+ * To initialize a tree you have to call tommy_tree_init() specifying a comparison
  * function that will define the order in the tree.
  *
  * \code
@@ -117,13 +117,13 @@ typedef tommy_node tommy_tree_node;
  */
 typedef struct tommy_tree_struct {
 	tommy_tree_node* root; /**< Root node. */
-	tommy_count_t count; /**< Number of elements. */
 	tommy_compare_func* cmp; /**< Comparison function. */
+	tommy_size_t count; /**< Number of elements. */
 } tommy_tree;
 
 /**
  * Initializes the tree.
- * \param cmp The comparison function that defines the orderin the tree.
+ * \param cmp The comparison function that defines the order in the tree.
  */
 void tommy_tree_init(tommy_tree* tree, tommy_compare_func* cmp);
 
@@ -213,7 +213,7 @@ void tommy_tree_foreach_arg(tommy_tree* tree, tommy_foreach_arg_func* func, void
 /**
  * Gets the number of elements.
  */
-tommy_inline tommy_count_t tommy_tree_count(tommy_tree* tree)
+tommy_inline tommy_size_t tommy_tree_count(tommy_tree* tree)
 {
 	return tree->count;
 }

tommyds/tommytypes.h

@@ -24,7 +24,6 @@
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
-
 /** \file
  * Generic types.
  */
@@ -37,17 +36,37 @@
 
 #include <stddef.h>
 
-#if defined(_MSC_VER)
+#ifdef _MSC_VER
 typedef unsigned tommy_uint32_t; /**< Generic uint32_t type. */
 typedef unsigned _int64 tommy_uint64_t; /**< Generic uint64_t type. */
 typedef size_t tommy_uintptr_t; /**< Generic uintptr_t type. */
+#ifdef _WIN64
+#define TOMMY_SIZE_BIT 64
+typedef unsigned _int64_t tommy_size_t; /**< Generic size_t type. */
+typedef _int64_t tommy_ssize_t; /**< Generic ssize_t type. */
+#else
+#define TOMMY_SIZE_BIT 32
+typedef unsigned tommy_size_t; /**< Generic size_t type. */
+typedef int tommy_ssize_t; /**< Generic ssize_t type. */
+#endif
 #else
 #include <stdint.h>
 typedef uint32_t tommy_uint32_t; /**< Generic uint32_t type. */
 typedef uint64_t tommy_uint64_t; /**< Generic uint64_t type. */
 typedef uintptr_t tommy_uintptr_t; /**< Generic uintptr_t type. */
+#if SIZE_MAX == UINT64_MAX
+#define TOMMY_SIZE_BIT 64
+typedef uint64_t tommy_size_t; /**< Generic size_t type. */
+typedef int64_t tommy_ssize_t; /**< Generic ssize_t type. */
+#elif SIZE_MAX == UINT32_MAX
+#define TOMMY_SIZE_BIT 32
+typedef uint32_t tommy_size_t; /**< Generic size_t type. */
+typedef int32_t tommy_ssize_t; /**< Generic ssize_t type. */
+#else
+#error Unsupported SIZE_MAX
 #endif
-typedef size_t tommy_size_t; /**< Generic size_t type. */
+#endif
+
 typedef ptrdiff_t tommy_ptrdiff_t; /**< Generic ptrdiff_t type. */
 typedef int tommy_bool_t; /**< Generic boolean type. */
 
@@ -59,13 +78,6 @@ typedef int tommy_bool_t; /**< Generic boolean type. */
  */
 typedef tommy_uint32_t tommy_uint_t;
 
-/**
- * Generic unsigned integer for counting objects.
- *
- * TommyDS doesn't support more than 2^32-1 objects.
- */
-typedef tommy_uint32_t tommy_count_t;
-
 /** \internal
  * Type cast required for the C++ compilation.
  * When compiling in C++ we cannot convert a void* pointer to another pointer.
@@ -152,17 +164,17 @@ typedef tommy_uint32_t tommy_count_t;
 #endif
 
 /******************************************************************************/
-/* key */
+/* key/hash */
 
 /**
- * Key type used in indexed data structures to store the key or the hash value.
+ * Type used in indexed data structures to store the key of a object.
  */
-typedef tommy_uint32_t tommy_key_t;
+typedef tommy_size_t tommy_key_t;
 
 /**
- * Bits into the ::tommy_key_t type.
+ * Type used in hashtables to store the hash of a object.
  */
-#define TOMMY_KEY_BIT (sizeof(tommy_key_t) * 8)
+typedef tommy_size_t tommy_hash_t;
 
 /******************************************************************************/
 /* node */
@@ -200,11 +212,12 @@ typedef struct tommy_node_struct {
 	void* data;
 
 	/**
-	 * Key used to store the node.
+	 * Index of the node.
+	 * With tries this field is used to store the key.
 	 * With hashtables this field is used to store the hash value.
 	 * With lists this field is not used.
 	 */
-	tommy_key_t key;
+	tommy_size_t index;
 } tommy_node;
 
 /******************************************************************************/
@@ -214,7 +227,7 @@ typedef struct tommy_node_struct {
  * Compare function for elements.
  * \param obj_a Pointer to the first object to compare.
  * \param obj_b Pointer to the second object to compare.
- * \return <0 if the first element is less than the second, ==0 equal, >0 if greather.
+ * \return <0 if the first element is less than the second, ==0 equal, >0 if greater.
  *
  * This function is like the C strcmp().
  *
@@ -302,6 +315,10 @@ typedef void tommy_foreach_arg_func(void* arg, void* obj);
 #include <intrin.h>
 #pragma intrinsic(_BitScanReverse)
 #pragma intrinsic(_BitScanForward)
+#if TOMMY_SIZE_BIT == 64
+#pragma intrinsic(_BitScanReverse64)
+#pragma intrinsic(_BitScanForward64)
+#endif
 #endif
 
 /** \internal
@@ -363,6 +380,29 @@ tommy_inline tommy_uint_t tommy_ilog2_u32(tommy_uint32_t value)
 #endif
 }
 
+#if TOMMY_SIZE_BIT == 64
+/**
+ * Bit scan reverse or integer log2 for 64 bits.
+ */
+tommy_inline tommy_uint_t tommy_ilog2_u64(tommy_uint64_t value)
+{
+#if defined(_MSC_VER)
+	unsigned long count;
+	_BitScanReverse64(&count, value);
+	return count;
+#elif defined(__GNUC__)
+	return __builtin_clzll(value) ^ 63;
+#else
+	uint32_t l = value & 0xFFFFFFFFU;
+	uint32_t h = value >> 32;
+	if (h)
+		return tommy_ilog2_u32(h) + 32;
+	else
+		return tommy_ilog2_u32(l);
+#endif
+}
+#endif
+
 /**
  * Bit scan forward or trailing zero count.
  * Return the bit index of the least significant 1 bit.
@@ -391,6 +431,29 @@ tommy_inline tommy_uint_t tommy_ctz_u32(tommy_uint32_t value)
 #endif
 }
 
+#if TOMMY_SIZE_BIT == 64
+/**
+ * Bit scan forward or trailing zero count for 64 bits.
+ */
+tommy_inline tommy_uint_t tommy_ctz_u64(tommy_uint64_t value)
+{
+#if defined(_MSC_VER)
+	unsigned long count;
+	_BitScanForward64(&count, value);
+	return count;
+#elif defined(__GNUC__)
+	return __builtin_ctzll(value);
+#else
+	uint32_t l = value & 0xFFFFFFFFU;
+	uint32_t h = value >> 32;
+	if (l)
+		return tommy_ctz_u32(l);
+	else
+		return tommy_ctz_u32(h) + 32;
+#endif
+}
+#endif
+
 /**
  * Rounds up to the next power of 2.
  * For the value 0, the result is undefined.
@@ -412,6 +475,23 @@ tommy_inline tommy_uint32_t tommy_roundup_pow2_u32(tommy_uint32_t value)
 	return value;
 }
 
+/**
+ * Rounds up to the next power of 2 for 64 bits.
+ */
+tommy_inline tommy_uint64_t tommy_roundup_pow2_u64(tommy_uint64_t value)
+{
+	--value;
+	value |= value >> 1;
+	value |= value >> 2;
+	value |= value >> 4;
+	value |= value >> 8;
+	value |= value >> 16;
+	value |= value >> 32;
+	++value;
+
+	return value;
+}
+
 /**
  * Check if the specified word has a byte at 0.
  * \return 0 or 1.
@@ -420,5 +500,19 @@ tommy_inline int tommy_haszero_u32(tommy_uint32_t value)
 {
 	return ((value - 0x01010101) & ~value & 0x80808080) != 0;
 }
+
+/*
+ * Bit depth mapping.
+ */
+#if TOMMY_SIZE_BIT == 64
+#define tommy_ilog2 tommy_ilog2_u64
+#define tommy_ctz tommy_ctz_u64
+#define tommy_roundup_pow2 tommy_roundup_pow2_u64
+#else
+#define tommy_ilog2 tommy_ilog2_u32
+#define tommy_ctz tommy_ctz_u32
+#define tommy_roundup_pow2 tommy_roundup_pow2_u32
+#endif
+
 #endif