Merge tag 'upstream/11.3'

Upstream version 11.3

.version

@@ -1 +1 @@
-11.2
+11.3

HISTORY

@@ -1,6 +1,16 @@
 SnapRAID HISTORY
 ================
 
+11.3 2018/11
+============
+ * Fixed handing of Linux devices that have multiple slaves. This affects
+   the smart/list/devices/down commands [Valentin Hilbig].
+ * The 'list' command in verbose mode prints the full nanosecond
+   timestamp precision.
+ * After writing content files also sync their directory.
+ * Fix a invalid time computation that could result in future scrub dates.
+   Such dates are fixed automatically at the next scrub or sync.
+
 11.2 2017/12
 ============
  * Fixed recognition of NTFS hardlinks. They behave differently than

TODO

@@ -17,6 +17,18 @@ Not checked disks should be allowed to be missing.
 * Add an option to ignore subsecond timestamp.
 Like when you copy data to a filesystem with less timestamp precision.
 
+* Use threads to scan all the disks at the same time.
+- After 7.0 Windows changes it seems fast enough even
+with a mono thread implementation with 100.0000 files.
++ But if you have millions of files, it could take minutes.
+
+* Support more parity levels
+It can be done with a generic computation function, using
+intrinsic for SSSE3 and AVX instructions.
+It would be intersting to compare performance with the hand-written
+assembler functions. Eventially we can convert them to use intrinsic also.
+https://sourceforge.net/p/snapraid/discussion/1677233/thread/9dbd7581/
+
 * Extend haspdeep to support the SnapRAID hash :
 https://github.com/jessek/hashdeep/
 https://sourceforge.net/p/snapraid/discussion/1677233/thread/90b0e9b2/?limit=25
@@ -222,10 +234,6 @@ and if we manage to keep it in the cache, we should save time.
 - We now hash first the faster disks, and this could
 reduce performance as we'll have to wait for all disks.
 
-* Use threads to scan all the disks at the same time.
-- After 7.0 Windows changes it seems fast enough even
-with a mono thread implementation.
-
 * Enable storing of creation time NTFS, crtime/birth time EXT4.
 But see: http://unix.stackexchange.com/questions/50177/birth-is-empty-on-ext4
 coreutils stat has an example, but it doesn't work in Linux (see lib/stat-time.h)

cmdline/io.c

@@ -889,7 +889,7 @@ void io_init(struct snapraid_io* io, struct snapraid_state* state,
 		allocated += state->block_size * buffer_max;
 	}
 
-	msg_progress("Using %u MiB of memory for %u blocks of IO cache.\n", (unsigned)(allocated / MEBI), io->io_max);
+	msg_progress("Using %u MiB of memory for %u cached blocks.\n", (unsigned)(allocated / MEBI), io->io_max);
 
 	if (parity_writer) {
 		io->reader_max = handle_max;

cmdline/list.c

@@ -74,7 +74,7 @@ void state_list(struct snapraid_state* state)
 			if (tm) {
 				printf("%04u/%02u/%02u %02u:%02u", tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min);
 				if (msg_level >= MSG_VERBOSE)
-					printf(":%02u.%03u", tm->tm_sec, file->mtime_nsec / 1000000);
+					printf(":%02u.%09u", tm->tm_sec, file->mtime_nsec);
 				printf(" ");
 			}
 			printf("%s\n", fmt_term(disk, file->sub, esc_buffer));
@@ -105,7 +105,7 @@ void state_list(struct snapraid_state* state)
 			printf("%12s ", type);
 			printf("                 ");
 			if (msg_level >= MSG_VERBOSE)
-				printf("       ");
+				printf("             ");
 			printf("%s -> %s\n", fmt_term(disk, slink->sub, esc_buffer), fmt_term(disk, slink->linkto, esc_buffer_alt));
 		}
 	}

cmdline/mingw.c

@@ -694,6 +694,7 @@ static void windows_errno(DWORD error)
 		errno = EPERM;
 		break;
 	case ERROR_IO_DEVICE : /* in ReadFile() and WriteFile() */
+	case ERROR_CRC : /* in ReadFile() */
 		errno = EIO;
 		break;
 	default :

cmdline/mkstream.c

@@ -41,6 +41,7 @@ void test(void)
 	s = sopen_multi_write(STREAM_MAX);
 	for (i = 0; i < STREAM_MAX; ++i) {
 		snprintf(file, sizeof(file), "stream%u.bin", i);
+		remove(file);
 		if (sopen_multi_file(s, i, file) != 0) {
 			/* LCOV_EXCL_START */
 			exit(EXIT_FAILURE);

cmdline/murmur3.c

@@ -136,25 +136,29 @@ void MurmurHash3_x86_128(const void* data, size_t size, const uint8_t* seed, voi
 		uint32_t k4 = 0;
 
 		switch (size_remainder) {
-		case 15 : k4 ^= (uint32_t)tail[14] << 16;
+		case 15 : k4 ^= (uint32_t)tail[14] << 16; /* fallthrough */
-		case 14 : k4 ^= (uint32_t)tail[13] << 8;
+		case 14 : k4 ^= (uint32_t)tail[13] << 8; /* fallthrough */
-		case 13 : k4 ^= (uint32_t)tail[12] << 0;
+		case 13 : k4 ^= (uint32_t)tail[12] << 0; /* fallthrough */
 			k4 *= c4; k4 = util_rotl32(k4, 18); k4 *= c1; h4 ^= k4;
-		case 12 : k3 ^= (uint32_t)tail[11] << 24;
+			/* fallthrough */
-		case 11 : k3 ^= (uint32_t)tail[10] << 16;
+		case 12 : k3 ^= (uint32_t)tail[11] << 24; /* fallthrough */
-		case 10 : k3 ^= (uint32_t)tail[ 9] << 8;
+		case 11 : k3 ^= (uint32_t)tail[10] << 16; /* fallthrough */
-		case 9 : k3 ^= (uint32_t)tail[ 8] << 0;
+		case 10 : k3 ^= (uint32_t)tail[ 9] << 8; /* fallthrough */
+		case 9 : k3 ^= (uint32_t)tail[ 8] << 0; /* fallthrough */
 			k3 *= c3; k3 = util_rotl32(k3, 17); k3 *= c4; h3 ^= k3;
-		case 8 : k2 ^= (uint32_t)tail[ 7] << 24;
+			/* fallthrough */
-		case 7 : k2 ^= (uint32_t)tail[ 6] << 16;
+		case 8 : k2 ^= (uint32_t)tail[ 7] << 24; /* fallthrough */
-		case 6 : k2 ^= (uint32_t)tail[ 5] << 8;
+		case 7 : k2 ^= (uint32_t)tail[ 6] << 16; /* fallthrough */
-		case 5 : k2 ^= (uint32_t)tail[ 4] << 0;
+		case 6 : k2 ^= (uint32_t)tail[ 5] << 8; /* fallthrough */
+		case 5 : k2 ^= (uint32_t)tail[ 4] << 0; /* fallthrough */
 			k2 *= c2; k2 = util_rotl32(k2, 16); k2 *= c3; h2 ^= k2;
-		case 4 : k1 ^= (uint32_t)tail[ 3] << 24;
+			/* fallthrough */
-		case 3 : k1 ^= (uint32_t)tail[ 2] << 16;
+		case 4 : k1 ^= (uint32_t)tail[ 3] << 24; /* fallthrough */
-		case 2 : k1 ^= (uint32_t)tail[ 1] << 8;
+		case 3 : k1 ^= (uint32_t)tail[ 2] << 16; /* fallthrough */
-		case 1 : k1 ^= (uint32_t)tail[ 0] << 0;
+		case 2 : k1 ^= (uint32_t)tail[ 1] << 8; /* fallthrough */
+		case 1 : k1 ^= (uint32_t)tail[ 0] << 0; /* fallthrough */
 			k1 *= c1; k1 = util_rotl32(k1, 15); k1 *= c2; h1 ^= k1;
+			/* fallthrough */
 		}
 	}
 

cmdline/snapraid.c

@@ -87,7 +87,7 @@ void memory(void)
 	log_tag("memory:link:%" PRIu64 "\n", (uint64_t)(sizeof(struct snapraid_link)));
 	log_tag("memory:dir:%" PRIu64 "\n", (uint64_t)(sizeof(struct snapraid_dir)));
 
-	msg_progress("Using %u MiB of memory for the FileSystem.\n", (unsigned)(malloc_counter_get() / MEBI));
+	msg_progress("Using %u MiB of memory for the file-system.\n", (unsigned)(malloc_counter_get() / MEBI));
 }
 
 void test(int argc, char* argv[])
@@ -1091,7 +1091,7 @@ int main(int argc, char* argv[])
 			exit(EXIT_FAILURE);
 			/* LCOV_EXCL_STOP */
 		}
-		/* follow */
+		/* fallthrough */
 	case OPERATION_SPINUP :
 	case OPERATION_SPINDOWN :
 		if (!tommy_list_empty(&filterlist_file)) {

cmdline/state.c

@@ -2872,6 +2872,7 @@ struct state_write_thread_context {
 	/* input */
 	block_off_t blockmax;
 	time_t info_oldest;
+	time_t info_now;
 	int info_has_rehash;
 	STREAM* f;
 	/* output */
@@ -2888,6 +2889,7 @@ static void* state_write_thread(void* arg)
 	struct snapraid_state* state = context->state;
 	block_off_t blockmax = context->blockmax;
 	time_t info_oldest = context->info_oldest;
+	time_t info_now = context->info_now;
 	int info_has_rehash = context->info_has_rehash;
 	STREAM* f = context->f;
 	uint32_t crc;
@@ -3278,7 +3280,18 @@ static void* state_write_thread(void* arg)
 				flag |= 8;
 			sputb32(flag, f);
 
-			t = info_get_time(info) - info_oldest;
+			t = info_get_time(info);
+
+			/* truncate any time that is in the future */
+			if (t > info_now)
+				t = info_now;
+
+			/* the oldest info is computed only on required blocks, so it may not be the absolute oldest */
+			if (t < info_oldest)
+				t = 0;
+			else
+				t -= info_oldest;
+
 			sputb32(t, f);
 		} else {
 			/* write a special 0 flag to mark missing info */
@@ -3353,6 +3366,7 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
 	tommy_node* i;
 	block_off_t blockmax;
 	time_t info_oldest;
+	time_t info_now;
 	int info_has_rehash;
 	int mapping_idx;
 	block_off_t idx;
@@ -3371,6 +3385,7 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
 	/* clear the info for unused blocks */
 	/* and get some other info */
 	info_oldest = 0; /* oldest time in info */
+	info_now = time(0); /* get the present time */
 	info_has_rehash = 0; /* if there is a rehash info */
 	for (idx = 0; idx < blockmax; ++idx) {
 		/* if the position is used */
@@ -3434,10 +3449,21 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
 		msg_progress("Saving state to %s...\n", content->content);
 
 		pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
+
+		/* ensure to delete a previous stale file */
+		if (remove(tmp) != 0) {
+			if (errno != ENOENT) {
+				/* LCOV_EXCL_START */
+				log_fatal("Error removing the stale content file '%s'. %s.\n", tmp, strerror(errno));
+				exit(EXIT_FAILURE);
+				/* LCOV_EXCL_STOP */
+			}
+		}
+
 		f = sopen_write(tmp);
 		if (f == 0) {
 			/* LCOV_EXCL_START */
-			log_fatal("Error opening the content file '%s'. %s.\n", tmp, strerror(errno));
+			log_fatal("Error opening the temporary content file '%s'. %s.\n", tmp, strerror(errno));
 			exit(EXIT_FAILURE);
 			/* LCOV_EXCL_STOP */
 		}
@@ -3450,6 +3476,7 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
 		context->state = state;
 		context->blockmax = blockmax;
 		context->info_oldest = info_oldest;
+		context->info_now = info_now;
 		context->info_has_rehash = info_has_rehash;
 		context->f = f;
 
@@ -3562,12 +3589,24 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
 		struct snapraid_content* content = i->data;
 		char tmp[PATH_MAX];
 		pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
+
+		/* ensure to delete a previous stale file */
+		if (remove(tmp) != 0) {
+			if (errno != ENOENT) {
+				/* LCOV_EXCL_START */
+				log_fatal("Error removing the stale content file '%s'. %s.\n", tmp, strerror(errno));
+				exit(EXIT_FAILURE);
+				/* LCOV_EXCL_STOP */
+			}
+		}
+
 		if (sopen_multi_file(f, k, tmp) != 0) {
 			/* LCOV_EXCL_START */
-			log_fatal("Error opening the content file '%s'. %s.\n", tmp, strerror(errno));
+			log_fatal("Error opening the temporary content file '%s'. %s.\n", tmp, strerror(errno));
 			exit(EXIT_FAILURE);
 			/* LCOV_EXCL_STOP */
 		}
+
 		++k;
 		i = i->next;
 	}
@@ -3579,6 +3618,7 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
 	context->state = state;
 	context->blockmax = blockmax;
 	context->info_oldest = info_oldest;
+	context->info_now = info_now;
 	context->info_has_rehash = info_has_rehash;
 	context->f = f;
 
@@ -3851,11 +3891,10 @@ static void state_verify_content(struct snapraid_state* state, uint32_t crc)
 		msg_progress("Verifying %s...\n", content->content);
 
 		pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
-
 		f = sopen_read(tmp);
 		if (f == 0) {
 			/* LCOV_EXCL_START */
-			log_fatal("Error reopening the content file '%s'. %s.\n", tmp, strerror(errno));
+			log_fatal("Error reopening the temporary content file '%s'. %s.\n", tmp, strerror(errno));
 			exit(EXIT_FAILURE);
 			/* LCOV_EXCL_STOP */
 		}
@@ -3924,6 +3963,59 @@ static void state_rename_content(struct snapraid_state* state)
 {
 	tommy_node* i;
 
+#if defined(_linux) /* this sequence is linux specific */
+	i = tommy_list_head(&state->contentlist);
+	while (i) {
+		struct snapraid_content* content = i->data;
+		char tmp[PATH_MAX];
+		char dir[PATH_MAX];
+		char* slash;
+		int handle;
+
+		pathcpy(dir, sizeof(dir), content->content);
+
+		slash = strrchr(tmp, '/');
+		if (slash)
+			*slash = 0;
+		else
+			pathcpy(dir, sizeof(dir), ".");
+
+		/* open the directory to get the handle */
+		handle = open(dir, O_RDONLY | O_DIRECTORY);
+		if (handle < 0) {
+			/* LCOV_EXCL_START */
+			log_fatal("Error opening the directory '%s'. %s.\n", dir, strerror(errno));
+			exit(EXIT_FAILURE);
+			/* LCOV_EXCL_STOP */
+		}
+
+		/* now rename the just written copy with the correct name */
+		pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
+		if (rename(tmp, content->content) != 0) {
+			/* LCOV_EXCL_START */
+			log_fatal("Error renaming the content file '%s' to '%s'. %s.\n", tmp, content->content, strerror(errno));
+			exit(EXIT_FAILURE);
+			/* LCOV_EXCL_STOP */
+		}
+
+		/* sync the directory */
+		if (fsync(handle) != 0) {
+			/* LCOV_EXCL_START */
+			log_fatal("Error syncing the directory '%s'. %s.\n", dir, strerror(errno));
+			exit(EXIT_FAILURE);
+			/* LCOV_EXCL_STOP */
+		}
+
+		if (close(handle) != 0) {
+			/* LCOV_EXCL_START */
+			log_fatal("Error closing the directory '%s'. %s.\n", dir, strerror(errno));
+			exit(EXIT_FAILURE);
+			/* LCOV_EXCL_STOP */
+		}
+
+		i = i->next;
+	}
+#else
 	i = tommy_list_head(&state->contentlist);
 	while (i) {
 		struct snapraid_content* content = i->data;
@@ -3940,6 +4032,7 @@ static void state_rename_content(struct snapraid_state* state)
 
 		i = i->next;
 	}
+#endif
 }
 
 void state_write(struct snapraid_state* state)

cmdline/status.c

@@ -29,6 +29,10 @@
 
 unsigned day_ago(time_t ref, time_t now)
 {
+	/* in case some dates is in the future */
+	if (now < ref)
+		return 0;
+
 	return (now - ref) / (24 * 3600);
 }
 
@@ -458,6 +462,10 @@ int state_status(struct snapraid_state* state)
 
 	printf("\n");
 
+	if (newest > now) {
+		printf("WARNING! You have scrub dates in the future! The next sync/scrub will truncate them!\n");
+	}
+
 	if (unsynced_blocks) {
 		printf("WARNING! The array is NOT fully synced.\n");
 		printf("You have a sync in progress at %u%%.\n", (blockmax - unsynced_blocks) * 100 / blockmax);

cmdline/stream.c

@@ -112,7 +112,8 @@ int sopen_multi_file(STREAM* s, unsigned i, const char* file)
 
 	pathcpy(s->handle[i].path, sizeof(s->handle[i].path), file);
 
-	f = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY | O_SEQUENTIAL, 0600);
+	/* O_EXCL to be resilent ensure to always create a new file and not use a stale link to the original file */
+	f = open(file, O_WRONLY | O_CREAT | O_EXCL | O_BINARY | O_SEQUENTIAL, 0600);
 	if (f == -1) {
 		/* LCOV_EXCL_START */
 		return -1;

cmdline/support.h

@@ -299,6 +299,8 @@ void pathslash(char* dst, size_t size);
 
 /**
  * Cut everything after the latest slash.
+ *
+ * If the string doesn't contain any slash, it returns the empty string.
  */
 void pathcut(char* dst);
 

cmdline/sync.c

@@ -1536,7 +1536,7 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
 				data_off_t out_size;
 				parity_size(&parity_handle[l], &out_size);
 				parity_overflow(state, out_size);
-				log_fatal("WARNING! Without an unsable %s file, it isn't possible to sync.\n", lev_name(l));
+				log_fatal("WARNING! Without an usable %s file, it isn't possible to sync.\n", lev_name(l));
 				exit(EXIT_FAILURE);
 				/* LCOV_EXCL_STOP */
 			}

cmdline/unix.c

@@ -917,18 +917,20 @@ static int devtree(const char* name, const char* custom, dev_t device, devinfo_t
 
 		while ((dd = readdir(d)) != 0) {
 			if (dd->d_name[0] != '.') {
+				dev_t subdev;
+
 				/* for each slave, expand the full potential tree */
 				pathprint(path, sizeof(path), "/sys/dev/block/%u:%u/slaves/%s/dev", major(device), minor(device), dd->d_name);
 
-				device = devread(path);
+				subdev = devread(path);
-				if (!device) {
+				if (!subdev) {
 					/* LCOV_EXCL_START */
 					closedir(d);
 					return -1;
 					/* LCOV_EXCL_STOP */
 				}
 
-				if (devtree(name, custom, device, parent, list) != 0) {
+				if (devtree(name, custom, subdev, parent, list) != 0) {
 					/* LCOV_EXCL_START */
 					closedir(d);
 					return -1;
@@ -1072,8 +1074,8 @@ static int devscan(tommy_list* list)
 #if HAVE_LINUX_DEVICE
 static int devsmart(dev_t device, const char* name, const char* custom, uint64_t* smart, char* serial, char* vendor, char* model)
 {
-	char cmd[128];
+	char cmd[PATH_MAX + 64];
-	char file[128];
+	char file[PATH_MAX];
 	FILE* f;
 	int ret;
 
@@ -1086,7 +1088,7 @@ static int devsmart(dev_t device, const char* name, const char* custom, uint64_t
 
 	/* if there is a custom command */
 	if (custom[0]) {
-		char option[128];
+		char option[PATH_MAX];
 		snprintf(option, sizeof(option), custom, file);
 		snprintf(cmd, sizeof(cmd), "smartctl -a %s", option);
 	} else {
@@ -1140,8 +1142,8 @@ static int devsmart(dev_t device, const char* name, const char* custom, uint64_t
 #if HAVE_LINUX_DEVICE
 static int devdown(dev_t device, const char* name, const char* custom)
 {
-	char cmd[128];
+	char cmd[PATH_MAX + 64];
-	char file[128];
+	char file[PATH_MAX];
 	FILE* f;
 	int ret;
 
@@ -1154,7 +1156,7 @@ static int devdown(dev_t device, const char* name, const char* custom)
 
 	/* if there is a custom command */
 	if (custom[0]) {
-		char option[128];
+		char option[PATH_MAX];
 		snprintf(option, sizeof(option), custom, file);
 		snprintf(cmd, sizeof(cmd), "smartctl -s standby,now %s", option);
 	} else {

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for snapraid 11.2.
+# Generated by GNU Autoconf 2.69 for snapraid 11.3.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -577,8 +577,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='snapraid'
 PACKAGE_TARNAME='snapraid'
-PACKAGE_VERSION='11.2'
+PACKAGE_VERSION='11.3'
-PACKAGE_STRING='snapraid 11.2'
+PACKAGE_STRING='snapraid 11.3'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL='http://www.snapraid.it'
 
@@ -1304,7 +1304,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures snapraid 11.2 to adapt to many kinds of systems.
+\`configure' configures snapraid 11.3 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1374,7 +1374,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of snapraid 11.2:";;
+     short | recursive ) echo "Configuration of snapraid 11.3:";;
    esac
   cat <<\_ACEOF
 
@@ -1496,7 +1496,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-snapraid configure 11.2
+snapraid configure 11.3
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2102,7 +2102,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by snapraid $as_me 11.2, which was
+It was created by snapraid $as_me 11.3, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -2965,7 +2965,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='snapraid'
- VERSION='11.2'
+ VERSION='11.3'
 
 
 cat >>confdefs.h <<_ACEOF
@@ -7373,7 +7373,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by snapraid $as_me 11.2, which was
+This file was extended by snapraid $as_me 11.3, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -7436,7 +7436,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-snapraid config.status 11.2
+snapraid config.status 11.3
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

snapraid.1

@@ -647,7 +647,7 @@ and ignore all the parity data.
 This command will take a long time, but if you are not paranoid,
 you can skip it.
 .SS STEP 4 \-> Sync 
-Run the \[dq]sync\[dq] command to resynchronize the array with the new disk.
+Run the \[dq]sync\[dq] command to re\-synchronize the array with the new disk.
 .PP
 .RS 4
 snapraid sync
@@ -745,7 +745,7 @@ Spins up all the disks of the array.
 .PP
 You can spin\-up only some specific disks using the \-d, \-\-filter\-disk option.
 .PP
-Take care that spinniup\-up all the disks at the same time needs a lot of power.
+Take care that spinning\-up all the disks at the same time needs a lot of power.
 Ensure that your power\-supply can sustain that.
 .PP
 Nothing is modified.
@@ -912,7 +912,7 @@ Verify all the files and the parity data.
 It works like \[dq]fix\[dq], but it only simulates a recovery and no change
 is written in the array.
 .PP
-This command is mostly intended for manual verifications,
+This command is mostly intended for manual verification,
 like after a recovery process or in other special conditions.
 For periodic and scheduled checks uses \[dq]scrub\[dq].
 .PP
@@ -931,7 +931,7 @@ Nothing is modified.
 .SS dup 
 Lists all the duplicate files. Two files are assumed equal if their
 hashes are matching. The file data is not read, but only the
-precomputed hashes are used.
+pre\-computed hashes are used.
 .PP
 Nothing is modified.
 .SS pool 
@@ -942,7 +942,7 @@ The files are not really copied here, but just linked using
 symbolic links.
 .PP
 When updating, all the present symbolic links and empty
-subdirectories are deleted and replaced with the new
+sub\-directories are deleted and replaced with the new
 view of the array. Any other regular file is left in place.
 .PP
 Nothing is modified outside the pool directory.
@@ -962,20 +962,20 @@ low level devices used by a single disk in the array.
 .PP
 Nothing is modified.
 .SS touch 
-Sets arbitrarely the sub\-second timestamp of all the files
+Sets arbitrarily the sub\-second time\-stamp of all the files
 that have it at zero.
 .PP
 This improves the SnapRAID capability to recognize moved
-and copied files as it makes the timestamp almost unique,
+and copied files as it makes the time\-stamp almost unique,
 removing possible duplicates.
 .PP
-More specifically, if the sub\-second timestamp is not zero,
+More specifically, if the sub\-second time\-stamp is not zero,
 a moved or copied file is identified as such if it matches
-the name, size and timestamp. If instead the sub\-second timestamp
+the name, size and time\-stamp. If instead the sub\-second time\-stamp
 is zero, it\'s considered a copy only if it matches the full path,
-size and timestamp.
+size and time\-stamp.
 .PP
-Note that the second precision timestamp is not modified,
+Note that the second precision time\-stamp is not modified,
 and all the dates and times of your files will be maintained.
 .SS rehash 
 Schedules a rehash of the whole array.
@@ -1090,7 +1090,7 @@ Imports from the specified directory any file that you deleted
 from the array after the last \[dq]sync\[dq].
 If you still have such files, they could be used by \[dq]check\[dq]
 and \[dq]fix\[dq] to improve the recover process.
-The files are read also in subdirectories and they are
+The files are read also in sub\-directories and they are
 identified regardless of their name.
 This option can be used only with \[dq]check\[dq] and \[dq]fix\[dq].
 .TP
@@ -1146,7 +1146,7 @@ This allows to identify copied or moved files from one disk
 to another, and to reuse the already computed hash information
 to detect silent errors or to recover missing files.
 This behavior, in some rare cases, may result in false positives,
-or in a slow process due the many hash verifications, and this
+or in a slow process due the many hash verification, and this
 option allows to resolve them.
 This option can be used only with \[dq]sync\[dq], \[dq]check\[dq] and \[dq]fix\[dq].
 .TP
@@ -1280,7 +1280,7 @@ is enabled:
 6\-parity enables hexa (six) parity
 .PD
 .PP
-Each parity level requires the precence of all the previous parity
+Each parity level requires the presence of all the previous parity
 levels.
 .PP
 The same considerations of the \'parity\' option apply.

snapraid.txt

@@ -391,7 +391,7 @@ This command will take a long time, but if you are not paranoid,
 you can skip it.
 
 ---- 4.4.4 STEP 4 -> Sync ----
-Run the "sync" command to resynchronize the array with the new disk.
+Run the "sync" command to re-synchronize the array with the new disk.
 
     snapraid sync
 
@@ -476,7 +476,7 @@ Spins up all the disks of the array.
 
 You can spin-up only some specific disks using the -d, --filter-disk option.
 
-Take care that spinniup-up all the disks at the same time needs a lot of power.
+Take care that spinning-up all the disks at the same time needs a lot of power.
 Ensure that your power-supply can sustain that.
 
 Nothing is modified.
@@ -643,7 +643,7 @@ Verify all the files and the parity data.
 It works like "fix", but it only simulates a recovery and no change
 is written in the array.
 
-This command is mostly intended for manual verifications,
+This command is mostly intended for manual verification,
 like after a recovery process or in other special conditions.
 For periodic and scheduled checks uses "scrub".
 
@@ -668,7 +668,7 @@ Nothing is modified.
 
 Lists all the duplicate files. Two files are assumed equal if their
 hashes are matching. The file data is not read, but only the
-precomputed hashes are used.
+pre-computed hashes are used.
 
 Nothing is modified.
 
@@ -682,7 +682,7 @@ The files are not really copied here, but just linked using
 symbolic links.
 
 When updating, all the present symbolic links and empty
-subdirectories are deleted and replaced with the new
+sub-directories are deleted and replaced with the new
 view of the array. Any other regular file is left in place.
 
 Nothing is modified outside the pool directory.
@@ -708,20 +708,20 @@ Nothing is modified.
 5.14 touch
 ----------
 
-Sets arbitrarely the sub-second timestamp of all the files
+Sets arbitrarily the sub-second time-stamp of all the files
 that have it at zero.
 
 This improves the SnapRAID capability to recognize moved
-and copied files as it makes the timestamp almost unique,
+and copied files as it makes the time-stamp almost unique,
 removing possible duplicates.
 
-More specifically, if the sub-second timestamp is not zero,
+More specifically, if the sub-second time-stamp is not zero,
 a moved or copied file is identified as such if it matches
-the name, size and timestamp. If instead the sub-second timestamp
+the name, size and time-stamp. If instead the sub-second time-stamp
 is zero, it's considered a copy only if it matches the full path,
-size and timestamp.
+size and time-stamp.
 
-Note that the second precision timestamp is not modified,
+Note that the second precision time-stamp is not modified,
 and all the dates and times of your files will be maintained.
 
 5.15 rehash
@@ -843,7 +843,7 @@ SnapRAID provides the following options:
         from the array after the last "sync".
         If you still have such files, they could be used by "check"
         and "fix" to improve the recover process.
-        The files are read also in subdirectories and they are
+        The files are read also in sub-directories and they are
         identified regardless of their name.
         This option can be used only with "check" and "fix".
 
@@ -899,7 +899,7 @@ SnapRAID provides the following options:
         to another, and to reuse the already computed hash information
         to detect silent errors or to recover missing files.
         This behavior, in some rare cases, may result in false positives,
-        or in a slow process due the many hash verifications, and this
+        or in a slow process due the many hash verification, and this
         option allows to resolve them.
         This option can be used only with "sync", "check" and "fix".
 
@@ -1037,7 +1037,7 @@ is enabled:
 * 5-parity enables penta (five) parity
 * 6-parity enables hexa (six) parity
 
-Each parity level requires the precence of all the previous parity
+Each parity level requires the presence of all the previous parity
 levels.
 
 The same considerations of the 'parity' option apply.

tommyds/tommyarray.c

@@ -36,7 +36,7 @@ void tommy_array_init(tommy_array* array)
 
 	/* fixed initial size */
 	array->bucket_bit = TOMMY_ARRAY_BIT;
-	array->bucket_max = 1 << array->bucket_bit;
+	array->bucket_max = (tommy_size_t)1 << array->bucket_bit;
 	array->bucket[0] = tommy_cast(void**, tommy_calloc(array->bucket_max, sizeof(void*)));
 	for (i = 1; i < TOMMY_ARRAY_BIT; ++i)
 		array->bucket[i] = array->bucket[0];
@@ -51,11 +51,11 @@ void tommy_array_done(tommy_array* array)
 	tommy_free(array->bucket[0]);
 	for (i = TOMMY_ARRAY_BIT; i < array->bucket_bit; ++i) {
 		void** segment = array->bucket[i];
-		tommy_free(&segment[((tommy_ptrdiff_t)1) << i]);
+		tommy_free(&segment[(tommy_ptrdiff_t)1 << i]);
 	}
 }
 
-void tommy_array_grow(tommy_array* array, tommy_count_t count)
+void tommy_array_grow(tommy_array* array, tommy_size_t count)
 {
 	if (array->count >= count)
 		return;
@@ -72,7 +72,7 @@ void tommy_array_grow(tommy_array* array, tommy_count_t count)
 		array->bucket[array->bucket_bit] = &segment[-(tommy_ptrdiff_t)array->bucket_max];
 
 		++array->bucket_bit;
-		array->bucket_max = 1 << array->bucket_bit;
+		array->bucket_max = (tommy_size_t)1 << array->bucket_bit;
 	}
 }
 

tommyds/tommyarray.h

@@ -53,20 +53,15 @@
  */
 #define TOMMY_ARRAY_BIT 6
 
-/** \internal
- * Max number of elements as a power of 2.
- */
-#define TOMMY_ARRAY_BIT_MAX 32
-
 /**
  * Array container type.
  * \note Don't use internal fields directly, but access the container only using functions.
  */
 typedef struct tommy_array_struct {
-	void** bucket[TOMMY_ARRAY_BIT_MAX]; /**< Dynamic array of buckets. */
+	void** bucket[TOMMY_SIZE_BIT]; /**< Dynamic array of buckets. */
+	tommy_size_t bucket_max; /**< Number of buckets. */
+	tommy_size_t count; /**< Number of initialized elements in the array. */
 	tommy_uint_t bucket_bit; /**< Bits used in the bit mask. */
-	tommy_count_t bucket_max; /**< Number of buckets. */
-	tommy_count_t count; /**< Number of initialized elements in the array. */
 } tommy_array;
 
 /**
@@ -83,21 +78,21 @@ void tommy_array_done(tommy_array* array);
  * Grows the size up to the specified value.
  * All the new elements in the array are initialized with the 0 value.
  */
-void tommy_array_grow(tommy_array* array, tommy_count_t size);
+void tommy_array_grow(tommy_array* array, tommy_size_t size);
 
 /**
  * Gets a reference of the element at the specified position.
  * You must be sure that space for this position is already
  * allocated calling tommy_array_grow().
  */
-tommy_inline void** tommy_array_ref(tommy_array* array, tommy_count_t pos)
+tommy_inline void** tommy_array_ref(tommy_array* array, tommy_size_t pos)
 {
 	tommy_uint_t bsr;
 
 	assert(pos < array->count);
 
 	/* get the highest bit set, in case of all 0, return 0 */
-	bsr = tommy_ilog2_u32(pos | 1);
+	bsr = tommy_ilog2(pos | 1);
 
 	return &array->bucket[bsr][pos];
 }
@@ -107,7 +102,7 @@ tommy_inline void** tommy_array_ref(tommy_array* array, tommy_count_t pos)
  * You must be sure that space for this position is already
  * allocated calling tommy_array_grow().
  */
-tommy_inline void tommy_array_set(tommy_array* array, tommy_count_t pos, void* element)
+tommy_inline void tommy_array_set(tommy_array* array, tommy_size_t pos, void* element)
 {
 	*tommy_array_ref(array, pos) = element;
 }
@@ -117,7 +112,7 @@ tommy_inline void tommy_array_set(tommy_array* array, tommy_count_t pos, void* e
  * You must be sure that space for this position is already
  * allocated calling tommy_array_grow().
  */
-tommy_inline void* tommy_array_get(tommy_array* array, tommy_count_t pos)
+tommy_inline void* tommy_array_get(tommy_array* array, tommy_size_t pos)
 {
 	return *tommy_array_ref(array, pos);
 }
@@ -127,7 +122,7 @@ tommy_inline void* tommy_array_get(tommy_array* array, tommy_count_t pos)
  */
 tommy_inline void tommy_array_insert(tommy_array* array, void* element)
 {
-	tommy_count_t pos = array->count;
+	tommy_size_t pos = array->count;
 
 	tommy_array_grow(array, pos + 1);
 
@@ -137,7 +132,7 @@ tommy_inline void tommy_array_insert(tommy_array* array, void* element)
 /**
  * Gets the initialized size of the array.
  */
-tommy_inline tommy_count_t tommy_array_size(tommy_array* array)
+tommy_inline tommy_size_t tommy_array_size(tommy_array* array)
 {
 	return array->count;
 }

tommyds/tommyarrayblkof.c

@@ -40,7 +40,7 @@ void tommy_arrayblkof_init(tommy_arrayblkof* array, tommy_size_t element_size)
 
 void tommy_arrayblkof_done(tommy_arrayblkof* array)
 {
-	tommy_count_t i;
+	tommy_size_t i;
 
 	for (i = 0; i < tommy_array_size(&array->block); ++i)
 		tommy_free(tommy_array_get(&array->block, i));
@@ -48,10 +48,10 @@ void tommy_arrayblkof_done(tommy_arrayblkof* array)
 	tommy_array_done(&array->block);
 }
 
-void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_count_t count)
+void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_size_t count)
 {
-	tommy_count_t block_max;
+	tommy_size_t block_max;
-	tommy_count_t block_mac;
+	tommy_size_t block_mac;
 
 	if (array->count >= count)
 		return;

tommyds/tommyarrayblkof.h

@@ -61,7 +61,7 @@
 typedef struct tommy_arrayblkof_struct {
 	tommy_array block; /**< Array of blocks. */
 	tommy_size_t element_size; /**< Size of the stored element in bytes. */
-	tommy_count_t count; /**< Number of initialized elements in the array. */
+	tommy_size_t count; /**< Number of initialized elements in the array. */
 } tommy_arrayblkof;
 
 /**
@@ -79,14 +79,14 @@ void tommy_arrayblkof_done(tommy_arrayblkof* array);
  * Grows the size up to the specified value.
  * All the new elements in the array are initialized with the 0 value.
  */
-void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_count_t size);
+void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_size_t size);
 
 /**
  * Gets a reference of the element at the specified position.
  * You must be sure that space for this position is already
  * allocated calling tommy_arrayblkof_grow().
  */
-tommy_inline void* tommy_arrayblkof_ref(tommy_arrayblkof* array, tommy_count_t pos)
+tommy_inline void* tommy_arrayblkof_ref(tommy_arrayblkof* array, tommy_size_t pos)
 {
 	unsigned char* base;
 
@@ -100,7 +100,7 @@ tommy_inline void* tommy_arrayblkof_ref(tommy_arrayblkof* array, tommy_count_t p
 /**
  * Gets the initialized size of the array.
  */
-tommy_inline tommy_count_t tommy_arrayblkof_size(tommy_arrayblkof* array)
+tommy_inline tommy_size_t tommy_arrayblkof_size(tommy_arrayblkof* array)
 {
 	return array->count;
 }

tommyds/tommychain.h

@@ -136,11 +136,6 @@ tommy_inline void tommy_chain_merge_degenerated(tommy_chain* first, tommy_chain*
 	tommy_chain_merge(first, second, cmp);
 }
 
-/**
- * Max number of elements as a power of 2.
- */
-#define TOMMY_CHAIN_BIT_MAX 32
-
 /**
  * Sorts a chain.
  * It's a stable merge sort using power of 2 buckets, with O(N*log(N)) complexity,
@@ -158,20 +153,20 @@ tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func*
 	/*
 	 * Bit buckets of chains.
 	 * Each bucket contains 2^i nodes or it's empty.
-	 * The chain at address TOMMY_CHAIN_BIT_MAX is an independet variable operating as "carry".
+	 * The chain at address TOMMY_BIT_MAX is an independet variable operating as "carry".
 	 * We keep it in the same "bit" vector to avoid reports from the valgrind tool sgcheck.
 	 */
-	tommy_chain bit[TOMMY_CHAIN_BIT_MAX + 1];
+	tommy_chain bit[TOMMY_SIZE_BIT + 1];
 
 	/**
 	 * Value stored inside the bit bucket.
 	 * It's used to know which bucket is empty of full.
 	 */
-	tommy_count_t counter;
+	tommy_size_t counter;
 	tommy_node* node = chain->head;
 	tommy_node* tail = chain->tail;
-	tommy_count_t mask;
+	tommy_size_t mask;
-	tommy_count_t i;
+	tommy_size_t i;
 
 	counter = 0;
 	while (1) {
@@ -179,9 +174,9 @@ tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func*
 		tommy_chain* last;
 
 		/* carry bit to add */
-		last = &bit[TOMMY_CHAIN_BIT_MAX];
+		last = &bit[TOMMY_SIZE_BIT];
-		bit[TOMMY_CHAIN_BIT_MAX].head = node;
+		bit[TOMMY_SIZE_BIT].head = node;
-		bit[TOMMY_CHAIN_BIT_MAX].tail = node;
+		bit[TOMMY_SIZE_BIT].tail = node;
 		next = node->next;
 
 		/* add the bit, propagating the carry */
@@ -206,7 +201,7 @@ tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func*
 	}
 
 	/* merge the buckets */
-	i = tommy_ctz_u32(counter);
+	i = tommy_ctz(counter);
 	mask = counter >> i;
 	while (mask != 1) {
 		mask >>= 1;

tommyds/tommyhash.c

@@ -92,22 +92,22 @@ tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tom
 		b += tommy_le_uint32_read(key + 4);
 		a += tommy_le_uint32_read(key + 0);
 		break;
-	case 11 : c += ((tommy_uint32_t)key[10]) << 16;
+	case 11 : c += ((tommy_uint32_t)key[10]) << 16; /* fallthrough */
-	case 10 : c += ((tommy_uint32_t)key[9]) << 8;
+	case 10 : c += ((tommy_uint32_t)key[9]) << 8; /* fallthrough */
-	case 9 : c += key[8];
+	case 9 : c += key[8]; /* fallthrough */
 	case 8 :
 		b += tommy_le_uint32_read(key + 4);
 		a += tommy_le_uint32_read(key + 0);
 		break;
-	case 7 : b += ((tommy_uint32_t)key[6]) << 16;
+	case 7 : b += ((tommy_uint32_t)key[6]) << 16; /* fallthrough */
-	case 6 : b += ((tommy_uint32_t)key[5]) << 8;
+	case 6 : b += ((tommy_uint32_t)key[5]) << 8; /* fallthrough */
-	case 5 : b += key[4];
+	case 5 : b += key[4]; /* fallthrough */
 	case 4 :
 		a += tommy_le_uint32_read(key + 0);
 		break;
-	case 3 : a += ((tommy_uint32_t)key[2]) << 16;
+	case 3 : a += ((tommy_uint32_t)key[2]) << 16; /* fallthrough */
-	case 2 : a += ((tommy_uint32_t)key[1]) << 8;
+	case 2 : a += ((tommy_uint32_t)key[1]) << 8; /* fallthrough */
-	case 1 : a += key[0];
+	case 1 : a += key[0]; /* fallthrough */
 	}
 
 	tommy_final(a, b, c);
@@ -142,22 +142,22 @@ tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tom
 		b += tommy_le_uint32_read(key + 4);
 		a += tommy_le_uint32_read(key + 0);
 		break;
-	case 11 : c += ((tommy_uint32_t)key[10]) << 16;
+	case 11 : c += ((tommy_uint32_t)key[10]) << 16; /* fallthrough */
-	case 10 : c += ((tommy_uint32_t)key[9]) << 8;
+	case 10 : c += ((tommy_uint32_t)key[9]) << 8; /* fallthrough */
-	case 9 : c += key[8];
+	case 9 : c += key[8]; /* fallthrough */
 	case 8 :
 		b += tommy_le_uint32_read(key + 4);
 		a += tommy_le_uint32_read(key + 0);
 		break;
-	case 7 : b += ((tommy_uint32_t)key[6]) << 16;
+	case 7 : b += ((tommy_uint32_t)key[6]) << 16; /* fallthrough */
-	case 6 : b += ((tommy_uint32_t)key[5]) << 8;
+	case 6 : b += ((tommy_uint32_t)key[5]) << 8; /* fallthrough */
-	case 5 : b += key[4];
+	case 5 : b += key[4]; /* fallthrough */
 	case 4 :
 		a += tommy_le_uint32_read(key + 0);
 		break;
-	case 3 : a += ((tommy_uint32_t)key[2]) << 16;
+	case 3 : a += ((tommy_uint32_t)key[2]) << 16; /* fallthrough */
-	case 2 : a += ((tommy_uint32_t)key[1]) << 8;
+	case 2 : a += ((tommy_uint32_t)key[1]) << 8; /* fallthrough */
-	case 1 : a += key[0];
+	case 1 : a += key[0]; /* fallthrough */
 	}
 
 	tommy_final(a, b, c);

tommyds/tommyhash.h

@@ -37,11 +37,6 @@
 /******************************************************************************/
 /* hash */
 
-/**
- * Hash type used in hashtables.
- */
-typedef tommy_key_t tommy_hash_t;
-
 /**
  * Hash function with a 32 bits result.
  * Implementation of the Robert Jenkins "lookup3" hash 32 bits version,

tommyds/tommyhashdyn.c

@@ -35,7 +35,7 @@ void tommy_hashdyn_init(tommy_hashdyn* hashdyn)
 {
 	/* fixed initial size */
 	hashdyn->bucket_bit = TOMMY_HASHDYN_BIT;
-	hashdyn->bucket_max = 1 << hashdyn->bucket_bit;
+	hashdyn->bucket_max = (tommy_size_t)1 << hashdyn->bucket_bit;
 	hashdyn->bucket_mask = hashdyn->bucket_max - 1;
 	hashdyn->bucket = tommy_cast(tommy_hashdyn_node**, tommy_calloc(hashdyn->bucket_max, sizeof(tommy_hashdyn_node*)));
 
@@ -50,18 +50,18 @@ void tommy_hashdyn_done(tommy_hashdyn* hashdyn)
 /**
  * Resize the bucket vector.
  */
-static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_count_t new_bucket_bit)
+static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_size_t new_bucket_bit)
 {
-	tommy_count_t bucket_bit;
+	tommy_size_t bucket_bit;
-	tommy_count_t bucket_max;
+	tommy_size_t bucket_max;
-	tommy_count_t new_bucket_max;
+	tommy_size_t new_bucket_max;
-	tommy_count_t new_bucket_mask;
+	tommy_size_t new_bucket_mask;
 	tommy_hashdyn_node** new_bucket;
 
 	bucket_bit = hashdyn->bucket_bit;
 	bucket_max = hashdyn->bucket_max;
 
-	new_bucket_max = 1 << new_bucket_bit;
+	new_bucket_max = (tommy_size_t)1 << new_bucket_bit;
 	new_bucket_mask = new_bucket_max - 1;
 
 	/* allocate the new vector using malloc() and not calloc() */
@@ -70,7 +70,7 @@ static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_count_t new_bucke
 
 	/* reinsert all the elements */
 	if (new_bucket_bit > bucket_bit) {
-		tommy_count_t i;
+		tommy_size_t i;
 
 		/* grow */
 		for (i = 0; i < bucket_max; ++i) {
@@ -84,7 +84,7 @@ static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_count_t new_bucke
 			j = hashdyn->bucket[i];
 			while (j) {
 				tommy_hashdyn_node* j_next = j->next;
-				tommy_count_t pos = j->key & new_bucket_mask;
+				tommy_size_t pos = j->index & new_bucket_mask;
 				if (new_bucket[pos])
 					tommy_list_insert_tail_not_empty(new_bucket[pos], j);
 				else
@@ -93,7 +93,7 @@ static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_count_t new_bucke
 			}
 		}
 	} else {
-		tommy_count_t i;
+		tommy_size_t i;
 
 		/* shrink */
 		for (i = 0; i < new_bucket_max; ++i) {
@@ -136,11 +136,11 @@ tommy_inline void hashdyn_shrink_step(tommy_hashdyn* hashdyn)
 
 void tommy_hashdyn_insert(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void* data, tommy_hash_t hash)
 {
-	tommy_count_t pos = hash & hashdyn->bucket_mask;
+	tommy_size_t pos = hash & hashdyn->bucket_mask;
 
 	tommy_list_insert_tail(&hashdyn->bucket[pos], node, data);
 
-	node->key = hash;
+	node->index = hash;
 
 	++hashdyn->count;
 
@@ -149,7 +149,7 @@ void tommy_hashdyn_insert(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void
 
 void* tommy_hashdyn_remove_existing(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node)
 {
-	tommy_count_t pos = node->key & hashdyn->bucket_mask;
+	tommy_size_t pos = node->index & hashdyn->bucket_mask;
 
 	tommy_list_remove_existing(&hashdyn->bucket[pos], node);
 
@@ -162,12 +162,12 @@ void* tommy_hashdyn_remove_existing(tommy_hashdyn* hashdyn, tommy_hashdyn_node*
 
 void* tommy_hashdyn_remove(tommy_hashdyn* hashdyn, tommy_search_func* cmp, const void* cmp_arg, tommy_hash_t hash)
 {
-	tommy_count_t pos = hash & hashdyn->bucket_mask;
+	tommy_size_t pos = hash & hashdyn->bucket_mask;
 	tommy_hashdyn_node* node = hashdyn->bucket[pos];
 
 	while (node) {
 		/* we first check if the hash matches, as in the same bucket we may have multiples hash values */
-		if (node->key == hash && cmp(cmp_arg, node->data) == 0) {
+		if (node->index == hash && cmp(cmp_arg, node->data) == 0) {
 			tommy_list_remove_existing(&hashdyn->bucket[pos], node);
 
 			--hashdyn->count;
@@ -184,9 +184,9 @@ void* tommy_hashdyn_remove(tommy_hashdyn* hashdyn, tommy_search_func* cmp, const
 
 void tommy_hashdyn_foreach(tommy_hashdyn* hashdyn, tommy_foreach_func* func)
 {
-	tommy_count_t bucket_max = hashdyn->bucket_max;
+	tommy_size_t bucket_max = hashdyn->bucket_max;
 	tommy_hashdyn_node** bucket = hashdyn->bucket;
-	tommy_count_t pos;
+	tommy_size_t pos;
 
 	for (pos = 0; pos < bucket_max; ++pos) {
 		tommy_hashdyn_node* node = bucket[pos];
@@ -201,9 +201,9 @@ void tommy_hashdyn_foreach(tommy_hashdyn* hashdyn, tommy_foreach_func* func)
 
 void tommy_hashdyn_foreach_arg(tommy_hashdyn* hashdyn, tommy_foreach_arg_func* func, void* arg)
 {
-	tommy_count_t bucket_max = hashdyn->bucket_max;
+	tommy_size_t bucket_max = hashdyn->bucket_max;
 	tommy_hashdyn_node** bucket = hashdyn->bucket;
-	tommy_count_t pos;
+	tommy_size_t pos;
 
 	for (pos = 0; pos < bucket_max; ++pos) {
 		tommy_hashdyn_node* node = bucket[pos];

tommyds/tommyhashdyn.h

@@ -160,10 +160,10 @@ typedef tommy_node tommy_hashdyn_node;
  */
 typedef struct tommy_hashdyn_struct {
 	tommy_hashdyn_node** bucket; /**< Hash buckets. One list for each hash modulus. */
+	tommy_size_t bucket_max; /**< Number of buckets. */
+	tommy_size_t bucket_mask; /**< Bit mask to access the buckets. */
+	tommy_size_t count; /**< Number of elements. */
 	tommy_uint_t bucket_bit; /**< Bits used in the bit mask. */
-	tommy_count_t bucket_max; /**< Number of buckets. */
-	tommy_count_t bucket_mask; /**< Bit mask to access the buckets. */
-	tommy_count_t count; /**< Number of elements. */
 } tommy_hashdyn;
 
 /**
@@ -226,7 +226,7 @@ tommy_inline void* tommy_hashdyn_search(tommy_hashdyn* hashdyn, tommy_search_fun
 
 	while (i) {
 		/* we first check if the hash matches, as in the same bucket we may have multiples hash values */
-		if (i->key == hash && cmp(cmp_arg, i->data) == 0)
+		if (i->index == hash && cmp(cmp_arg, i->data) == 0)
 			return i->data;
 		i = i->next;
 	}
@@ -281,7 +281,7 @@ void tommy_hashdyn_foreach_arg(tommy_hashdyn* hashdyn, tommy_foreach_arg_func* f
 /**
  * Gets the number of elements.
  */
-tommy_inline tommy_count_t tommy_hashdyn_count(tommy_hashdyn* hashdyn)
+tommy_inline tommy_size_t tommy_hashdyn_count(tommy_hashdyn* hashdyn)
 {
 	return hashdyn->count;
 }

tommyds/tommylist.h

@@ -226,24 +226,6 @@ tommy_inline void tommy_list_insert_tail(tommy_list* list, tommy_node* node, voi
 	node->data = data;
 }
 
-/** \internal
- * Removes an element from the head of a not empty list.
- * \param list The list. The list cannot be empty.
- * \return The node removed.
- */
-tommy_inline tommy_node* tommy_list_remove_head_not_empty(tommy_list* list)
-{
-	tommy_node* head = tommy_list_head(list);
-
-	/* remove from the "circular" prev list */
-	head->next->prev = head->prev;
-
-	/* remove from the "0 terminated" next list */
-	*list = head->next; /* the new head, in case 0 */
-
-	return head;
-}
-
 /**
  * Removes an element from the list.
  * You must already have the address of the element to remove.
@@ -329,9 +311,9 @@ tommy_inline tommy_bool_t tommy_list_empty(tommy_list* list)
  * Gets the number of elements.
  * \note This operation is O(n).
  */
-tommy_inline tommy_count_t tommy_list_count(tommy_list* list)
+tommy_inline tommy_size_t tommy_list_count(tommy_list* list)
 {
-	tommy_count_t count = 0;
+	tommy_size_t count = 0;
 	tommy_node* i = tommy_list_head(list);
 
 	while (i) {

tommyds/tommytree.c

@@ -39,10 +39,10 @@ void tommy_tree_init(tommy_tree* tree, tommy_compare_func* cmp)
 	tree->cmp = cmp;
 }
 
-static int tommy_tree_delta(tommy_tree_node* root)
+static tommy_ssize_t tommy_tree_delta(tommy_tree_node* root)
 {
-	int left_height = root->prev ? root->prev->key : 0;
+	tommy_ssize_t left_height = root->prev ? root->prev->index : 0;
-	int right_height = root->next ? root->next->key : 0;
+	tommy_ssize_t right_height = root->next ? root->next->index : 0;
 
 	return left_height - right_height;
 }
@@ -84,7 +84,7 @@ static tommy_tree_node* tommy_tree_move_right(tommy_tree_node* root, tommy_tree_
 
 static tommy_tree_node* tommy_tree_balance(tommy_tree_node* root)
 {
-	int delta = tommy_tree_delta(root);
+	tommy_ssize_t delta = tommy_tree_delta(root);
 
 	if (delta < -1) {
 		if (tommy_tree_delta(root->next) > 0)
@@ -99,16 +99,16 @@ static tommy_tree_node* tommy_tree_balance(tommy_tree_node* root)
 	}
 
 	/* recompute key */
-	root->key = 0;
+	root->index = 0;
 
-	if (root->prev && root->prev->key > root->key)
+	if (root->prev && root->prev->index > root->index)
-		root->key = root->prev->key;
+		root->index = root->prev->index;
 
-	if (root->next && root->next->key > root->key)
+	if (root->next && root->next->index > root->index)
-		root->key = root->next->key;
+		root->index = root->next->index;
 
 	/* count itself */
-	root->key += 1;
+	root->index += 1;
 
 	return root;
 }
@@ -145,7 +145,7 @@ void* tommy_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data)
 	insert->data = data;
 	insert->prev = 0;
 	insert->next = 0;
-	insert->key = 0;
+	insert->index = 0;
 
 	tree->root = tommy_tree_insert_node(tree->cmp, tree->root, &insert);
 

tommyds/tommytree.h

@@ -117,8 +117,8 @@ typedef tommy_node tommy_tree_node;
  */
 typedef struct tommy_tree_struct {
 	tommy_tree_node* root; /**< Root node. */
-	tommy_count_t count; /**< Number of elements. */
 	tommy_compare_func* cmp; /**< Comparison function. */
+	tommy_size_t count; /**< Number of elements. */
 } tommy_tree;
 
 /**
@@ -213,7 +213,7 @@ void tommy_tree_foreach_arg(tommy_tree* tree, tommy_foreach_arg_func* func, void
 /**
  * Gets the number of elements.
  */
-tommy_inline tommy_count_t tommy_tree_count(tommy_tree* tree)
+tommy_inline tommy_size_t tommy_tree_count(tommy_tree* tree)
 {
 	return tree->count;
 }

tommyds/tommytypes.h

@@ -24,7 +24,6 @@
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
-
 /** \file
  * Generic types.
  */
@@ -37,17 +36,37 @@
 
 #include <stddef.h>
 
-#if defined(_MSC_VER)
+#ifdef _MSC_VER
 typedef unsigned tommy_uint32_t; /**< Generic uint32_t type. */
 typedef unsigned _int64 tommy_uint64_t; /**< Generic uint64_t type. */
 typedef size_t tommy_uintptr_t; /**< Generic uintptr_t type. */
+#ifdef _WIN64
+#define TOMMY_SIZE_BIT 64
+typedef unsigned _int64_t tommy_size_t; /**< Generic size_t type. */
+typedef _int64_t tommy_ssize_t; /**< Generic ssize_t type. */
+#else
+#define TOMMY_SIZE_BIT 32
+typedef unsigned tommy_size_t; /**< Generic size_t type. */
+typedef int tommy_ssize_t; /**< Generic ssize_t type. */
+#endif
 #else
 #include <stdint.h>
 typedef uint32_t tommy_uint32_t; /**< Generic uint32_t type. */
 typedef uint64_t tommy_uint64_t; /**< Generic uint64_t type. */
 typedef uintptr_t tommy_uintptr_t; /**< Generic uintptr_t type. */
+#if SIZE_MAX == UINT64_MAX
+#define TOMMY_SIZE_BIT 64
+typedef uint64_t tommy_size_t; /**< Generic size_t type. */
+typedef int64_t tommy_ssize_t; /**< Generic ssize_t type. */
+#elif SIZE_MAX == UINT32_MAX
+#define TOMMY_SIZE_BIT 32
+typedef uint32_t tommy_size_t; /**< Generic size_t type. */
+typedef int32_t tommy_ssize_t; /**< Generic ssize_t type. */
+#else
+#error Unsupported SIZE_MAX
 #endif
-typedef size_t tommy_size_t; /**< Generic size_t type. */
+#endif
+
 typedef ptrdiff_t tommy_ptrdiff_t; /**< Generic ptrdiff_t type. */
 typedef int tommy_bool_t; /**< Generic boolean type. */
 
@@ -59,13 +78,6 @@ typedef int tommy_bool_t; /**< Generic boolean type. */
  */
 typedef tommy_uint32_t tommy_uint_t;
 
-/**
- * Generic unsigned integer for counting objects.
- *
- * TommyDS doesn't support more than 2^32-1 objects.
- */
-typedef tommy_uint32_t tommy_count_t;
-
 /** \internal
  * Type cast required for the C++ compilation.
  * When compiling in C++ we cannot convert a void* pointer to another pointer.
@@ -152,17 +164,17 @@ typedef tommy_uint32_t tommy_count_t;
 #endif
 
 /******************************************************************************/
-/* key */
+/* key/hash */
 
 /**
- * Key type used in indexed data structures to store the key or the hash value.
+ * Type used in indexed data structures to store the key of a object.
  */
-typedef tommy_uint32_t tommy_key_t;
+typedef tommy_size_t tommy_key_t;
 
 /**
- * Bits into the ::tommy_key_t type.
+ * Type used in hashtables to store the hash of a object.
  */
-#define TOMMY_KEY_BIT (sizeof(tommy_key_t) * 8)
+typedef tommy_size_t tommy_hash_t;
 
 /******************************************************************************/
 /* node */
@@ -200,11 +212,12 @@ typedef struct tommy_node_struct {
 	void* data;
 
 	/**
-	 * Key used to store the node.
+	 * Index of the node.
+	 * With tries this field is used to store the key.
 	 * With hashtables this field is used to store the hash value.
 	 * With lists this field is not used.
 	 */
-	tommy_key_t key;
+	tommy_size_t index;
 } tommy_node;
 
 /******************************************************************************/
@@ -302,6 +315,10 @@ typedef void tommy_foreach_arg_func(void* arg, void* obj);
 #include <intrin.h>
 #pragma intrinsic(_BitScanReverse)
 #pragma intrinsic(_BitScanForward)
+#if TOMMY_SIZE_BIT == 64
+#pragma intrinsic(_BitScanReverse64)
+#pragma intrinsic(_BitScanForward64)
+#endif
 #endif
 
 /** \internal
@@ -363,6 +380,29 @@ tommy_inline tommy_uint_t tommy_ilog2_u32(tommy_uint32_t value)
 #endif
 }
 
+#if TOMMY_SIZE_BIT == 64
+/**
+ * Bit scan reverse or integer log2 for 64 bits.
+ */
+tommy_inline tommy_uint_t tommy_ilog2_u64(tommy_uint64_t value)
+{
+#if defined(_MSC_VER)
+	unsigned long count;
+	_BitScanReverse64(&count, value);
+	return count;
+#elif defined(__GNUC__)
+	return __builtin_clzll(value) ^ 63;
+#else
+	uint32_t l = value & 0xFFFFFFFFU;
+	uint32_t h = value >> 32;
+	if (h)
+		return tommy_ilog2_u32(h) + 32;
+	else
+		return tommy_ilog2_u32(l);
+#endif
+}
+#endif
+
 /**
  * Bit scan forward or trailing zero count.
  * Return the bit index of the least significant 1 bit.
@@ -391,6 +431,29 @@ tommy_inline tommy_uint_t tommy_ctz_u32(tommy_uint32_t value)
 #endif
 }
 
+#if TOMMY_SIZE_BIT == 64
+/**
+ * Bit scan forward or trailing zero count for 64 bits.
+ */
+tommy_inline tommy_uint_t tommy_ctz_u64(tommy_uint64_t value)
+{
+#if defined(_MSC_VER)
+	unsigned long count;
+	_BitScanForward64(&count, value);
+	return count;
+#elif defined(__GNUC__)
+	return __builtin_ctzll(value);
+#else
+	uint32_t l = value & 0xFFFFFFFFU;
+	uint32_t h = value >> 32;
+	if (l)
+		return tommy_ctz_u32(l);
+	else
+		return tommy_ctz_u32(h) + 32;
+#endif
+}
+#endif
+
 /**
  * Rounds up to the next power of 2.
  * For the value 0, the result is undefined.
@@ -412,6 +475,23 @@ tommy_inline tommy_uint32_t tommy_roundup_pow2_u32(tommy_uint32_t value)
 	return value;
 }
 
+/**
+ * Rounds up to the next power of 2 for 64 bits.
+ */
+tommy_inline tommy_uint64_t tommy_roundup_pow2_u64(tommy_uint64_t value)
+{
+	--value;
+	value |= value >> 1;
+	value |= value >> 2;
+	value |= value >> 4;
+	value |= value >> 8;
+	value |= value >> 16;
+	value |= value >> 32;
+	++value;
+
+	return value;
+}
+
 /**
  * Check if the specified word has a byte at 0.
  * \return 0 or 1.
@@ -420,5 +500,19 @@ tommy_inline int tommy_haszero_u32(tommy_uint32_t value)
 {
 	return ((value - 0x01010101) & ~value & 0x80808080) != 0;
 }
+
+/*
+ * Bit depth mapping.
+ */
+#if TOMMY_SIZE_BIT == 64
+#define tommy_ilog2 tommy_ilog2_u64
+#define tommy_ctz tommy_ctz_u64
+#define tommy_roundup_pow2 tommy_roundup_pow2_u64
+#else
+#define tommy_ilog2 tommy_ilog2_u32
+#define tommy_ctz tommy_ctz_u32
+#define tommy_roundup_pow2 tommy_roundup_pow2_u32
+#endif
+
 #endif