From c540e242170c4bccd3421621de31614f8f3a5c9e Mon Sep 17 00:00:00 2001 From: Mario Fetka Date: Mon, 7 Jan 2019 14:41:48 +0100 Subject: [PATCH] Imported Upstream version 11.3 --- .version | 2 +- HISTORY | 10 +++ TODO | 16 +++-- cmdline/io.c | 2 +- cmdline/list.c | 4 +- cmdline/mingw.c | 1 + cmdline/mkstream.c | 1 + cmdline/murmur3.c | 34 +++++----- cmdline/snapraid.c | 4 +- cmdline/state.c | 103 ++++++++++++++++++++++++++++-- cmdline/status.c | 8 +++ cmdline/stream.c | 3 +- cmdline/support.h | 2 + cmdline/sync.c | 2 +- cmdline/unix.c | 20 +++--- configure | 20 +++--- snapraid.1 | 28 ++++----- snapraid.txt | 28 ++++----- tommyds/tommyarray.c | 8 +-- tommyds/tommyarray.h | 25 +++----- tommyds/tommyarrayblkof.c | 8 +-- tommyds/tommyarrayblkof.h | 8 +-- tommyds/tommychain.h | 23 +++---- tommyds/tommyhash.c | 36 +++++------ tommyds/tommyhash.h | 5 -- tommyds/tommyhashdyn.c | 38 +++++------ tommyds/tommyhashdyn.h | 10 +-- tommyds/tommylist.h | 22 +------ tommyds/tommytree.c | 22 +++---- tommyds/tommytree.h | 4 +- tommyds/tommytypes.h | 128 +++++++++++++++++++++++++++++++++----- 31 files changed, 408 insertions(+), 217 deletions(-) diff --git a/.version b/.version index 3913853..f2f3fcc 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -11.2 \ No newline at end of file +11.3 \ No newline at end of file diff --git a/HISTORY b/HISTORY index 2cb15d9..ad7411f 100644 --- a/HISTORY +++ b/HISTORY @@ -1,6 +1,16 @@ SnapRAID HISTORY ================ +11.3 2018/11 +============ + * Fixed handing of Linux devices that have multiple slaves. This affects + the smart/list/devices/down commands [Valentin Hilbig]. + * The 'list' command in verbose mode prints the full nanosecond + timestamp precision. + * After writing content files also sync their directory. + * Fix a invalid time computation that could result in future scrub dates. + Such dates are fixed automatically at the next scrub or sync. + 11.2 2017/12 ============ * Fixed recognition of NTFS hardlinks. They behave differently than diff --git a/TODO b/TODO index ad2a21c..f02c44c 100644 --- a/TODO +++ b/TODO @@ -17,6 +17,18 @@ Not checked disks should be allowed to be missing. * Add an option to ignore subsecond timestamp. Like when you copy data to a filesystem with less timestamp precision. +* Use threads to scan all the disks at the same time. +- After 7.0 Windows changes it seems fast enough even +with a mono thread implementation with 100.0000 files. ++ But if you have millions of files, it could take minutes. + +* Support more parity levels +It can be done with a generic computation function, using +intrinsic for SSSE3 and AVX instructions. +It would be intersting to compare performance with the hand-written +assembler functions. Eventially we can convert them to use intrinsic also. +https://sourceforge.net/p/snapraid/discussion/1677233/thread/9dbd7581/ + * Extend haspdeep to support the SnapRAID hash : https://github.com/jessek/hashdeep/ https://sourceforge.net/p/snapraid/discussion/1677233/thread/90b0e9b2/?limit=25 @@ -222,10 +234,6 @@ and if we manage to keep it in the cache, we should save time. - We now hash first the faster disks, and this could reduce performance as we'll have to wait for all disks. -* Use threads to scan all the disks at the same time. -- After 7.0 Windows changes it seems fast enough even -with a mono thread implementation. - * Enable storing of creation time NTFS, crtime/birth time EXT4. But see: http://unix.stackexchange.com/questions/50177/birth-is-empty-on-ext4 coreutils stat has an example, but it doesn't work in Linux (see lib/stat-time.h) diff --git a/cmdline/io.c b/cmdline/io.c index bafea0d..c71ba41 100644 --- a/cmdline/io.c +++ b/cmdline/io.c @@ -889,7 +889,7 @@ void io_init(struct snapraid_io* io, struct snapraid_state* state, allocated += state->block_size * buffer_max; } - msg_progress("Using %u MiB of memory for %u blocks of IO cache.\n", (unsigned)(allocated / MEBI), io->io_max); + msg_progress("Using %u MiB of memory for %u cached blocks.\n", (unsigned)(allocated / MEBI), io->io_max); if (parity_writer) { io->reader_max = handle_max; diff --git a/cmdline/list.c b/cmdline/list.c index a73cca4..421c8c5 100644 --- a/cmdline/list.c +++ b/cmdline/list.c @@ -74,7 +74,7 @@ void state_list(struct snapraid_state* state) if (tm) { printf("%04u/%02u/%02u %02u:%02u", tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour, tm->tm_min); if (msg_level >= MSG_VERBOSE) - printf(":%02u.%03u", tm->tm_sec, file->mtime_nsec / 1000000); + printf(":%02u.%09u", tm->tm_sec, file->mtime_nsec); printf(" "); } printf("%s\n", fmt_term(disk, file->sub, esc_buffer)); @@ -105,7 +105,7 @@ void state_list(struct snapraid_state* state) printf("%12s ", type); printf(" "); if (msg_level >= MSG_VERBOSE) - printf(" "); + printf(" "); printf("%s -> %s\n", fmt_term(disk, slink->sub, esc_buffer), fmt_term(disk, slink->linkto, esc_buffer_alt)); } } diff --git a/cmdline/mingw.c b/cmdline/mingw.c index 2805378..ef10ee8 100644 --- a/cmdline/mingw.c +++ b/cmdline/mingw.c @@ -694,6 +694,7 @@ static void windows_errno(DWORD error) errno = EPERM; break; case ERROR_IO_DEVICE : /* in ReadFile() and WriteFile() */ + case ERROR_CRC : /* in ReadFile() */ errno = EIO; break; default : diff --git a/cmdline/mkstream.c b/cmdline/mkstream.c index 8f852bd..5a2f804 100644 --- a/cmdline/mkstream.c +++ b/cmdline/mkstream.c @@ -41,6 +41,7 @@ void test(void) s = sopen_multi_write(STREAM_MAX); for (i = 0; i < STREAM_MAX; ++i) { snprintf(file, sizeof(file), "stream%u.bin", i); + remove(file); if (sopen_multi_file(s, i, file) != 0) { /* LCOV_EXCL_START */ exit(EXIT_FAILURE); diff --git a/cmdline/murmur3.c b/cmdline/murmur3.c index 2731cd9..71b61da 100644 --- a/cmdline/murmur3.c +++ b/cmdline/murmur3.c @@ -136,25 +136,29 @@ void MurmurHash3_x86_128(const void* data, size_t size, const uint8_t* seed, voi uint32_t k4 = 0; switch (size_remainder) { - case 15 : k4 ^= (uint32_t)tail[14] << 16; - case 14 : k4 ^= (uint32_t)tail[13] << 8; - case 13 : k4 ^= (uint32_t)tail[12] << 0; + case 15 : k4 ^= (uint32_t)tail[14] << 16; /* fallthrough */ + case 14 : k4 ^= (uint32_t)tail[13] << 8; /* fallthrough */ + case 13 : k4 ^= (uint32_t)tail[12] << 0; /* fallthrough */ k4 *= c4; k4 = util_rotl32(k4, 18); k4 *= c1; h4 ^= k4; - case 12 : k3 ^= (uint32_t)tail[11] << 24; - case 11 : k3 ^= (uint32_t)tail[10] << 16; - case 10 : k3 ^= (uint32_t)tail[ 9] << 8; - case 9 : k3 ^= (uint32_t)tail[ 8] << 0; + /* fallthrough */ + case 12 : k3 ^= (uint32_t)tail[11] << 24; /* fallthrough */ + case 11 : k3 ^= (uint32_t)tail[10] << 16; /* fallthrough */ + case 10 : k3 ^= (uint32_t)tail[ 9] << 8; /* fallthrough */ + case 9 : k3 ^= (uint32_t)tail[ 8] << 0; /* fallthrough */ k3 *= c3; k3 = util_rotl32(k3, 17); k3 *= c4; h3 ^= k3; - case 8 : k2 ^= (uint32_t)tail[ 7] << 24; - case 7 : k2 ^= (uint32_t)tail[ 6] << 16; - case 6 : k2 ^= (uint32_t)tail[ 5] << 8; - case 5 : k2 ^= (uint32_t)tail[ 4] << 0; + /* fallthrough */ + case 8 : k2 ^= (uint32_t)tail[ 7] << 24; /* fallthrough */ + case 7 : k2 ^= (uint32_t)tail[ 6] << 16; /* fallthrough */ + case 6 : k2 ^= (uint32_t)tail[ 5] << 8; /* fallthrough */ + case 5 : k2 ^= (uint32_t)tail[ 4] << 0; /* fallthrough */ k2 *= c2; k2 = util_rotl32(k2, 16); k2 *= c3; h2 ^= k2; - case 4 : k1 ^= (uint32_t)tail[ 3] << 24; - case 3 : k1 ^= (uint32_t)tail[ 2] << 16; - case 2 : k1 ^= (uint32_t)tail[ 1] << 8; - case 1 : k1 ^= (uint32_t)tail[ 0] << 0; + /* fallthrough */ + case 4 : k1 ^= (uint32_t)tail[ 3] << 24; /* fallthrough */ + case 3 : k1 ^= (uint32_t)tail[ 2] << 16; /* fallthrough */ + case 2 : k1 ^= (uint32_t)tail[ 1] << 8; /* fallthrough */ + case 1 : k1 ^= (uint32_t)tail[ 0] << 0; /* fallthrough */ k1 *= c1; k1 = util_rotl32(k1, 15); k1 *= c2; h1 ^= k1; + /* fallthrough */ } } diff --git a/cmdline/snapraid.c b/cmdline/snapraid.c index 1221872..07103a6 100644 --- a/cmdline/snapraid.c +++ b/cmdline/snapraid.c @@ -87,7 +87,7 @@ void memory(void) log_tag("memory:link:%" PRIu64 "\n", (uint64_t)(sizeof(struct snapraid_link))); log_tag("memory:dir:%" PRIu64 "\n", (uint64_t)(sizeof(struct snapraid_dir))); - msg_progress("Using %u MiB of memory for the FileSystem.\n", (unsigned)(malloc_counter_get() / MEBI)); + msg_progress("Using %u MiB of memory for the file-system.\n", (unsigned)(malloc_counter_get() / MEBI)); } void test(int argc, char* argv[]) @@ -1091,7 +1091,7 @@ int main(int argc, char* argv[]) exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } - /* follow */ + /* fallthrough */ case OPERATION_SPINUP : case OPERATION_SPINDOWN : if (!tommy_list_empty(&filterlist_file)) { diff --git a/cmdline/state.c b/cmdline/state.c index cff523e..b09fba2 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -2872,6 +2872,7 @@ struct state_write_thread_context { /* input */ block_off_t blockmax; time_t info_oldest; + time_t info_now; int info_has_rehash; STREAM* f; /* output */ @@ -2888,6 +2889,7 @@ static void* state_write_thread(void* arg) struct snapraid_state* state = context->state; block_off_t blockmax = context->blockmax; time_t info_oldest = context->info_oldest; + time_t info_now = context->info_now; int info_has_rehash = context->info_has_rehash; STREAM* f = context->f; uint32_t crc; @@ -3278,7 +3280,18 @@ static void* state_write_thread(void* arg) flag |= 8; sputb32(flag, f); - t = info_get_time(info) - info_oldest; + t = info_get_time(info); + + /* truncate any time that is in the future */ + if (t > info_now) + t = info_now; + + /* the oldest info is computed only on required blocks, so it may not be the absolute oldest */ + if (t < info_oldest) + t = 0; + else + t -= info_oldest; + sputb32(t, f); } else { /* write a special 0 flag to mark missing info */ @@ -3353,6 +3366,7 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc) tommy_node* i; block_off_t blockmax; time_t info_oldest; + time_t info_now; int info_has_rehash; int mapping_idx; block_off_t idx; @@ -3371,6 +3385,7 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc) /* clear the info for unused blocks */ /* and get some other info */ info_oldest = 0; /* oldest time in info */ + info_now = time(0); /* get the present time */ info_has_rehash = 0; /* if there is a rehash info */ for (idx = 0; idx < blockmax; ++idx) { /* if the position is used */ @@ -3434,10 +3449,21 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc) msg_progress("Saving state to %s...\n", content->content); pathprint(tmp, sizeof(tmp), "%s.tmp", content->content); + + /* ensure to delete a previous stale file */ + if (remove(tmp) != 0) { + if (errno != ENOENT) { + /* LCOV_EXCL_START */ + log_fatal("Error removing the stale content file '%s'. %s.\n", tmp, strerror(errno)); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + } + f = sopen_write(tmp); if (f == 0) { /* LCOV_EXCL_START */ - log_fatal("Error opening the content file '%s'. %s.\n", tmp, strerror(errno)); + log_fatal("Error opening the temporary content file '%s'. %s.\n", tmp, strerror(errno)); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -3450,6 +3476,7 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc) context->state = state; context->blockmax = blockmax; context->info_oldest = info_oldest; + context->info_now = info_now; context->info_has_rehash = info_has_rehash; context->f = f; @@ -3562,12 +3589,24 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc) struct snapraid_content* content = i->data; char tmp[PATH_MAX]; pathprint(tmp, sizeof(tmp), "%s.tmp", content->content); + + /* ensure to delete a previous stale file */ + if (remove(tmp) != 0) { + if (errno != ENOENT) { + /* LCOV_EXCL_START */ + log_fatal("Error removing the stale content file '%s'. %s.\n", tmp, strerror(errno)); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + } + if (sopen_multi_file(f, k, tmp) != 0) { /* LCOV_EXCL_START */ - log_fatal("Error opening the content file '%s'. %s.\n", tmp, strerror(errno)); + log_fatal("Error opening the temporary content file '%s'. %s.\n", tmp, strerror(errno)); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } + ++k; i = i->next; } @@ -3579,6 +3618,7 @@ static void state_write_content(struct snapraid_state* state, uint32_t* out_crc) context->state = state; context->blockmax = blockmax; context->info_oldest = info_oldest; + context->info_now = info_now; context->info_has_rehash = info_has_rehash; context->f = f; @@ -3851,11 +3891,10 @@ static void state_verify_content(struct snapraid_state* state, uint32_t crc) msg_progress("Verifying %s...\n", content->content); pathprint(tmp, sizeof(tmp), "%s.tmp", content->content); - f = sopen_read(tmp); if (f == 0) { /* LCOV_EXCL_START */ - log_fatal("Error reopening the content file '%s'. %s.\n", tmp, strerror(errno)); + log_fatal("Error reopening the temporary content file '%s'. %s.\n", tmp, strerror(errno)); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -3924,6 +3963,59 @@ static void state_rename_content(struct snapraid_state* state) { tommy_node* i; +#if defined(_linux) /* this sequence is linux specific */ + i = tommy_list_head(&state->contentlist); + while (i) { + struct snapraid_content* content = i->data; + char tmp[PATH_MAX]; + char dir[PATH_MAX]; + char* slash; + int handle; + + pathcpy(dir, sizeof(dir), content->content); + + slash = strrchr(tmp, '/'); + if (slash) + *slash = 0; + else + pathcpy(dir, sizeof(dir), "."); + + /* open the directory to get the handle */ + handle = open(dir, O_RDONLY | O_DIRECTORY); + if (handle < 0) { + /* LCOV_EXCL_START */ + log_fatal("Error opening the directory '%s'. %s.\n", dir, strerror(errno)); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + /* now rename the just written copy with the correct name */ + pathprint(tmp, sizeof(tmp), "%s.tmp", content->content); + if (rename(tmp, content->content) != 0) { + /* LCOV_EXCL_START */ + log_fatal("Error renaming the content file '%s' to '%s'. %s.\n", tmp, content->content, strerror(errno)); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + /* sync the directory */ + if (fsync(handle) != 0) { + /* LCOV_EXCL_START */ + log_fatal("Error syncing the directory '%s'. %s.\n", dir, strerror(errno)); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + if (close(handle) != 0) { + /* LCOV_EXCL_START */ + log_fatal("Error closing the directory '%s'. %s.\n", dir, strerror(errno)); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + i = i->next; + } +#else i = tommy_list_head(&state->contentlist); while (i) { struct snapraid_content* content = i->data; @@ -3940,6 +4032,7 @@ static void state_rename_content(struct snapraid_state* state) i = i->next; } +#endif } void state_write(struct snapraid_state* state) diff --git a/cmdline/status.c b/cmdline/status.c index a6d769c..9f621de 100644 --- a/cmdline/status.c +++ b/cmdline/status.c @@ -29,6 +29,10 @@ unsigned day_ago(time_t ref, time_t now) { + /* in case some dates is in the future */ + if (now < ref) + return 0; + return (now - ref) / (24 * 3600); } @@ -458,6 +462,10 @@ int state_status(struct snapraid_state* state) printf("\n"); + if (newest > now) { + printf("WARNING! You have scrub dates in the future! The next sync/scrub will truncate them!\n"); + } + if (unsynced_blocks) { printf("WARNING! The array is NOT fully synced.\n"); printf("You have a sync in progress at %u%%.\n", (blockmax - unsynced_blocks) * 100 / blockmax); diff --git a/cmdline/stream.c b/cmdline/stream.c index eeac131..b7bb2eb 100644 --- a/cmdline/stream.c +++ b/cmdline/stream.c @@ -112,7 +112,8 @@ int sopen_multi_file(STREAM* s, unsigned i, const char* file) pathcpy(s->handle[i].path, sizeof(s->handle[i].path), file); - f = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY | O_SEQUENTIAL, 0600); + /* O_EXCL to be resilent ensure to always create a new file and not use a stale link to the original file */ + f = open(file, O_WRONLY | O_CREAT | O_EXCL | O_BINARY | O_SEQUENTIAL, 0600); if (f == -1) { /* LCOV_EXCL_START */ return -1; diff --git a/cmdline/support.h b/cmdline/support.h index 9cdf220..3b75b38 100644 --- a/cmdline/support.h +++ b/cmdline/support.h @@ -299,6 +299,8 @@ void pathslash(char* dst, size_t size); /** * Cut everything after the latest slash. + * + * If the string doesn't contain any slash, it returns the empty string. */ void pathcut(char* dst); diff --git a/cmdline/sync.c b/cmdline/sync.c index dbcce38..0d12590 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -1536,7 +1536,7 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t data_off_t out_size; parity_size(&parity_handle[l], &out_size); parity_overflow(state, out_size); - log_fatal("WARNING! Without an unsable %s file, it isn't possible to sync.\n", lev_name(l)); + log_fatal("WARNING! Without an usable %s file, it isn't possible to sync.\n", lev_name(l)); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } diff --git a/cmdline/unix.c b/cmdline/unix.c index cf3f162..351cfba 100644 --- a/cmdline/unix.c +++ b/cmdline/unix.c @@ -917,18 +917,20 @@ static int devtree(const char* name, const char* custom, dev_t device, devinfo_t while ((dd = readdir(d)) != 0) { if (dd->d_name[0] != '.') { + dev_t subdev; + /* for each slave, expand the full potential tree */ pathprint(path, sizeof(path), "/sys/dev/block/%u:%u/slaves/%s/dev", major(device), minor(device), dd->d_name); - device = devread(path); - if (!device) { + subdev = devread(path); + if (!subdev) { /* LCOV_EXCL_START */ closedir(d); return -1; /* LCOV_EXCL_STOP */ } - if (devtree(name, custom, device, parent, list) != 0) { + if (devtree(name, custom, subdev, parent, list) != 0) { /* LCOV_EXCL_START */ closedir(d); return -1; @@ -1072,8 +1074,8 @@ static int devscan(tommy_list* list) #if HAVE_LINUX_DEVICE static int devsmart(dev_t device, const char* name, const char* custom, uint64_t* smart, char* serial, char* vendor, char* model) { - char cmd[128]; - char file[128]; + char cmd[PATH_MAX + 64]; + char file[PATH_MAX]; FILE* f; int ret; @@ -1086,7 +1088,7 @@ static int devsmart(dev_t device, const char* name, const char* custom, uint64_t /* if there is a custom command */ if (custom[0]) { - char option[128]; + char option[PATH_MAX]; snprintf(option, sizeof(option), custom, file); snprintf(cmd, sizeof(cmd), "smartctl -a %s", option); } else { @@ -1140,8 +1142,8 @@ static int devsmart(dev_t device, const char* name, const char* custom, uint64_t #if HAVE_LINUX_DEVICE static int devdown(dev_t device, const char* name, const char* custom) { - char cmd[128]; - char file[128]; + char cmd[PATH_MAX + 64]; + char file[PATH_MAX]; FILE* f; int ret; @@ -1154,7 +1156,7 @@ static int devdown(dev_t device, const char* name, const char* custom) /* if there is a custom command */ if (custom[0]) { - char option[128]; + char option[PATH_MAX]; snprintf(option, sizeof(option), custom, file); snprintf(cmd, sizeof(cmd), "smartctl -s standby,now %s", option); } else { diff --git a/configure b/configure index 8492d01..62f8255 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for snapraid 11.2. +# Generated by GNU Autoconf 2.69 for snapraid 11.3. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -577,8 +577,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='snapraid' PACKAGE_TARNAME='snapraid' -PACKAGE_VERSION='11.2' -PACKAGE_STRING='snapraid 11.2' +PACKAGE_VERSION='11.3' +PACKAGE_STRING='snapraid 11.3' PACKAGE_BUGREPORT='' PACKAGE_URL='http://www.snapraid.it' @@ -1304,7 +1304,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures snapraid 11.2 to adapt to many kinds of systems. +\`configure' configures snapraid 11.3 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1374,7 +1374,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of snapraid 11.2:";; + short | recursive ) echo "Configuration of snapraid 11.3:";; esac cat <<\_ACEOF @@ -1496,7 +1496,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -snapraid configure 11.2 +snapraid configure 11.3 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2102,7 +2102,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by snapraid $as_me 11.2, which was +It was created by snapraid $as_me 11.3, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2965,7 +2965,7 @@ fi # Define the identity of the package. PACKAGE='snapraid' - VERSION='11.2' + VERSION='11.3' cat >>confdefs.h <<_ACEOF @@ -7373,7 +7373,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by snapraid $as_me 11.2, which was +This file was extended by snapraid $as_me 11.3, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -7436,7 +7436,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -snapraid config.status 11.2 +snapraid config.status 11.3 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/snapraid.1 b/snapraid.1 index 915df9c..0fb9b7b 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -647,7 +647,7 @@ and ignore all the parity data. This command will take a long time, but if you are not paranoid, you can skip it. .SS STEP 4 \-> Sync -Run the \[dq]sync\[dq] command to resynchronize the array with the new disk. +Run the \[dq]sync\[dq] command to re\-synchronize the array with the new disk. .PP .RS 4 snapraid sync @@ -745,7 +745,7 @@ Spins up all the disks of the array. .PP You can spin\-up only some specific disks using the \-d, \-\-filter\-disk option. .PP -Take care that spinniup\-up all the disks at the same time needs a lot of power. +Take care that spinning\-up all the disks at the same time needs a lot of power. Ensure that your power\-supply can sustain that. .PP Nothing is modified. @@ -912,7 +912,7 @@ Verify all the files and the parity data. It works like \[dq]fix\[dq], but it only simulates a recovery and no change is written in the array. .PP -This command is mostly intended for manual verifications, +This command is mostly intended for manual verification, like after a recovery process or in other special conditions. For periodic and scheduled checks uses \[dq]scrub\[dq]. .PP @@ -931,7 +931,7 @@ Nothing is modified. .SS dup Lists all the duplicate files. Two files are assumed equal if their hashes are matching. The file data is not read, but only the -precomputed hashes are used. +pre\-computed hashes are used. .PP Nothing is modified. .SS pool @@ -942,7 +942,7 @@ The files are not really copied here, but just linked using symbolic links. .PP When updating, all the present symbolic links and empty -subdirectories are deleted and replaced with the new +sub\-directories are deleted and replaced with the new view of the array. Any other regular file is left in place. .PP Nothing is modified outside the pool directory. @@ -962,20 +962,20 @@ low level devices used by a single disk in the array. .PP Nothing is modified. .SS touch -Sets arbitrarely the sub\-second timestamp of all the files +Sets arbitrarily the sub\-second time\-stamp of all the files that have it at zero. .PP This improves the SnapRAID capability to recognize moved -and copied files as it makes the timestamp almost unique, +and copied files as it makes the time\-stamp almost unique, removing possible duplicates. .PP -More specifically, if the sub\-second timestamp is not zero, +More specifically, if the sub\-second time\-stamp is not zero, a moved or copied file is identified as such if it matches -the name, size and timestamp. If instead the sub\-second timestamp +the name, size and time\-stamp. If instead the sub\-second time\-stamp is zero, it\'s considered a copy only if it matches the full path, -size and timestamp. +size and time\-stamp. .PP -Note that the second precision timestamp is not modified, +Note that the second precision time\-stamp is not modified, and all the dates and times of your files will be maintained. .SS rehash Schedules a rehash of the whole array. @@ -1090,7 +1090,7 @@ Imports from the specified directory any file that you deleted from the array after the last \[dq]sync\[dq]. If you still have such files, they could be used by \[dq]check\[dq] and \[dq]fix\[dq] to improve the recover process. -The files are read also in subdirectories and they are +The files are read also in sub\-directories and they are identified regardless of their name. This option can be used only with \[dq]check\[dq] and \[dq]fix\[dq]. .TP @@ -1146,7 +1146,7 @@ This allows to identify copied or moved files from one disk to another, and to reuse the already computed hash information to detect silent errors or to recover missing files. This behavior, in some rare cases, may result in false positives, -or in a slow process due the many hash verifications, and this +or in a slow process due the many hash verification, and this option allows to resolve them. This option can be used only with \[dq]sync\[dq], \[dq]check\[dq] and \[dq]fix\[dq]. .TP @@ -1280,7 +1280,7 @@ is enabled: 6\-parity enables hexa (six) parity .PD .PP -Each parity level requires the precence of all the previous parity +Each parity level requires the presence of all the previous parity levels. .PP The same considerations of the \'parity\' option apply. diff --git a/snapraid.txt b/snapraid.txt index 863d8c6..9621e04 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -391,7 +391,7 @@ This command will take a long time, but if you are not paranoid, you can skip it. ---- 4.4.4 STEP 4 -> Sync ---- -Run the "sync" command to resynchronize the array with the new disk. +Run the "sync" command to re-synchronize the array with the new disk. snapraid sync @@ -476,7 +476,7 @@ Spins up all the disks of the array. You can spin-up only some specific disks using the -d, --filter-disk option. -Take care that spinniup-up all the disks at the same time needs a lot of power. +Take care that spinning-up all the disks at the same time needs a lot of power. Ensure that your power-supply can sustain that. Nothing is modified. @@ -643,7 +643,7 @@ Verify all the files and the parity data. It works like "fix", but it only simulates a recovery and no change is written in the array. -This command is mostly intended for manual verifications, +This command is mostly intended for manual verification, like after a recovery process or in other special conditions. For periodic and scheduled checks uses "scrub". @@ -668,7 +668,7 @@ Nothing is modified. Lists all the duplicate files. Two files are assumed equal if their hashes are matching. The file data is not read, but only the -precomputed hashes are used. +pre-computed hashes are used. Nothing is modified. @@ -682,7 +682,7 @@ The files are not really copied here, but just linked using symbolic links. When updating, all the present symbolic links and empty -subdirectories are deleted and replaced with the new +sub-directories are deleted and replaced with the new view of the array. Any other regular file is left in place. Nothing is modified outside the pool directory. @@ -708,20 +708,20 @@ Nothing is modified. 5.14 touch ---------- -Sets arbitrarely the sub-second timestamp of all the files +Sets arbitrarily the sub-second time-stamp of all the files that have it at zero. This improves the SnapRAID capability to recognize moved -and copied files as it makes the timestamp almost unique, +and copied files as it makes the time-stamp almost unique, removing possible duplicates. -More specifically, if the sub-second timestamp is not zero, +More specifically, if the sub-second time-stamp is not zero, a moved or copied file is identified as such if it matches -the name, size and timestamp. If instead the sub-second timestamp +the name, size and time-stamp. If instead the sub-second time-stamp is zero, it's considered a copy only if it matches the full path, -size and timestamp. +size and time-stamp. -Note that the second precision timestamp is not modified, +Note that the second precision time-stamp is not modified, and all the dates and times of your files will be maintained. 5.15 rehash @@ -843,7 +843,7 @@ SnapRAID provides the following options: from the array after the last "sync". If you still have such files, they could be used by "check" and "fix" to improve the recover process. - The files are read also in subdirectories and they are + The files are read also in sub-directories and they are identified regardless of their name. This option can be used only with "check" and "fix". @@ -899,7 +899,7 @@ SnapRAID provides the following options: to another, and to reuse the already computed hash information to detect silent errors or to recover missing files. This behavior, in some rare cases, may result in false positives, - or in a slow process due the many hash verifications, and this + or in a slow process due the many hash verification, and this option allows to resolve them. This option can be used only with "sync", "check" and "fix". @@ -1037,7 +1037,7 @@ is enabled: * 5-parity enables penta (five) parity * 6-parity enables hexa (six) parity -Each parity level requires the precence of all the previous parity +Each parity level requires the presence of all the previous parity levels. The same considerations of the 'parity' option apply. diff --git a/tommyds/tommyarray.c b/tommyds/tommyarray.c index a8d946b..d77db04 100644 --- a/tommyds/tommyarray.c +++ b/tommyds/tommyarray.c @@ -36,7 +36,7 @@ void tommy_array_init(tommy_array* array) /* fixed initial size */ array->bucket_bit = TOMMY_ARRAY_BIT; - array->bucket_max = 1 << array->bucket_bit; + array->bucket_max = (tommy_size_t)1 << array->bucket_bit; array->bucket[0] = tommy_cast(void**, tommy_calloc(array->bucket_max, sizeof(void*))); for (i = 1; i < TOMMY_ARRAY_BIT; ++i) array->bucket[i] = array->bucket[0]; @@ -51,11 +51,11 @@ void tommy_array_done(tommy_array* array) tommy_free(array->bucket[0]); for (i = TOMMY_ARRAY_BIT; i < array->bucket_bit; ++i) { void** segment = array->bucket[i]; - tommy_free(&segment[((tommy_ptrdiff_t)1) << i]); + tommy_free(&segment[(tommy_ptrdiff_t)1 << i]); } } -void tommy_array_grow(tommy_array* array, tommy_count_t count) +void tommy_array_grow(tommy_array* array, tommy_size_t count) { if (array->count >= count) return; @@ -72,7 +72,7 @@ void tommy_array_grow(tommy_array* array, tommy_count_t count) array->bucket[array->bucket_bit] = &segment[-(tommy_ptrdiff_t)array->bucket_max]; ++array->bucket_bit; - array->bucket_max = 1 << array->bucket_bit; + array->bucket_max = (tommy_size_t)1 << array->bucket_bit; } } diff --git a/tommyds/tommyarray.h b/tommyds/tommyarray.h index c04193f..bc124de 100644 --- a/tommyds/tommyarray.h +++ b/tommyds/tommyarray.h @@ -53,20 +53,15 @@ */ #define TOMMY_ARRAY_BIT 6 -/** \internal - * Max number of elements as a power of 2. - */ -#define TOMMY_ARRAY_BIT_MAX 32 - /** * Array container type. * \note Don't use internal fields directly, but access the container only using functions. */ typedef struct tommy_array_struct { - void** bucket[TOMMY_ARRAY_BIT_MAX]; /**< Dynamic array of buckets. */ + void** bucket[TOMMY_SIZE_BIT]; /**< Dynamic array of buckets. */ + tommy_size_t bucket_max; /**< Number of buckets. */ + tommy_size_t count; /**< Number of initialized elements in the array. */ tommy_uint_t bucket_bit; /**< Bits used in the bit mask. */ - tommy_count_t bucket_max; /**< Number of buckets. */ - tommy_count_t count; /**< Number of initialized elements in the array. */ } tommy_array; /** @@ -83,21 +78,21 @@ void tommy_array_done(tommy_array* array); * Grows the size up to the specified value. * All the new elements in the array are initialized with the 0 value. */ -void tommy_array_grow(tommy_array* array, tommy_count_t size); +void tommy_array_grow(tommy_array* array, tommy_size_t size); /** * Gets a reference of the element at the specified position. * You must be sure that space for this position is already * allocated calling tommy_array_grow(). */ -tommy_inline void** tommy_array_ref(tommy_array* array, tommy_count_t pos) +tommy_inline void** tommy_array_ref(tommy_array* array, tommy_size_t pos) { tommy_uint_t bsr; assert(pos < array->count); /* get the highest bit set, in case of all 0, return 0 */ - bsr = tommy_ilog2_u32(pos | 1); + bsr = tommy_ilog2(pos | 1); return &array->bucket[bsr][pos]; } @@ -107,7 +102,7 @@ tommy_inline void** tommy_array_ref(tommy_array* array, tommy_count_t pos) * You must be sure that space for this position is already * allocated calling tommy_array_grow(). */ -tommy_inline void tommy_array_set(tommy_array* array, tommy_count_t pos, void* element) +tommy_inline void tommy_array_set(tommy_array* array, tommy_size_t pos, void* element) { *tommy_array_ref(array, pos) = element; } @@ -117,7 +112,7 @@ tommy_inline void tommy_array_set(tommy_array* array, tommy_count_t pos, void* e * You must be sure that space for this position is already * allocated calling tommy_array_grow(). */ -tommy_inline void* tommy_array_get(tommy_array* array, tommy_count_t pos) +tommy_inline void* tommy_array_get(tommy_array* array, tommy_size_t pos) { return *tommy_array_ref(array, pos); } @@ -127,7 +122,7 @@ tommy_inline void* tommy_array_get(tommy_array* array, tommy_count_t pos) */ tommy_inline void tommy_array_insert(tommy_array* array, void* element) { - tommy_count_t pos = array->count; + tommy_size_t pos = array->count; tommy_array_grow(array, pos + 1); @@ -137,7 +132,7 @@ tommy_inline void tommy_array_insert(tommy_array* array, void* element) /** * Gets the initialized size of the array. */ -tommy_inline tommy_count_t tommy_array_size(tommy_array* array) +tommy_inline tommy_size_t tommy_array_size(tommy_array* array) { return array->count; } diff --git a/tommyds/tommyarrayblkof.c b/tommyds/tommyarrayblkof.c index 8a491a0..11187be 100644 --- a/tommyds/tommyarrayblkof.c +++ b/tommyds/tommyarrayblkof.c @@ -40,7 +40,7 @@ void tommy_arrayblkof_init(tommy_arrayblkof* array, tommy_size_t element_size) void tommy_arrayblkof_done(tommy_arrayblkof* array) { - tommy_count_t i; + tommy_size_t i; for (i = 0; i < tommy_array_size(&array->block); ++i) tommy_free(tommy_array_get(&array->block, i)); @@ -48,10 +48,10 @@ void tommy_arrayblkof_done(tommy_arrayblkof* array) tommy_array_done(&array->block); } -void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_count_t count) +void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_size_t count) { - tommy_count_t block_max; - tommy_count_t block_mac; + tommy_size_t block_max; + tommy_size_t block_mac; if (array->count >= count) return; diff --git a/tommyds/tommyarrayblkof.h b/tommyds/tommyarrayblkof.h index fce6840..16d70cb 100644 --- a/tommyds/tommyarrayblkof.h +++ b/tommyds/tommyarrayblkof.h @@ -61,7 +61,7 @@ typedef struct tommy_arrayblkof_struct { tommy_array block; /**< Array of blocks. */ tommy_size_t element_size; /**< Size of the stored element in bytes. */ - tommy_count_t count; /**< Number of initialized elements in the array. */ + tommy_size_t count; /**< Number of initialized elements in the array. */ } tommy_arrayblkof; /** @@ -79,14 +79,14 @@ void tommy_arrayblkof_done(tommy_arrayblkof* array); * Grows the size up to the specified value. * All the new elements in the array are initialized with the 0 value. */ -void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_count_t size); +void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_size_t size); /** * Gets a reference of the element at the specified position. * You must be sure that space for this position is already * allocated calling tommy_arrayblkof_grow(). */ -tommy_inline void* tommy_arrayblkof_ref(tommy_arrayblkof* array, tommy_count_t pos) +tommy_inline void* tommy_arrayblkof_ref(tommy_arrayblkof* array, tommy_size_t pos) { unsigned char* base; @@ -100,7 +100,7 @@ tommy_inline void* tommy_arrayblkof_ref(tommy_arrayblkof* array, tommy_count_t p /** * Gets the initialized size of the array. */ -tommy_inline tommy_count_t tommy_arrayblkof_size(tommy_arrayblkof* array) +tommy_inline tommy_size_t tommy_arrayblkof_size(tommy_arrayblkof* array) { return array->count; } diff --git a/tommyds/tommychain.h b/tommyds/tommychain.h index b15bc6a..12d2514 100644 --- a/tommyds/tommychain.h +++ b/tommyds/tommychain.h @@ -136,11 +136,6 @@ tommy_inline void tommy_chain_merge_degenerated(tommy_chain* first, tommy_chain* tommy_chain_merge(first, second, cmp); } -/** - * Max number of elements as a power of 2. - */ -#define TOMMY_CHAIN_BIT_MAX 32 - /** * Sorts a chain. * It's a stable merge sort using power of 2 buckets, with O(N*log(N)) complexity, @@ -158,20 +153,20 @@ tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func* /* * Bit buckets of chains. * Each bucket contains 2^i nodes or it's empty. - * The chain at address TOMMY_CHAIN_BIT_MAX is an independet variable operating as "carry". + * The chain at address TOMMY_BIT_MAX is an independet variable operating as "carry". * We keep it in the same "bit" vector to avoid reports from the valgrind tool sgcheck. */ - tommy_chain bit[TOMMY_CHAIN_BIT_MAX + 1]; + tommy_chain bit[TOMMY_SIZE_BIT + 1]; /** * Value stored inside the bit bucket. * It's used to know which bucket is empty of full. */ - tommy_count_t counter; + tommy_size_t counter; tommy_node* node = chain->head; tommy_node* tail = chain->tail; - tommy_count_t mask; - tommy_count_t i; + tommy_size_t mask; + tommy_size_t i; counter = 0; while (1) { @@ -179,9 +174,9 @@ tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func* tommy_chain* last; /* carry bit to add */ - last = &bit[TOMMY_CHAIN_BIT_MAX]; - bit[TOMMY_CHAIN_BIT_MAX].head = node; - bit[TOMMY_CHAIN_BIT_MAX].tail = node; + last = &bit[TOMMY_SIZE_BIT]; + bit[TOMMY_SIZE_BIT].head = node; + bit[TOMMY_SIZE_BIT].tail = node; next = node->next; /* add the bit, propagating the carry */ @@ -206,7 +201,7 @@ tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func* } /* merge the buckets */ - i = tommy_ctz_u32(counter); + i = tommy_ctz(counter); mask = counter >> i; while (mask != 1) { mask >>= 1; diff --git a/tommyds/tommyhash.c b/tommyds/tommyhash.c index 05d78d2..cc7495d 100644 --- a/tommyds/tommyhash.c +++ b/tommyds/tommyhash.c @@ -92,22 +92,22 @@ tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tom b += tommy_le_uint32_read(key + 4); a += tommy_le_uint32_read(key + 0); break; - case 11 : c += ((tommy_uint32_t)key[10]) << 16; - case 10 : c += ((tommy_uint32_t)key[9]) << 8; - case 9 : c += key[8]; + case 11 : c += ((tommy_uint32_t)key[10]) << 16; /* fallthrough */ + case 10 : c += ((tommy_uint32_t)key[9]) << 8; /* fallthrough */ + case 9 : c += key[8]; /* fallthrough */ case 8 : b += tommy_le_uint32_read(key + 4); a += tommy_le_uint32_read(key + 0); break; - case 7 : b += ((tommy_uint32_t)key[6]) << 16; - case 6 : b += ((tommy_uint32_t)key[5]) << 8; - case 5 : b += key[4]; + case 7 : b += ((tommy_uint32_t)key[6]) << 16; /* fallthrough */ + case 6 : b += ((tommy_uint32_t)key[5]) << 8; /* fallthrough */ + case 5 : b += key[4]; /* fallthrough */ case 4 : a += tommy_le_uint32_read(key + 0); break; - case 3 : a += ((tommy_uint32_t)key[2]) << 16; - case 2 : a += ((tommy_uint32_t)key[1]) << 8; - case 1 : a += key[0]; + case 3 : a += ((tommy_uint32_t)key[2]) << 16; /* fallthrough */ + case 2 : a += ((tommy_uint32_t)key[1]) << 8; /* fallthrough */ + case 1 : a += key[0]; /* fallthrough */ } tommy_final(a, b, c); @@ -142,22 +142,22 @@ tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tom b += tommy_le_uint32_read(key + 4); a += tommy_le_uint32_read(key + 0); break; - case 11 : c += ((tommy_uint32_t)key[10]) << 16; - case 10 : c += ((tommy_uint32_t)key[9]) << 8; - case 9 : c += key[8]; + case 11 : c += ((tommy_uint32_t)key[10]) << 16; /* fallthrough */ + case 10 : c += ((tommy_uint32_t)key[9]) << 8; /* fallthrough */ + case 9 : c += key[8]; /* fallthrough */ case 8 : b += tommy_le_uint32_read(key + 4); a += tommy_le_uint32_read(key + 0); break; - case 7 : b += ((tommy_uint32_t)key[6]) << 16; - case 6 : b += ((tommy_uint32_t)key[5]) << 8; - case 5 : b += key[4]; + case 7 : b += ((tommy_uint32_t)key[6]) << 16; /* fallthrough */ + case 6 : b += ((tommy_uint32_t)key[5]) << 8; /* fallthrough */ + case 5 : b += key[4]; /* fallthrough */ case 4 : a += tommy_le_uint32_read(key + 0); break; - case 3 : a += ((tommy_uint32_t)key[2]) << 16; - case 2 : a += ((tommy_uint32_t)key[1]) << 8; - case 1 : a += key[0]; + case 3 : a += ((tommy_uint32_t)key[2]) << 16; /* fallthrough */ + case 2 : a += ((tommy_uint32_t)key[1]) << 8; /* fallthrough */ + case 1 : a += key[0]; /* fallthrough */ } tommy_final(a, b, c); diff --git a/tommyds/tommyhash.h b/tommyds/tommyhash.h index 738e33d..50d595c 100644 --- a/tommyds/tommyhash.h +++ b/tommyds/tommyhash.h @@ -37,11 +37,6 @@ /******************************************************************************/ /* hash */ -/** - * Hash type used in hashtables. - */ -typedef tommy_key_t tommy_hash_t; - /** * Hash function with a 32 bits result. * Implementation of the Robert Jenkins "lookup3" hash 32 bits version, diff --git a/tommyds/tommyhashdyn.c b/tommyds/tommyhashdyn.c index 02ef404..a00737e 100644 --- a/tommyds/tommyhashdyn.c +++ b/tommyds/tommyhashdyn.c @@ -35,7 +35,7 @@ void tommy_hashdyn_init(tommy_hashdyn* hashdyn) { /* fixed initial size */ hashdyn->bucket_bit = TOMMY_HASHDYN_BIT; - hashdyn->bucket_max = 1 << hashdyn->bucket_bit; + hashdyn->bucket_max = (tommy_size_t)1 << hashdyn->bucket_bit; hashdyn->bucket_mask = hashdyn->bucket_max - 1; hashdyn->bucket = tommy_cast(tommy_hashdyn_node**, tommy_calloc(hashdyn->bucket_max, sizeof(tommy_hashdyn_node*))); @@ -50,18 +50,18 @@ void tommy_hashdyn_done(tommy_hashdyn* hashdyn) /** * Resize the bucket vector. */ -static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_count_t new_bucket_bit) +static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_size_t new_bucket_bit) { - tommy_count_t bucket_bit; - tommy_count_t bucket_max; - tommy_count_t new_bucket_max; - tommy_count_t new_bucket_mask; + tommy_size_t bucket_bit; + tommy_size_t bucket_max; + tommy_size_t new_bucket_max; + tommy_size_t new_bucket_mask; tommy_hashdyn_node** new_bucket; bucket_bit = hashdyn->bucket_bit; bucket_max = hashdyn->bucket_max; - new_bucket_max = 1 << new_bucket_bit; + new_bucket_max = (tommy_size_t)1 << new_bucket_bit; new_bucket_mask = new_bucket_max - 1; /* allocate the new vector using malloc() and not calloc() */ @@ -70,7 +70,7 @@ static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_count_t new_bucke /* reinsert all the elements */ if (new_bucket_bit > bucket_bit) { - tommy_count_t i; + tommy_size_t i; /* grow */ for (i = 0; i < bucket_max; ++i) { @@ -84,7 +84,7 @@ static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_count_t new_bucke j = hashdyn->bucket[i]; while (j) { tommy_hashdyn_node* j_next = j->next; - tommy_count_t pos = j->key & new_bucket_mask; + tommy_size_t pos = j->index & new_bucket_mask; if (new_bucket[pos]) tommy_list_insert_tail_not_empty(new_bucket[pos], j); else @@ -93,7 +93,7 @@ static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_count_t new_bucke } } } else { - tommy_count_t i; + tommy_size_t i; /* shrink */ for (i = 0; i < new_bucket_max; ++i) { @@ -136,11 +136,11 @@ tommy_inline void hashdyn_shrink_step(tommy_hashdyn* hashdyn) void tommy_hashdyn_insert(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void* data, tommy_hash_t hash) { - tommy_count_t pos = hash & hashdyn->bucket_mask; + tommy_size_t pos = hash & hashdyn->bucket_mask; tommy_list_insert_tail(&hashdyn->bucket[pos], node, data); - node->key = hash; + node->index = hash; ++hashdyn->count; @@ -149,7 +149,7 @@ void tommy_hashdyn_insert(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void void* tommy_hashdyn_remove_existing(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node) { - tommy_count_t pos = node->key & hashdyn->bucket_mask; + tommy_size_t pos = node->index & hashdyn->bucket_mask; tommy_list_remove_existing(&hashdyn->bucket[pos], node); @@ -162,12 +162,12 @@ void* tommy_hashdyn_remove_existing(tommy_hashdyn* hashdyn, tommy_hashdyn_node* void* tommy_hashdyn_remove(tommy_hashdyn* hashdyn, tommy_search_func* cmp, const void* cmp_arg, tommy_hash_t hash) { - tommy_count_t pos = hash & hashdyn->bucket_mask; + tommy_size_t pos = hash & hashdyn->bucket_mask; tommy_hashdyn_node* node = hashdyn->bucket[pos]; while (node) { /* we first check if the hash matches, as in the same bucket we may have multiples hash values */ - if (node->key == hash && cmp(cmp_arg, node->data) == 0) { + if (node->index == hash && cmp(cmp_arg, node->data) == 0) { tommy_list_remove_existing(&hashdyn->bucket[pos], node); --hashdyn->count; @@ -184,9 +184,9 @@ void* tommy_hashdyn_remove(tommy_hashdyn* hashdyn, tommy_search_func* cmp, const void tommy_hashdyn_foreach(tommy_hashdyn* hashdyn, tommy_foreach_func* func) { - tommy_count_t bucket_max = hashdyn->bucket_max; + tommy_size_t bucket_max = hashdyn->bucket_max; tommy_hashdyn_node** bucket = hashdyn->bucket; - tommy_count_t pos; + tommy_size_t pos; for (pos = 0; pos < bucket_max; ++pos) { tommy_hashdyn_node* node = bucket[pos]; @@ -201,9 +201,9 @@ void tommy_hashdyn_foreach(tommy_hashdyn* hashdyn, tommy_foreach_func* func) void tommy_hashdyn_foreach_arg(tommy_hashdyn* hashdyn, tommy_foreach_arg_func* func, void* arg) { - tommy_count_t bucket_max = hashdyn->bucket_max; + tommy_size_t bucket_max = hashdyn->bucket_max; tommy_hashdyn_node** bucket = hashdyn->bucket; - tommy_count_t pos; + tommy_size_t pos; for (pos = 0; pos < bucket_max; ++pos) { tommy_hashdyn_node* node = bucket[pos]; diff --git a/tommyds/tommyhashdyn.h b/tommyds/tommyhashdyn.h index ed4a607..73baba9 100644 --- a/tommyds/tommyhashdyn.h +++ b/tommyds/tommyhashdyn.h @@ -160,10 +160,10 @@ typedef tommy_node tommy_hashdyn_node; */ typedef struct tommy_hashdyn_struct { tommy_hashdyn_node** bucket; /**< Hash buckets. One list for each hash modulus. */ + tommy_size_t bucket_max; /**< Number of buckets. */ + tommy_size_t bucket_mask; /**< Bit mask to access the buckets. */ + tommy_size_t count; /**< Number of elements. */ tommy_uint_t bucket_bit; /**< Bits used in the bit mask. */ - tommy_count_t bucket_max; /**< Number of buckets. */ - tommy_count_t bucket_mask; /**< Bit mask to access the buckets. */ - tommy_count_t count; /**< Number of elements. */ } tommy_hashdyn; /** @@ -226,7 +226,7 @@ tommy_inline void* tommy_hashdyn_search(tommy_hashdyn* hashdyn, tommy_search_fun while (i) { /* we first check if the hash matches, as in the same bucket we may have multiples hash values */ - if (i->key == hash && cmp(cmp_arg, i->data) == 0) + if (i->index == hash && cmp(cmp_arg, i->data) == 0) return i->data; i = i->next; } @@ -281,7 +281,7 @@ void tommy_hashdyn_foreach_arg(tommy_hashdyn* hashdyn, tommy_foreach_arg_func* f /** * Gets the number of elements. */ -tommy_inline tommy_count_t tommy_hashdyn_count(tommy_hashdyn* hashdyn) +tommy_inline tommy_size_t tommy_hashdyn_count(tommy_hashdyn* hashdyn) { return hashdyn->count; } diff --git a/tommyds/tommylist.h b/tommyds/tommylist.h index d88cee7..42ef3ad 100644 --- a/tommyds/tommylist.h +++ b/tommyds/tommylist.h @@ -226,24 +226,6 @@ tommy_inline void tommy_list_insert_tail(tommy_list* list, tommy_node* node, voi node->data = data; } -/** \internal - * Removes an element from the head of a not empty list. - * \param list The list. The list cannot be empty. - * \return The node removed. - */ -tommy_inline tommy_node* tommy_list_remove_head_not_empty(tommy_list* list) -{ - tommy_node* head = tommy_list_head(list); - - /* remove from the "circular" prev list */ - head->next->prev = head->prev; - - /* remove from the "0 terminated" next list */ - *list = head->next; /* the new head, in case 0 */ - - return head; -} - /** * Removes an element from the list. * You must already have the address of the element to remove. @@ -329,9 +311,9 @@ tommy_inline tommy_bool_t tommy_list_empty(tommy_list* list) * Gets the number of elements. * \note This operation is O(n). */ -tommy_inline tommy_count_t tommy_list_count(tommy_list* list) +tommy_inline tommy_size_t tommy_list_count(tommy_list* list) { - tommy_count_t count = 0; + tommy_size_t count = 0; tommy_node* i = tommy_list_head(list); while (i) { diff --git a/tommyds/tommytree.c b/tommyds/tommytree.c index 6b2f1a9..8ef8478 100644 --- a/tommyds/tommytree.c +++ b/tommyds/tommytree.c @@ -39,10 +39,10 @@ void tommy_tree_init(tommy_tree* tree, tommy_compare_func* cmp) tree->cmp = cmp; } -static int tommy_tree_delta(tommy_tree_node* root) +static tommy_ssize_t tommy_tree_delta(tommy_tree_node* root) { - int left_height = root->prev ? root->prev->key : 0; - int right_height = root->next ? root->next->key : 0; + tommy_ssize_t left_height = root->prev ? root->prev->index : 0; + tommy_ssize_t right_height = root->next ? root->next->index : 0; return left_height - right_height; } @@ -84,7 +84,7 @@ static tommy_tree_node* tommy_tree_move_right(tommy_tree_node* root, tommy_tree_ static tommy_tree_node* tommy_tree_balance(tommy_tree_node* root) { - int delta = tommy_tree_delta(root); + tommy_ssize_t delta = tommy_tree_delta(root); if (delta < -1) { if (tommy_tree_delta(root->next) > 0) @@ -99,16 +99,16 @@ static tommy_tree_node* tommy_tree_balance(tommy_tree_node* root) } /* recompute key */ - root->key = 0; + root->index = 0; - if (root->prev && root->prev->key > root->key) - root->key = root->prev->key; + if (root->prev && root->prev->index > root->index) + root->index = root->prev->index; - if (root->next && root->next->key > root->key) - root->key = root->next->key; + if (root->next && root->next->index > root->index) + root->index = root->next->index; /* count itself */ - root->key += 1; + root->index += 1; return root; } @@ -145,7 +145,7 @@ void* tommy_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data) insert->data = data; insert->prev = 0; insert->next = 0; - insert->key = 0; + insert->index = 0; tree->root = tommy_tree_insert_node(tree->cmp, tree->root, &insert); diff --git a/tommyds/tommytree.h b/tommyds/tommytree.h index 55e4006..ada12c0 100644 --- a/tommyds/tommytree.h +++ b/tommyds/tommytree.h @@ -117,8 +117,8 @@ typedef tommy_node tommy_tree_node; */ typedef struct tommy_tree_struct { tommy_tree_node* root; /**< Root node. */ - tommy_count_t count; /**< Number of elements. */ tommy_compare_func* cmp; /**< Comparison function. */ + tommy_size_t count; /**< Number of elements. */ } tommy_tree; /** @@ -213,7 +213,7 @@ void tommy_tree_foreach_arg(tommy_tree* tree, tommy_foreach_arg_func* func, void /** * Gets the number of elements. */ -tommy_inline tommy_count_t tommy_tree_count(tommy_tree* tree) +tommy_inline tommy_size_t tommy_tree_count(tommy_tree* tree) { return tree->count; } diff --git a/tommyds/tommytypes.h b/tommyds/tommytypes.h index ff834cf..0608b15 100644 --- a/tommyds/tommytypes.h +++ b/tommyds/tommytypes.h @@ -24,7 +24,6 @@ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ - /** \file * Generic types. */ @@ -37,17 +36,37 @@ #include -#if defined(_MSC_VER) +#ifdef _MSC_VER typedef unsigned tommy_uint32_t; /**< Generic uint32_t type. */ typedef unsigned _int64 tommy_uint64_t; /**< Generic uint64_t type. */ typedef size_t tommy_uintptr_t; /**< Generic uintptr_t type. */ +#ifdef _WIN64 +#define TOMMY_SIZE_BIT 64 +typedef unsigned _int64_t tommy_size_t; /**< Generic size_t type. */ +typedef _int64_t tommy_ssize_t; /**< Generic ssize_t type. */ +#else +#define TOMMY_SIZE_BIT 32 +typedef unsigned tommy_size_t; /**< Generic size_t type. */ +typedef int tommy_ssize_t; /**< Generic ssize_t type. */ +#endif #else #include typedef uint32_t tommy_uint32_t; /**< Generic uint32_t type. */ typedef uint64_t tommy_uint64_t; /**< Generic uint64_t type. */ typedef uintptr_t tommy_uintptr_t; /**< Generic uintptr_t type. */ +#if SIZE_MAX == UINT64_MAX +#define TOMMY_SIZE_BIT 64 +typedef uint64_t tommy_size_t; /**< Generic size_t type. */ +typedef int64_t tommy_ssize_t; /**< Generic ssize_t type. */ +#elif SIZE_MAX == UINT32_MAX +#define TOMMY_SIZE_BIT 32 +typedef uint32_t tommy_size_t; /**< Generic size_t type. */ +typedef int32_t tommy_ssize_t; /**< Generic ssize_t type. */ +#else +#error Unsupported SIZE_MAX #endif -typedef size_t tommy_size_t; /**< Generic size_t type. */ +#endif + typedef ptrdiff_t tommy_ptrdiff_t; /**< Generic ptrdiff_t type. */ typedef int tommy_bool_t; /**< Generic boolean type. */ @@ -59,13 +78,6 @@ typedef int tommy_bool_t; /**< Generic boolean type. */ */ typedef tommy_uint32_t tommy_uint_t; -/** - * Generic unsigned integer for counting objects. - * - * TommyDS doesn't support more than 2^32-1 objects. - */ -typedef tommy_uint32_t tommy_count_t; - /** \internal * Type cast required for the C++ compilation. * When compiling in C++ we cannot convert a void* pointer to another pointer. @@ -152,17 +164,17 @@ typedef tommy_uint32_t tommy_count_t; #endif /******************************************************************************/ -/* key */ +/* key/hash */ /** - * Key type used in indexed data structures to store the key or the hash value. + * Type used in indexed data structures to store the key of a object. */ -typedef tommy_uint32_t tommy_key_t; +typedef tommy_size_t tommy_key_t; /** - * Bits into the ::tommy_key_t type. + * Type used in hashtables to store the hash of a object. */ -#define TOMMY_KEY_BIT (sizeof(tommy_key_t) * 8) +typedef tommy_size_t tommy_hash_t; /******************************************************************************/ /* node */ @@ -200,11 +212,12 @@ typedef struct tommy_node_struct { void* data; /** - * Key used to store the node. + * Index of the node. + * With tries this field is used to store the key. * With hashtables this field is used to store the hash value. * With lists this field is not used. */ - tommy_key_t key; + tommy_size_t index; } tommy_node; /******************************************************************************/ @@ -302,6 +315,10 @@ typedef void tommy_foreach_arg_func(void* arg, void* obj); #include #pragma intrinsic(_BitScanReverse) #pragma intrinsic(_BitScanForward) +#if TOMMY_SIZE_BIT == 64 +#pragma intrinsic(_BitScanReverse64) +#pragma intrinsic(_BitScanForward64) +#endif #endif /** \internal @@ -363,6 +380,29 @@ tommy_inline tommy_uint_t tommy_ilog2_u32(tommy_uint32_t value) #endif } +#if TOMMY_SIZE_BIT == 64 +/** + * Bit scan reverse or integer log2 for 64 bits. + */ +tommy_inline tommy_uint_t tommy_ilog2_u64(tommy_uint64_t value) +{ +#if defined(_MSC_VER) + unsigned long count; + _BitScanReverse64(&count, value); + return count; +#elif defined(__GNUC__) + return __builtin_clzll(value) ^ 63; +#else + uint32_t l = value & 0xFFFFFFFFU; + uint32_t h = value >> 32; + if (h) + return tommy_ilog2_u32(h) + 32; + else + return tommy_ilog2_u32(l); +#endif +} +#endif + /** * Bit scan forward or trailing zero count. * Return the bit index of the least significant 1 bit. @@ -391,6 +431,29 @@ tommy_inline tommy_uint_t tommy_ctz_u32(tommy_uint32_t value) #endif } +#if TOMMY_SIZE_BIT == 64 +/** + * Bit scan forward or trailing zero count for 64 bits. + */ +tommy_inline tommy_uint_t tommy_ctz_u64(tommy_uint64_t value) +{ +#if defined(_MSC_VER) + unsigned long count; + _BitScanForward64(&count, value); + return count; +#elif defined(__GNUC__) + return __builtin_ctzll(value); +#else + uint32_t l = value & 0xFFFFFFFFU; + uint32_t h = value >> 32; + if (l) + return tommy_ctz_u32(l); + else + return tommy_ctz_u32(h) + 32; +#endif +} +#endif + /** * Rounds up to the next power of 2. * For the value 0, the result is undefined. @@ -412,6 +475,23 @@ tommy_inline tommy_uint32_t tommy_roundup_pow2_u32(tommy_uint32_t value) return value; } +/** + * Rounds up to the next power of 2 for 64 bits. + */ +tommy_inline tommy_uint64_t tommy_roundup_pow2_u64(tommy_uint64_t value) +{ + --value; + value |= value >> 1; + value |= value >> 2; + value |= value >> 4; + value |= value >> 8; + value |= value >> 16; + value |= value >> 32; + ++value; + + return value; +} + /** * Check if the specified word has a byte at 0. * \return 0 or 1. @@ -420,5 +500,19 @@ tommy_inline int tommy_haszero_u32(tommy_uint32_t value) { return ((value - 0x01010101) & ~value & 0x80808080) != 0; } + +/* + * Bit depth mapping. + */ +#if TOMMY_SIZE_BIT == 64 +#define tommy_ilog2 tommy_ilog2_u64 +#define tommy_ctz tommy_ctz_u64 +#define tommy_roundup_pow2 tommy_roundup_pow2_u64 +#else +#define tommy_ilog2 tommy_ilog2_u32 +#define tommy_ctz tommy_ctz_u32 +#define tommy_roundup_pow2 tommy_roundup_pow2_u32 +#endif + #endif