deb/snapraid
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Imported Upstream version 12.2

Browse Source
This commit is contained in:
Mario Fetka
2023-07-01 12:07:26 +02:00
parent 1f7c89fb07
commit 1aa8d4d499
21 changed files with 611 additions and 273 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
cmdline/elem.h
View File

@@ -369,6 +369,13 @@ struct snapraid_disk {
*/
thread_mutex_t fs_mutex;
int fs_mutex_enabled; /*< If the lock has to be used. */
/**
* Mutex for protecting the scan process.
*
* It's used during the scan process to protect the stampset to identity copy of files
*/
thread_mutex_t stamp_mutex;
#endif
/**

4
cmdline/io.c
View File

@@ -879,9 +879,9 @@ void io_init(struct snapraid_io* io, struct snapraid_state* state,
#if HAVE_THREAD
if (io_cache == 0) {
/* default is 8 MiB of cache */
/* default is 16 MiB of cache */
/* this seems to be a good tradeoff between speed and memory usage */
io->io_max = 8 * 1024 * 1024 / state->block_size;
io->io_max = 16 * 1024 * 1024 / state->block_size;
if (io->io_max < IO_MIN)
io->io_max = IO_MIN;
if (io->io_max > IO_MAX)

4
cmdline/io.h
View File

@@ -35,8 +35,8 @@
* 4 - 452 MB/s, CPU 54%, speed 118%
* 8 - 487 MB/s, CPU 60%, speed 128%
* 16 - 505 MB/s, CPU 63%, speed 132%
* 32 - 520 MB/s, CPU 64%, speed 136%
* 64 - 524 MB/s, CPU 65%, speed 137%
* 32 - 520 MB/s, CPU 64%, speed 136% [SnapRAID <= 12.0]
* 64 - 524 MB/s, CPU 65%, speed 137% [SnapRAID > 12.0]
* 128 - 525 MB/s, CPU 66%, speed 138%
*/
#define IO_MIN 3 /* required by writers, readers can work also with 2 */

34
cmdline/mingw.c
View File

@@ -31,14 +31,26 @@ int exit_sync_needed = 2;
/* Add missing Windows declaration */
/* For SetThreadExecutionState */
#ifndef WIN32_ES_SYSTEM_REQUIRED
#define WIN32_ES_SYSTEM_REQUIRED 0x00000001L
#endif
#ifndef WIN32_ES_DISPLAY_REQUIRED
#define WIN32_ES_DISPLAY_REQUIRED 0x00000002L
#endif
#ifndef WIN32_ES_USER_PRESENT
#define WIN32_ES_USER_PRESENT 0x00000004L
#endif
#ifndef WIN32_ES_AWAYMODE_REQUIRED
#define WIN32_ES_AWAYMODE_REQUIRED 0x00000040L
#endif
#ifndef WIN32_ES_CONTINUOUS
#define WIN32_ES_CONTINUOUS 0x80000000L
#endif
/* File Index */
#ifndef FILE_INVALID_FILE_ID
#define FILE_INVALID_FILE_ID ((ULONGLONG)-1LL)
#endif
/**
* Direct access to RtlGenRandom().
@@ -475,7 +487,7 @@ static void windows_attr2stat(DWORD FileAttributes, DWORD ReparseTag, struct win
*/
static int windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE_ATTRIBUTE_TAG_INFO* tag, struct windows_stat* st)
{
uint64_t mtime;
int64_t mtime;
windows_attr2stat(info->dwFileAttributes, tag->ReparseTag, st);
@@ -518,7 +530,7 @@ static int windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE_
* "that includes 128-bit file identifiers. If GetFileInformationByHandle returns"
* "FILE_INVALID_FILE_ID, the identifier may only be described in 128 bit form."
*/
if (st->st_ino == FILE_INVALID_FILE_ID) {
if (st->st_ino == (uint64_t)FILE_INVALID_FILE_ID) {
log_fatal("Invalid inode number! Is this ReFS?\n");
errno = EINVAL;
return -1;
@@ -532,7 +544,7 @@ static int windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE_
*/
static int windows_stream2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE_ID_BOTH_DIR_INFO* stream, struct windows_stat* st)
{
uint64_t mtime;
int64_t mtime;
/* The FILE_ID_BOTH_DIR_INFO doesn't have the ReparseTag information */
/* we could use instead FILE_ID_EXTD_DIR_INFO, but it's available only */
@@ -563,7 +575,7 @@ static int windows_stream2stat(const BY_HANDLE_FILE_INFORMATION* info, const FIL
st->st_sync = 0;
/* in ReFS the IDs are 128 bit, and the 64 bit interface may fail */
if (st->st_ino == FILE_INVALID_FILE_ID) {
if (st->st_ino == (uint64_t)FILE_INVALID_FILE_ID) {
log_fatal("Invalid inode number! Is this ReFS?\n");
errno = EINVAL;
return -1;
@@ -577,7 +589,7 @@ static int windows_stream2stat(const BY_HANDLE_FILE_INFORMATION* info, const FIL
*/
static void windows_finddata2stat(const WIN32_FIND_DATAW* info, struct windows_stat* st)
{
uint64_t mtime;
int64_t mtime;
windows_attr2stat(info->dwFileAttributes, info->dwReserved0, st);
@@ -789,7 +801,7 @@ static BOOL GetFilePhysicalOffset(HANDLE h, uint64_t* physical)
BOOL ret;
DWORD n;
/* in Wine FSCTL_GET_RETRIVIAL_POINTERS is not supported */
/* in Wine FSCTL_GET_RETRIEVAL_POINTERS is not supported */
if (is_wine) {
*physical = FILEPHY_UNREPORTED_OFFSET;
return TRUE;
@@ -1039,7 +1051,7 @@ int windows_futimens(int fd, struct windows_timespec tv[2])
{
HANDLE h;
FILETIME ft;
uint64_t mtime;
int64_t mtime;
if (fd == -1) {
errno = EBADF;
@@ -1061,7 +1073,7 @@ int windows_futimens(int fd, struct windows_timespec tv[2])
mtime = tv[0].tv_sec;
mtime *= 10000000;
mtime += tv[0].tv_nsec / 100;
mtime += 116444736000000000;
mtime += 116444736000000000LL;
ft.dwHighDateTime = mtime >> 32;
ft.dwLowDateTime = mtime;
@@ -1079,7 +1091,7 @@ int windows_utimensat(int fd, const char* file, struct windows_timespec tv[2], i
wchar_t conv_buf[CONV_MAX];
HANDLE h;
FILETIME ft;
uint64_t mtime;
int64_t mtime;
DWORD wflags;
/*
@@ -1119,7 +1131,7 @@ int windows_utimensat(int fd, const char* file, struct windows_timespec tv[2], i
mtime = tv[0].tv_sec;
mtime *= 10000000;
mtime += tv[0].tv_nsec / 100;
mtime += 116444736000000000;
mtime += 116444736000000000LL;
ft.dwHighDateTime = mtime >> 32;
ft.dwLowDateTime = mtime;
@@ -1556,7 +1568,7 @@ int windows_link(const char* existing, const char* file)
}
/**
* In Windows 10 allow creation of symblink by not privileged user.
* In Windows 10 allow creation of symlink by not privileged user.
*
* See: Symlinks in Windows 10!
* https://blogs.windows.com/buildingapps/2016/12/02/symlinks-windows-10/#cQG7cx48oGH86lkI.97

2
cmdline/portable.h
View File

@@ -362,7 +362,7 @@ int devuuid(uint64_t device, char* uuid, size_t size);
#define FILEPHY_REAL_OFFSET 3
/**
* Get the physcal address of the specified file.
* Get the physical address of the specified file.
* This is expected to be just a hint and not necessarily correct or unique.
* Return 0 on success.
*/

353
cmdline/scan.c
View File

@@ -25,6 +25,17 @@
struct snapraid_scan {
struct snapraid_state* state; /**< State used. */
struct snapraid_disk* disk; /**< Disk used. */
thread_id_t thread; /**< Thread used for scanning the disk */
int is_diff; /**< If it's a diff command or a scanning */
int need_write; /**< If a state write is required */
#if HAVE_THREAD
/**
* Mutex for protecting the disk stampset table
*/
thread_mutex_t mutex;
#endif
/**
* Counters of changes.
@@ -45,16 +56,69 @@ struct snapraid_scan {
tommy_node node;
};
static struct snapraid_scan* scan_alloc(struct snapraid_state* state, struct snapraid_disk* disk, int is_diff)
{
struct snapraid_scan* scan;
scan = malloc_nofail(sizeof(struct snapraid_scan));
scan->state = state;
scan->disk = disk;
scan->count_equal = 0;
scan->count_move = 0;
scan->count_copy = 0;
scan->count_restore = 0;
scan->count_change = 0;
scan->count_remove = 0;
scan->count_insert = 0;
tommy_list_init(&scan->file_insert_list);
tommy_list_init(&scan->link_insert_list);
tommy_list_init(&scan->dir_insert_list);
scan->is_diff = is_diff;
scan->need_write = 0;
#if HAVE_THREAD
thread_mutex_init(&disk->stamp_mutex);
#endif
return scan;
}
static void scan_free(struct snapraid_scan* scan)
{
#if HAVE_THREAD
thread_mutex_destroy(&scan->disk->stamp_mutex);
#endif
free(scan);
}
static void stamp_lock(struct snapraid_disk* disk)
{
#if HAVE_THREAD
thread_mutex_lock(&disk->stamp_mutex);
#else
(void)disk;
#endif
}
static void stamp_unlock(struct snapraid_disk* disk)
{
#if HAVE_THREAD
thread_mutex_unlock(&disk->stamp_mutex);
#else
(void)disk;
#endif
}
/**
* Remove the specified link from the data set.
*/
static void scan_link_remove(struct snapraid_scan* scan, struct snapraid_link* slink)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
/* state changed */
state->need_write = 1;
scan->need_write = 1;
/* remove the file from the link containers */
tommy_hashdyn_remove_existing(&disk->linkset, &slink->nodeset);
@@ -69,11 +133,10 @@ static void scan_link_remove(struct snapraid_scan* scan, struct snapraid_link* s
*/
static void scan_link_insert(struct snapraid_scan* scan, struct snapraid_link* slink)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
/* state changed */
state->need_write = 1;
scan->need_write = 1;
/* insert the link in the link containers */
tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
@@ -116,13 +179,13 @@ static void scan_link(struct snapraid_scan* scan, int is_diff, const char* sub,
/* it's an update */
/* we have to save the linkto/type */
state->need_write = 1;
scan->need_write = 1;
++scan->count_change;
log_tag("scan:update:%s:%s\n", disk->name, esc_tag(slink->sub, esc_buffer));
if (is_diff) {
printf("update %s\n", fmt_term(disk, slink->sub, esc_buffer));
msg_info("update %s\n", fmt_term(disk, slink->sub, esc_buffer));
}
/* update it */
@@ -139,7 +202,7 @@ static void scan_link(struct snapraid_scan* scan, int is_diff, const char* sub,
log_tag("scan:add:%s:%s\n", disk->name, esc_tag(sub, esc_buffer));
if (is_diff) {
printf("add %s\n", fmt_term(disk, sub, esc_buffer));
msg_info("add %s\n", fmt_term(disk, sub, esc_buffer));
}
/* and continue to insert it */
@@ -166,7 +229,7 @@ static void scan_file_allocate(struct snapraid_scan* scan, struct snapraid_file*
block_off_t parity_pos;
/* state changed */
state->need_write = 1;
scan->need_write = 1;
/* allocate the blocks of the file */
parity_pos = disk->first_free_block;
@@ -267,7 +330,7 @@ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_fil
tommy_list_remove_existing(&disk->filelist, &file->nodelist);
/* state changed */
state->need_write = 1;
scan->need_write = 1;
/* here we are supposed to adjust the ::first_free_block position */
/* with the parity position we are deleting */
@@ -563,8 +626,11 @@ static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* f
/* insert the file in the containers */
if (!file_flag_has(file, FILE_IS_WITHOUT_INODE))
tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
stamp_lock(disk);
tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
stamp_unlock(disk);
/* delayed allocation of the parity */
scan_file_delayed_allocate(scan, file);
@@ -583,7 +649,10 @@ static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* f
if (!file_flag_has(file, FILE_IS_WITHOUT_INODE))
tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
stamp_lock(disk);
tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset);
stamp_unlock(disk);
/* deallocate the file from the parity */
scan_file_deallocate(scan, file);
@@ -706,7 +775,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
file->mtime_nsec = STAT_NSEC(st);
/* we have to save the new mtime */
state->need_write = 1;
scan->need_write = 1;
}
if (strcmp(file->sub, sub) != 0) {
@@ -715,7 +784,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
log_tag("scan:move:%s:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer), esc_tag(sub, esc_buffer_alt));
if (is_diff) {
printf("move %s -> %s\n", fmt_term(disk, file->sub, esc_buffer), fmt_term(disk, sub, esc_buffer_alt));
msg_info("move %s -> %s\n", fmt_term(disk, file->sub, esc_buffer), fmt_term(disk, sub, esc_buffer_alt));
}
/* remove from the name set */
@@ -728,7 +797,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
/* we have to save the new name */
state->need_write = 1;
scan->need_write = 1;
} else {
/* otherwise it's equal */
++scan->count_equal;
@@ -864,7 +933,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
file->mtime_nsec = STAT_NSEC(st);
/* we have to save the new mtime */
state->need_write = 1;
scan->need_write = 1;
}
/* if when processing the disk we used the past inodes values */
@@ -878,7 +947,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
log_tag("scan:restore:%s:%s\n", disk->name, esc_tag(sub, esc_buffer));
if (is_diff) {
printf("restore %s\n", fmt_term(disk, sub, esc_buffer));
msg_info("restore %s\n", fmt_term(disk, sub, esc_buffer));
}
/* remove from the inode set */
@@ -891,7 +960,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
/* we have to save the new inode */
state->need_write = 1;
scan->need_write = 1;
} else {
/* otherwise it's the case of not persistent inode, where doesn't */
/* matter if the inode is different or equal, because they have no */
@@ -976,12 +1045,14 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
struct snapraid_disk* other_disk = i->data;
struct snapraid_file* other_file;
stamp_lock(other_disk);
/* if the nanosecond part of the time stamp is valid, search */
/* for name and stamp, otherwise for path and stamp */
if (file->mtime_nsec != 0 && file->mtime_nsec != STAT_NSEC_INVALID)
other_file = tommy_hashdyn_search(&other_disk->stampset, file_namestamp_compare, file, hash);
else
other_file = tommy_hashdyn_search(&other_disk->stampset, file_pathstamp_compare, file, hash);
stamp_unlock(other_disk);
/* if found, and it's a fully hashed file */
if (other_file && file_is_full_hashed_and_stable(scan->state, other_disk, other_file)) {
@@ -993,7 +1064,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
log_tag("scan:copy:%s:%s:%s:%s\n", other_disk->name, esc_tag(other_file->sub, esc_buffer), disk->name, esc_tag(file->sub, esc_buffer_alt));
if (is_diff) {
printf("copy %s -> %s\n", fmt_term(other_disk, other_file->sub, esc_buffer), fmt_term(disk, file->sub, esc_buffer_alt));
msg_info("copy %s -> %s\n", fmt_term(other_disk, other_file->sub, esc_buffer), fmt_term(disk, file->sub, esc_buffer_alt));
}
/* mark it as reported */
@@ -1017,14 +1088,14 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
);
if (is_diff) {
printf("update %s\n", fmt_term(disk, sub, esc_buffer));
msg_info("update %s\n", fmt_term(disk, sub, esc_buffer));
}
} else {
++scan->count_insert;
log_tag("scan:add:%s:%s\n", disk->name, esc_tag(sub, esc_buffer));
if (is_diff) {
printf("add %s\n", fmt_term(disk, sub, esc_buffer));
msg_info("add %s\n", fmt_term(disk, sub, esc_buffer));
}
}
}
@@ -1038,11 +1109,10 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
*/
static void scan_emptydir_remove(struct snapraid_scan* scan, struct snapraid_dir* dir)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
/* state changed */
state->need_write = 1;
scan->need_write = 1;
/* remove the file from the dir containers */
tommy_hashdyn_remove_existing(&disk->dirset, &dir->nodeset);
@@ -1057,11 +1127,10 @@ static void scan_emptydir_remove(struct snapraid_scan* scan, struct snapraid_dir
*/
static void scan_emptydir_insert(struct snapraid_scan* scan, struct snapraid_dir* dir)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
/* state changed */
state->need_write = 1;
scan->need_write = 1;
/* insert the dir in the dir containers */
tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash(dir->sub));
@@ -1172,7 +1241,7 @@ struct stat* dstat(const char* file, struct stat* st)
* Process a directory.
* Return != 0 if at least one file or link is processed.
*/
static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const char* dir, const char* sub)
static int scan_sub(struct snapraid_scan* scan, int level, int is_diff, char* path_next, char* sub_next, char* tmp)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
@@ -1180,25 +1249,28 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
DIR* d;
tommy_list list;
tommy_node* node;
size_t path_len;
size_t sub_len;
path_len = strlen(path_next);
sub_len = strlen(sub_next);
tommy_list_init(&list);
d = opendir(dir);
d = opendir(path_next);
if (!d) {
/* LCOV_EXCL_START */
log_fatal("Error opening directory '%s'. %s.\n", dir, strerror(errno));
log_fatal("Error opening directory '%s'. %s.\n", path_next, strerror(errno));
if (level == 0)
log_fatal("If this is the disk mount point, remember to create it manually\n");
else
log_fatal("If it's a permission problem, you can exclude it in the config file with:\n\texclude /%s\n", sub);
log_fatal("If it's a permission problem, you can exclude it in the config file with:\n\texclude /%s\n", sub_next);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* read the full directory */
while (1) {
char path_next[PATH_MAX];
char sub_next[PATH_MAX];
struct dirent_sorted* entry;
const char* name;
struct dirent* dd;
@@ -1215,8 +1287,11 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
dd = readdir(d);
if (dd == 0 && errno != 0) {
/* LCOV_EXCL_START */
log_fatal("Error reading directory '%s'. %s.\n", dir, strerror(errno));
log_fatal("You can exclude it in the config file with:\n\texclude /%s\n", sub);
/* restore removing additions */
path_next[path_len] = 0;
sub_next[sub_len] = 0;
log_fatal("Error reading directory '%s'. %s.\n", path_next, strerror(errno));
log_fatal("You can exclude it in the config file with:\n\texclude /%s\n", sub_next);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
@@ -1229,8 +1304,7 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
if (name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0)))
continue;
pathprint(path_next, sizeof(path_next), "%s%s", dir, name);
pathprint(sub_next, sizeof(sub_next), "%s%s", sub, name);
pathcatl(path_next, path_len, PATH_MAX, name);
/* check for not supported file names */
if (name[0] == 0) {
@@ -1276,7 +1350,9 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
if (closedir(d) != 0) {
/* LCOV_EXCL_START */
log_fatal("Error closing directory '%s'. %s.\n", dir, strerror(errno));
/* restore removing additions */
path_next[path_len] = 0;
log_fatal("Error closing directory '%s'. %s.\n", path_next, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
@@ -1299,9 +1375,6 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
/* process the sorted dir entries */
node = list;
while (node != 0) {
char path_next[PATH_MAX];
char sub_next[PATH_MAX];
char out[PATH_MAX];
struct snapraid_filter* reason = 0;
struct dirent_sorted* dd = node->data;
const char* name = dd->d_name;
@@ -1311,8 +1384,8 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
struct stat st_buf;
#endif
pathprint(path_next, sizeof(path_next), "%s%s", dir, name);
pathprint(sub_next, sizeof(sub_next), "%s%s", sub, name);
pathcatl(path_next, path_len, PATH_MAX, name);
pathcatl(sub_next, sub_len, PATH_MAX, name);
/* start with an unknown type */
type = -1;
@@ -1383,14 +1456,13 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
scan_file(scan, is_diff, sub_next, st, FILEPHY_UNREAD_OFFSET);
processed = 1;
} else {
msg_verbose("Excluding file '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
msg_verbose("Excluding file '%s' for rule '%s'\n", path_next, filter_type(reason, tmp, PATH_MAX));
}
} else if (type == 1) { /* LNK */
if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0) {
char subnew[PATH_MAX];
int ret;
ret = readlink(path_next, subnew, sizeof(subnew));
ret = readlink(path_next, tmp, PATH_MAX);
if (ret >= PATH_MAX) {
/* LCOV_EXCL_START */
log_fatal("Error in readlink file '%s'. Symlink too long.\n", path_next);
@@ -1407,13 +1479,13 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
log_fatal("WARNING! Empty symbolic link '%s'.\n", path_next);
/* readlink doesn't put the final 0 */
subnew[ret] = 0;
tmp[ret] = 0;
/* process as a symbolic link */
scan_link(scan, is_diff, sub_next, subnew, FILE_IS_SYMLINK);
scan_link(scan, is_diff, sub_next, tmp, FILE_IS_SYMLINK);
processed = 1;
} else {
msg_verbose("Excluding link '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
msg_verbose("Excluding link '%s' for rule '%s'\n", path_next, filter_type(reason, tmp, PATH_MAX));
}
} else if (type == 2) { /* DIR */
if (filter_subdir(&state->filterlist, &reason, disk->name, sub_next) == 0) {
@@ -1429,13 +1501,12 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
} else
#endif
{
char sub_dir[PATH_MAX];
/* recurse */
pathslash(path_next, sizeof(path_next));
pathcpy(sub_dir, sizeof(sub_dir), sub_next);
pathslash(sub_dir, sizeof(sub_dir));
if (scan_dir(scan, level + 1, is_diff, path_next, sub_dir) == 0) {
pathslash(path_next, PATH_MAX);
pathslash(sub_next, PATH_MAX);
if (scan_sub(scan, level + 1, is_diff, path_next, sub_next, tmp) == 0) {
/* restore removing additions */
pathcatl(sub_next, sub_len, PATH_MAX, name);
/* scan the directory as empty dir */
scan_emptydir(scan, sub_next);
}
@@ -1443,7 +1514,7 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
processed = 1;
}
} else {
msg_verbose("Excluding directory '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
msg_verbose("Excluding directory '%s' for rule '%s'\n", path_next, filter_type(reason, tmp, PATH_MAX));
}
} else {
if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0) {
@@ -1453,7 +1524,7 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
log_fatal("WARNING! Ignoring special '%s' file '%s'\n", stat_desc(st), path_next);
} else {
msg_verbose("Excluding special file '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
msg_verbose("Excluding special file '%s' for rule '%s'\n", path_next, filter_type(reason, tmp, PATH_MAX));
}
}
@@ -1467,6 +1538,80 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
return processed;
}
/**
* Process a directory.
* Return != 0 if at least one file or link is processed.
*/
static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const char* dir, const char* sub)
{
/* working buffers used by scan_sub() */
char path_next[PATH_MAX];
char sub_next[PATH_MAX];
char tmp[PATH_MAX];
pathcpy(path_next, sizeof(path_next), dir);
pathcpy(sub_next, sizeof(sub_next), sub);
return scan_sub(scan, level, is_diff, path_next, sub_next, tmp);
}
static void* scan_disk(void* arg)
{
struct snapraid_scan* scan = arg;
struct snapraid_disk* disk = scan->disk;
int ret;
int has_persistent_inodes;
int has_syncronized_hardlinks;
uint64_t start;
/* check if the disk supports persistent inodes */
ret = fsinfo(disk->dir, &has_persistent_inodes, &has_syncronized_hardlinks, 0, 0);
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing disk '%s' to get file-system info. %s.\n", disk->dir, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (!has_persistent_inodes) {
disk->has_volatile_inodes = 1;
}
if (!has_syncronized_hardlinks) {
disk->has_volatile_hardlinks = 1;
}
/* if inodes or UUID are not persistent/changed/unsupported */
if (disk->has_volatile_inodes || disk->has_different_uuid || disk->has_unsupported_uuid) {
/* remove all the inodes from the inode collection */
/* if they are not persistent, all of them could be changed now */
/* and we don't want to find false matching ones */
/* see scan_file() for more details */
tommy_node* node = disk->filelist;
while (node) {
struct snapraid_file* file = node->data;
node = node->next;
/* remove from the inode set */
tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
/* clear the inode */
file->inode = 0;
/* mark as missing inode */
file_flag_set(file, FILE_IS_WITHOUT_INODE);
}
}
start = tick_ms();
scan_dir(scan, 0, scan->is_diff, disk->dir, "");
if (!scan->is_diff)
msg_progress("Scanned %s in %" PRIu64 " seconds\n", disk->name, (tick_ms() - start) / 1000);
return 0;
}
static int state_diffscan(struct snapraid_state* state, int is_diff)
{
tommy_node* i;
@@ -1482,76 +1627,44 @@ static int state_diffscan(struct snapraid_state* state, int is_diff)
if (is_diff)
msg_progress("Comparing...\n");
else
msg_progress("Scanning...\n");
/* first scan all the directory and find new and deleted files */
/* allocate all the scan data */
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
struct snapraid_scan* scan;
tommy_node* node;
int ret;
int has_persistent_inodes;
int has_syncronized_hardlinks;
scan = malloc_nofail(sizeof(struct snapraid_scan));
scan->state = state;
scan->disk = disk;
scan->count_equal = 0;
scan->count_move = 0;
scan->count_copy = 0;
scan->count_restore = 0;
scan->count_change = 0;
scan->count_remove = 0;
scan->count_insert = 0;
tommy_list_init(&scan->file_insert_list);
tommy_list_init(&scan->link_insert_list);
tommy_list_init(&scan->dir_insert_list);
scan = scan_alloc(state, disk, is_diff);
tommy_list_insert_tail(&scanlist, &scan->node, scan);
if (!is_diff)
msg_progress("Scanning disk %s...\n", disk->name);
/* check if the disk supports persistent inodes */
ret = fsinfo(disk->dir, &has_persistent_inodes, &has_syncronized_hardlinks, 0, 0);
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing disk '%s' to get file-system info. %s.\n", disk->dir, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (!has_persistent_inodes) {
disk->has_volatile_inodes = 1;
}
if (!has_syncronized_hardlinks) {
disk->has_volatile_hardlinks = 1;
}
/* if inodes or UUID are not persistent/changed/unsupported */
if (disk->has_volatile_inodes || disk->has_different_uuid || disk->has_unsupported_uuid) {
/* remove all the inodes from the inode collection */
/* if they are not persistent, all of them could be changed now */
/* and we don't want to find false matching ones */
/* see scan_file() for more details */
node = disk->filelist;
while (node) {
struct snapraid_file* file = node->data;
node = node->next;
/* remove from the inode set */
tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
/* clear the inode */
file->inode = 0;
/* mark as missing inode */
file_flag_set(file, FILE_IS_WITHOUT_INODE);
}
}
scan_dir(scan, 0, is_diff, disk->dir, "");
}
/* first scan all the directory and find new and deleted files */
for (i = scanlist; i != 0; i = i->next) {
struct snapraid_scan* scan = i->data;
#if HAVE_THREAD
if (state->opt.skip_multi_scan)
scan_disk(scan);
else
thread_create(&scan->thread, scan_disk, scan);
#else
scan_disk(scan);
#endif
}
#if HAVE_THREAD
/* wait for all threads to terminate */
for (i = scanlist; i != 0; i = i->next) {
struct snapraid_scan* scan = i->data;
void* retval;
/* wait for thread termination */
if (!state->opt.skip_multi_scan)
thread_join(scan->thread, &retval);
}
#endif
/* we split the search in two phases because to detect files */
/* moved from one disk to another we have to start deletion */
/* only when all disks have all the new files found */
@@ -1580,7 +1693,7 @@ static int state_diffscan(struct snapraid_state* state, int is_diff)
log_tag("scan:remove:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
if (is_diff) {
printf("remove %s\n", fmt_term(disk, file->sub, esc_buffer));
msg_info("remove %s\n", fmt_term(disk, file->sub, esc_buffer));
}
scan_file_remove(scan, file);
@@ -1601,7 +1714,7 @@ static int state_diffscan(struct snapraid_state* state, int is_diff)
log_tag("scan:remove:%s:%s\n", disk->name, esc_tag(slink->sub, esc_buffer));
if (is_diff) {
printf("remove %s\n", fmt_term(disk, slink->sub, esc_buffer));
msg_info("remove %s\n", fmt_term(disk, slink->sub, esc_buffer));
}
scan_link_remove(scan, slink);
@@ -1707,6 +1820,14 @@ static int state_diffscan(struct snapraid_state* state, int is_diff)
}
}
/* propagate the state change (after all the scan operations are called) */
for (i = scanlist; i != 0; i = i->next) {
struct snapraid_scan* scan = i->data;
if (scan->need_write) {
state->need_write = 1;
}
}
/* check for disks where all the previously existing files where removed */
if (!state->opt.force_empty) {
int all_missing = 0;
@@ -1896,7 +2017,7 @@ static int state_diffscan(struct snapraid_state* state, int is_diff)
}
log_flush();
tommy_list_foreach(&scanlist, (tommy_foreach_func*)free);
tommy_list_foreach(&scanlist, (tommy_foreach_func*)scan_free);
/* check the file-system on all disks */
state_fscheck(state, "after scan");

7
cmdline/snapraid.c
View File

@@ -307,6 +307,7 @@ void config(char* conf, size_t conf_size, const char* argv0)
#define OPT_TEST_SKIP_CONTENT_WRITE 302
#define OPT_TEST_SKIP_SPACE_HOLDER 303
#define OPT_TEST_FORMAT 304
#define OPT_TEST_SKIP_MULTI_SCAN 305
#if HAVE_GETOPT_LONG
struct option long_options[] = {
@@ -475,6 +476,9 @@ struct option long_options[] = {
/* Set the output format */
{ "test-fmt", 1, 0, OPT_TEST_FORMAT },
/* Skip thread in disk scan */
{ "test-skip-multi-scan", 0, 0, OPT_TEST_SKIP_MULTI_SCAN },
{ 0, 0, 0, 0 }
};
#endif
@@ -929,6 +933,9 @@ int main(int argc, char* argv[])
/* LCOV_EXCL_STOP */
}
break;
case OPT_TEST_SKIP_MULTI_SCAN :
opt.skip_multi_scan = 1;
break;
default :
/* LCOV_EXCL_START */
log_fatal("Unknown option '%c'\n", (char)c);

8
cmdline/state.c
View File

@@ -37,10 +37,6 @@
* writing to disk, but you'll need to access multiple times the same data,
* being potentially slower.
*
* For upcoming SnapRAID version it's planned to add a mutex protection
* at the file-system structure, slowing down multiple data access,
* so we disable it.
*
* Multi thread for verify is instead always generally faster,
* so we enable it if possible.
*/
@@ -3904,6 +3900,8 @@ static void state_verify_content(struct snapraid_state* state, uint32_t crc)
tommy_node* i;
int fail;
msg_progress("Verifying...\n");
/* start all reading threads */
i = tommy_list_head(&state->contentlist);
while (i) {
@@ -3912,8 +3910,6 @@ static void state_verify_content(struct snapraid_state* state, uint32_t crc)
char tmp[PATH_MAX];
STREAM* f;
msg_progress("Verifying %s...\n", content->content);
pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
f = sopen_read(tmp);
if (f == 0) {

1
cmdline/state.h
View File

@@ -119,6 +119,7 @@ struct snapraid_option {
int auto_conf; /**< Allow to run without configuration file. */
int force_stats; /**< Force stats print during process. */
uint64_t parity_limit_size; /**< Test limit for parity files. */
int skip_multi_scan; /**< Don't use threads in scan. */
};
struct snapraid_state {

14
cmdline/support.c
View File

@@ -663,6 +663,20 @@ void pathcat(char* dst, size_t size, const char* src)
memcpy(dst + dst_len, src, src_len + 1);
}
void pathcatl(char* dst, size_t dst_len, size_t size, const char* src)
{
size_t src_len = strlen(src);
if (dst_len + src_len + 1 > size) {
/* LCOV_EXCL_START */
log_fatal("Path too long '%s%s'\n", dst, src);
os_abort();
/* LCOV_EXCL_STOP */
}
memcpy(dst + dst_len, src, src_len + 1);
}
void pathcatc(char* dst, size_t size, char c)
{
size_t dst_len = strlen(dst);

6
cmdline/support.h
View File

@@ -265,6 +265,12 @@ void pathcpy(char* dst, size_t size, const char* src);
*/
void pathcat(char* dst, size_t size, const char* src);
/**
* Concatenate a path limiting the size knowing the length.
* Abort if too long.
*/
void pathcatl(char* dst, size_t dst_len, size_t size, const char* src);
/**
* Concatenate a path limiting the size.
* Abort if too long.

2
cmdline/unix.c
View File

@@ -474,7 +474,7 @@ static int devuuid_dev(uint64_t device, char* uuid, size_t uuid_size)
#endif
/**
* Get the UUID using liblkid.
* Get the UUID using libblkid.
* It uses a cache to work without root permission, resulting in UUID
* not necessarily recent.
* We could call blkid_probe_all() to refresh the UUID, but it would