Skip to content

Implemented force-realloc-tail option for sync #31

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Ralf1108 wants to merge 6 commits into from
Closed

Implemented force-realloc-tail option for sync #31

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
47768f6
implemented force-realloc-tail
Ralf1108 Feb 3, 2026
9a00971
fixed comment
Ralf1108 Feb 6, 2026
0477426
check if block is in normal state BLOCK_STATE_BLK
Ralf1108 Feb 6, 2026
c1f8e25
fixed comment
Ralf1108 Feb 6, 2026
fe6e85c
added check to prevent using force_realloc_tail with force_full
Ralf1108 Feb 6, 2026
2f7a040
dont allow force-realloc-tail with --force-nocopy
Ralf1108 Feb 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 52 additions & 10 deletions cmdline/locate.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,13 @@ struct snapraid_parity_entry {
tommy_node node;
};

struct snapraid_locate_info{
data_off_t block_max;
data_off_t parity_size;
block_off_t tail_block;
block_off_t min_occupied_block_number;
};

static int parity_entry_compare(const void* void_a, const void* void_b)
{
const struct snapraid_parity_entry* entry_a = void_a;
Expand Down Expand Up @@ -112,6 +119,16 @@ static void collect_parity_block_file(uint32_t block_size, struct snapraid_disk*
tommy_list_insert_tail(file_list, &entry->node, entry);
}

void state_locate_info(struct snapraid_state* state, uint64_t parity_tail, struct snapraid_locate_info* info)
{
uint64_t block_size = state->block_size;

info->block_max = parity_allocated_size(state);
info->parity_size = info->block_max * block_size;
info->tail_block = (parity_tail + block_size - 1) / block_size;
info->min_occupied_block_number = info->block_max - info->tail_block;
}

void state_locate(struct snapraid_state* state, uint64_t parity_tail)
{
char buf[64];
Expand All @@ -127,20 +144,16 @@ void state_locate(struct snapraid_state* state, uint64_t parity_tail)
min_occupied_block_number = 0;
} else {
printf("Locate files within the tail of %sB of the parity\n\n", fmt_size(parity_tail, buf, sizeof(buf)));

data_off_t block_max = parity_allocated_size(state);
data_off_t parity_size = block_max * block_size;

printf("Current parity size is %sB\n", fmt_size(parity_size, buf, sizeof(buf)));

block_off_t tail_block = (parity_tail + block_size - 1) / block_size;

if (tail_block >= block_max) {
struct snapraid_locate_info info;
state_locate_info(state, parity_tail, &info);

printf("Current parity size is %sB\n", fmt_size(info.parity_size, buf, sizeof(buf)));
if (info.tail_block >= info.block_max) {
printf("Specified tail greater than the parity size!\n");
return;
}

min_occupied_block_number = block_max - tail_block;
min_occupied_block_number = info.min_occupied_block_number;
}

msg_progress("Collecting files with offset greater or equal to %" PRIu64 "\n", min_occupied_block_number * block_size);
Expand Down Expand Up @@ -179,3 +192,32 @@ void state_locate(struct snapraid_state* state, uint64_t parity_tail)
tommy_list_foreach(&files, free);
}

void state_locate_mark_tail_blocks_for_resync(struct snapraid_state* state, uint64_t parity_tail)
{
struct snapraid_locate_info info;
state_locate_info(state, parity_tail, &info);
block_off_t min_occupied_block_number = info.min_occupied_block_number;
printf("Forcing reallocation of all tail blocks from block number %u onwards\n", min_occupied_block_number);

for (tommy_node* i = tommy_list_head(&state->disklist); i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
for (tommy_node* j = tommy_list_head(&disk->filelist); j != 0; j = j->next) {
struct snapraid_file* file = j->data;
for (block_off_t f = 0; f < file->blockmax; ++f) {
block_off_t parity_pos = fs_file2par_find(disk, file, f);
if (parity_pos == POS_NULL)
continue; /* not allocated */
if (parity_pos < min_occupied_block_number)
continue; /* not relevant */

/* mark the file for reallocation */
struct snapraid_block* block = fs_file2block_get(file, f);

// TODO: check: is condition correct or should we always set BLOCK_STATE_REP?
if (block_state_get(block) == BLOCK_STATE_BLK) {
block_state_set(block, BLOCK_STATE_REP);
}
}
}
}
}
3 changes: 2 additions & 1 deletion cmdline/locate.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,6 @@

void state_locate(struct snapraid_state* state, uint64_t parity_tail);

#endif
void state_locate_mark_tail_blocks_for_resync(struct snapraid_state* state, uint64_t parity_tail);

#endif
89 changes: 67 additions & 22 deletions cmdline/snapraid.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,26 +58,27 @@ void usage(const char* conf)
printf(" fix Fix the array\n");
printf("\n");
printf("Options:\n");
printf(" " SWITCH_GETOPT_LONG("-c, --conf FILE ", "-c") " Configuration file\n");
printf(" " SWITCH_GETOPT_LONG("-f, --filter PATTERN ", "-f") " Process only files matching the pattern\n");
printf(" " SWITCH_GETOPT_LONG("-d, --filter-disk NAME", "-d") " Process only files in the specified disk\n");
printf(" " SWITCH_GETOPT_LONG("-m, --filter-missing ", "-m") " Process only missing/deleted files\n");
printf(" " SWITCH_GETOPT_LONG("-e, --filter-error ", "-e") " Process only files with errors\n");
printf(" " SWITCH_GETOPT_LONG("-p, --plan PLAN ", "-p") " Define a scrub plan or percentage\n");
printf(" " SWITCH_GETOPT_LONG("-o, --older-than DAYS ", "-o") " Process only the older part of the array\n");
printf(" " SWITCH_GETOPT_LONG("-i, --import DIR ", "-i") " Import deleted files\n");
printf(" " SWITCH_GETOPT_LONG("-l, --log FILE ", "-l") " Log file. Default none\n");
printf(" " SWITCH_GETOPT_LONG("-a, --audit-only ", "-a") " Check only file data and not parity\n");
printf(" " SWITCH_GETOPT_LONG("-h, --pre-hash ", "-h") " Pre-hash all the new data\n");
printf(" " SWITCH_GETOPT_LONG("-Z, --force-zero ", "-Z") " Force syncing of files that get zero size\n");
printf(" " SWITCH_GETOPT_LONG("-E, --force-empty ", "-E") " Force syncing of disks that get empty\n");
printf(" " SWITCH_GETOPT_LONG("-U, --force-uuid ", "-U") " Force commands on disks with uuid changed\n");
printf(" " SWITCH_GETOPT_LONG("-D, --force-device ", "-D") " Force commands with inaccessible/shared disks\n");
printf(" " SWITCH_GETOPT_LONG("-N, --force-nocopy ", "-N") " Force commands disabling the copy detection\n");
printf(" " SWITCH_GETOPT_LONG("-F, --force-full ", "-F") " Force a full parity computation in sync\n");
printf(" " SWITCH_GETOPT_LONG("-R, --force-realloc ", "-R") " Force a full parity reallocation in sync\n");
printf(" " SWITCH_GETOPT_LONG("-w, --bw-limit RATE ", "-w") " Limit IO bandwidth (M|G)\n");
printf(" " SWITCH_GETOPT_LONG("-v, --verbose ", "-v") " Verbose\n");
printf(" " SWITCH_GETOPT_LONG("-c, --conf FILE ", "-c") " Configuration file\n");
printf(" " SWITCH_GETOPT_LONG("-f, --filter PATTERN ", "-f") " Process only files matching the pattern\n");
printf(" " SWITCH_GETOPT_LONG("-d, --filter-disk NAME ", "-d") " Process only files in the specified disk\n");
printf(" " SWITCH_GETOPT_LONG("-m, --filter-missing ", "-m") " Process only missing/deleted files\n");
printf(" " SWITCH_GETOPT_LONG("-e, --filter-error ", "-e") " Process only files with errors\n");
printf(" " SWITCH_GETOPT_LONG("-p, --plan PLAN ", "-p") " Define a scrub plan or percentage\n");
printf(" " SWITCH_GETOPT_LONG("-o, --older-than DAYS ", "-o") " Process only the older part of the array\n");
printf(" " SWITCH_GETOPT_LONG("-i, --import DIR ", "-i") " Import deleted files\n");
printf(" " SWITCH_GETOPT_LONG("-l, --log FILE ", "-l") " Log file. Default none\n");
printf(" " SWITCH_GETOPT_LONG("-a, --audit-only ", "-a") " Check only file data and not parity\n");
printf(" " SWITCH_GETOPT_LONG("-h, --pre-hash ", "-h") " Pre-hash all the new data\n");
printf(" " SWITCH_GETOPT_LONG("-Z, --force-zero ", "-Z") " Force syncing of files that get zero size\n");
printf(" " SWITCH_GETOPT_LONG("-E, --force-empty ", "-E") " Force syncing of disks that get empty\n");
printf(" " SWITCH_GETOPT_LONG("-U, --force-uuid ", "-U") " Force commands on disks with uuid changed\n");
printf(" " SWITCH_GETOPT_LONG("-D, --force-device ", "-D") " Force commands with inaccessible/shared disks\n");
printf(" " SWITCH_GETOPT_LONG("-N, --force-nocopy ", "-N") " Force commands disabling the copy detection\n");
printf(" " SWITCH_GETOPT_LONG("-F, --force-full ", "-F") " Force a full parity computation in sync\n");
printf(" " SWITCH_GETOPT_LONG("-R, --force-realloc ", "-R") " Force a full parity reallocation in sync\n");
printf(" " SWITCH_GETOPT_LONG("-X, --force-realloc-tail ", "-X") " Force a tail parity reallocation in sync\n");
printf(" " SWITCH_GETOPT_LONG("-w, --bw-limit RATE ", "-w") " Limit IO bandwidth (M|G)\n");
printf(" " SWITCH_GETOPT_LONG("-v, --verbose ", "-v") " Verbose\n");
printf("\n");
printf("Configuration file: %s\n", conf);
printf("\n");
Expand Down Expand Up @@ -712,6 +713,7 @@ static struct option long_options[] = {
{ "force-nocopy", 0, 0, 'N' },
{ "force-full", 0, 0, 'F' },
{ "force-realloc", 0, 0, 'R' },
{ "force-realloc-tail", 0, 0, 'X' },
{ "bw-limit", 1, 0, 'w' },
{ "audit-only", 0, 0, 'a' },
{ "pre-hash", 0, 0, 'h' },
Expand Down Expand Up @@ -870,12 +872,12 @@ static struct option long_options[] = {
#endif

/*
* Free letters: gIjJkKMnPQruxXWz
* Free letters: gIjJkKMnPQruxWz
*
* The 's' letter is used in main.c
* The 'G' letter is free but only from 14.0
*/
#define OPTIONS "t:c:f:d:mebp:o:S:B:L:i:l:AZEUDNFRahTC:vqHVw:"
#define OPTIONS "X:t:c:f:d:mebp:o:S:B:L:i:l:AZEUDNFRahTC:vqHVw:"

int parse_option_size(const char* arg, uint64_t* out_size)
{
Expand Down Expand Up @@ -1197,6 +1199,9 @@ int snapraid_main(int argc, char* argv[])
case 'R' :
opt.force_realloc = 1;
break;
case 'X' :
opt.force_realloc_tail = 1;
break;
case 'a' :
opt.auditonly = 1;
break;
Expand Down Expand Up @@ -1548,6 +1553,13 @@ int snapraid_main(int argc, char* argv[])
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}

if (opt.force_realloc_tail) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "You cannot use -X, --force_realloc_tail with the '%s' command\n", command);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}

if (opt.force_full && opt.force_nocopy) {
Expand All @@ -1571,6 +1583,34 @@ int snapraid_main(int argc, char* argv[])
/* LCOV_EXCL_STOP */
}

if (opt.force_realloc && opt.force_realloc_tail) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "You cannot use the -R, --force-realloc and -X, --force-realloc-tail options simultaneously\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}

if (opt.force_realloc_tail && opt.force_nocopy) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "You cannot use the -X, --force-realloc-tail and -N, --force-nocopy options simultaneously\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}

if (opt.force_realloc_tail && opt.force_full) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "You cannot use the -X, --force-realloc-tail and -F, --force-full options simultaneously\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}

if (opt.force_realloc_tail && parity_tail == 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "You cannot use the -X, --force-realloc-tail without option 't' == 0 \n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}

if (opt.prehash && opt.force_nocopy) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "You cannot use the -h, --pre-hash and -N, --force-nocopy options simultaneously\n");
Expand Down Expand Up @@ -1813,6 +1853,11 @@ int snapraid_main(int argc, char* argv[])

state_read(&state);

// TODO: check if this is the correct place for marking, before or after regular scan?
// "scan_file_keep()"" will be called inside the following "state_scan()" so we have to mark the file before
if(state.opt.force_realloc_tail)
state_locate_mark_tail_blocks_for_resync(&state, parity_tail);

state_scan(&state);

/* refresh the size info before the content write */
Expand Down
1 change: 1 addition & 0 deletions cmdline/state.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ struct snapraid_option {
int force_nocopy; /**< Force dangerous operations of syncing files without using copy detection. */
int force_full; /**< Force a full parity update. */
int force_realloc; /**< Force a full reallocation and parity update. */
int force_realloc_tail; /**< Force a partial reallocation and parity update for tail space. */
int expect_unrecoverable; /**< Expect presence of unrecoverable error in checking or fixing. */
int expect_recoverable; /**< Expect presence of recoverable error in checking. */
int skip_device; /**< Skip devices matching checks. */
Expand Down