From 8a0bedafa10e7898db06494c98b06ad205d8672e Mon Sep 17 00:00:00 2001 From: Sean Ellingham Date: Wed, 15 Jan 2020 00:19:24 +0000 Subject: [PATCH 1/3] Added support for waiting for other processes Detect lock files from other processes that are/may be modifying the data disks and wait for them to be removed before continuing with the diff/sync --- snapraid-runner.conf.example | 9 +++++++++ snapraid-runner.py | 26 +++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/snapraid-runner.conf.example b/snapraid-runner.conf.example index ed64810..bbabbe9 100644 --- a/snapraid-runner.conf.example +++ b/snapraid-runner.conf.example @@ -8,6 +8,15 @@ deletethreshold = 40 ; if you want touch to be ran each time touch = false +[wait] +enabled = false +delay = 60 +maxdelays = 9 + +[waitfiles] +file-1 = wait-for.file +file-2 = also-wait-for.file + [logging] ; logfile to write to, leave empty to disable file = snapraid.log diff --git a/snapraid-runner.py b/snapraid-runner.py index 71d87c4..d2f7516 100644 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -142,7 +142,7 @@ def load_config(args): global config parser = configparser.RawConfigParser() parser.read(args.conf) - sections = ["snapraid", "logging", "email", "smtp", "scrub"] + sections = ["snapraid", "logging", "email", "smtp", "scrub", "wait", "waitfiles"] config = dict((x, defaultdict(lambda: "")) for x in sections) for section in parser.sections(): for (k, v) in parser.items(section): @@ -150,6 +150,7 @@ def load_config(args): int_options = [ ("snapraid", "deletethreshold"), ("logging", "maxsize"), + ("wait", "delay"), ("wait", "maxdelays"), ("scrub", "percentage"), ("scrub", "older-than"), ("email", "maxsize"), ] for section, option in int_options: @@ -163,10 +164,14 @@ def load_config(args): config["scrub"]["enabled"] = (config["scrub"]["enabled"].lower() == "true") config["email"]["short"] = (config["email"]["short"].lower() == "true") config["snapraid"]["touch"] = (config["snapraid"]["touch"].lower() == "true") + config["wait"]["enabled"] = (config["wait"]["enabled"].lower() == "true") if args.scrub is not None: config["scrub"]["enabled"] = args.scrub + if args.ignorediff is not None: + config["snapraid"]["deletethreshold"] = -1 + def setup_logger(): log_format = logging.Formatter( @@ -210,6 +215,9 @@ def main(): parser.add_argument("--no-scrub", action='store_false', dest='scrub', default=None, help="Do not scrub (overrides config)") + parser.add_argument("--ignore-diff-threshold", action='store_true', + dest='ignorediff', default=None, + help="Sync if diff threshold exceeded (overrides config)") args = parser.parse_args() if not os.path.exists(args.conf): @@ -253,6 +261,22 @@ def run(): config["snapraid"]["config"]) finish(False) + if config["wait"]["enabled"]: + wait_files = [] + for key, path in config["waitfiles"].items(): + wait_files.append(path) + + wait_delays = 0 + while [f for f in wait_files if os.path.isfile(f)]: + if wait_delays > config["wait"]["maxdelays"]: + logging.error("Timed out waiting for other processes") + finish(False); + + logging.info("Waiting for other processes...") + + time.sleep(config["wait"]["delay"]) + wait_delays += 1 + if config["snapraid"]["touch"]: logging.info("Running touch...") snapraid_command("touch") From 76e69278629210a91c9513a72979dd769eb7d19e Mon Sep 17 00:00:00 2001 From: Sean Ellingham Date: Mon, 20 Jan 2020 10:03:54 +0000 Subject: [PATCH 2/3] Show which process is being waited for --- snapraid-runner.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/snapraid-runner.py b/snapraid-runner.py index d2f7516..9d0c799 100644 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -267,12 +267,12 @@ def run(): wait_files.append(path) wait_delays = 0 - while [f for f in wait_files if os.path.isfile(f)]: + while [path for key, path in config["waitfiles"].items() if os.path.isfile(path)]: if wait_delays > config["wait"]["maxdelays"]: - logging.error("Timed out waiting for other processes") + logging.error("Timed out waiting for {}".format(key)) finish(False); - logging.info("Waiting for other processes...") + logging.info("Waiting for {}...".format(key)) time.sleep(config["wait"]["delay"]) wait_delays += 1 From 2e27752ba8ee213d08b6fd9a82a42eb9df24b95e Mon Sep 17 00:00:00 2001 From: Sean Ellingham Date: Mon, 20 Jan 2020 10:56:58 +0000 Subject: [PATCH 3/3] Fixed bug where last file was always waited for Fixed bug where script would wait for last defined file even if it didn't exist --- snapraid-runner.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/snapraid-runner.py b/snapraid-runner.py index 9d0c799..c267a05 100644 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -262,20 +262,23 @@ def run(): finish(False) if config["wait"]["enabled"]: - wait_files = [] - for key, path in config["waitfiles"].items(): - wait_files.append(path) - wait_delays = 0 - while [path for key, path in config["waitfiles"].items() if os.path.isfile(path)]: - if wait_delays > config["wait"]["maxdelays"]: - logging.error("Timed out waiting for {}".format(key)) - finish(False); - - logging.info("Waiting for {}...".format(key)) - - time.sleep(config["wait"]["delay"]) - wait_delays += 1 + + while wait_delays <= config["wait"]["maxdelays"]: + wait = False + for key, path in config["waitfiles"].items(): + if os.path.isfile(path): + logging.info("Waiting for {}...".format(key)) + + time.sleep(config["wait"]["delay"]) + wait_delays += 1 + wait = True + break + if not wait: + break + else: + logging.error("Timed out waiting for {}".format(key)) + finish(False); if config["snapraid"]["touch"]: logging.info("Running touch...")