From 4164600e936903eb07a52e6ce6b1a5c739c733ae Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 11:32:58 -0600 Subject: [PATCH 01/23] setup for my test snapraid env --- .gitignore | 2 +- snapraid-runner.conf | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 snapraid-runner.conf diff --git a/.gitignore b/.gitignore index c0fde79..9b90ff6 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -snapraid-runner.conf +#snapraid-runner.conf diff --git a/snapraid-runner.conf b/snapraid-runner.conf new file mode 100644 index 0000000..2838afa --- /dev/null +++ b/snapraid-runner.conf @@ -0,0 +1,44 @@ +[snapraid] +; path to the snapraid executable (e.g. /bin/snapraid) +executable = C:\Users\Andy\Dropbox\Projects\2022_snapraid\SnapRaid_test_area\snapraid.exe +; path to the snapraid config to be used +config = C:\Users\Andy\Dropbox\Projects\2022_snapraid\SnapRaid_test_area\snapraid.conf +; abort operation if there are more deletes than this, set to -1 to disable +deletethreshold = 40 +; if you want touch to be ran each time +touch = false + +[logging] +; logfile to write to, leave empty to disable +file = snapraid.log +; maximum logfile size in KiB, leave empty for infinite +maxsize = 5000 + +[email] +; when to send an email, comma-separated list of [success, error] +sendon = +; set to false to get full programm output via email +short = true +subject = [SnapRAID] Status Report: +from = +to = +; maximum email size in KiB +maxsize = 500 + +[smtp] +host = +; leave empty for default port +port = +; set to "true" to activate +ssl = false +tls = false +user = +password = + +[scrub] +; set to true to run scrub after sync +enabled = false +; scrub plan - either a percentage or one of [bad, new, full] +plan = 12 +; minimum block age (in days) for scrubbing. Only used with percentage plans +older-than = 10 From da16feffb57eb7d079717f1dadf466fca600d5bd Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 11:34:42 -0600 Subject: [PATCH 02/23] add a modify threshold to the config --- snapraid-runner.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/snapraid-runner.conf b/snapraid-runner.conf index 2838afa..b69c2d6 100644 --- a/snapraid-runner.conf +++ b/snapraid-runner.conf @@ -5,6 +5,9 @@ executable = C:\Users\Andy\Dropbox\Projects\2022_snapraid\SnapRaid_test_area\sna config = C:\Users\Andy\Dropbox\Projects\2022_snapraid\SnapRaid_test_area\snapraid.conf ; abort operation if there are more deletes than this, set to -1 to disable deletethreshold = 40 +; abort operation if there are more modify's than this, set to -1 to disable, for rudimenary +; ransomware protection +modifythreshold = 40 ; if you want touch to be ran each time touch = false From 7e73eff8d162e9789dca5e86d0a8182708cc424f Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 11:37:19 -0600 Subject: [PATCH 03/23] int verification for modify threshold --- snapraid-runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/snapraid-runner.py b/snapraid-runner.py index 56b8e9e..826ee90 100755 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -140,8 +140,8 @@ def load_config(args): config[section][k] = v.strip() int_options = [ - ("snapraid", "deletethreshold"), ("logging", "maxsize"), - ("scrub", "older-than"), ("email", "maxsize"), + ("snapraid", "deletethreshold"), ("snapraid", "modifythreshold"), + ("logging", "maxsize"), ("scrub", "older-than"), ("email", "maxsize"), ] for section, option in int_options: try: From 9b422ff67a1d4c5609a6e53cc85a66d846077a06 Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 11:39:31 -0600 Subject: [PATCH 04/23] added parser argument for ignore-modifythreshold --- snapraid-runner.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/snapraid-runner.py b/snapraid-runner.py index 826ee90..8127141 100755 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -165,6 +165,9 @@ def load_config(args): if args.ignore_deletethreshold: config["snapraid"]["deletethreshold"] = -1 + if args.ignore_modifythreshold: + config["snapraid"]["modifythreshold"] = -1 + def setup_logger(): log_format = logging.Formatter( @@ -210,6 +213,8 @@ def main(): help="Do not scrub (overrides config)") parser.add_argument("--ignore-deletethreshold", action='store_true', help="Sync even if configured delete threshold is exceeded") + parser.add_argument("--ignore-modifythreshold", action='store_true', + help="Sync even if configured modify threshold is exceeded") args = parser.parse_args() if not os.path.exists(args.conf): From e67a86e53635cb782b44bb55805de8f4351e30da Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 11:43:22 -0600 Subject: [PATCH 05/23] add check for modifythreshold into run logic --- snapraid-runner.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/snapraid-runner.py b/snapraid-runner.py index 8127141..46eeabc 100755 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -281,6 +281,14 @@ def run(): logging.error("Run again with --ignore-deletethreshold to sync anyways") finish(False) + if (config["snapraid"]["modifythreshold"] >= 0 and + diff_results["update"] > config["snapraid"]["modifythreshold"]): + logging.error( + "Modified files exceed modify threshold of {}, aborting".format( + config["snapraid"]["modifythreshold"])) + logging.error("Run again with --ignore-modifythreshold to sync anyways") + finish(False) + if (diff_results["remove"] + diff_results["add"] + diff_results["move"] + diff_results["update"] == 0): logging.info("No changes detected, no sync required") From 0247877db7471290a2347a51cedbf3d5f3af61ad Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 11:43:42 -0600 Subject: [PATCH 06/23] Update snapraid-runner.conf --- snapraid-runner.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snapraid-runner.conf b/snapraid-runner.conf index b69c2d6..55b9d87 100644 --- a/snapraid-runner.conf +++ b/snapraid-runner.conf @@ -7,7 +7,7 @@ config = C:\Users\Andy\Dropbox\Projects\2022_snapraid\SnapRaid_test_area\snaprai deletethreshold = 40 ; abort operation if there are more modify's than this, set to -1 to disable, for rudimenary ; ransomware protection -modifythreshold = 40 +modifythreshold = 1 ; if you want touch to be ran each time touch = false From ede29dd196648682893d6216a6b5fa3b1abc0f13 Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 11:46:55 -0600 Subject: [PATCH 07/23] Update snapraid-runner.conf --- snapraid-runner.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/snapraid-runner.conf b/snapraid-runner.conf index 55b9d87..7ba2f13 100644 --- a/snapraid-runner.conf +++ b/snapraid-runner.conf @@ -9,7 +9,7 @@ deletethreshold = 40 ; ransomware protection modifythreshold = 1 ; if you want touch to be ran each time -touch = false +touch = true [logging] ; logfile to write to, leave empty to disable @@ -40,8 +40,8 @@ password = [scrub] ; set to true to run scrub after sync -enabled = false +enabled = true ; scrub plan - either a percentage or one of [bad, new, full] -plan = 12 +plan = full ; minimum block age (in days) for scrubbing. Only used with percentage plans older-than = 10 From 6af314ad25cdb6954262d269c659bf7372a4b5eb Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 11:55:30 -0600 Subject: [PATCH 08/23] add yagmail dependency --- snapraid-runner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/snapraid-runner.py b/snapraid-runner.py index 46eeabc..04db66f 100755 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import yagmail import argparse import configparser import logging From 29347644f562efd00b17a559acdd37881a53b2a0 Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 12:09:52 -0600 Subject: [PATCH 09/23] add oauth settings in config --- snapraid-runner.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/snapraid-runner.conf b/snapraid-runner.conf index 7ba2f13..f7945d7 100644 --- a/snapraid-runner.conf +++ b/snapraid-runner.conf @@ -18,6 +18,8 @@ file = snapraid.log maxsize = 5000 [email] +;set true to use yagmail's oauth2 for gmail insead of smtp for sending the email +use_oauth = true ; when to send an email, comma-separated list of [success, error] sendon = ; set to false to get full programm output via email @@ -28,6 +30,11 @@ to = ; maximum email size in KiB maxsize = 500 +[oauth] +; for use only if use_oauth = true, sets the oauth2 credentials file +; location, if no file exists then one will be created when the program is ran +oauth2_file = C:\Users\Andy\Dropbox\Projects\2022_snapraid\snapraid-runner\snapraid-runner\oauth2_creds.json + [smtp] host = ; leave empty for default port From 1869188760eee11db356dda820574874d0ab03d2 Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 12:46:02 -0600 Subject: [PATCH 10/23] Add oauth/yagmail to send_email --- snapraid-runner.py | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/snapraid-runner.py b/snapraid-runner.py index 04db66f..7730f6a 100755 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -import yagmail import argparse import configparser import logging @@ -67,11 +66,10 @@ def snapraid_command(command, args={}, *, allow_statuscodes=[]): def send_email(success): - import smtplib from email.mime.text import MIMEText from email import charset - - if len(config["smtp"]["host"]) == 0: + + if len(config["smtp"]["host"]) == 0 and not config["email"]["use_oauth"]: logging.error("Failed to send email because smtp host is not set") return @@ -99,22 +97,31 @@ def send_email(success): (" SUCCESS" if success else " ERROR") msg["From"] = config["email"]["from"] msg["To"] = config["email"]["to"] - smtp = {"host": config["smtp"]["host"]} - if config["smtp"]["port"]: - smtp["port"] = config["smtp"]["port"] - if config["smtp"]["ssl"]: - server = smtplib.SMTP_SSL(**smtp) + + if config["email"]["use_oauth"]: + import yagmail + + yag = yagmail.SMTP(msg["From"], oauth2_file=config["oauth"]["oauth2_file"]) + yag.send(to=msg["To"], subject=msg["Subject"], contents=body) else: - server = smtplib.SMTP(**smtp) - if config["smtp"]["tls"]: - server.starttls() - if config["smtp"]["user"]: - server.login(config["smtp"]["user"], config["smtp"]["password"]) - server.sendmail( - config["email"]["from"], - [config["email"]["to"]], - msg.as_string()) - server.quit() + import smtplib + + smtp = {"host": config["smtp"]["host"]} + if config["smtp"]["port"]: + smtp["port"] = config["smtp"]["port"] + if config["smtp"]["ssl"]: + server = smtplib.SMTP_SSL(**smtp) + else: + server = smtplib.SMTP(**smtp) + if config["smtp"]["tls"]: + server.starttls() + if config["smtp"]["user"]: + server.login(config["smtp"]["user"], config["smtp"]["password"]) + server.sendmail( + config["email"]["from"], + [config["email"]["to"]], + msg.as_string()) + server.quit() def finish(is_success): From ff2fdbdd103ad3e41c26325bb93fa605cbd88966 Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 12:53:21 -0600 Subject: [PATCH 11/23] add oauth to example config --- .gitignore | 2 +- snapraid-runner.conf.example | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 9b90ff6..c0fde79 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -#snapraid-runner.conf +snapraid-runner.conf diff --git a/snapraid-runner.conf.example b/snapraid-runner.conf.example index bc8fca4..de963ab 100644 --- a/snapraid-runner.conf.example +++ b/snapraid-runner.conf.example @@ -1,12 +1,15 @@ [snapraid] ; path to the snapraid executable (e.g. /bin/snapraid) -executable = snapraid +executable = snapraid.exe ; path to the snapraid config to be used config = snapraid.conf ; abort operation if there are more deletes than this, set to -1 to disable deletethreshold = 40 +; abort operation if there are more modify's than this, set to -1 to disable, for rudimenary +; ransomware protection +modifythreshold = 1 ; if you want touch to be ran each time -touch = false +touch = true [logging] ; logfile to write to, leave empty to disable @@ -15,16 +18,23 @@ file = snapraid.log maxsize = 5000 [email] +;set true to use yagmail's oauth2 for gmail insead of smtp for sending the email +use_oauth = true ; when to send an email, comma-separated list of [success, error] sendon = success,error ; set to false to get full programm output via email short = true subject = [SnapRAID] Status Report: -from = -to = +from = +to = ; maximum email size in KiB maxsize = 500 +[oauth] +; for use only if use_oauth = true, sets the oauth2 credentials file +; location, if no file exists then one will be created when the program is ran +oauth2_file = oauth2_creds.json + [smtp] host = ; leave empty for default port From e300ec2f7c761fcf99ef6a1fde7d3ca98980b5be Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 12:54:46 -0600 Subject: [PATCH 12/23] Update .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9b90ff6..c0fde79 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -#snapraid-runner.conf +snapraid-runner.conf From 07e98449a04d07d7f7a2d66b98671d7da5fd5b38 Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 12:55:26 -0600 Subject: [PATCH 13/23] Update snapraid-runner.conf --- snapraid-runner.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/snapraid-runner.conf b/snapraid-runner.conf index f7945d7..1a940bc 100644 --- a/snapraid-runner.conf +++ b/snapraid-runner.conf @@ -21,12 +21,12 @@ maxsize = 5000 ;set true to use yagmail's oauth2 for gmail insead of smtp for sending the email use_oauth = true ; when to send an email, comma-separated list of [success, error] -sendon = +sendon = success ; set to false to get full programm output via email short = true subject = [SnapRAID] Status Report: -from = -to = +from = @gmail.com +to = @gmail.com ; maximum email size in KiB maxsize = 500 From 623e564803049811ed256cd0ab2e38ec6509f593 Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 13:02:50 -0600 Subject: [PATCH 14/23] Delete snapraid-runner.conf --- snapraid-runner.conf | 54 -------------------------------------------- 1 file changed, 54 deletions(-) delete mode 100644 snapraid-runner.conf diff --git a/snapraid-runner.conf b/snapraid-runner.conf deleted file mode 100644 index 1a940bc..0000000 --- a/snapraid-runner.conf +++ /dev/null @@ -1,54 +0,0 @@ -[snapraid] -; path to the snapraid executable (e.g. /bin/snapraid) -executable = C:\Users\Andy\Dropbox\Projects\2022_snapraid\SnapRaid_test_area\snapraid.exe -; path to the snapraid config to be used -config = C:\Users\Andy\Dropbox\Projects\2022_snapraid\SnapRaid_test_area\snapraid.conf -; abort operation if there are more deletes than this, set to -1 to disable -deletethreshold = 40 -; abort operation if there are more modify's than this, set to -1 to disable, for rudimenary -; ransomware protection -modifythreshold = 1 -; if you want touch to be ran each time -touch = true - -[logging] -; logfile to write to, leave empty to disable -file = snapraid.log -; maximum logfile size in KiB, leave empty for infinite -maxsize = 5000 - -[email] -;set true to use yagmail's oauth2 for gmail insead of smtp for sending the email -use_oauth = true -; when to send an email, comma-separated list of [success, error] -sendon = success -; set to false to get full programm output via email -short = true -subject = [SnapRAID] Status Report: -from = @gmail.com -to = @gmail.com -; maximum email size in KiB -maxsize = 500 - -[oauth] -; for use only if use_oauth = true, sets the oauth2 credentials file -; location, if no file exists then one will be created when the program is ran -oauth2_file = C:\Users\Andy\Dropbox\Projects\2022_snapraid\snapraid-runner\snapraid-runner\oauth2_creds.json - -[smtp] -host = -; leave empty for default port -port = -; set to "true" to activate -ssl = false -tls = false -user = -password = - -[scrub] -; set to true to run scrub after sync -enabled = true -; scrub plan - either a percentage or one of [bad, new, full] -plan = full -; minimum block age (in days) for scrubbing. Only used with percentage plans -older-than = 10 From fc53ed89c62aa7943e36b620fd37c6ee7654492e Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 13:04:06 -0600 Subject: [PATCH 15/23] Delete snapraid-runner.conf --- snapraid-runner.conf | 47 -------------------------------------------- 1 file changed, 47 deletions(-) delete mode 100644 snapraid-runner.conf diff --git a/snapraid-runner.conf b/snapraid-runner.conf deleted file mode 100644 index 7ba2f13..0000000 --- a/snapraid-runner.conf +++ /dev/null @@ -1,47 +0,0 @@ -[snapraid] -; path to the snapraid executable (e.g. /bin/snapraid) -executable = C:\Users\Andy\Dropbox\Projects\2022_snapraid\SnapRaid_test_area\snapraid.exe -; path to the snapraid config to be used -config = C:\Users\Andy\Dropbox\Projects\2022_snapraid\SnapRaid_test_area\snapraid.conf -; abort operation if there are more deletes than this, set to -1 to disable -deletethreshold = 40 -; abort operation if there are more modify's than this, set to -1 to disable, for rudimenary -; ransomware protection -modifythreshold = 1 -; if you want touch to be ran each time -touch = true - -[logging] -; logfile to write to, leave empty to disable -file = snapraid.log -; maximum logfile size in KiB, leave empty for infinite -maxsize = 5000 - -[email] -; when to send an email, comma-separated list of [success, error] -sendon = -; set to false to get full programm output via email -short = true -subject = [SnapRAID] Status Report: -from = -to = -; maximum email size in KiB -maxsize = 500 - -[smtp] -host = -; leave empty for default port -port = -; set to "true" to activate -ssl = false -tls = false -user = -password = - -[scrub] -; set to true to run scrub after sync -enabled = true -; scrub plan - either a percentage or one of [bad, new, full] -plan = full -; minimum block age (in days) for scrubbing. Only used with percentage plans -older-than = 10 From ba30e8d77d9d1411f404de4739391124a0c2c7fb Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 13:15:16 -0600 Subject: [PATCH 16/23] update config setup --- .gitignore | 1 + snapraid-runner.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c0fde79..2446be6 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ snapraid-runner.conf +oauth2_creds.json diff --git a/snapraid-runner.py b/snapraid-runner.py index 7730f6a..544448a 100755 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -141,7 +141,7 @@ def load_config(args): global config parser = configparser.RawConfigParser() parser.read(args.conf) - sections = ["snapraid", "logging", "email", "smtp", "scrub"] + sections = ["snapraid", "logging", "email", "oauth", "smtp", "scrub"] config = dict((x, defaultdict(lambda: "")) for x in sections) for section in parser.sections(): for (k, v) in parser.items(section): From a63a2ada6bb3d7e586ec7d9ae69abedd797c5247 Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 13:17:04 -0600 Subject: [PATCH 17/23] add send log on error to config file --- snapraid-runner.conf.example | 2 ++ 1 file changed, 2 insertions(+) diff --git a/snapraid-runner.conf.example b/snapraid-runner.conf.example index de963ab..bb75739 100644 --- a/snapraid-runner.conf.example +++ b/snapraid-runner.conf.example @@ -34,6 +34,8 @@ maxsize = 500 ; for use only if use_oauth = true, sets the oauth2 credentials file ; location, if no file exists then one will be created when the program is ran oauth2_file = oauth2_creds.json +;set true to attach the log file to email if there is an error +send_log_on_error = false [smtp] host = From 7a063026363c000e12045f5cd2a112ec48b96225 Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 13:27:35 -0600 Subject: [PATCH 18/23] add send log on error to send_email --- snapraid-runner.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/snapraid-runner.py b/snapraid-runner.py index 544448a..862feb4 100755 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -101,8 +101,12 @@ def send_email(success): if config["email"]["use_oauth"]: import yagmail + attachment_path = [] + if config["oauth"]["send_log_on_error"] and not success: + attachment_path = config["logging"]["file"] + yag = yagmail.SMTP(msg["From"], oauth2_file=config["oauth"]["oauth2_file"]) - yag.send(to=msg["To"], subject=msg["Subject"], contents=body) + yag.send(to=msg["To"], subject=msg["Subject"], contents=body, attachments=attachment_path) else: import smtplib From bf661c2ff1d329f385b71b408c150d42d6a0cf6e Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 13:28:22 -0600 Subject: [PATCH 19/23] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c0fde79..2446be6 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ snapraid-runner.conf +oauth2_creds.json From 2ba7d2c80ed682defd167b38a44c522af0316a7d Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 13:35:37 -0600 Subject: [PATCH 20/23] clarify settings, changed oauth to yagmail --- snapraid-runner.conf.example | 4 ++-- snapraid-runner.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/snapraid-runner.conf.example b/snapraid-runner.conf.example index bb75739..84f14af 100644 --- a/snapraid-runner.conf.example +++ b/snapraid-runner.conf.example @@ -19,7 +19,7 @@ maxsize = 5000 [email] ;set true to use yagmail's oauth2 for gmail insead of smtp for sending the email -use_oauth = true +use_yagmail = true ; when to send an email, comma-separated list of [success, error] sendon = success,error ; set to false to get full programm output via email @@ -30,7 +30,7 @@ to = ; maximum email size in KiB maxsize = 500 -[oauth] +[yagmail] ; for use only if use_oauth = true, sets the oauth2 credentials file ; location, if no file exists then one will be created when the program is ran oauth2_file = oauth2_creds.json diff --git a/snapraid-runner.py b/snapraid-runner.py index 862feb4..07baebd 100755 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -69,7 +69,7 @@ def send_email(success): from email.mime.text import MIMEText from email import charset - if len(config["smtp"]["host"]) == 0 and not config["email"]["use_oauth"]: + if len(config["smtp"]["host"]) == 0 and not config["email"]["use_yagmail"]: logging.error("Failed to send email because smtp host is not set") return @@ -98,14 +98,14 @@ def send_email(success): msg["From"] = config["email"]["from"] msg["To"] = config["email"]["to"] - if config["email"]["use_oauth"]: + if config["email"]["use_yagmail"]: import yagmail attachment_path = [] - if config["oauth"]["send_log_on_error"] and not success: + if config["yagmail"]["send_log_on_error"] and not success: attachment_path = config["logging"]["file"] - yag = yagmail.SMTP(msg["From"], oauth2_file=config["oauth"]["oauth2_file"]) + yag = yagmail.SMTP(msg["From"], oauth2_file=config["yagmail"]["oauth2_file"]) yag.send(to=msg["To"], subject=msg["Subject"], contents=body, attachments=attachment_path) else: import smtplib @@ -145,7 +145,7 @@ def load_config(args): global config parser = configparser.RawConfigParser() parser.read(args.conf) - sections = ["snapraid", "logging", "email", "oauth", "smtp", "scrub"] + sections = ["snapraid", "logging", "email", "yagmail", "smtp", "scrub"] config = dict((x, defaultdict(lambda: "")) for x in sections) for section in parser.sections(): for (k, v) in parser.items(section): From ade0f7bfd75d3b804f5bc1d0f9ea0e668125addc Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 13:39:13 -0600 Subject: [PATCH 21/23] Update snapraid-runner.conf.example --- snapraid-runner.conf.example | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/snapraid-runner.conf.example b/snapraid-runner.conf.example index 84f14af..46a155c 100644 --- a/snapraid-runner.conf.example +++ b/snapraid-runner.conf.example @@ -34,8 +34,10 @@ maxsize = 500 ; for use only if use_oauth = true, sets the oauth2 credentials file ; location, if no file exists then one will be created when the program is ran oauth2_file = oauth2_creds.json -;set true to attach the log file to email if there is an error -send_log_on_error = false +;set true to attach the log file to email +send_log = false +;set true to attach the log file to email only if there is an error +send_log_only_on_error = false [smtp] host = From e0d2a7282110416e500d6c79e2e0e65f43401730 Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 14:13:27 -0600 Subject: [PATCH 22/23] fixed send log only on error --- snapraid-runner.conf.example | 2 +- snapraid-runner.py | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/snapraid-runner.conf.example b/snapraid-runner.conf.example index 46a155c..f9252e5 100644 --- a/snapraid-runner.conf.example +++ b/snapraid-runner.conf.example @@ -37,7 +37,7 @@ oauth2_file = oauth2_creds.json ;set true to attach the log file to email send_log = false ;set true to attach the log file to email only if there is an error -send_log_only_on_error = false +iff_error = false [smtp] host = diff --git a/snapraid-runner.py b/snapraid-runner.py index 07baebd..23f8368 100755 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -102,7 +102,7 @@ def send_email(success): import yagmail attachment_path = [] - if config["yagmail"]["send_log_on_error"] and not success: + if config["yagmail"]["send_log"] and not (config["yagmail"]["iff_error"] and success): attachment_path = config["logging"]["file"] yag = yagmail.SMTP(msg["From"], oauth2_file=config["yagmail"]["oauth2_file"]) @@ -159,13 +159,15 @@ def load_config(args): try: config[section][option] = int(config[section][option]) except ValueError: - config[section][option] = 0 - + config[section][option] = 0 + + config["snapraid"]["touch"] = (config["snapraid"]["touch"].lower() == "true") + config["email"]["short"] = (config["email"]["short"].lower() == "true") + config["yagmail"]["send_log"] = (config["yagmail"]["send_log"].lower() == "true") + config["yagmail"]["iff_error"] = (config["yagmail"]["iff_error"].lower() == "true") config["smtp"]["ssl"] = (config["smtp"]["ssl"].lower() == "true") config["smtp"]["tls"] = (config["smtp"]["tls"].lower() == "true") config["scrub"]["enabled"] = (config["scrub"]["enabled"].lower() == "true") - config["email"]["short"] = (config["email"]["short"].lower() == "true") - config["snapraid"]["touch"] = (config["snapraid"]["touch"].lower() == "true") # Migration if config["scrub"]["percentage"]: From 31fb9118d065b15f3ed4d188d10ee97c23870c15 Mon Sep 17 00:00:00 2001 From: AndyHegemann Date: Sun, 24 Jul 2022 14:46:17 -0600 Subject: [PATCH 23/23] Update README.md --- README.md | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index ca28549..3bf98de 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Snapraid Runner Script +# Snapraid Runner Script - Yagmail Fork This script runs snapraid and sends its output to the console, a log file and via email. All this is configurable. @@ -8,10 +8,17 @@ scheduler. It supports Windows, Linux and macOS and requires at least python3.7. +**This is a fork** of the original script that supports sending from new **gmail accounts that can't enable smtp access**. Yagmail is used to authenticate via oauth2, this requires some manual setup of the sending account and of the oauth2 credentials on the first run of the script. + +[This](https://blog.macuyiko.com/post/2016/how-to-send-html-mails-with-oauth2-and-gmail-in-python.html) shows how to setup your sending google account via the Google API Console. And [This](https://github.com/kootenpv/yagmail#oauth2) walks through the prompts that the console will ask about on the first run. + +This fork also adds an optional check on the number of modified files to abort if it is over a configurable threshold, this is a rudimentary method of stopping some ransomware. **This is a very poor method of ransomeware detection and mitigation**, offline and or immutable backups are highly suggested. + ## How to use * If you don’t already have it, download and install [the latest python version](https://www.python.org/downloads/). -* Download [the latest release](https://github.com/Chronial/snapraid-runner/releases) +* Install yagmail via pip if you are planning on using it +* Download [the latest release](https://github.com/AndyHegemann/snapraid-runner/releases) of this script and extract it anywhere or clone this repository via git. * Copy/rename the `snapraid-runner.conf.example` to `snapraid-runner.conf` and edit its contents. You need to at least configure `snapraid.executable` and @@ -20,23 +27,34 @@ It supports Windows, Linux and macOS and requires at least python3.7. `py -3 snapraid-runner.py` on Windows. ## Features -* Runs `diff` before `sync` to see how many files were deleted and aborts if +* Runs `diff` before `sync` to see how many files were deleted and or modified and aborts if that number exceeds a set threshold. * Can create a size-limited rotated logfile. * Can send notification emails after each run or only for failures. + * Can attach log after each run or only for failures * Can run `scrub` after `sync` ## Scope of this project and contributions -Snapraid-runner is supposed to be a small tool with clear focus. It should not +~Snapraid-runner is supposed to be a small tool with clear focus. It should not have any dependencies to keep installation trivial. I always welcome bugfixes and contributions, but be aware that I will not merge new features that I feel -do not fit the core purpose of this tool. +do not fit the core purpose of this tool.~ -I keep the PRs for features I do not plan on merging open, so if there's a +~I keep the PRs for features I do not plan on merging open, so if there's a feature you are missing, you can have a look -[at the open PRs](https://github.com/Chronial/snapraid-runner/pulls). +[at the open PRs](https://github.com/Chronial/snapraid-runner/pulls).~ + +I added features to the original that I wanted, please feel free to do the same to my fork. There is a very good chance I broke something, but if I'm not going to use it then I probably won't get around to fix it. PRs for fixes and features will probably get merged if you feel like opening one. ## Changelog + +### vA0.6 (24 Jul 2022) +* Add Yagmail (oauth2) support +* Add attaching log file to email report + * Add attach log file only on error +* Add abort on too many modified files +* Add --ignore-modifythreshold + ### Unreleased * Add --ignore-deletethreshold (by exterrestris, #25) * Add support for scrub --plan, replacing --percentage (thanks to fmoledina)