From 8e86392cbc088b91198fb4c8637dc964e60aae07 Mon Sep 17 00:00:00 2001 From: Blake Dewey Date: Sun, 30 Jan 2022 14:58:19 -0500 Subject: [PATCH 1/3] refactor to use object for runner --- snapraid-runner.py | 574 +++++++++++++++++++++++---------------------- 1 file changed, 295 insertions(+), 279 deletions(-) diff --git a/snapraid-runner.py b/snapraid-runner.py index 56b8e9e..d49cecf 100755 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -1,314 +1,330 @@ #!/usr/bin/env python3 -import argparse -import configparser + +from argparse import ArgumentParser +# noinspection PyUnresolvedReferences, PyProtectedMember +from configparser import RawConfigParser, _UNSET import logging -import logging.handlers +from logging.handlers import RotatingFileHandler import os.path import subprocess import sys -import threading +from threading import Thread import time -import traceback -from collections import Counter, defaultdict +from collections import Counter from io import StringIO -# Global variables -config = None -email_log = None + +class RunnerConfigParser(RawConfigParser): + def getstring(self, section, option, *, fallback=_UNSET): + # noinspection PyProtectedMember + value = super()._get_conv(section, option, str, fallback=fallback) + if value is None: + return value + value = value.strip() + if len(value) == 0: + if fallback is _UNSET: + raise ValueError('Option "%s" in section "%s" cannot be an empty string.' + % (option, section)) + return fallback + return value -def tee_log(infile, out_lines, log_level): +def tee_log(infile, log_level): """ Create a thread that saves all the output on infile to out_lines and logs every line with log_level """ + out_lines = [] + def tee_thread(): - for line in iter(infile.readline, ""): + for line in iter(infile.readline, ''): logging.log(log_level, line.rstrip()) out_lines.append(line) infile.close() - t = threading.Thread(target=tee_thread) + + t = Thread(target=tee_thread) t.daemon = True t.start() - return t - - -def snapraid_command(command, args={}, *, allow_statuscodes=[]): - """ - Run snapraid command - Raises subprocess.CalledProcessError if errorlevel != 0 - """ - arguments = ["--conf", config["snapraid"]["config"], - "--quiet"] - for (k, v) in args.items(): - arguments.extend(["--" + k, str(v)]) - p = subprocess.Popen( - [config["snapraid"]["executable"], command] + arguments, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - # Snapraid always outputs utf-8 on windows. On linux, utf-8 - # also seems a sensible assumption. - encoding="utf-8", - errors="replace") - out = [] - threads = [ - tee_log(p.stdout, out, logging.OUTPUT), - tee_log(p.stderr, [], logging.OUTERR)] - for t in threads: - t.join() - ret = p.wait() - # sleep for a while to make pervent output mixup - time.sleep(0.3) - if ret == 0 or ret in allow_statuscodes: - return out - else: - raise subprocess.CalledProcessError(ret, "snapraid " + command) - - -def send_email(success): - import smtplib - from email.mime.text import MIMEText - from email import charset - - if len(config["smtp"]["host"]) == 0: - logging.error("Failed to send email because smtp host is not set") - return - - # use quoted-printable instead of the default base64 - charset.add_charset("utf-8", charset.SHORTEST, charset.QP) - if success: - body = "SnapRAID job completed successfully:\n\n\n" - else: - body = "Error during SnapRAID job:\n\n\n" - - log = email_log.getvalue() - maxsize = config['email'].get('maxsize', 500) * 1024 - if maxsize and len(log) > maxsize: - cut_lines = log.count("\n", maxsize // 2, -maxsize // 2) - log = ( - "NOTE: Log was too big for email and was shortened\n\n" + - log[:maxsize // 2] + - "[...]\n\n\n --- LOG WAS TOO BIG - {} LINES REMOVED --\n\n\n[...]".format( - cut_lines) + - log[-maxsize // 2:]) - body += log - - msg = MIMEText(body, "plain", "utf-8") - msg["Subject"] = config["email"]["subject"] + \ - (" SUCCESS" if success else " ERROR") - msg["From"] = config["email"]["from"] - msg["To"] = config["email"]["to"] - smtp = {"host": config["smtp"]["host"]} - if config["smtp"]["port"]: - smtp["port"] = config["smtp"]["port"] - if config["smtp"]["ssl"]: - server = smtplib.SMTP_SSL(**smtp) - else: - server = smtplib.SMTP(**smtp) - if config["smtp"]["tls"]: - server.starttls() - if config["smtp"]["user"]: - server.login(config["smtp"]["user"], config["smtp"]["password"]) - server.sendmail( - config["email"]["from"], - [config["email"]["to"]], - msg.as_string()) - server.quit() - - -def finish(is_success): - if ("error", "success")[is_success] in config["email"]["sendon"]: - try: - send_email(is_success) - except Exception: - logging.exception("Failed to send email") - if is_success: - logging.info("Run finished successfully") - else: - logging.error("Run failed") - sys.exit(0 if is_success else 1) - - -def load_config(args): - global config - parser = configparser.RawConfigParser() - parser.read(args.conf) - sections = ["snapraid", "logging", "email", "smtp", "scrub"] - config = dict((x, defaultdict(lambda: "")) for x in sections) - for section in parser.sections(): - for (k, v) in parser.items(section): - config[section][k] = v.strip() - - int_options = [ - ("snapraid", "deletethreshold"), ("logging", "maxsize"), - ("scrub", "older-than"), ("email", "maxsize"), - ] - for section, option in int_options: - try: - config[section][option] = int(config[section][option]) - except ValueError: - config[section][option] = 0 - - config["smtp"]["ssl"] = (config["smtp"]["ssl"].lower() == "true") - config["smtp"]["tls"] = (config["smtp"]["tls"].lower() == "true") - config["scrub"]["enabled"] = (config["scrub"]["enabled"].lower() == "true") - config["email"]["short"] = (config["email"]["short"].lower() == "true") - config["snapraid"]["touch"] = (config["snapraid"]["touch"].lower() == "true") - - # Migration - if config["scrub"]["percentage"]: - config["scrub"]["plan"] = config["scrub"]["percentage"] - - if args.scrub is not None: - config["scrub"]["enabled"] = args.scrub - - if args.ignore_deletethreshold: - config["snapraid"]["deletethreshold"] = -1 - - -def setup_logger(): - log_format = logging.Formatter( - "%(asctime)s [%(levelname)-6.6s] %(message)s") - root_logger = logging.getLogger() - logging.OUTPUT = 15 - logging.addLevelName(logging.OUTPUT, "OUTPUT") - logging.OUTERR = 25 - logging.addLevelName(logging.OUTERR, "OUTERR") - root_logger.setLevel(logging.OUTPUT) - console_logger = logging.StreamHandler(sys.stdout) - console_logger.setFormatter(log_format) - root_logger.addHandler(console_logger) - - if config["logging"]["file"]: - max_log_size = max(config["logging"]["maxsize"], 0) * 1024 - file_logger = logging.handlers.RotatingFileHandler( - config["logging"]["file"], - maxBytes=max_log_size, - backupCount=9) - file_logger.setFormatter(log_format) - root_logger.addHandler(file_logger) - - if config["email"]["sendon"]: - global email_log - email_log = StringIO() - email_logger = logging.StreamHandler(email_log) - email_logger.setFormatter(log_format) - if config["email"]["short"]: - # Don't send programm stdout in email - email_logger.setLevel(logging.INFO) - root_logger.addHandler(email_logger) + return t, out_lines + + +class SnapraidRunner: + OUTPUT = 15 + OUTERR = 25 + + def __init__(self, config_file, scrub=True, ignore_deletethreshold=False, dry_run=False): + self.dry_run = dry_run + + if not os.path.exists(config_file): + raise ValueError('Configuration file does not exist.') + + config = RunnerConfigParser() + config.read(config_file) + + # SnapRaid Options + self.snapraid_exe = config.getstring('snapraid', 'executable') + self.snapraid_config = config.getstring('snapraid', 'config') + self.delete_threshold = config.getint('snapraid', 'deletethreshold', fallback=0) + if ignore_deletethreshold: + self.delete_threshold = -1 + self.snapraid_touch = config.getboolean('snapraid', 'touch', fallback=False) + + # Logging Options + self.log_maxsize = config.getint('logging', 'maxsize', fallback=0) + self.log_file = config.getstring('logging', 'file', fallback=None) + + # Scrub Options + self.scrub_older_than = config.getint('scrub', 'older-than', fallback=None) + self.scrub_enabled = scrub if scrub is not None else \ + config.getboolean('scrub', 'enabled', fallback=False) + self.scrub_plan = config.getstring('scrub', 'plan', fallback=None) + scrub_percentage = config.getint('scrub', 'percentage', fallback=None) + if scrub_percentage is not None: + self.scrub_plan = scrub_percentage + + # Email Options + self.email_maxsize = config.getint('email', 'maxsize', fallback=500) + self.email_short = config.getboolean('email', 'short', fallback=False) + self.email_subject = config.getstring('email', 'subject', fallback=None) + self.email_to = config.getstring('email', 'to', fallback=None) + self.email_from = config.getstring('email', 'from', fallback=None) + self.email_sendon = config.getstring('email', 'sendon', fallback=None) + if self.email_sendon is not None: + self.email_sendon = self.email_sendon.split(',') + + # SMTP Options + self.smtp_ssl = config.getboolean('smtp', 'ssl', fallback=False) + self.smtp_tls = config.getboolean('smtp', 'tls', fallback=False) + self.smtp_host = config.getstring('smtp', 'host', fallback=None) + self.smtp_port = config.getstring('smtp', 'port', fallback=None) + self.smtp_user = config.getstring('smtp', 'user', fallback=None) + self.smtp_password = config.getstring('smtp', 'password', fallback=None) + + # Global Variables + self.email_stream = None + self.logger = None + + def snapraid_command(self, command, allow_statuscodes=None, **kwargs): + cli_args = ['--conf', self.snapraid_config, '--quiet'] + for k, v in kwargs.items(): + cli_args.extend(['--' + k, str(v)]) + + if self.dry_run: + logging.info(' '.join([self.snapraid_exe, command] + cli_args)) + return [] + + p = subprocess.Popen( + [self.snapraid_exe, command] + cli_args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + # Snapraid always outputs utf-8 on windows. On linux, utf-8 + # also seems a sensible assumption. + encoding='utf-8', + errors='replace' + ) + + stdout, out_lines = tee_log(p.stdout, self.OUTPUT) + stderr, _ = tee_log(p.stderr, self.OUTERR) + for t in [stdout, stderr]: + t.join() + + return_code = p.wait() + # sleep for a while to prevent output mixup + time.sleep(0.3) + if return_code != 0: + if allow_statuscodes is None or return_code not in allow_statuscodes: + raise subprocess.CalledProcessError(return_code, 'snapraid ' + command) + + return out_lines + + def create_logger(self): + log_format = logging.Formatter('%(asctime)s [%(levelname)-6.6s] %(message)s') + logging.addLevelName(self.OUTPUT, 'OUTPUT') + logging.addLevelName(self.OUTERR, 'OUTERR') + logger = logging.getLogger() + logger.setLevel(self.OUTPUT) + + console_logger = logging.StreamHandler(sys.stdout) + console_logger.setFormatter(log_format) + logger.addHandler(console_logger) + + if self.log_file: + max_log_size = max(self.log_maxsize, 0) * 1024 + file_logger = RotatingFileHandler( + self.log_file, + maxBytes=max_log_size, + backupCount=9 + ) + file_logger.setFormatter(log_format) + logger.addHandler(file_logger) + + if self.email_sendon is not None: + self.email_stream = StringIO() + email_logger = logging.StreamHandler(self.email_stream) + email_logger.setFormatter(log_format) + if self.email_short: + # Don't send programm stdout in email + email_logger.setLevel(logging.INFO) + logger.addHandler(email_logger) + + def send_email(self, success): + import smtplib + from email.mime.text import MIMEText + from email import charset + + for varname in ['smtp_host', 'email_subject', 'email_to', 'email_from']: + if getattr(self, varname) is None: + logging.error('Failed to send email because option "%s" is not set' % varname) + return + + # use quoted-printable instead of the default base64 + charset.add_charset('utf-8', charset.SHORTEST, charset.QP) + if success: + body = 'SnapRAID job completed successfully:\n\n\n' + else: + body = 'Error during SnapRAID job:\n\n\n' + + log = self.email_stream.getvalue() + maxsize = self.email_maxsize * 1024 + if maxsize and len(log) > maxsize: + cut_lines = log.count('\n', maxsize // 2, -maxsize // 2) + log = ( + 'NOTE: Log was too big for email and was shortened\n\n' + + log[:maxsize // 2] + + '[...]\n\n\n --- LOG WAS TOO BIG - {} LINES REMOVED --\n\n\n[...]'.format(cut_lines) + + log[-maxsize // 2:] + ) + body += log + + msg = MIMEText(body, 'plain', 'utf-8') + msg['Subject'] = self.email_subject + (' SUCCESS' if success else ' ERROR') + msg['From'] = self.email_from + msg['To'] = self.email_to + smtp = {'host': self.smtp_host} + if self.smtp_port is not None: + smtp['port'] = self.smtp_port + if self.smtp_ssl: + server = smtplib.SMTP_SSL(**smtp) + else: + server = smtplib.SMTP(**smtp) + if self.smtp_tls: + server.starttls() + if self.smtp_user: + server.login(self.smtp_user, '' if self.smtp_password is None else self.smtp_password) + server.sendmail(self.email_from, [self.email_to], msg.as_string()) + server.quit() + + def run(self): + logging.info('=' * 60) + logging.info('Run started') + logging.info('=' * 60) + + if not os.path.isfile(self.snapraid_exe): + logging.error('The configured snapraid executable "%s" does not ' + 'exist or is not a file' % self.snapraid_exe) + self.finish(False) + if not os.path.isfile(self.snapraid_config): + logging.error('Snapraid config "%s" does not exist or is not a file.' % self.snapraid_config) + self.finish(False) + + if self.snapraid_touch: + logging.info('Running touch...') + self.snapraid_command('touch') + logging.info('*' * 60) + + logging.info('Running diff...') + diff_out = self.snapraid_command('diff', allow_statuscodes=[2]) + logging.info('*' * 60) + + diff_results = Counter(line.split(' ')[0] for line in diff_out) + diff_results = {x: diff_results[x] for x in ['add', 'remove', 'move', 'update']} + logging.info(('Diff results: {add} added, {remove} removed, ' + + '{move} moved, {update} modified').format(**diff_results)) + + if 0 <= self.delete_threshold < diff_results['remove']: + logging.error('Deleted files exceed delete threshold of %d, aborting' + % self.delete_threshold) + logging.error('Run again with --ignore-deletethreshold to sync anyways') + self.finish(False) + + if sum([diff_results[x] for x in ['remove', 'add', 'move', 'update']]) == 0: + logging.info('No changes detected, no sync required') + else: + logging.info('Running sync...') + try: + self.snapraid_command('sync') + except subprocess.CalledProcessError as e: + logging.error(e) + self.finish(False) + logging.info('*' * 60) + + if self.scrub_enabled: + logging.info('Running scrub...') + scrub_args = {} + if self.scrub_plan is not None: + try: + self.scrub_plan = int(self.scrub_plan) # Check if a percentage plan was given + except ValueError: + pass + scrub_args.update({'plan': self.scrub_plan}) + if self.scrub_plan is None or isinstance(self.scrub_plan, int): + scrub_args.update({'older-than': self.scrub_older_than}) + try: + self.snapraid_command('scrub', **scrub_args) + except subprocess.CalledProcessError as e: + logging.error(e) + self.finish(False) + logging.info('*' * 60) + + logging.info('All done') + self.finish(True) + + def finish(self, is_success): + if ('error', 'success')[is_success] in self.email_sendon: + try: + self.send_email(is_success) + except Exception: + logging.exception('Failed to send email') + if is_success: + logging.info('Run finished successfully') + else: + logging.error('Run failed') + sys.exit(0 if is_success else 1) def main(): - parser = argparse.ArgumentParser() - parser.add_argument("-c", "--conf", - default="snapraid-runner.conf", - metavar="CONFIG", - help="Configuration file (default: %(default)s)") - parser.add_argument("--no-scrub", action='store_false', + parser = ArgumentParser() + parser.add_argument('-c', '--conf', + default='snapraid-runner.conf', + dest='config_file', + metavar='CONFIG', + help='Configuration file (default: %(default)s)') + parser.add_argument('--no-scrub', action='store_false', dest='scrub', default=None, - help="Do not scrub (overrides config)") - parser.add_argument("--ignore-deletethreshold", action='store_true', - help="Sync even if configured delete threshold is exceeded") + help='Do not scrub (overrides config)') + parser.add_argument('--ignore-deletethreshold', action='store_true', default=False, + help='Sync even if configured delete threshold is exceeded') + parser.add_argument('--dry-run', action='store_true', default=False, + help='Display commands but do not run them') args = parser.parse_args() - if not os.path.exists(args.conf): - print("snapraid-runner configuration file not found") - parser.print_help() - sys.exit(2) - try: - load_config(args) - except Exception: - print("unexpected exception while loading config") - print(traceback.format_exc()) - sys.exit(2) + runner = SnapraidRunner(**vars(args)) + except Exception as e: + raise Exception('Unexpected exception while loading config.')\ + .with_traceback(e.__traceback__) try: - setup_logger() - except Exception: - print("unexpected exception while setting up logging") - print(traceback.format_exc()) - sys.exit(2) + runner.create_logger() + except Exception as e: + raise Exception('Unexpected exception while setting up logging')\ + .with_traceback(e.__traceback__) try: - run() + runner.run() except Exception: - logging.exception("Run failed due to unexpected exception:") - finish(False) - - -def run(): - logging.info("=" * 60) - logging.info("Run started") - logging.info("=" * 60) - - if not os.path.isfile(config["snapraid"]["executable"]): - logging.error("The configured snapraid executable \"{}\" does not " - "exist or is not a file".format( - config["snapraid"]["executable"])) - finish(False) - if not os.path.isfile(config["snapraid"]["config"]): - logging.error("Snapraid config does not exist at " + - config["snapraid"]["config"]) - finish(False) - - if config["snapraid"]["touch"]: - logging.info("Running touch...") - snapraid_command("touch") - logging.info("*" * 60) - - logging.info("Running diff...") - diff_out = snapraid_command("diff", allow_statuscodes=[2]) - logging.info("*" * 60) - - diff_results = Counter(line.split(" ")[0] for line in diff_out) - diff_results = dict((x, diff_results[x]) for x in - ["add", "remove", "move", "update"]) - logging.info(("Diff results: {add} added, {remove} removed, " + - "{move} moved, {update} modified").format(**diff_results)) - - if (config["snapraid"]["deletethreshold"] >= 0 and - diff_results["remove"] > config["snapraid"]["deletethreshold"]): - logging.error( - "Deleted files exceed delete threshold of {}, aborting".format( - config["snapraid"]["deletethreshold"])) - logging.error("Run again with --ignore-deletethreshold to sync anyways") - finish(False) - - if (diff_results["remove"] + diff_results["add"] + diff_results["move"] + - diff_results["update"] == 0): - logging.info("No changes detected, no sync required") - else: - logging.info("Running sync...") - try: - snapraid_command("sync") - except subprocess.CalledProcessError as e: - logging.error(e) - finish(False) - logging.info("*" * 60) - - if config["scrub"]["enabled"]: - logging.info("Running scrub...") - try: - # Check if a percentage plan was given - int(config["scrub"]["plan"]) - except ValueError: - scrub_args = {"plan": config["scrub"]["plan"]} - else: - scrub_args = { - "plan": config["scrub"]["plan"], - "older-than": config["scrub"]["older-than"], - } - try: - snapraid_command("scrub", scrub_args) - except subprocess.CalledProcessError as e: - logging.error(e) - finish(False) - logging.info("*" * 60) - - logging.info("All done") - finish(True) + logging.exception('Run failed due to unexpected exception:') + runner.finish(False) main() From 7ae69d30bb4146f32d391846a07b6b83e449068c Mon Sep 17 00:00:00 2001 From: Blake Dewey Date: Sun, 30 Jan 2022 15:05:36 -0500 Subject: [PATCH 2/3] update .gitignore for Pycharm/MacOS --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index c0fde79..c1a57e7 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ snapraid-runner.conf +.idea +.DS_Store \ No newline at end of file From 3785738ad221a7e46c11ecb2b36d9f339f1bb001 Mon Sep 17 00:00:00 2001 From: Blake Dewey Date: Sun, 30 Jan 2022 15:24:09 -0500 Subject: [PATCH 3/3] allow for nothing in email_sendon --- snapraid-runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/snapraid-runner.py b/snapraid-runner.py index d49cecf..ce69472 100755 --- a/snapraid-runner.py +++ b/snapraid-runner.py @@ -280,7 +280,8 @@ def run(self): self.finish(True) def finish(self, is_success): - if ('error', 'success')[is_success] in self.email_sendon: + status = ('error', 'success')[is_success] + if self.email_sendon is not None and status in self.email_sendon: try: self.send_email(is_success) except Exception: