From 78350b9ddf3e871e885ad6cf684f3f3f955f2728 Mon Sep 17 00:00:00 2001 From: Graeme Seaton Date: Wed, 1 Feb 2023 15:25:00 +0000 Subject: [PATCH 1/5] Add check for sufficient free space in dst before starting new plot --- src/plotman/manager.py | 183 +++++++++++++++++++++-------------------- 1 file changed, 93 insertions(+), 90 deletions(-) diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 6066249e..99e6d710 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -183,105 +183,108 @@ def key(key: str) -> job.Phase: dstdir = max(dir2ph, key=key) - log_file_path = log_cfg.create_plot_log_path(time=pendulum.now()) - - plot_args: typing.List[str] - if plotting_cfg.type == "bladebit": - if plotting_cfg.bladebit is None: - raise Exception( - "bladebit plotter selected but not configured, report this as a plotman bug", + if plot_util.df_b(dstdir) > plot_util.get_plotsize(32): + log_file_path = log_cfg.create_plot_log_path(time=pendulum.now()) + + plot_args: typing.List[str] + if plotting_cfg.type == "bladebit": + if plotting_cfg.bladebit is None: + raise Exception( + "bladebit plotter selected but not configured, report this as a plotman bug", + ) + plot_args = plotman.plotters.bladebit.create_command_line( + options=plotting_cfg.bladebit, + tmpdir=tmpdir, + tmp2dir=dir_cfg.tmp2, + dstdir=dstdir, + farmer_public_key=plotting_cfg.farmer_pk, + pool_public_key=plotting_cfg.pool_pk, + pool_contract_address=plotting_cfg.pool_contract_address, ) - plot_args = plotman.plotters.bladebit.create_command_line( - options=plotting_cfg.bladebit, - tmpdir=tmpdir, - tmp2dir=dir_cfg.tmp2, - dstdir=dstdir, - farmer_public_key=plotting_cfg.farmer_pk, - pool_public_key=plotting_cfg.pool_pk, - pool_contract_address=plotting_cfg.pool_contract_address, - ) - elif plotting_cfg.type == "madmax": - if plotting_cfg.madmax is None: - raise Exception( - "madmax plotter selected but not configured, report this as a plotman bug", + elif plotting_cfg.type == "madmax": + if plotting_cfg.madmax is None: + raise Exception( + "madmax plotter selected but not configured, report this as a plotman bug", + ) + plot_args = plotman.plotters.madmax.create_command_line( + options=plotting_cfg.madmax, + tmpdir=tmpdir, + tmp2dir=dir_cfg.tmp2, + dstdir=dstdir, + farmer_public_key=plotting_cfg.farmer_pk, + pool_public_key=plotting_cfg.pool_pk, + pool_contract_address=plotting_cfg.pool_contract_address, ) - plot_args = plotman.plotters.madmax.create_command_line( - options=plotting_cfg.madmax, - tmpdir=tmpdir, - tmp2dir=dir_cfg.tmp2, - dstdir=dstdir, - farmer_public_key=plotting_cfg.farmer_pk, - pool_public_key=plotting_cfg.pool_pk, - pool_contract_address=plotting_cfg.pool_contract_address, - ) - else: - if plotting_cfg.chia is None: - raise Exception( - "chia plotter selected but not configured, report this as a plotman bug", + else: + if plotting_cfg.chia is None: + raise Exception( + "chia plotter selected but not configured, report this as a plotman bug", + ) + plot_args = plotman.plotters.chianetwork.create_command_line( + options=plotting_cfg.chia, + tmpdir=tmpdir, + tmp2dir=dir_cfg.tmp2, + dstdir=dstdir, + farmer_public_key=plotting_cfg.farmer_pk, + pool_public_key=plotting_cfg.pool_pk, + pool_contract_address=plotting_cfg.pool_contract_address, ) - plot_args = plotman.plotters.chianetwork.create_command_line( - options=plotting_cfg.chia, - tmpdir=tmpdir, - tmp2dir=dir_cfg.tmp2, - dstdir=dstdir, - farmer_public_key=plotting_cfg.farmer_pk, - pool_public_key=plotting_cfg.pool_pk, - pool_contract_address=plotting_cfg.pool_contract_address, + + logmsg = "Starting plot job: %s ; logging to %s" % ( + " ".join(plot_args), + log_file_path, ) - logmsg = "Starting plot job: %s ; logging to %s" % ( - " ".join(plot_args), - log_file_path, - ) + # TODO: CAMPid 09840103109429840981397487498131 + try: + open_log_file = open(log_file_path, "x") + except FileExistsError: + # The desired log file name already exists. Most likely another + # plotman process already launched a new process in response to + # the same scenario that triggered us. Let's at least not + # confuse things further by having two plotting processes + # logging to the same file. If we really should launch another + # plotting process, we'll get it at the next check cycle anyways. + message = ( + f"Plot log file already exists, skipping attempt to start a" + f" new plot: {log_file_path!r}" + ) + return (False, logmsg) + except FileNotFoundError as e: + message = ( + f"Unable to open log file. Verify that the directory exists" + f" and has proper write permissions: {log_file_path!r}" + ) + raise Exception(message) from e - # TODO: CAMPid 09840103109429840981397487498131 - try: - open_log_file = open(log_file_path, "x") - except FileExistsError: - # The desired log file name already exists. Most likely another - # plotman process already launched a new process in response to - # the same scenario that triggered us. Let's at least not - # confuse things further by having two plotting processes - # logging to the same file. If we really should launch another - # plotting process, we'll get it at the next check cycle anyways. - message = ( - f"Plot log file already exists, skipping attempt to start a" - f" new plot: {log_file_path!r}" - ) - return (False, logmsg) - except FileNotFoundError as e: - message = ( - f"Unable to open log file. Verify that the directory exists" - f" and has proper write permissions: {log_file_path!r}" - ) - raise Exception(message) from e + # Preferably, do not add any code between the try block above + # and the with block below. IOW, this space intentionally left + # blank... As is, this provides a good chance that our handle + # of the log file will get closed explicitly while still + # allowing handling of just the log file opening error. - # Preferably, do not add any code between the try block above - # and the with block below. IOW, this space intentionally left - # blank... As is, this provides a good chance that our handle - # of the log file will get closed explicitly while still - # allowing handling of just the log file opening error. + if sys.platform == "win32": + creationflags = subprocess.CREATE_NO_WINDOW + nice = psutil.BELOW_NORMAL_PRIORITY_CLASS + else: + creationflags = 0 + nice = 15 + + with open_log_file: + # start_new_sessions to make the job independent of this controlling tty (POSIX only). + # subprocess.CREATE_NO_WINDOW to make the process independent of this controlling tty and have no console window on Windows. + p = subprocess.Popen( + plot_args, + stdout=open_log_file, + stderr=subprocess.STDOUT, + start_new_session=True, + creationflags=creationflags, + ) - if sys.platform == "win32": - creationflags = subprocess.CREATE_NO_WINDOW - nice = psutil.BELOW_NORMAL_PRIORITY_CLASS + psutil.Process(p.pid).nice(nice) + return (True, logmsg) else: - creationflags = 0 - nice = 15 - - with open_log_file: - # start_new_sessions to make the job independent of this controlling tty (POSIX only). - # subprocess.CREATE_NO_WINDOW to make the process independent of this controlling tty and have no console window on Windows. - p = subprocess.Popen( - plot_args, - stdout=open_log_file, - stderr=subprocess.STDOUT, - start_new_session=True, - creationflags=creationflags, - ) - - psutil.Process(p.pid).nice(nice) - return (True, logmsg) + wait_reason = "Insufficient space in dst" return (False, wait_reason) From 187dca9b5bf3193cb5342305469c5d4b2b6cd2a4 Mon Sep 17 00:00:00 2001 From: Graeme Seaton Date: Wed, 8 Feb 2023 12:04:14 +0000 Subject: [PATCH 2/5] Alter direction of test and return immediately (avoid indentation of following code) --- src/plotman/manager.py | 188 +++++++++++++++++++++-------------------- 1 file changed, 95 insertions(+), 93 deletions(-) diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 99e6d710..58b23338 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -182,109 +182,111 @@ def key(key: str) -> job.Phase: return dir2ph[key] dstdir = max(dir2ph, key=key) + + # Check available space with a 10G margin for error + if plot_util.df_b(dstdir) < (plot_util.get_plotsize(32) + 10737418240): + wait_reason = "Insufficient space in dst" + return (False, wait_reason) + + log_file_path = log_cfg.create_plot_log_path(time=pendulum.now()) - if plot_util.df_b(dstdir) > plot_util.get_plotsize(32): - log_file_path = log_cfg.create_plot_log_path(time=pendulum.now()) - - plot_args: typing.List[str] - if plotting_cfg.type == "bladebit": - if plotting_cfg.bladebit is None: - raise Exception( - "bladebit plotter selected but not configured, report this as a plotman bug", - ) - plot_args = plotman.plotters.bladebit.create_command_line( - options=plotting_cfg.bladebit, - tmpdir=tmpdir, - tmp2dir=dir_cfg.tmp2, - dstdir=dstdir, - farmer_public_key=plotting_cfg.farmer_pk, - pool_public_key=plotting_cfg.pool_pk, - pool_contract_address=plotting_cfg.pool_contract_address, + plot_args: typing.List[str] + if plotting_cfg.type == "bladebit": + if plotting_cfg.bladebit is None: + raise Exception( + "bladebit plotter selected but not configured, report this as a plotman bug", ) - elif plotting_cfg.type == "madmax": - if plotting_cfg.madmax is None: - raise Exception( - "madmax plotter selected but not configured, report this as a plotman bug", - ) - plot_args = plotman.plotters.madmax.create_command_line( - options=plotting_cfg.madmax, - tmpdir=tmpdir, - tmp2dir=dir_cfg.tmp2, - dstdir=dstdir, - farmer_public_key=plotting_cfg.farmer_pk, - pool_public_key=plotting_cfg.pool_pk, - pool_contract_address=plotting_cfg.pool_contract_address, + plot_args = plotman.plotters.bladebit.create_command_line( + options=plotting_cfg.bladebit, + tmpdir=tmpdir, + tmp2dir=dir_cfg.tmp2, + dstdir=dstdir, + farmer_public_key=plotting_cfg.farmer_pk, + pool_public_key=plotting_cfg.pool_pk, + pool_contract_address=plotting_cfg.pool_contract_address, + ) + elif plotting_cfg.type == "madmax": + if plotting_cfg.madmax is None: + raise Exception( + "madmax plotter selected but not configured, report this as a plotman bug", ) - else: - if plotting_cfg.chia is None: - raise Exception( - "chia plotter selected but not configured, report this as a plotman bug", - ) - plot_args = plotman.plotters.chianetwork.create_command_line( - options=plotting_cfg.chia, - tmpdir=tmpdir, - tmp2dir=dir_cfg.tmp2, - dstdir=dstdir, - farmer_public_key=plotting_cfg.farmer_pk, - pool_public_key=plotting_cfg.pool_pk, - pool_contract_address=plotting_cfg.pool_contract_address, + plot_args = plotman.plotters.madmax.create_command_line( + options=plotting_cfg.madmax, + tmpdir=tmpdir, + tmp2dir=dir_cfg.tmp2, + dstdir=dstdir, + farmer_public_key=plotting_cfg.farmer_pk, + pool_public_key=plotting_cfg.pool_pk, + pool_contract_address=plotting_cfg.pool_contract_address, + ) + else: + if plotting_cfg.chia is None: + raise Exception( + "chia plotter selected but not configured, report this as a plotman bug", ) - - logmsg = "Starting plot job: %s ; logging to %s" % ( - " ".join(plot_args), - log_file_path, + plot_args = plotman.plotters.chianetwork.create_command_line( + options=plotting_cfg.chia, + tmpdir=tmpdir, + tmp2dir=dir_cfg.tmp2, + dstdir=dstdir, + farmer_public_key=plotting_cfg.farmer_pk, + pool_public_key=plotting_cfg.pool_pk, + pool_contract_address=plotting_cfg.pool_contract_address, ) - # TODO: CAMPid 09840103109429840981397487498131 - try: - open_log_file = open(log_file_path, "x") - except FileExistsError: - # The desired log file name already exists. Most likely another - # plotman process already launched a new process in response to - # the same scenario that triggered us. Let's at least not - # confuse things further by having two plotting processes - # logging to the same file. If we really should launch another - # plotting process, we'll get it at the next check cycle anyways. - message = ( - f"Plot log file already exists, skipping attempt to start a" - f" new plot: {log_file_path!r}" - ) - return (False, logmsg) - except FileNotFoundError as e: - message = ( - f"Unable to open log file. Verify that the directory exists" - f" and has proper write permissions: {log_file_path!r}" - ) - raise Exception(message) from e + logmsg = "Starting plot job: %s ; logging to %s" % ( + " ".join(plot_args), + log_file_path, + ) - # Preferably, do not add any code between the try block above - # and the with block below. IOW, this space intentionally left - # blank... As is, this provides a good chance that our handle - # of the log file will get closed explicitly while still - # allowing handling of just the log file opening error. + # TODO: CAMPid 09840103109429840981397487498131 + try: + open_log_file = open(log_file_path, "x") + except FileExistsError: + # The desired log file name already exists. Most likely another + # plotman process already launched a new process in response to + # the same scenario that triggered us. Let's at least not + # confuse things further by having two plotting processes + # logging to the same file. If we really should launch another + # plotting process, we'll get it at the next check cycle anyways. + message = ( + f"Plot log file already exists, skipping attempt to start a" + f" new plot: {log_file_path!r}" + ) + return (False, logmsg) + except FileNotFoundError as e: + message = ( + f"Unable to open log file. Verify that the directory exists" + f" and has proper write permissions: {log_file_path!r}" + ) + raise Exception(message) from e - if sys.platform == "win32": - creationflags = subprocess.CREATE_NO_WINDOW - nice = psutil.BELOW_NORMAL_PRIORITY_CLASS - else: - creationflags = 0 - nice = 15 - - with open_log_file: - # start_new_sessions to make the job independent of this controlling tty (POSIX only). - # subprocess.CREATE_NO_WINDOW to make the process independent of this controlling tty and have no console window on Windows. - p = subprocess.Popen( - plot_args, - stdout=open_log_file, - stderr=subprocess.STDOUT, - start_new_session=True, - creationflags=creationflags, - ) + # Preferably, do not add any code between the try block above + # and the with block below. IOW, this space intentionally left + # blank... As is, this provides a good chance that our handle + # of the log file will get closed explicitly while still + # allowing handling of just the log file opening error. - psutil.Process(p.pid).nice(nice) - return (True, logmsg) + if sys.platform == "win32": + creationflags = subprocess.CREATE_NO_WINDOW + nice = psutil.BELOW_NORMAL_PRIORITY_CLASS else: - wait_reason = "Insufficient space in dst" + creationflags = 0 + nice = 15 + + with open_log_file: + # start_new_sessions to make the job independent of this controlling tty (POSIX only). + # subprocess.CREATE_NO_WINDOW to make the process independent of this controlling tty and have no console window on Windows. + p = subprocess.Popen( + plot_args, + stdout=open_log_file, + stderr=subprocess.STDOUT, + start_new_session=True, + creationflags=creationflags, + ) + + psutil.Process(p.pid).nice(nice) + return (True, logmsg) return (False, wait_reason) From af0ed020a1af92446dca142537766f92bb94ebca Mon Sep 17 00:00:00 2001 From: Graeme Seaton Date: Wed, 8 Feb 2023 14:59:20 +0000 Subject: [PATCH 3/5] Include shared dst/tmp in check --- src/plotman/manager.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 58b23338..18cbc58b 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -183,8 +183,19 @@ def key(key: str) -> job.Phase: dstdir = max(dir2ph, key=key) + required_plot_space = plot_util.get_plotsize(32) + 10737418240 + # If tmp & dst share directory and 'worst case' tmp space required + if plotting_cfg.type == "bladebit": + if dstdir == tmpdir: + required_plot_space = required_plot_space + 64424509440 # 480 GiB + elif plotting_cfg.type == "madmax": + if dstdir == tmpdir: + required_plot_space = required_plot_space + 44291850240 # 330 GiB + elif dstdir == tmpdir: + required_plot_space = required_plot_space + 32078036992 # 239 GiB + # Check available space with a 10G margin for error - if plot_util.df_b(dstdir) < (plot_util.get_plotsize(32) + 10737418240): + if plot_util.df_b(dstdir) < required_plot_space: wait_reason = "Insufficient space in dst" return (False, wait_reason) From 4de528869b432904e97911849e4be466536a1c28 Mon Sep 17 00:00:00 2001 From: graemes Date: Fri, 10 Feb 2023 19:31:50 +0000 Subject: [PATCH 4/5] Update manager.py Fix indentation --- src/plotman/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/manager.py b/src/plotman/manager.py index fca54da9..7b6f3176 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -192,7 +192,7 @@ def key(key: str) -> job.Phase: if dstdir == tmpdir: required_plot_space = required_plot_space + 44291850240 # 330 GiB elif dstdir == tmpdir: - required_plot_space = required_plot_space + 32078036992 # 239 GiB + required_plot_space = required_plot_space + 32078036992 # 239 GiB # Check available space with a 10G margin for error if plot_util.df_b(dstdir) < required_plot_space: From 79e116d027abe79e330b52dd4bc6d5bee3bf0624 Mon Sep 17 00:00:00 2001 From: Graeme Seaton Date: Sat, 11 Feb 2023 07:23:34 +0000 Subject: [PATCH 5/5] Remove extra tabs --- src/plotman/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 7b6f3176..5d3b49b7 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -182,7 +182,7 @@ def key(key: str) -> job.Phase: return dir2ph[key] dstdir = max(dir2ph, key=key) - + required_plot_space = plot_util.get_plotsize(32) + 10737418240 # If tmp & dst share directory and 'worst case' tmp space required if plotting_cfg.type == "bladebit":