From c453d38ba9f43bd6e091b85ec4b08c801a0db75d Mon Sep 17 00:00:00 2001 From: tcinbis Date: Tue, 18 May 2021 08:44:59 +0200 Subject: [PATCH 1/4] Adding json flag to status command This allows for printing the status report in a more machine readable json format. --- src/plotman/job.py | 21 +++++++++++++++++++++ src/plotman/plotman.py | 18 ++++++++++++------ src/plotman/reporting.py | 11 ++++++++++- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index 6f19c755..66377917 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -19,6 +19,7 @@ import psutil from plotman import chia +from plotman.reporting import phase_str def job_phases_for_tmpdir(d: str, all_jobs: typing.List["Job"]) -> typing.List["Phase"]: @@ -407,6 +408,26 @@ def status_str_long(self) -> str: dst = self.dstdir, logfile = self.logfile ) + + def to_dict(self): + '''Exports important information as dictionary.''' + # TODO: Check if being called in oneshot context to improve performance + return dict( + plot_id=self.plot_id[:8], + k=self.k, + tmp_dir=self.tmpdir, + dst_dir=self.dstdir, + progress=phase_str(self.progress()), + tmp_usage=self.get_tmp_usage(), + pid=self.proc.pid, + run_status=self.get_run_status(), + mem_usage=self.get_mem_usage(), + time_wall=self.get_time_wall(), + time_user=self.get_time_user(), + time_sys=self.get_time_sys(), + time_iowait=self.get_time_iowait() + ) + def get_mem_usage(self) -> int: # Total, inc swapped diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py index 04863491..a844905a 100755 --- a/src/plotman/plotman.py +++ b/src/plotman/plotman.py @@ -33,7 +33,9 @@ def parse_args(self) -> typing.Any: sp.add_parser('version', help='print the version') - sp.add_parser('status', help='show current plotting status') + p_status = sp.add_parser('status', help='show current plotting status') + p_status.add_argument("--json", action="store_true", + help="export status report in json format") sp.add_parser('dirs', help='show directories info') @@ -210,11 +212,15 @@ def main() -> None: # Status report if args.cmd == 'status': - result = "{0}\n\n{1}\n\nUpdated at: {2}".format( - reporting.status_report(jobs, get_term_width()), - reporting.summary(jobs), - datetime.datetime.today().strftime("%c"), - ) + if args.json: + # convert jobs list into json + result = reporting.json_report(jobs) + else: + result = "{0}\n\n{1}\n\nUpdated at: {2}".format( + reporting.status_report(jobs, get_term_width()), + reporting.summary(jobs), + datetime.datetime.today().strftime("%c"), + ) print(result) # Directories report diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 20ce8940..3cd5cb53 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -1,3 +1,4 @@ +import json import math import os import typing @@ -5,7 +6,7 @@ import psutil import texttable as tt # from somewhere? from itertools import groupby - +from collections import defaultdict from plotman import archive, configuration, job, manager, plot_util @@ -222,3 +223,11 @@ def dirs_report(jobs: typing.List[job.Job], dir_cfg: configuration.Directories, ]) return '\n'.join(reports) + '\n' + +def json_report(jobs): + jobs_dict = defaultdict(list) + for _, j in enumerate(sorted(jobs, key=job.Job.get_time_wall)): + with j.proc.oneshot(): + jobs_dict["jobs"].append(j.to_dict()) + return json.dumps(jobs_dict) + From 808666afdfc6ae3f80a23b6d78cfc8bdab9b2135 Mon Sep 17 00:00:00 2001 From: tcinbis Date: Tue, 18 May 2021 09:11:56 +0200 Subject: [PATCH 2/4] Adding latest status fields to json status --- src/plotman/reporting.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 3cd5cb53..2f4c7655 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -1,3 +1,4 @@ +import time import json import math import os @@ -229,5 +230,7 @@ def json_report(jobs): for _, j in enumerate(sorted(jobs, key=job.Job.get_time_wall)): with j.proc.oneshot(): jobs_dict["jobs"].append(j.to_dict()) + jobs_dict["total_jobs"] = len(jobs) + jobs_dict["updated"] = time.time() return json.dumps(jobs_dict) From 0d6412b4ee194f9d4cda3df139837ca91bbd196d Mon Sep 17 00:00:00 2001 From: tcinbis Date: Mon, 14 Jun 2021 10:08:34 +0200 Subject: [PATCH 3/4] Moving phase_str into Phase class as __str__ --- src/plotman/job.py | 9 ++++++--- src/plotman/reporting.py | 14 ++++---------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index 66377917..9d987095 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -19,8 +19,6 @@ import psutil from plotman import chia -from plotman.reporting import phase_str - def job_phases_for_tmpdir(d: str, all_jobs: typing.List["Job"]) -> typing.List["Phase"]: '''Return phase 2-tuples for jobs running on tmpdir d''' @@ -133,6 +131,11 @@ def list_from_tuples( ) -> typing.List["Phase"]: return [cls.from_tuple(t) for t in l] + def __str__(self): + if not self.known: + return '?:?' + return f'{self.major}:{self.minor}' + # TODO: be more principled and explicit about what we cache vs. what we look up # dynamically from the logfile class Job: @@ -417,7 +420,7 @@ def to_dict(self): k=self.k, tmp_dir=self.tmpdir, dst_dir=self.dstdir, - progress=phase_str(self.progress()), + progress=str(self.progress()), tmp_usage=self.get_tmp_usage(), pid=self.proc.pid, run_status=self.get_run_status(), diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 2f4c7655..b668d7ff 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -17,23 +17,17 @@ def abbr_path(path: str, putative_prefix: str) -> str: else: return path -def phase_str(phase: job.Phase) -> str: - if not phase.known: - return '?:?' - - return f'{phase.major}:{phase.minor}' - def phases_str(phases: typing.List[job.Phase], max_num: typing.Optional[int] = None) -> str: '''Take a list of phase-subphase pairs and return them as a compact string''' if not max_num or len(phases) <= max_num: - return ' '.join([phase_str(pair) for pair in phases]) + return ' '.join([str(pair) for pair in phases]) else: n_first = math.floor(max_num / 2) n_last = max_num - n_first n_elided = len(phases) - (n_first + n_last) - first = ' '.join([phase_str(pair) for pair in phases[:n_first]]) + first = ' '.join([str(pair) for pair in phases[:n_first]]) elided = " [+%d] " % n_elided - last = ' '.join([phase_str(pair) for pair in phases[n_first + n_elided:]]) + last = ' '.join([str(pair) for pair in phases[n_first + n_elided:]]) return first + elided + last def n_at_ph(jobs: typing.List[job.Job], ph: job.Phase) -> int: @@ -108,7 +102,7 @@ def status_report(jobs: typing.List[job.Job], width: int, height: typing.Optiona abbr_path(j.tmpdir, tmp_prefix), # Temp directory abbr_path(j.dstdir, dst_prefix), # Destination directory plot_util.time_format(j.get_time_wall()), # Time wall - phase_str(j.progress()), # Overall progress (major:minor) + str(j.progress()), # Overall progress (major:minor) plot_util.human_format(j.get_tmp_usage(), 0), # Current temp file size j.proc.pid, # System pid j.get_run_status(), # OS status for the job process From 873f8b0a8b6a9d68d1cd6c6bfaa5e6ad6271907b Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 14 Jun 2021 20:56:40 -0400 Subject: [PATCH 4/4] changelog, type hints, and tweaks --- CHANGELOG.md | 1 + src/plotman/job.py | 6 +++--- src/plotman/reporting.py | 20 ++++++++++++-------- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5979d90a..4fb4b02a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] ### Added - `plotman export` command to output summaries from plot logs in `.csv` format. ([#557](https://github.com/ericaltendorf/plotman/pull/557)) +- `--json` option for `plotman status`. ([#549](https://github.com/ericaltendorf/plotman/pull/549)) ## [0.4.1] - 2021-06-11 ### Fixed diff --git a/src/plotman/job.py b/src/plotman/job.py index 9d987095..b5c4a838 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -20,6 +20,7 @@ from plotman import chia + def job_phases_for_tmpdir(d: str, all_jobs: typing.List["Job"]) -> typing.List["Phase"]: '''Return phase 2-tuples for jobs running on tmpdir d''' return sorted([j.progress() for j in all_jobs if j.tmpdir == d]) @@ -131,7 +132,7 @@ def list_from_tuples( ) -> typing.List["Phase"]: return [cls.from_tuple(t) for t in l] - def __str__(self): + def __str__(self) -> str: if not self.known: return '?:?' return f'{self.major}:{self.minor}' @@ -412,9 +413,8 @@ def status_str_long(self) -> str: logfile = self.logfile ) - def to_dict(self): + def to_dict(self) -> typing.Dict[str, object]: '''Exports important information as dictionary.''' - # TODO: Check if being called in oneshot context to improve performance return dict( plot_id=self.plot_id[:8], k=self.k, diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index b668d7ff..3e6e0bfd 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -7,7 +7,6 @@ import psutil import texttable as tt # from somewhere? from itertools import groupby -from collections import defaultdict from plotman import archive, configuration, job, manager, plot_util @@ -219,12 +218,17 @@ def dirs_report(jobs: typing.List[job.Job], dir_cfg: configuration.Directories, return '\n'.join(reports) + '\n' -def json_report(jobs): - jobs_dict = defaultdict(list) - for _, j in enumerate(sorted(jobs, key=job.Job.get_time_wall)): +def json_report(jobs: typing.List[job.Job]) -> str: + jobs_dicts = [] + for j in sorted(jobs, key=job.Job.get_time_wall): with j.proc.oneshot(): - jobs_dict["jobs"].append(j.to_dict()) - jobs_dict["total_jobs"] = len(jobs) - jobs_dict["updated"] = time.time() - return json.dumps(jobs_dict) + jobs_dicts.append(j.to_dict()) + + stuff = { + "jobs": jobs_dicts, + "total_jobs": len(jobs), + "updated": time.time(), + } + + return json.dumps(stuff)