From 778783db98c010c545de3fee83fb5379648ff64e Mon Sep 17 00:00:00 2001 From: Cristian Ruiz Date: Fri, 15 Aug 2025 13:19:59 +0200 Subject: [PATCH 1/4] Add Python packaging --- .../Screenshot HPC 2024-08-20.png | Bin .../example_output_workloadManager.tsv | 0 .../example_sacctOutput_raw.txt | 0 .../example_sacctOutput_raw_asDF.tsv | 0 .../__init__.py | 22 +++++++++------- .../backend}/__init__.py | 6 ++--- .../backend}/helpers.py | 0 .../backend}/slurm_extract.py | 0 .../data}/cluster_info.yaml | 0 .../data}/fixed_parameters.yaml | 0 .../frontend}/__init__.py | 4 +-- .../frontend}/dashboard_output.py | 2 +- .../frontend}/helpers.py | 0 .../frontend}/templates/_user.html | 0 .../frontend}/templates/report_blank.html | 0 .../frontend}/templates/styles.css | 0 .../frontend}/terminal_output.py | 2 +- pyproject.toml | 24 ++++++++++++++++++ 18 files changed, 44 insertions(+), 16 deletions(-) rename {example_files => examples}/Screenshot HPC 2024-08-20.png (100%) rename {example_files => examples}/example_output_workloadManager.tsv (100%) rename {example_files => examples}/example_sacctOutput_raw.txt (100%) rename {example_files => examples}/example_sacctOutput_raw_asDF.tsv (100%) rename __init__.py => greenalgorithms4HPC/__init__.py (92%) rename {backend => greenalgorithms4HPC/backend}/__init__.py (97%) rename {backend => greenalgorithms4HPC/backend}/helpers.py (100%) rename {backend => greenalgorithms4HPC/backend}/slurm_extract.py (100%) rename {data => greenalgorithms4HPC/data}/cluster_info.yaml (100%) rename {data => greenalgorithms4HPC/data}/fixed_parameters.yaml (100%) rename {frontend => greenalgorithms4HPC/frontend}/__init__.py (93%) rename {frontend => greenalgorithms4HPC/frontend}/dashboard_output.py (98%) rename {frontend => greenalgorithms4HPC/frontend}/helpers.py (100%) rename {frontend => greenalgorithms4HPC/frontend}/templates/_user.html (100%) rename {frontend => greenalgorithms4HPC/frontend}/templates/report_blank.html (100%) rename {frontend => greenalgorithms4HPC/frontend}/templates/styles.css (100%) rename {frontend => greenalgorithms4HPC/frontend}/terminal_output.py (98%) create mode 100644 pyproject.toml diff --git a/example_files/Screenshot HPC 2024-08-20.png b/examples/Screenshot HPC 2024-08-20.png similarity index 100% rename from example_files/Screenshot HPC 2024-08-20.png rename to examples/Screenshot HPC 2024-08-20.png diff --git a/example_files/example_output_workloadManager.tsv b/examples/example_output_workloadManager.tsv similarity index 100% rename from example_files/example_output_workloadManager.tsv rename to examples/example_output_workloadManager.tsv diff --git a/example_files/example_sacctOutput_raw.txt b/examples/example_sacctOutput_raw.txt similarity index 100% rename from example_files/example_sacctOutput_raw.txt rename to examples/example_sacctOutput_raw.txt diff --git a/example_files/example_sacctOutput_raw_asDF.tsv b/examples/example_sacctOutput_raw_asDF.tsv similarity index 100% rename from example_files/example_sacctOutput_raw_asDF.tsv rename to examples/example_sacctOutput_raw_asDF.tsv diff --git a/__init__.py b/greenalgorithms4HPC/__init__.py similarity index 92% rename from __init__.py rename to greenalgorithms4HPC/__init__.py index ff4e03d..f3cf9da 100644 --- a/__init__.py +++ b/greenalgorithms4HPC/__init__.py @@ -3,9 +3,10 @@ import datetime import os -from backend import main_backend -from frontend import main_frontend +from greenalgorithms4HPC.backend import main_backend +from greenalgorithms4HPC.frontend import main_frontend +SYSTEM_CONF="/etc/GreenAlgorithms4HPC.conf" def create_arguments(): """ Command line arguments for the tool. @@ -108,17 +109,20 @@ def all(self, args): self._validate_dates(args) self._validate_output(args) -if __name__ == "__main__": +def main(): # print("Working dir0: ", os.getcwd()) # DEBUGONLY - args = create_arguments() ## Decide which infrastructure info to use - if args.useOtherInfrastuctureInfo != '': - args.path_infrastucture_info = args.useOtherInfrastuctureInfo - print(f"Overriding infrastructure info with: {args.path_infrastucture_info}") + # Load cluster specific info + cluster_config = os.environ.get("GA4HPC_CONF", None) + + if not cluster_config and os.path.exists(SYSTEM_CONF): + cluster_config = SYSTEM_CONF else: - args.path_infrastucture_info = 'data' + cluster_config = os.path.join(os.path.dirname(__file__), 'data') + + args.path_infrastucture_info = cluster_config ## Organise the unique output directory (used for output report and logs export for debugging) ## creating a uniquely named subdirectory in whatever @@ -149,5 +153,5 @@ def all(self, args): ### Run backend to get data extracted_data = main_backend(args) + main_frontend(extracted_data, args) - main_frontend(extracted_data, args) \ No newline at end of file diff --git a/backend/__init__.py b/greenalgorithms4HPC/backend/__init__.py similarity index 97% rename from backend/__init__.py rename to greenalgorithms4HPC/backend/__init__.py index feb6dcd..c03da31 100644 --- a/backend/__init__.py +++ b/greenalgorithms4HPC/backend/__init__.py @@ -4,8 +4,8 @@ import pandas as pd import numpy as np -from backend.helpers import check_empty_results, simulate_mock_jobs -from backend.slurm_extract import WorkloadManager +from greenalgorithms4HPC.backend.helpers import check_empty_results, simulate_mock_jobs +from greenalgorithms4HPC.backend.slurm_extract import WorkloadManager # print("Working dir1: ", os.getcwd()) # DEBUGONLY @@ -220,7 +220,7 @@ def main_backend(args): print(exc) ### Load fixed parameters - with open("data/fixed_parameters.yaml", "r") as stream: + with open(os.path.join(args.path_infrastucture_info,"fixed_parameters.yaml"), "r") as stream: try: fParams = yaml.safe_load(stream) except yaml.YAMLError as exc: diff --git a/backend/helpers.py b/greenalgorithms4HPC/backend/helpers.py similarity index 100% rename from backend/helpers.py rename to greenalgorithms4HPC/backend/helpers.py diff --git a/backend/slurm_extract.py b/greenalgorithms4HPC/backend/slurm_extract.py similarity index 100% rename from backend/slurm_extract.py rename to greenalgorithms4HPC/backend/slurm_extract.py diff --git a/data/cluster_info.yaml b/greenalgorithms4HPC/data/cluster_info.yaml similarity index 100% rename from data/cluster_info.yaml rename to greenalgorithms4HPC/data/cluster_info.yaml diff --git a/data/fixed_parameters.yaml b/greenalgorithms4HPC/data/fixed_parameters.yaml similarity index 100% rename from data/fixed_parameters.yaml rename to greenalgorithms4HPC/data/fixed_parameters.yaml diff --git a/frontend/__init__.py b/greenalgorithms4HPC/frontend/__init__.py similarity index 93% rename from frontend/__init__.py rename to greenalgorithms4HPC/frontend/__init__.py index ab81868..72ecfe3 100644 --- a/frontend/__init__.py +++ b/greenalgorithms4HPC/frontend/__init__.py @@ -2,8 +2,8 @@ import yaml import os -from frontend.terminal_output import generate_terminal_view -from frontend.dashboard_output import dashboard_html +from greenalgorithms4HPC.frontend.terminal_output import generate_terminal_view +from greenalgorithms4HPC.frontend.dashboard_output import dashboard_html def main_frontend(dict_stats, args): ### Load cluster specific info diff --git a/frontend/dashboard_output.py b/greenalgorithms4HPC/frontend/dashboard_output.py similarity index 98% rename from frontend/dashboard_output.py rename to greenalgorithms4HPC/frontend/dashboard_output.py index 0e2f966..f7984d2 100644 --- a/frontend/dashboard_output.py +++ b/greenalgorithms4HPC/frontend/dashboard_output.py @@ -8,7 +8,7 @@ import numpy as np import plotly.express as px -from frontend.helpers import formatText_footprint, formatText_treemonths, formatText_flying +from greenalgorithms4HPC.frontend.helpers import formatText_footprint, formatText_treemonths, formatText_flying # class SilentUndefined(Undefined): # DEBUGONLY # def _fail_with_undefined_error(self, *args, **kwargs): diff --git a/frontend/helpers.py b/greenalgorithms4HPC/frontend/helpers.py similarity index 100% rename from frontend/helpers.py rename to greenalgorithms4HPC/frontend/helpers.py diff --git a/frontend/templates/_user.html b/greenalgorithms4HPC/frontend/templates/_user.html similarity index 100% rename from frontend/templates/_user.html rename to greenalgorithms4HPC/frontend/templates/_user.html diff --git a/frontend/templates/report_blank.html b/greenalgorithms4HPC/frontend/templates/report_blank.html similarity index 100% rename from frontend/templates/report_blank.html rename to greenalgorithms4HPC/frontend/templates/report_blank.html diff --git a/frontend/templates/styles.css b/greenalgorithms4HPC/frontend/templates/styles.css similarity index 100% rename from frontend/templates/styles.css rename to greenalgorithms4HPC/frontend/templates/styles.css diff --git a/frontend/terminal_output.py b/greenalgorithms4HPC/frontend/terminal_output.py similarity index 98% rename from frontend/terminal_output.py rename to greenalgorithms4HPC/frontend/terminal_output.py index e798673..92c5f22 100644 --- a/frontend/terminal_output.py +++ b/greenalgorithms4HPC/frontend/terminal_output.py @@ -1,6 +1,6 @@ import math -from frontend.helpers import formatText_footprint, formatText_treemonths, formatText_flying +from greenalgorithms4HPC.frontend.helpers import formatText_footprint, formatText_treemonths, formatText_flying import pandas as pd import os diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2284f3e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,24 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "greenalgorithms4HPC" +version = "0.4" +dependencies = [ + "numpy","pandas>=2.0", "PyYAML", "jinja2", "plotly" +] +requires-python = ">=3.6" +authors = [ + {name = " Loïc Lannelongue", email = "loic.lannelongue@posteo.net"}, +] +maintainers = [ + {name = " Loïc Lannelongue", email = "loic.lannelongue@posteo.net"}, +] +description = "Green Algorithms for High Performance Computing" + +[project.scripts] +myCarbonFootprint = "greenalgorithms4HPC:main" + +[tool.setuptools.package-data] +"*" = ["*.yaml"] \ No newline at end of file From 794bc710ba21d49c8132e4cdde927d51f3cac742 Mon Sep 17 00:00:00 2001 From: Cristian Ruiz Date: Sun, 17 Aug 2025 13:28:50 +0200 Subject: [PATCH 2/4] Adapt main bash entry point --- myCarbonFootprint.sh | 14 +++++++------- requirements.txt | 5 ----- 2 files changed, 7 insertions(+), 12 deletions(-) delete mode 100644 requirements.txt diff --git a/myCarbonFootprint.sh b/myCarbonFootprint.sh index bc2e45f..37c461d 100755 --- a/myCarbonFootprint.sh +++ b/myCarbonFootprint.sh @@ -11,16 +11,16 @@ userCWD="$(pwd)" # Cd into the directory where the GA files are located parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) cd "$parent_path" - +export GA4HPC_CONFDIR=$parent_path/greenalgorithms4HPC/data # Test if the virtualenv GA_env already exists, and if not, creates it. Download python 3.8 or higher for better results. -if [ ! -f GA_env/bin/activate ]; then +if [ ! -f .venv/bin/activate ]; then echo "Need to create virtualenv" - /usr/bin/python3.8 -m venv GA_env # this line needs updating to load python on your server - source GA_env/bin/activate - pip3 install -r requirements.txt + /usr/bin/python3.8 -m venv .venv # this line needs updating to load python on your server + source .venv/bin/activate + pip3 install . else echo "Virtualenv: OK" - source GA_env/bin/activate + source .venv/bin/activate fi # Test if the python version is at least 3.8 @@ -40,4 +40,4 @@ echo "Python versions: OK" # Run the python code and pass on the arguments #userCWD="/home/ll582/ with space" # DEBUGONLY -python __init__.py "$@" --userCWD "$userCWD" +myCarbonFootprint "$@" --userCWD "$userCWD" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 3a9207f..0000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -numpy==1.24 -pandas==2.0 -PyYAML==6.0 -jinja2==3.1 -plotly==5.18 \ No newline at end of file From 1bad5f03e50dc3a95a6f3597ae44442c698c1bcb Mon Sep 17 00:00:00 2001 From: Cristian Ruiz Date: Sun, 17 Aug 2025 15:05:38 +0200 Subject: [PATCH 3/4] Update doc --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4f9a3a6..7edb1d8 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ By default, it's under `GreenAlgorithms4HPC/outputs/`, but this can be changed. ## Full list of options ``` -usage: __init__.py [-h] [-S STARTDAY] [-E ENDDAY] [-o OUTPUT] [--outputDir OUTPUTDIR] [--filterCWD] [--filterJobIDs FILTERJOBIDS] [--filterAccount FILTERACCOUNT] [--customSuccessStates CUSTOMSUCCESSSTATES] +usage: myCarbonFootprint [-h] [-S STARTDAY] [-E ENDDAY] [-o OUTPUT] [--outputDir OUTPUTDIR] [--filterCWD] [--filterJobIDs FILTERJOBIDS] [--filterAccount FILTERACCOUNT] [--customSuccessStates CUSTOMSUCCESSSTATES] [--reportBug | --reportBugHere] [--useCustomLogs USECUSTOMLOGS] Calculate your carbon footprint on the server. @@ -105,12 +105,12 @@ optional arguments: 2. Edit `myCarbonFootprint.sh` line 20 to create the virtual environment with Python 3.8 or later. The default line is: ```bash - /usr/bin/python3.8 -m venv GA_env + /usr/bin/python3.8 -m venv .venv ``` But it may be something else on your server, for example: ```bash module load python/3.11.7 - python -m venv GA_env + python -m venv .venv ``` 3. Make the bash script executable: From c586fcdab499c6a6feecded5df2b51ee8898892f Mon Sep 17 00:00:00 2001 From: Cristian Ruiz Date: Fri, 15 Aug 2025 14:27:21 +0200 Subject: [PATCH 4/4] Fix data path --- greenalgorithms4HPC/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/greenalgorithms4HPC/__init__.py b/greenalgorithms4HPC/__init__.py index f3cf9da..1ca3c9b 100644 --- a/greenalgorithms4HPC/__init__.py +++ b/greenalgorithms4HPC/__init__.py @@ -115,12 +115,13 @@ def main(): ## Decide which infrastructure info to use # Load cluster specific info - cluster_config = os.environ.get("GA4HPC_CONF", None) + cluster_config = os.environ.get("GA4HPC_CONFDIR", None) - if not cluster_config and os.path.exists(SYSTEM_CONF): - cluster_config = SYSTEM_CONF - else: - cluster_config = os.path.join(os.path.dirname(__file__), 'data') + if not cluster_config: + if os.path.exists(SYSTEM_CONF): + cluster_config = SYSTEM_CONF + else: + cluster_config = os.path.join(os.path.dirname(__file__), 'data') args.path_infrastucture_info = cluster_config