From 2dbe8fc541f621053241fb3a6224e087a1ba3978 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Thu, 5 Feb 2026 13:32:39 -0500 Subject: [PATCH 1/9] update pixi.toml with explicit addition of python-dotenv * already included in lock file, so no update necessary --- pixi.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pixi.toml b/pixi.toml index 50e3351..6438d0d 100644 --- a/pixi.toml +++ b/pixi.toml @@ -8,6 +8,7 @@ prefect = "3.*" python = "<3.14" tiled-client = ">=0.2.3" bluesky-tiled-plugins = ">=2" +python-dotenv = ">=1.2.1,<2" [pypi-dependencies] lixtools = "==2023.1.23.0" From 52c46fcdf4775ab17b206181ec029bf76d335722 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Thu, 5 Feb 2026 13:33:25 -0500 Subject: [PATCH 2/9] update code to use dotenv * read TILED_API_KEY in from env.secrets --- data_validation.py | 9 +++++++-- utils.py | 12 ++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/data_validation.py b/data_validation.py index 60ad578..7da4597 100644 --- a/data_validation.py +++ b/data_validation.py @@ -1,5 +1,7 @@ +import os + +from dotenv import load_dotenv from prefect import task, flow, get_run_logger -from prefect.blocks.system import Secret import time as ttime from tiled.client import from_profile @@ -7,7 +9,10 @@ @task(retries=2, retry_delay_seconds=10) def read_all_streams(uid, beamline_acronym): logger = get_run_logger() - api_key = Secret.load("tiled-tst-api-key").get() + with open("/srv/env.secrets", "r") as secrets: + load_dotenv(stream=secrets) + api_key = os.environ["TILED_API_KEY"] + logger.info(f"first 4 characters of key: {api_key[:4]}") cl = from_profile("nsls2", api_key=api_key) run = cl["tst"]["raw"][uid] logger.info(f"Validating uid {run.start['uid']}") diff --git a/utils.py b/utils.py index 1cd0072..cbd4935 100644 --- a/utils.py +++ b/utils.py @@ -1,5 +1,6 @@ +from dotenv import load_dotenv +from prefect import get_run_logger from tiled.client import from_profile -from prefect.blocks.system import Secret import os @@ -7,7 +8,10 @@ def get_tiled_client(): - os.environ["TILED_API_KEY"] = Secret.load(f"tiled-{LOCATION}-api-key").get() - tiled_client = from_profile("nsls2")[LOCATION] - os.environ.pop("TILED_API_KEY") + logger = get_run_logger() + with open("/srv/env.secrets", "r") as secrets: + load_dotenv(stream=secrets) + api_key = os.environ["TILED_API_KEY"] + logger.info(f"first 4 characters of key: {api_key:4}") + tiled_client = from_profile("nsls2", api_key=api_key)[LOCATION] return tiled_client From 1f6cb2094241016a46713b8f29fd79453e9bb74a Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Thu, 5 Feb 2026 13:34:01 -0500 Subject: [PATCH 3/9] add env.secrets file into container --- prefect.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/prefect.yaml b/prefect.yaml index 065a601..79412b5 100644 --- a/prefect.yaml +++ b/prefect.yaml @@ -33,5 +33,6 @@ deployments: volumes: - /nsls2/data/tst/proposals:/nsls2/data/tst/proposals - /nsls2/software/etc/tiled:/nsls2/software/etc/tiled + - /srv/prefect3-docker-worker-tst/env.secret:/srv/env.secret auto_remove: true name: tst-work-pool-docker From 95ad0462907dc27277601d920c34cd7918029732 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Thu, 5 Feb 2026 13:41:46 -0500 Subject: [PATCH 4/9] build container, use update-dotenv code for deployment --- .github/workflows/publish-ghcr.yml | 2 +- prefect.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-ghcr.yml b/.github/workflows/publish-ghcr.yml index 45e4f05..110ad75 100644 --- a/.github/workflows/publish-ghcr.yml +++ b/.github/workflows/publish-ghcr.yml @@ -4,7 +4,7 @@ name: Create and publish a Docker image # Configures this workflow to run every time a change is pushed to the branch called `release`. on: push: - branches: ["main"] + branches: ["update-dotenv"] # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: diff --git a/prefect.yaml b/prefect.yaml index 79412b5..a733b21 100644 --- a/prefect.yaml +++ b/prefect.yaml @@ -10,7 +10,7 @@ pull: directory: /repo - prefect.deployments.steps.git_clone: repository: https://github.com/nsls2/tst-workflows.git - branch: main + branch: update-dotenv deployments: - name: tst-end-of-run-workflow-docker @@ -26,7 +26,7 @@ deployments: job_variables: env: TILED_SITE_PROFILES: /nsls2/software/etc/tiled/profiles - image: ghcr.io/nsls2/tst-workflows:main + image: ghcr.io/nsls2/tst-workflows:update-dotenv image_pull_policy: Always network_mode: slirp4netns userns: "keep-id:uid=402974,gid=402974" # workflow-tst:workflow-tst From ff41adb50098223d53b9036e1de63084fa2e311a Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Thu, 5 Feb 2026 13:59:24 -0500 Subject: [PATCH 5/9] update deployment version --- prefect.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prefect.yaml b/prefect.yaml index a733b21..9f2b159 100644 --- a/prefect.yaml +++ b/prefect.yaml @@ -14,7 +14,7 @@ pull: deployments: - name: tst-end-of-run-workflow-docker - version: 0.1.1 + version: 0.1.2 tags: - tst - main From 1e0fd69223007a6a4936e99d9c34eceec2d840f2 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Thu, 5 Feb 2026 14:11:49 -0500 Subject: [PATCH 6/9] fix filename --- data_validation.py | 2 +- prefect.yaml | 2 +- utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/data_validation.py b/data_validation.py index 7da4597..c8fb636 100644 --- a/data_validation.py +++ b/data_validation.py @@ -9,7 +9,7 @@ @task(retries=2, retry_delay_seconds=10) def read_all_streams(uid, beamline_acronym): logger = get_run_logger() - with open("/srv/env.secrets", "r") as secrets: + with open("/srv/env.secret", "r") as secrets: load_dotenv(stream=secrets) api_key = os.environ["TILED_API_KEY"] logger.info(f"first 4 characters of key: {api_key[:4]}") diff --git a/prefect.yaml b/prefect.yaml index 9f2b159..70a420d 100644 --- a/prefect.yaml +++ b/prefect.yaml @@ -33,6 +33,6 @@ deployments: volumes: - /nsls2/data/tst/proposals:/nsls2/data/tst/proposals - /nsls2/software/etc/tiled:/nsls2/software/etc/tiled - - /srv/prefect3-docker-worker-tst/env.secret:/srv/env.secret + - /srv/prefect3-docker-worker-tst:/srv auto_remove: true name: tst-work-pool-docker diff --git a/utils.py b/utils.py index cbd4935..71209bf 100644 --- a/utils.py +++ b/utils.py @@ -9,7 +9,7 @@ def get_tiled_client(): logger = get_run_logger() - with open("/srv/env.secrets", "r") as secrets: + with open("/srv/env.secret", "r") as secrets: load_dotenv(stream=secrets) api_key = os.environ["TILED_API_KEY"] logger.info(f"first 4 characters of key: {api_key:4}") From 706b6eb416944b193233487b028375f5140b30dd Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Thu, 5 Feb 2026 14:42:35 -0500 Subject: [PATCH 7/9] update location of Tiled secret * corresponding changes in prefect3_worker role --- data_validation.py | 2 +- prefect.yaml | 2 +- utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/data_validation.py b/data_validation.py index c8fb636..2a62240 100644 --- a/data_validation.py +++ b/data_validation.py @@ -9,7 +9,7 @@ @task(retries=2, retry_delay_seconds=10) def read_all_streams(uid, beamline_acronym): logger = get_run_logger() - with open("/srv/env.secret", "r") as secrets: + with open("/srv/tiled.secret", "r") as secrets: load_dotenv(stream=secrets) api_key = os.environ["TILED_API_KEY"] logger.info(f"first 4 characters of key: {api_key[:4]}") diff --git a/prefect.yaml b/prefect.yaml index 70a420d..25a6220 100644 --- a/prefect.yaml +++ b/prefect.yaml @@ -33,6 +33,6 @@ deployments: volumes: - /nsls2/data/tst/proposals:/nsls2/data/tst/proposals - /nsls2/software/etc/tiled:/nsls2/software/etc/tiled - - /srv/prefect3-docker-worker-tst:/srv + - /srv/prefect3-docker-worker-tst/tiled:/srv auto_remove: true name: tst-work-pool-docker diff --git a/utils.py b/utils.py index 71209bf..2b477ea 100644 --- a/utils.py +++ b/utils.py @@ -9,7 +9,7 @@ def get_tiled_client(): logger = get_run_logger() - with open("/srv/env.secret", "r") as secrets: + with open("/srv/tiled.secret", "r") as secrets: load_dotenv(stream=secrets) api_key = os.environ["TILED_API_KEY"] logger.info(f"first 4 characters of key: {api_key:4}") From ed405f32158fcade504ed4ab4fa19e7c41e0d49a Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Thu, 5 Feb 2026 15:19:37 -0500 Subject: [PATCH 8/9] remove logging (testing dotenv mechanism) --- data_validation.py | 1 - utils.py | 1 - 2 files changed, 2 deletions(-) diff --git a/data_validation.py b/data_validation.py index 2a62240..186d369 100644 --- a/data_validation.py +++ b/data_validation.py @@ -12,7 +12,6 @@ def read_all_streams(uid, beamline_acronym): with open("/srv/tiled.secret", "r") as secrets: load_dotenv(stream=secrets) api_key = os.environ["TILED_API_KEY"] - logger.info(f"first 4 characters of key: {api_key[:4]}") cl = from_profile("nsls2", api_key=api_key) run = cl["tst"]["raw"][uid] logger.info(f"Validating uid {run.start['uid']}") diff --git a/utils.py b/utils.py index 2b477ea..732c67c 100644 --- a/utils.py +++ b/utils.py @@ -12,6 +12,5 @@ def get_tiled_client(): with open("/srv/tiled.secret", "r") as secrets: load_dotenv(stream=secrets) api_key = os.environ["TILED_API_KEY"] - logger.info(f"first 4 characters of key: {api_key:4}") tiled_client = from_profile("nsls2", api_key=api_key)[LOCATION] return tiled_client From eea5a2feebee0f3f779b796caebbaf687ef9caa9 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Thu, 5 Feb 2026 15:20:50 -0500 Subject: [PATCH 9/9] LINT: remove unused logging, and thus its import --- utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/utils.py b/utils.py index 732c67c..ecefd2d 100644 --- a/utils.py +++ b/utils.py @@ -1,5 +1,4 @@ from dotenv import load_dotenv -from prefect import get_run_logger from tiled.client import from_profile import os @@ -8,7 +7,6 @@ def get_tiled_client(): - logger = get_run_logger() with open("/srv/tiled.secret", "r") as secrets: load_dotenv(stream=secrets) api_key = os.environ["TILED_API_KEY"]