diff --git a/.github/workflows/publish-ghcr.yml b/.github/workflows/publish-ghcr.yml index 45e4f05..110ad75 100644 --- a/.github/workflows/publish-ghcr.yml +++ b/.github/workflows/publish-ghcr.yml @@ -4,7 +4,7 @@ name: Create and publish a Docker image # Configures this workflow to run every time a change is pushed to the branch called `release`. on: push: - branches: ["main"] + branches: ["update-dotenv"] # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: diff --git a/data_validation.py b/data_validation.py index 60ad578..186d369 100644 --- a/data_validation.py +++ b/data_validation.py @@ -1,5 +1,7 @@ +import os + +from dotenv import load_dotenv from prefect import task, flow, get_run_logger -from prefect.blocks.system import Secret import time as ttime from tiled.client import from_profile @@ -7,7 +9,9 @@ @task(retries=2, retry_delay_seconds=10) def read_all_streams(uid, beamline_acronym): logger = get_run_logger() - api_key = Secret.load("tiled-tst-api-key").get() + with open("/srv/tiled.secret", "r") as secrets: + load_dotenv(stream=secrets) + api_key = os.environ["TILED_API_KEY"] cl = from_profile("nsls2", api_key=api_key) run = cl["tst"]["raw"][uid] logger.info(f"Validating uid {run.start['uid']}") diff --git a/pixi.toml b/pixi.toml index 50e3351..6438d0d 100644 --- a/pixi.toml +++ b/pixi.toml @@ -8,6 +8,7 @@ prefect = "3.*" python = "<3.14" tiled-client = ">=0.2.3" bluesky-tiled-plugins = ">=2" +python-dotenv = ">=1.2.1,<2" [pypi-dependencies] lixtools = "==2023.1.23.0" diff --git a/prefect.yaml b/prefect.yaml index 065a601..25a6220 100644 --- a/prefect.yaml +++ b/prefect.yaml @@ -10,11 +10,11 @@ pull: directory: /repo - prefect.deployments.steps.git_clone: repository: https://github.com/nsls2/tst-workflows.git - branch: main + branch: update-dotenv deployments: - name: tst-end-of-run-workflow-docker - version: 0.1.1 + version: 0.1.2 tags: - tst - main @@ -26,12 +26,13 @@ deployments: job_variables: env: TILED_SITE_PROFILES: /nsls2/software/etc/tiled/profiles - image: ghcr.io/nsls2/tst-workflows:main + image: ghcr.io/nsls2/tst-workflows:update-dotenv image_pull_policy: Always network_mode: slirp4netns userns: "keep-id:uid=402974,gid=402974" # workflow-tst:workflow-tst volumes: - /nsls2/data/tst/proposals:/nsls2/data/tst/proposals - /nsls2/software/etc/tiled:/nsls2/software/etc/tiled + - /srv/prefect3-docker-worker-tst/tiled:/srv auto_remove: true name: tst-work-pool-docker diff --git a/utils.py b/utils.py index 1cd0072..ecefd2d 100644 --- a/utils.py +++ b/utils.py @@ -1,5 +1,5 @@ +from dotenv import load_dotenv from tiled.client import from_profile -from prefect.blocks.system import Secret import os @@ -7,7 +7,8 @@ def get_tiled_client(): - os.environ["TILED_API_KEY"] = Secret.load(f"tiled-{LOCATION}-api-key").get() - tiled_client = from_profile("nsls2")[LOCATION] - os.environ.pop("TILED_API_KEY") + with open("/srv/tiled.secret", "r") as secrets: + load_dotenv(stream=secrets) + api_key = os.environ["TILED_API_KEY"] + tiled_client = from_profile("nsls2", api_key=api_key)[LOCATION] return tiled_client