Standardize workflows dt1 1282

.github/workflows/linting.yml

@@ -0,0 +1,14 @@
+name: pre-commit
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4.3.0
+      - uses: actions/setup-python@v5.6.0
+      - uses: pre-commit/action@v3.0.1

.github/workflows/publish-ghcr.yml

@@ -0,0 +1,59 @@
+#
+name: Create and publish a Docker image
+
+# Configures this workflow to run every time a change is pushed to the branch called `release`.
+on:
+  push:
+    branches: ["main"]
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+      #
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
+      # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see [Usage](https://github.com/docker/build-push-action#usage) in the README of the `docker/build-push-action` repository.
+      # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see [Using artifact attestations to establish provenance for builds](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds).
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v3
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true

.gitignore

@@ -80,4 +80,3 @@ target/
 
 #Ipython Notebook
 .ipynb_checkpoints
-

.pre-commit-config.yaml

@@ -0,0 +1,92 @@
+ci:
+  autoupdate_commit_msg: "chore: update pre-commit hooks"
+  autofix_commit_msg: "style: pre-commit fixes"
+
+exclude: ^.cruft.json|.copier-answers.yml$
+
+repos:
+  - repo: https://github.com/adamchainz/blacken-docs
+    rev: "1.19.1"
+    hooks:
+      - id: blacken-docs
+        additional_dependencies: [black==24.*]
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: "v5.0.0"
+    hooks:
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: check-yaml
+      - id: debug-statements
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: name-tests-test
+        args: ["--pytest-test-first"]
+      - id: requirements-txt-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/pre-commit/pygrep-hooks
+    rev: "v1.10.0"
+    hooks:
+      - id: rst-backticks
+      - id: rst-directive-colons
+      - id: rst-inline-touching-normal
+
+  - repo: https://github.com/rbubley/mirrors-prettier
+    rev: "v3.5.3"
+    hooks:
+      - id: prettier
+        types_or: [yaml, markdown, html, css, scss, javascript, json]
+        args: [--prose-wrap=always]
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: "v0.11.11"
+    hooks:
+      - id: ruff
+        args: ["--fix", "--show-fixes"]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: "v1.15.0"
+    hooks:
+      - id: mypy
+        files: src|tests
+        args: []
+        additional_dependencies:
+          - pytest
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: "v2.4.1"
+    hooks:
+      - id: codespell
+        additional_dependencies:
+          - tomli; python_version<'3.11'
+        exclude: "pixi.*"
+
+  - repo: https://github.com/shellcheck-py/shellcheck-py
+    rev: "v0.10.0.1"
+    hooks:
+      - id: shellcheck
+
+  - repo: local
+    hooks:
+      - id: disallow-caps
+        name: Disallow improper capitalization
+        language: pygrep
+        entry: PyBind|Numpy|Cmake|CCache|Github|PyTest
+        exclude: .pre-commit-config.yaml
+
+  - repo: https://github.com/abravalheri/validate-pyproject
+    rev: "v0.24.1"
+    hooks:
+      - id: validate-pyproject
+        additional_dependencies: ["validate-pyproject-schema-store[all]"]
+
+  - repo: https://github.com/python-jsonschema/check-jsonschema
+    rev: "0.33.0"
+    hooks:
+      - id: check-dependabot
+      - id: check-github-workflows
+      - id: check-readthedocs

Dockerfile

@@ -0,0 +1,27 @@
+FROM ghcr.io/prefix-dev/pixi:0.57.0
+
+ENV TZ="America/New_York"
+
+RUN apt-get -y update && \
+    apt-get -y install git tzdata
+
+COPY pixi.toml .
+COPY pixi.lock .
+# use `--locked` to ensure the lockfile is up to date with pixi.toml
+RUN pixi install --locked
+# create the shell-hook bash script to activate the environment
+RUN pixi shell-hook -s bash > /shell-hook
+
+ENV PYTHONUNBUFFERED=1
+
+COPY default.py .
+
+RUN mkdir /etc/tiled
+RUN mkdir /.prefect -m 0777
+RUN mkdir /repo -m 0777
+
+RUN /bin/bash /shell-hook
+
+#now reapply deployment to push the image that is being created
+ENTRYPOINT ["pixi", "run"]
+CMD ["python", "-m", "default", "arg"]

LICENSE

@@ -27,4 +27,3 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-

README.md

@@ -1,4 +1,3 @@
 # Workflows
 
 Repository of workflows for the TST beamline.
-

data_validation.py

@@ -1,13 +1,15 @@
 from prefect import task, flow, get_run_logger
+from prefect.blocks.system import Secret
 import time as ttime
 from tiled.client import from_profile
 
 
 @task(retries=2, retry_delay_seconds=10)
 def read_all_streams(uid, beamline_acronym):
     logger = get_run_logger()
-    tiled_client = from_profile("nsls2")
-    run = tiled_client[beamline_acronym]["raw"][uid]
+    api_key = Secret.load("tiled-tst-api-key").get()
+    cl = from_profile("nsls2", api_key=api_key)
+    run = cl["tst"]["raw"][uid]
     logger.info(f"Validating uid {run.start['uid']}")
     start_time = ttime.monotonic()
     for stream in run:

default.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+import prefect
+import subprocess
+import sys
+import tiled
+
+
+def print_argument(argument_to_print=""):
+    if argument_to_print:
+        print(f"argument to print: {argument_to_print}")  # noqa: T201
+    else:
+        print("argument to print: EMPTY")  # noqa: T201
+
+
+def info():
+    print(f"Prefect info: {prefect.__version_info__}")
+    print(f"Tiled info: {tiled.__version__}")
+    output = subprocess.check_output(["pixi", "--version"])
+    print(f"Pixi info: {output.decode().strip()}")
+
+
+if __name__ == "__main__":
+    info()
+    if len(sys.argv) > 1:
+        print_argument(sys.argv[1])
+    else:
+        print_argument()

end_of_run_workflow.py

@@ -1,6 +1,6 @@
 from prefect import task, flow, get_run_logger
 from data_validation import data_validation
-from  prefect2_test_flow import hello_world
+from test_extra_client import get_other_docs
 # from long_flow import long_flow
 
 
@@ -13,8 +13,8 @@ def log_completion():
 @flow
 def end_of_run_workflow(stop_doc):
     uid = stop_doc["run_start"]
-    hello_world()
+    # hello_world()
     data_validation(uid, return_state=True)
+    get_other_docs(uid)
     # long_flow(iterations=100, sleep_length=10)
     log_completion()
-

long_flow.py

@@ -1,4 +1,3 @@
-import numpy as np
 from prefect import task, flow
 import time as ttime