diff --git a/.github/workflows/github-actions-check-pydantic.yaml b/.github/workflows/check-python-code.yaml similarity index 81% rename from .github/workflows/github-actions-check-pydantic.yaml rename to .github/workflows/check-python-code.yaml index 406d77f76..97d37eb4b 100644 --- a/.github/workflows/github-actions-check-pydantic.yaml +++ b/.github/workflows/check-python-code.yaml @@ -1,4 +1,4 @@ -name: Check Pydantic models +name: Check Python package code on: pull_request_target: @@ -6,6 +6,12 @@ on: - 'packages/**' - 'pyproject.toml' - 'uv.lock' + push: + branches: [main, dev] + paths: + - 'packages/**' + - 'pyproject.toml' + - 'uv.lock' jobs: check: diff --git a/.github/workflows/check-python-package-versions.yaml b/.github/workflows/check-python-package-versions.yaml new file mode 100644 index 000000000..4b0fdf41e --- /dev/null +++ b/.github/workflows/check-python-package-versions.yaml @@ -0,0 +1,15 @@ +name: Check Python package version numbers + +on: + pull_request_target: + paths: + - '**/pyproject.toml' + - 'packages/**/__about__.py' + +jobs: + check: + if: github.event.pull_request.head.repo.full_name == github.repository + uses: ./.github/workflows/reusable/check-python-package-versions.yaml' + with: + before_commit: ${{ github.event.pull_request.base.sha }} + after_commit: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/github-actions-copy-latest-docs-to-staging.yaml b/.github/workflows/copy-latest-docs-to-staging.yaml similarity index 100% rename from .github/workflows/github-actions-copy-latest-docs-to-staging.yaml rename to .github/workflows/copy-latest-docs-to-staging.yaml diff --git a/.github/workflows/github-actions-copy-pr-docs-to-staging.yaml b/.github/workflows/copy-pr-docs-to-staging.yaml similarity index 100% rename from .github/workflows/github-actions-copy-pr-docs-to-staging.yaml rename to .github/workflows/copy-pr-docs-to-staging.yaml diff --git a/.github/workflows/github-actions-enforce-change-type-label.yaml b/.github/workflows/enforce-change-type-label.yaml similarity index 100% rename from .github/workflows/github-actions-enforce-change-type-label.yaml rename to .github/workflows/enforce-change-type-label.yaml diff --git a/.github/workflows/github-actions-publish-docs-gh-pages.yaml b/.github/workflows/publish-docs-gh-pages.yaml similarity index 100% rename from .github/workflows/github-actions-publish-docs-gh-pages.yaml rename to .github/workflows/publish-docs-gh-pages.yaml diff --git a/.github/workflows/reusable/check-python-package-versions.yaml b/.github/workflows/reusable/check-python-package-versions.yaml new file mode 100644 index 000000000..7c88177b2 --- /dev/null +++ b/.github/workflows/reusable/check-python-package-versions.yaml @@ -0,0 +1,84 @@ +name: Check Python package versions + +on: + workflow_call: + inputs: + before_commit: + description: >- + The base Git commit to compare against, i.e., the base of the PR or the previous commit + in a push. + type: string + required: true + after_commit: + description: >- + The Git commit representing the head of the change to be checked, i.e. the head of the + PR or the latest commit in a push. + type: string + required: true + +jobs: + get-index-url: + uses: ./.github/workflows/reusable/get-code-artifact-index-url.yaml + + check-python-package-versions: + needs: get-index-url + runs-on: ubuntu-latest + steps: + - name: Install jq + run: sudo apt-get update && sudo apt-get install -y jq + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Check out code before change + uses: actions/checkout@v4 + with: + ref: ${{ inputs.before_commit }} + + - name: Capture package versions before change + run: uv run python scripts/package-versions.py collect > /tmp/package-versions-before.json + + - name: Check out code after change + uses: actions/checkout@v4 + with: + ref: ${{ inputs.after_commit }} + + - name: Capture package versions after change + run: uv run python scripts/package-versions.py collect > /tmp/package-versions-after.json + + - name: Compare package versions before and after change + run: | + uv run python scripts/package-versions.py compare \ + /tmp/package-versions-before.json \ + /tmp/package-versions-after.json \ + >/tmp/package-version-diff.json + + - name: Print changed versions + run: cat /tmp/package-version-diff.json + + - name: Fail if any of the new versions already exist in the repo + run: | + jq -c '.[]' /tmp/package-version-diff.json | while read -r entry; do + package=$(echo "$entry" | jq -r '.package') + after=$(echo "$entry" | jq -r '.after') + exit_code=0 + output=$(uv run pip download "${package}==${after}" --index-url "${{ needs.get-index-url.outputs.index_url }}simple/" --no-deps -d /tmp --quiet 2>&1) | exit_code=$? + if [[ $exit_code -eq 0 || ( + "${output,,}" != *"could not find a version"* && + "${output,,}" != *"no matching distributions"* + ) ]]; then + echo "Package ${package} version ${after} already exists in the repository. Failing the workflow." + echo " pip exit code: ${exit_code}." + echo " pip stderr: ${output}." + exit 1 + else + echo "Package ${package} version ${after} is new, as expected. Continuing." + fi + done diff --git a/.github/workflows/reusable/get-code-artifact-index-url.yaml b/.github/workflows/reusable/get-code-artifact-index-url.yaml new file mode 100644 index 000000000..e8dec13f0 --- /dev/null +++ b/.github/workflows/reusable/get-code-artifact-index-url.yaml @@ -0,0 +1,57 @@ +name: Get CodeArtifact Python package index URL + +on: + workflow_call: + inputs: + account_id: + description: The AWS account ID that owns the CodeArtifact domain + type: string + required: false + default: 505071440022 + aws_region: + description: The AWS region where the CodeArtifact repository is hosted + type: string + required: false + default: us-west-2 + role_name: + description: The name of the IAM role to assume for accessing CodeArtifact + type: string + required: false + default: GithubActions_Schema_CodeArtifact_ReadOnly + domain: + description: The CodeArtifact domain name + type: string + required: false + default: overture-pypi + repository: + description: The CodeArtifact repository name + type: string + required: false + default: overture + outputs: + index_url: + description: The CodeArtifact Python index URL + value: ${{ jobs.get-code-artifact-index-url.outputs.index_url }} + +jobs: + get-code-artifact-index-url: + runs-on: ubuntu-latest + outputs: + index_url: ${{ steps.get-code-artifact-index-url.outputs.index_url }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ inputs.aws_region }} + role-to-assume: arn:aws:iam::${{ inputs.account_id }}:role/${{ inputs.role_name }} + role-session-name: GitHubActions_${{github.workflow}}_${{github.job}}_${{github.run_id}} + + - name: Get CodeArtifact authorization token + id: get-code-artifact-auth-token + run: | + AUTH_TOKEN=$(aws codeartifact get-authorization-token \ + --domain ${{ inputs.domain }} \ + --domain-owner ${{ inputs.account_id }} \ + --query authorizationToken \ + --output text) + echo "https://aws:${AUTH_TOKEN}@$${{ inputs.domain }}-${{ inputs.account_id }}.d.codeartifact.${{ inputs.aws_region }}.amazonaws.com/simple/" >> $GITHUB_OUTPUT diff --git a/.github/workflows/scripts/package-versions.py b/.github/workflows/scripts/package-versions.py new file mode 100755 index 000000000..4507d2672 --- /dev/null +++ b/.github/workflows/scripts/package-versions.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 + +from importlib import metadata +from pathlib import Path +import json +import sys + + +def collect(): + """ + Collect Python package versions and print them as a JSON array. + + Form of the JSON array: + + [ {"package": "p1", "version": "v1"}, {"package": "p2", "version": "v2"}, ... ] + """ + packages_dir = Path("packages") + + packages = sorted( + d.name + for d in packages_dir.iterdir() + if d.is_dir() and d.name.startswith("overture-schema") + ) + + package_versions = [ + {"package": p, "version": metadata.version(p.replace("-", "."))} + for p in packages + ] + + print(json.dumps(package_versions, indent=2)) + + +def compare(before_file: str, after_file: str): + """ + Compare two JSON files containing package versions and print the packages that have a version + number change as a JSON array. + + Form of the JSON array: + + [ {"package": "p1", "before": "v1", "after": "v2"}, ... ] + + Note that `before` will be `null` if the package did not exist in the "before" file, and `after` + will be `null` if the package did not exist in the "after" file. + """ + before_array = load(before_file) + after_array = load(after_file) + + before_dict = {item["package"]: item["version"] for item in before_array} + after_dict = {item["package"]: item["version"] for item in after_array} + + combined_keys = sorted(list(set(before_dict.keys()) | set(after_dict.keys()))) + + changed_packages = [] + for package in combined_keys: + before_version = before_dict.get(package) + after_version = after_dict.get(package) + if before_version != after_version: + changed_packages.append( + { + "package": package, + "before": before_version, + "after": after_version, + } + ) + + print(json.dumps(changed_packages, indent=2)) + + +def load(file_path: str) -> list[dict[str, str]]: + path = Path(file_path) + if not path.exists(): + print(f"File not found: {file_path}") + sys.exit(1) + + with path.open() as f: + value = json.load(f) + + if not isinstance(value, list): + print( + f"File {file_path} contains unexpected root value: expected a `list` but got value {repr(value)} of type `{type(value).__name__}`" + ) + sys.exit(1) + + for i, item in enumerate(value): + if not isinstance(item, dict): + print( + f"File {file_path} contains unexpected item at index {i}: expected `dict` but got value {repr(item)} of type `{type(item).__name__}`" + ) + sys.exit(1) + elif sorted(item.keys()) != ["package", "version"]: + print( + f"File {file_path} contains unexpected item at index {i}: expected keys `['package', 'version']` but got keys {sorted(item.keys())}" + ) + sys.exit(1) + elif not isinstance(item["package"], str): + print( + f"File {file_path} contains unexpected item at index {i}: expected `package` to be of type `str` but got value {repr(item['package'])} of type `{type(item['package']).__name__}`" + ) + sys.exit(1) + elif not isinstance(item["version"], str): + print( + f"File {file_path} contains unexpected item at index {i}: expected `version` to be of type `str` but got value {repr(item['version'])} of type `{type(item['version']).__name__}`" + ) + sys.exit(1) + + return value + + +def usage(): + print("Usage:") + print(f" ./{sys.argv[0]} collect") + print(f" ./{sys.argv[0]} compare BEFORE_FILE AFTER_FILE") + sys.exit(1) + + +def main(): + if len(sys.argv) < 2: + usage() + + cmd = sys.argv[1] + + if cmd == "collect": + collect() + elif cmd == "compare": + if len(sys.argv) != 4: + usage() + compare(sys.argv[2], sys.argv[3]) + else: + print(f"Unknown command: {cmd}") + usage() + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/github-actions-test-schema.yaml b/.github/workflows/test-schema.yaml similarity index 100% rename from .github/workflows/github-actions-test-schema.yaml rename to .github/workflows/test-schema.yaml