diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000..9cab3c5c --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,12 @@ +# Configuration related to self-hosted runners. +self-hosted-runner: + # Labels of self-hosted runner in array of strings. + labels: + - linux-amd64-* + - linux-arm64-* + +# Configuration variables in array of strings defined in your repository or organization. +config-variables: + - AWS_REGION + - AWS_ROLE_ARN + - TELEMETRY_ENABLED diff --git a/.github/workflows/breaking-change-alert.yaml b/.github/workflows/breaking-change-alert.yaml index fa106f8b..4df27965 100644 --- a/.github/workflows/breaking-change-alert.yaml +++ b/.github/workflows/breaking-change-alert.yaml @@ -73,21 +73,23 @@ jobs: echo "$1" | jq ${flag} -c . | grep -E '^".*"$' | awk '{print substr($0, 2, length($0) - 2)}' && [ "${PIPESTATUS[2]}" -eq 0 ] } - # Escape all input variables - echo "sender_login=$(escape_json "${SENDER_LOGIN}")" >> $GITHUB_OUTPUT - echo "sender_avatar=$(escape_json "${SENDER_AVATAR}")" >> $GITHUB_OUTPUT - echo "repo=$(escape_json "${REPO}")" >> $GITHUB_OUTPUT - echo "pr_number=${PR_NUMBER}" >> $GITHUB_OUTPUT - echo "pr_title=$(escape_json "${PR_TITLE}")" >> $GITHUB_OUTPUT - echo "pr_body=$(escape_json "${PR_BODY}" slurp)" >> $GITHUB_OUTPUT - echo "pr_base_ref=$(escape_json "${PR_BASE_REF}")" >> $GITHUB_OUTPUT - echo "pr_author=$(escape_json "${PR_AUTHOR}")" >> $GITHUB_OUTPUT + { + # Escape all input variables + echo "sender_login=$(escape_json "${SENDER_LOGIN}")" + echo "sender_avatar=$(escape_json "${SENDER_AVATAR}")" + echo "repo=$(escape_json "${REPO}")" + echo "pr_number=${PR_NUMBER}" + echo "pr_title=$(escape_json "${PR_TITLE}")" + echo "pr_body=$(escape_json "${PR_BODY}" slurp)" + echo "pr_base_ref=$(escape_json "${PR_BASE_REF}")" + echo "pr_author=$(escape_json "${PR_AUTHOR}")" - # Create escaped URLs - echo "repo_url=$(escape_json "https://github.com/${REPO}")" >> $GITHUB_OUTPUT - echo "pr_url=$(escape_json "https://github.com/${REPO}/pull/${PR_NUMBER}")" >> $GITHUB_OUTPUT - echo "branch_url=$(escape_json "https://github.com/${REPO}/tree/${PR_BASE_REF}")" >> $GITHUB_OUTPUT - echo "author_url=$(escape_json "https://github.com/${PR_AUTHOR}")" >> $GITHUB_OUTPUT + # Create escaped URLs + echo "repo_url=$(escape_json "https://github.com/${REPO}")" + echo "pr_url=$(escape_json "https://github.com/${REPO}/pull/${PR_NUMBER}")" + echo "branch_url=$(escape_json "https://github.com/${REPO}/tree/${PR_BASE_REF}")" + echo "author_url=$(escape_json "https://github.com/${PR_AUTHOR}")" + } >> "${GITHUB_OUTPUT}" - name: Determine notification parameters id: notification @@ -96,25 +98,33 @@ jobs: PR_MERGED: ${{ inputs.pr_merged }} run: | if [[ "$EVENT_ACTION" == "closed" && "$PR_MERGED" == "true" ]]; then - echo "action=Merged" >> $GITHUB_OUTPUT - echo "color=#D00000" >> $GITHUB_OUTPUT - echo "icon=:rocket:" >> $GITHUB_OUTPUT + { + echo "action=Merged" + echo "color=#D00000" + echo "icon=:rocket:" + } >> "${GITHUB_OUTPUT}" elif [[ "$EVENT_ACTION" == "closed" ]]; then - echo "action=Closed" >> $GITHUB_OUTPUT - echo "color=#1d9bd1" >> $GITHUB_OUTPUT - echo "icon=:information_source:" >> $GITHUB_OUTPUT + { + echo "action=Closed" + echo "color=#1d9bd1" + echo "icon=:information_source:" + } >> "${GITHUB_OUTPUT}" elif [[ "$EVENT_ACTION" == "reopened" ]]; then - echo "action=Reopened" >> $GITHUB_OUTPUT - echo "color=warning" >> $GITHUB_OUTPUT - echo "icon=:warning:" >> $GITHUB_OUTPUT + { + echo "action=Reopened" + echo "color=warning" + echo "icon=:warning:" + } >> "${GITHUB_OUTPUT}" else - echo "action=Modified" >> $GITHUB_OUTPUT - echo "color=good" >> $GITHUB_OUTPUT - echo "icon=:information_source:" >> $GITHUB_OUTPUT + { + echo "action=Modified" + echo "color=good" + echo "icon=:information_source:" + } >> "${GITHUB_OUTPUT}" fi - name: Send Slack notification - uses: slackapi/slack-github-action@v2.0.0 + uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 with: payload: | { diff --git a/.github/workflows/build-in-devcontainer.yaml b/.github/workflows/build-in-devcontainer.yaml index 0e509f80..e56b9d06 100644 --- a/.github/workflows/build-in-devcontainer.yaml +++ b/.github/workflows/build-in-devcontainer.yaml @@ -1,40 +1,76 @@ on: workflow_call: inputs: + env: + description: | + Additional environment variables to be set inside the devcontainer. + Should be a space-delimited string in the form "KEY=value". + type: string sha: + description: "Full git commit SHA to check out" type: string arch: + description: "One of [amd64, arm64]. CPU architecture to run on." type: string default: '["amd64"]' cuda: + description: | + Stringified JSON array of CUDA versions to run this workflow for. + This is used to select .devcontainer/ directories local to wherever this workflow is invoked from. + For example, if a repository has directories '.devcontainer/cuda12.0-pip/' and '.devcontainer/cuda12.8-pip/', + '["12.0", "12.8"]' could be passed here to build both of those devcontainers in CI. type: string default: '["12.0"]' python_package_manager: + description: | + Stringified JSON array of Python package managers to run devcontainer builds for. + One of: '["conda"]', '["pip"]', '["conda", "pip"]'. type: string default: '["conda", "pip"]' repo: + description: "Git repo to check out, in '{org}/{repo}' form, e.g. 'rapidsai/cudf'" type: string + timeout-minutes: + description: "Maximum time (in minutes) allowed for a run of this workflow." + type: number + default: 360 node_type: + description: | + Suffix, without leading '-', indicating the type of machine to run jobs on (e.g., 'cpu4' or 'gpu-l4-latest-1'). + Runner labels are of the form '{operating_system}-{arch}-{node_type}'. + See https://github.com/nv-gha-runners/enterprise-runner-configuration/blob/main/docs/runner-groups.md for a list + of valid values. type: string default: "cpu8" build_command: + description: | + Shell commands to run inside the devcontainer after it's built and started. + This is almost always some form of 'build-all --verbose;'. + See https://github.com/rapidsai/devcontainers for details. type: string required: true - # Note that this is the _name_ of a secret containing the key, not the key itself. extra-repo-deploy-key: + description: | + The NAME (not value) of a GitHub secret in the calling repo, containing a repo deploy key. + This is here to allow the use of additional private repos in runs of this workflow. required: false type: string default: '' - # Note that this is the _name_ of a secret containing the key, not the key itself. extra-repo-deploy-key-2: + description: | + The NAME (not value) of a GitHub secret in the calling repo, containing a repo deploy key. + This is here to allow the use of additional private repos in runs of this workflow. required: false type: string default: '' - # the use of secrets in shared-workflows is discouraged, especially for public repositories. - # these values were added for situations where the use of secrets is unavoidable. - secrets: - RAPIDS_AUX_SECRET_1: + rapids-aux-secret-1: + description: | + The NAME (not value) of a GitHub secret in the calling repo. + This allows callers of the workflow to make a single secret available in the devcontainer's + environment, via environment variable `RAPIDS_AUX_SECRET_1`. required: false + type: string + default: '' permissions: actions: read @@ -53,6 +89,7 @@ permissions: jobs: build: + timeout-minutes: ${{ inputs.timeout-minutes }} strategy: fail-fast: false matrix: @@ -67,69 +104,102 @@ jobs: repository: ${{ inputs.repo }} ref: ${{ inputs.sha }} fetch-depth: 0 + path: repo + persist-credentials: true # This provides an initial set of metadata tags. Jobs are free to add to the RAPIDS_JOB_ATTRIBUTES # environment variable as they see fit - but remember to export the variable to ${GITHUB_ENV} - name: Telemetry setup uses: rapidsai/shared-actions/telemetry-dispatch-setup@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + GH_TOKEN: ${{ github.token }} with: - extra_attributes: "rapids.PACKAGER=${{ matrix.PACKAGER }},rapids.CUDA_VER=${{ matrix.CUDA_VER }},rapids.ARCH=${{ matrix.ARCH }}" + extra_attributes: "rapids.PACKAGER=${{ matrix.PACKAGER }},rapids.CUDA_VER=${{ matrix.CUDA_VER }},rapids.ARCH=${{ matrix.ARCH }}" - name: Check if repo has devcontainer + env: + ARCH: ${{ matrix.ARCH }} + CUDA_VER: ${{ matrix.CUDA_VER }} + PACKAGER: ${{ matrix.PACKAGER }} + RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} run: | - echo "REPOSITORY=$(basename $(pwd))" | tee -a "${GITHUB_ENV}"; - if test -f .devcontainer/cuda${{ matrix.CUDA_VER }}-${{ matrix.PACKAGER }}/devcontainer.json; then - echo "HAS_DEVCONTAINER=true" >> "${GITHUB_ENV}"; - else - echo "HAS_DEVCONTAINER=false" >> "${GITHUB_ENV}"; + HAS_DEVCONTAINER=false + if test -f "repo/.devcontainer/cuda${CUDA_VER}-${PACKAGER}/devcontainer.json"; then + HAS_DEVCONTAINER=true fi + cat <> ~/.config/pip/pip.conf - [global] - extra-index-url = https://pypi.anaconda.org/rapidsai-wheels-nightly/simple - EOF - - rapids-make-${PYTHON_PACKAGE_MANAGER}-env; + set -euo pipefail; if test -n '${{ inputs.extra-repo-deploy-key }}' \ || test -n '${{ inputs.extra-repo-deploy-key-2 }}'; then @@ -143,9 +213,29 @@ jobs: devcontainer-utils-init-ssh-deploy-keys || true; fi + mkdir -p ~/.config/pip/; + cat <> ~/.config/pip/pip.conf + [global] + extra-index-url = https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + EOF + + rapids-make-${PYTHON_PACKAGE_MANAGER}-env; + cd ~/"${REPOSITORY}"; + mkdir -p telemetry-artifacts; ${{ inputs.build_command }} - - name: Telemetry upload attributes + + - if: ${{ !cancelled() && env.HAS_DEVCONTAINER == 'true' }} + name: Upload sccache logs + uses: actions/upload-artifact@v4 + with: + name: sccache-client-logs-${{ env.BUILD_SLUG }}-${{ env.ARTIFACT_SLUG }} + path: repo/sccache*.log + compression-level: 9 + + - if: ${{ always() && vars.TELEMETRY_ENABLED == 'true' }} + name: Telemetry upload attributes uses: rapidsai/shared-actions/telemetry-dispatch-stash-job-artifacts@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/changed-files.yaml b/.github/workflows/changed-files.yaml index b3c27af3..1de3c317 100644 --- a/.github/workflows/changed-files.yaml +++ b/.github/workflows/changed-files.yaml @@ -2,9 +2,19 @@ on: workflow_call: inputs: files_yaml: + description: | + YAML string containing mappings of the form `{key}: [{glob1}, {glob2}, etc.]`. + Where `{key}` is an arbitrary identifier for grouping a set of changed files + (e.g. "test_cpp" for "if these files change, C++ tests should be re-run") and + each `{glob}` is a glob expression matching file paths in the calling repo. + For example, "re-run all C++ tests on any changes EXCEPT docs" might look like this: + 'test_cpp: ["**", "!docs/**"]'. type: string required: true transform_expr: + description: | + jq expression for post-processing results to create this workflow's outputs. + Provided for backwards compatibility; should not need to be updated with normal use. type: string required: false default: | @@ -50,7 +60,9 @@ jobs: - name: Telemetry setup uses: rapidsai/shared-actions/telemetry-dispatch-setup@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + GH_TOKEN: ${{ github.token }} - name: Get PR info id: get-pr-info uses: nv-gha-runners/get-pr-info@main @@ -65,10 +77,10 @@ jobs: PR_SHA: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).head.sha }} BASE_SHA: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }} run: | - (echo -n "merge-base="; git merge-base "$BASE_SHA" "$PR_SHA") | tee --append "$GITHUB_OUTPUT" + (echo -n "merge-base="; git merge-base "$BASE_SHA" "$PR_SHA") | tee --append "${GITHUB_OUTPUT}" - name: Get changed files id: changed-files - uses: step-security/changed-files@3dbe17c78367e7d60f00d78ae6781a35be47b4a1 # v45.0.1 + uses: step-security/changed-files@95b56dadb92a30ca9036f16423fd3c088a71ee94 # v46.0.5 with: base_sha: ${{ steps.calculate-merge-base.outputs.merge-base }} sha: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).head.sha }} @@ -85,8 +97,10 @@ jobs: key="$(echo "$file" | sed -E 's/^\.github\/outputs\/(.*)\.json$/\1/g')" file_args=("${file_args[@]}" --slurpfile "$key" "$file") done - (echo -n "transformed-output="; jq -c -n "\$ARGS.named | to_entries | map({"key": .key, "value": .value[]}) | from_entries | $TRANSFORM_EXPR" "${file_args[@]}") | tee --append "$GITHUB_OUTPUT" + (echo -n "transformed-output="; jq -c -n "\$ARGS.named | to_entries | map({\"key\": .key, \"value\": .value[]}) | from_entries | $TRANSFORM_EXPR" "${file_args[@]}") | tee --append "${GITHUB_OUTPUT}" - name: Telemetry upload attributes uses: rapidsai/shared-actions/telemetry-dispatch-stash-job-artifacts@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index 39301479..2a97a599 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -40,16 +40,20 @@ jobs: other-checks: runs-on: ubuntu-latest container: - image: rapidsai/ci-conda:latest + image: rapidsai/ci-conda:25.10-latest # zizmor: ignore[unpinned-images] env: RAPIDS_GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: Checkout code uses: actions/checkout@v4 + with: + persist-credentials: true - name: Telemetry setup uses: rapidsai/shared-actions/telemetry-dispatch-setup@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + GH_TOKEN: ${{ github.token }} - name: Get PR Info id: get-pr-info uses: nv-gha-runners/get-pr-info@main @@ -66,22 +70,27 @@ jobs: - name: Telemetry upload attributes uses: rapidsai/shared-actions/telemetry-dispatch-stash-job-artifacts@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + GH_TOKEN: ${{ github.token }} check-style: if: ${{ inputs.enable_check_style }} runs-on: ubuntu-latest container: - image: rapidsai/ci-conda:latest + image: rapidsai/ci-conda:25.10-latest # zizmor: ignore[unpinned-images] steps: - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 0 + persist-credentials: true - name: Telemetry setup uses: rapidsai/shared-actions/telemetry-dispatch-setup@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + GH_TOKEN: ${{ github.token }} - name: Get PR Info id: get-pr-info uses: nv-gha-runners/get-pr-info@main @@ -96,4 +105,6 @@ jobs: - name: Telemetry upload attributes uses: rapidsai/shared-actions/telemetry-dispatch-stash-job-artifacts@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/conda-cpp-build.yaml b/.github/workflows/conda-cpp-build.yaml index 58ca161a..ce583a3a 100644 --- a/.github/workflows/conda-cpp-build.yaml +++ b/.github/workflows/conda-cpp-build.yaml @@ -6,22 +6,50 @@ on: required: true type: string branch: + description: | + Git branch the workflow run targets. + This is required even when 'sha' is provided because it is also used for organizing artifacts. type: string date: + description: "Date (YYYY-MM-DD) this run is for. Used to organize artifacts produced by nightly builds" type: string sha: + description: "Full git commit SHA to check out" type: string repo: + description: "Git repo to check out, in '{org}/{repo}' form, e.g. 'rapidsai/cudf'" type: string node_type: + description: | + Suffix, without leading '-', indicating the type of machine to run jobs on (e.g., 'cpu4' or 'gpu-l4-latest-1'). + Runner labels are of the form '{operating_system}-{arch}-{node_type}'. + See https://github.com/nv-gha-runners/enterprise-runner-configuration/blob/main/docs/runner-groups.md for a list + of valid values. type: string default: "cpu8" script: type: string - default: "ci/build_cpp.sh" + required: true + description: "Shell code to be executed in a step. Ideally this should just invoke a script managed in the repo the workflow runs from, like 'ci/build_cpp.sh'." + upload-artifacts: + type: boolean + default: true + required: false + description: "One of [true, false], true if artifacts should be uploaded to GitHub's artifact store" matrix_filter: + description: | + jq expression which modifies the matrix. + For example, 'map(select(.ARCH == "amd64"))' to achieve "only run amd64 jobs". type: string default: "." + alternative-gh-token-secret-name: + type: string + required: false + description: | + If provided, should contain the name of a secret in the repo which holds a GitHub API token. + When this is non-empty, that secret's value is used in place of the default repo-level token + anywhere that environment variable GH_TOKEN is set. This is especially useful for downloading + artifacts from other private repos, which repo tokens do not have access to. defaults: run: @@ -61,11 +89,9 @@ jobs: # export MATRIX=" # amd64 - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '11.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '12.8.0', LINUX_VER: 'rockylinux8' } + - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8' } # arm64 - - { ARCH: 'arm64', PY_VER: '3.10', CUDA_VER: '11.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'arm64', PY_VER: '3.10', CUDA_VER: '12.8.0', LINUX_VER: 'rockylinux8' } + - { ARCH: 'arm64', PY_VER: '3.10', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8' } " MATRIX="$( @@ -85,11 +111,11 @@ jobs: env: RAPIDS_ARTIFACTS_DIR: ${{ github.workspace }}/artifacts container: - image: rapidsai/ci-conda:cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }} + image: rapidsai/ci-conda:25.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }} env: RAPIDS_BUILD_TYPE: ${{ inputs.build_type }} steps: - - uses: aws-actions/configure-aws-credentials@v4 + - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: role-to-assume: ${{ vars.AWS_ROLE_ARN }} aws-region: ${{ vars.AWS_REGION }} @@ -99,12 +125,19 @@ jobs: repository: ${{ inputs.repo }} ref: ${{ inputs.sha }} fetch-depth: 0 + persist-credentials: true - name: Standardize repository information + env: + RAPIDS_REPOSITORY: ${{ inputs.repo || github.repository }} + RAPIDS_REF_NAME: ${{ inputs.branch || github.ref_name }} + RAPIDS_NIGHTLY_DATE: ${{ inputs.date }} run: | - echo "RAPIDS_REPOSITORY=${{ inputs.repo || github.repository }}" >> "${GITHUB_ENV}" - echo "RAPIDS_SHA=$(git rev-parse HEAD)" >> "${GITHUB_ENV}" - echo "RAPIDS_REF_NAME=${{ inputs.branch || github.ref_name }}" >> "${GITHUB_ENV}" - echo "RAPIDS_NIGHTLY_DATE=${{ inputs.date }}" >> "${GITHUB_ENV}" + { + echo "RAPIDS_REPOSITORY=${RAPIDS_REPOSITORY}" + echo "RAPIDS_SHA=$(git rev-parse HEAD)" + echo "RAPIDS_REF_NAME=${RAPIDS_REF_NAME}" + echo "RAPIDS_NIGHTLY_DATE=${RAPIDS_NIGHTLY_DATE}" + } >> "${GITHUB_ENV}" - name: Setup proxy cache uses: nv-gha-runners/setup-proxy-cache@main @@ -115,32 +148,60 @@ jobs: - name: Telemetry setup uses: rapidsai/shared-actions/telemetry-dispatch-setup@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + # DOES NOT NEED alternative-gh-token-secret-name - github.token is enough and more limited + GH_TOKEN: ${{ github.token }} with: - extra_attributes: "rapids.PACKAGER=conda,rapids.CUDA_VER=${{ matrix.CUDA_VER }},rapids.PY_VER=${{ matrix.PY_VER }},rapids.ARCH=${{ matrix.ARCH }},rapids.LINUX_VER=${{ matrix.LINUX_VER }}" + extra_attributes: "rapids.PACKAGER=conda,rapids.CUDA_VER=${{ matrix.CUDA_VER }},rapids.PY_VER=${{ matrix.PY_VER }},rapids.ARCH=${{ matrix.ARCH }},rapids.LINUX_VER=${{ matrix.LINUX_VER }}" + + # Per the docs at https://docs.github.com/en/rest/rate-limit/rate-limit?apiVersion=2022-11-28#get-rate-limit-status-for-the-authenticated-user, + # checking '/rate_limit | jq .' should not itself count against any rate limits. + # + # gh CLI is pre-installed on Github-hosted runners, but may not be on self-hosted runners. + - name: Check GitHub API rate limits + run: | + if ! type gh >/dev/null; then + echo "'gh' CLI is not installed... skipping rate-limits check" + else + gh api /rate_limit | jq . + fi + env: + # NEEDS alternative-gh-token-secret-name - API limits need to be for whatever token is used for upload/download. Repo token may be a different pool for rate limits. + GH_TOKEN: ${{ inputs.alternative-gh-token-secret-name && secrets[inputs.alternative-gh-token-secret-name] || github.token }} # zizmor: ignore[overprovisioned-secrets] - name: C++ build - run: ${{ inputs.script }} + run: ${{ inputs.script }} # zizmor: ignore[template-injection] env: STEP_NAME: "C++ build" - GH_TOKEN: ${{ github.token }} + # NEEDS alternative-gh-token-secret-name - may require a token with more permissions + GH_TOKEN: ${{ inputs.alternative-gh-token-secret-name && secrets[inputs.alternative-gh-token-secret-name] || github.token }} # zizmor: ignore[overprovisioned-secrets] - name: Get Package Name and Location - run: | + if: ${{ inputs.upload-artifacts }} + run: | echo "RAPIDS_PACKAGE_NAME=$(RAPIDS_NO_PKG_EXTENSION=true rapids-package-name conda_cpp)" >> "${GITHUB_OUTPUT}" echo "CONDA_OUTPUT_DIR=${RAPIDS_CONDA_BLD_OUTPUT_DIR}" >> "${GITHUB_OUTPUT}" id: package-name - name: Show files to be uploaded + if: ${{ inputs.upload-artifacts }} + env: + CONDA_OUTPUT_DIR: ${{ steps.package-name.outputs.CONDA_OUTPUT_DIR }} run: | echo "Contents of directory to be uploaded:" - ls -R ${{ steps.package-name.outputs.CONDA_OUTPUT_DIR }} + ls -R "${CONDA_OUTPUT_DIR}" - uses: actions/upload-artifact@v4 + if: ${{ inputs.upload-artifacts }} with: + if-no-files-found: 'error' name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} path: ${{ steps.package-name.outputs.CONDA_OUTPUT_DIR }} - name: Upload additional artifacts if: "!cancelled()" - run: rapids-upload-artifacts-dir cuda${RAPIDS_CUDA_VERSION%%.*}_$(arch) + run: rapids-upload-artifacts-dir "cuda${RAPIDS_CUDA_VERSION%%.*}_$(arch)" - name: Telemetry upload attributes uses: rapidsai/shared-actions/telemetry-dispatch-stash-job-artifacts@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + # DOES NOT NEED alternative-gh-token-secret-name - github.token is enough and more limited + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/conda-cpp-post-build-checks.yaml b/.github/workflows/conda-cpp-post-build-checks.yaml index 1f72ac22..ce0a9aba 100644 --- a/.github/workflows/conda-cpp-post-build-checks.yaml +++ b/.github/workflows/conda-cpp-post-build-checks.yaml @@ -6,18 +6,25 @@ on: required: true type: string branch: + description: | + Git branch the workflow run targets. + This is required even when 'sha' is provided because it is also used for organizing artifacts. type: string date: + description: "Date (YYYY-MM-DD) this run is for. Used to organize artifacts produced by nightly builds" type: string sha: + description: "Full git commit SHA to check out" type: string repo: + description: "Git repo to check out, in '{org}/{repo}' form, e.g. 'rapidsai/cudf'" type: string - enable_check_symbols: - default: false - type: boolean - required: false symbol_exclusions: + description: | + Regular expression matching unmangled symbol names to be ignored by this check. + Should be wrapped in '()' to form a capture group. + For example, to ignore any function symbols coming from the 'thrust' or 'cub' namespaces, + you might provide '(void (thrust::|cub::))'. type: string defaults: @@ -41,14 +48,13 @@ permissions: jobs: check-symbols: - if: ${{ inputs.enable_check_symbols }} runs-on: linux-amd64-cpu4 container: - image: rapidsai/ci-wheel:latest + image: rapidsai/ci-wheel:25.10-latest # zizmor: ignore[unpinned-images] env: RAPIDS_BUILD_TYPE: ${{ inputs.build_type }} steps: - - uses: aws-actions/configure-aws-credentials@v4 + - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: role-to-assume: ${{ vars.AWS_ROLE_ARN }} aws-region: ${{ vars.AWS_REGION }} @@ -59,19 +65,41 @@ jobs: ref: ${{ inputs.sha }} path: "./src/" fetch-depth: 0 + persist-credentials: true - name: Telemetry setup uses: rapidsai/shared-actions/telemetry-dispatch-setup@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + GH_TOKEN: ${{ github.token }} - name: Standardize repository information + env: + RAPIDS_REPOSITORY: ${{ inputs.repo || github.repository }} + RAPIDS_REF_NAME: ${{ inputs.branch || github.ref_name }} + RAPIDS_NIGHTLY_DATE: ${{ inputs.date }} run: | - echo "RAPIDS_REPOSITORY=${{ inputs.repo || github.repository }}" >> "${GITHUB_ENV}" - echo "RAPIDS_SHA=$(cd ./src && git rev-parse HEAD)" >> "${GITHUB_ENV}" - echo "RAPIDS_REF_NAME=${{ inputs.branch || github.ref_name }}" >> "${GITHUB_ENV}" - echo "RAPIDS_NIGHTLY_DATE=${{ inputs.date }}" >> "${GITHUB_ENV}" + { + echo "RAPIDS_REPOSITORY=${RAPIDS_REPOSITORY}" + echo "RAPIDS_SHA=$(cd ./src && git rev-parse HEAD)" + echo "RAPIDS_REF_NAME=${RAPIDS_REF_NAME}" + echo "RAPIDS_NIGHTLY_DATE=${RAPIDS_NIGHTLY_DATE}" + } >> "${GITHUB_ENV}" + # Per the docs at https://docs.github.com/en/rest/rate-limit/rate-limit?apiVersion=2022-11-28#get-rate-limit-status-for-the-authenticated-user, + # checking '/rate_limit | jq .' should not itself count against any rate limits. + - name: Check GitHub API rate limits + run: | + if ! type gh >/dev/null; then + echo "'gh' CLI is not installed... skipping rate-limits check" + else + gh api /rate_limit | jq . + fi + env: + GH_TOKEN: ${{ github.token }} - name: Download conda C++ build artifacts + env: + GH_TOKEN: ${{ github.token }} run: | - CPP_DIR=$(rapids-download-conda-from-s3 cpp) + CPP_DIR=$(rapids-download-conda-from-github cpp) EXTRACTED_DIR=$(rapids-extract-conda-files "${CPP_DIR}") echo "RAPIDS_EXTRACTED_DIR=${EXTRACTED_DIR}" >> "${GITHUB_ENV}" - name: Get weak detection tool @@ -81,16 +109,19 @@ jobs: ref: refs/heads/main path: "./tool/" fetch-depth: 0 + persist-credentials: true - name: Verify CUDA libraries have no public kernel entry points env: SYMBOL_EXCLUSIONS: ${{ inputs.symbol_exclusions }} run: | if [ -n "${SYMBOL_EXCLUSIONS}" ]; then - python ./tool/detect.py ${RAPIDS_EXTRACTED_DIR}/lib -e "${SYMBOL_EXCLUSIONS}" + python ./tool/detect.py "${RAPIDS_EXTRACTED_DIR}"/lib -e "${SYMBOL_EXCLUSIONS}" else - python ./tool/detect.py ${RAPIDS_EXTRACTED_DIR}/lib + python ./tool/detect.py "${RAPIDS_EXTRACTED_DIR}"/lib fi - name: Telemetry upload attributes uses: rapidsai/shared-actions/telemetry-dispatch-stash-job-artifacts@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/conda-cpp-tests.yaml b/.github/workflows/conda-cpp-tests.yaml index a582f0d8..48efbfee 100644 --- a/.github/workflows/conda-cpp-tests.yaml +++ b/.github/workflows/conda-cpp-tests.yaml @@ -11,23 +11,51 @@ on: type: string default: "auto" branch: + description: | + Git branch the workflow run targets. + This is required even when 'sha' is provided because it is also used for organizing artifacts. type: string date: + description: "Date (YYYY-MM-DD) this run is for. Used to organize artifacts produced by nightly builds" type: string sha: + description: "Full git commit SHA to check out" type: string repo: + description: "Git repo to check out, in '{org}/{repo}' form, e.g. 'rapidsai/cudf'" type: string script: type: string - default: "ci/test_cpp.sh" + required: true + description: "Shell code to be executed in a step. Ideally this should just invoke a script managed in the repo the workflow runs from, like 'ci/test_cpp.sh'." matrix_filter: + description: | + jq expression which modifies the matrix. + For example, 'map(select(.ARCH == "amd64"))' to achieve "only run amd64 jobs". type: string default: "." container-options: + description: | + Command-line arguments passed to 'docker run' when starting the container this workflow runs in. + This should be provided as a single string to be inlined into 'docker run', not an array. + For example, '--quiet --ulimit nofile=2048'. required: false type: string default: "-e _NOOP" + build_workflow_name: + description: | + Name of a workflow file that produced artifacts to be downloaded in this run. + If not set (the default), artifact-handling scripts use RAPIDS-conventional defaults (like "build.yaml" when "build_type == nightly"). + required: false + type: string + alternative-gh-token-secret-name: + type: string + required: false + description: | + If provided, should contain the name of a secret in the repo which holds a GitHub API token. + When this is non-empty, that secret's value is used in place of the default repo-level token + anywhere that environment variable GH_TOKEN is set. This is especially useful for downloading + artifacts from other private repos, which repo tokens do not have access to. defaults: run: @@ -81,21 +109,20 @@ jobs: export MATRICES=" pull-request: # amd64 - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '11.4.3', LINUX_VER: 'rockylinux8', GPU: 'l4', DRIVER: 'earliest', DEPENDENCIES: 'oldest' } - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '11.8.0', LINUX_VER: 'rockylinux8', GPU: 'l4', DRIVER: 'earliest', DEPENDENCIES: 'oldest' } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.8.0', LINUX_VER: 'ubuntu24.04', GPU: 'h100', DRIVER: 'latest', DEPENDENCIES: 'latest' } + - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '12.2.2', LINUX_VER: 'rockylinux8', GPU: 'l4', DRIVER: 'earliest', DEPENDENCIES: 'oldest' } + - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '12.9.1', LINUX_VER: 'ubuntu24.04', GPU: 'h100', DRIVER: 'latest', DEPENDENCIES: 'latest' } # arm64 - - { ARCH: 'arm64', PY_VER: '3.11', CUDA_VER: '12.0.1', LINUX_VER: 'ubuntu20.04', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'latest' } + - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '12.0.1', LINUX_VER: 'ubuntu22.04', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'latest' } nightly: # amd64 - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '11.4.3', LINUX_VER: 'rockylinux8', GPU: 'l4', DRIVER: 'earliest', DEPENDENCIES: 'oldest' } - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '11.8.0', LINUX_VER: 'ubuntu20.04', GPU: 'h100', DRIVER: 'latest', DEPENDENCIES: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '12.0.1', LINUX_VER: 'rockylinux8', GPU: 'l4', DRIVER: 'latest', DEPENDENCIES: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.8.0', LINUX_VER: 'ubuntu24.04', GPU: 'h100', DRIVER: 'latest', DEPENDENCIES: 'latest' } + - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '12.0.1', LINUX_VER: 'rockylinux8', GPU: 'l4', DRIVER: 'earliest', DEPENDENCIES: 'oldest' } + - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '12.2.2', LINUX_VER: 'ubuntu22.04', GPU: 'h100', DRIVER: 'latest', DEPENDENCIES: 'latest' } + - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.0.1', LINUX_VER: 'rockylinux8', GPU: 'l4', DRIVER: 'latest', DEPENDENCIES: 'latest' } + - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '12.9.1', LINUX_VER: 'ubuntu24.04', GPU: 'h100', DRIVER: 'latest', DEPENDENCIES: 'latest' } # arm64 - { ARCH: 'arm64', PY_VER: '3.10', CUDA_VER: '12.2.2', LINUX_VER: 'ubuntu22.04', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'oldest' } - - { ARCH: 'arm64', PY_VER: '3.11', CUDA_VER: '11.8.0', LINUX_VER: 'ubuntu20.04', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'latest' } - - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '12.8.0', LINUX_VER: 'rockylinux8', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'latest' } + - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '12.2.2', LINUX_VER: 'ubuntu22.04', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'latest' } + - { ARCH: 'arm64', PY_VER: '3.13', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'latest' } " # only overwrite MATRIX_TYPE if it was set to 'auto' @@ -129,13 +156,13 @@ jobs: RAPIDS_DEPENDENCIES: ${{ matrix.DEPENDENCIES }} RAPIDS_TESTS_DIR: ${{ github.workspace }}/test-results container: - image: rapidsai/ci-conda:cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }} + image: rapidsai/ci-conda:25.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }} options: ${{ inputs.container-options }} env: RAPIDS_BUILD_TYPE: ${{ inputs.build_type }} NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} steps: - - uses: aws-actions/configure-aws-credentials@v4 + - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: role-to-assume: ${{ vars.AWS_ROLE_ARN }} aws-region: ${{ vars.AWS_REGION }} @@ -146,40 +173,63 @@ jobs: repository: ${{ inputs.repo }} ref: ${{ inputs.sha }} fetch-depth: 0 + persist-credentials: true # This has to be AFTER the checkout step. It creates a telemetry-artifacts directory, # and the checkout step would destroy it. - name: Telemetry setup uses: rapidsai/shared-actions/telemetry-dispatch-setup@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1'}} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} with: extra_attributes: "rapids.PACKAGER=conda,rapids.CUDA_VER=${{ matrix.CUDA_VER }},rapids.PY_VER=${{ matrix.PY_VER }},rapids.ARCH=${{ matrix.ARCH }},rapids.LINUX_VER=${{ matrix.LINUX_VER }},rapids.GPU=${{ matrix.GPU }},rapids.DRIVER=${{ matrix.DRIVER }},rapids.DEPENDENCIES=${{ matrix.DEPENDENCIES }}" + env: + # DOES NOT NEED alternative-gh-token-secret_name - github.token is enough and more limited + GH_TOKEN: ${{ github.token }} + - name: Standardize repository information - run: | - echo "RAPIDS_REPOSITORY=${{ inputs.repo || github.repository }}" >> "${GITHUB_ENV}" - echo "RAPIDS_SHA=$(git rev-parse HEAD)" >> "${GITHUB_ENV}" - echo "RAPIDS_REF_NAME=${{ inputs.branch || github.ref_name }}" >> "${GITHUB_ENV}" - echo "RAPIDS_NIGHTLY_DATE=${{ inputs.date }}" >> "${GITHUB_ENV}" + uses: rapidsai/shared-actions/rapids-github-info@main + with: + repo: ${{ inputs.repo }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + build_workflow_name: ${{ inputs.build_workflow_name }} - name: Setup proxy cache uses: nv-gha-runners/setup-proxy-cache@main continue-on-error: true - # Skip the cache on RDS Lab nodes - if: ${{ matrix.GPU != 'v100' }} + # Per the docs at https://docs.github.com/en/rest/rate-limit/rate-limit?apiVersion=2022-11-28#get-rate-limit-status-for-the-authenticated-user, + # checking '/rate_limit | jq .' should not itself count against any rate limits. + # + # gh CLI is pre-installed on Github-hosted runners, but may not be on self-hosted runners. + - name: Check GitHub API rate limits + run: | + if ! type gh >/dev/null; then + echo "'gh' CLI is not installed... skipping rate-limits check" + else + gh api /rate_limit | jq . + fi + env: + # NEEDS alternative-gh-token-secret_name - API limits need to be for whatever token is used for upload/download. Repo token may be a different pool for rate limits. + GH_TOKEN: ${{ inputs.alternative-gh-token-secret-name && secrets[inputs.alternative-gh-token-secret-name] || github.token }} # zizmor: ignore[overprovisioned-secrets] - name: C++ tests - run: ${{ inputs.script }} + run: ${{ inputs.script }} # zizmor: ignore[template-injection] env: - GH_TOKEN: ${{ github.token }} + # NEEDS alternative-gh-token-secret-name - may require a token with more permissions + GH_TOKEN: ${{ inputs.alternative-gh-token-secret-name && secrets[inputs.alternative-gh-token-secret-name] || github.token }} # zizmor: ignore[overprovisioned-secrets] - name: Generate test report - uses: test-summary/action@v2.4 + uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 with: paths: "${{ env.RAPIDS_TESTS_DIR }}/*.xml" if: always() - name: Upload additional artifacts if: "!cancelled()" - run: rapids-upload-artifacts-dir cuda${RAPIDS_CUDA_VERSION%%.*}_$(arch) + run: rapids-upload-artifacts-dir "cuda${RAPIDS_CUDA_VERSION%%.*}_$(arch)" - name: Telemetry upload attributes uses: rapidsai/shared-actions/telemetry-dispatch-stash-job-artifacts@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + # DOES NOT NEED alternative-gh-token-secret-name - github.token is enough and more limited + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/conda-python-build.yaml b/.github/workflows/conda-python-build.yaml index 47c45646..8b3d4f60 100644 --- a/.github/workflows/conda-python-build.yaml +++ b/.github/workflows/conda-python-build.yaml @@ -6,22 +6,50 @@ on: required: true type: string branch: + description: | + Git branch the workflow run targets. + This is required even when 'sha' is provided because it is also used for organizing artifacts. type: string date: + description: "Date (YYYY-MM-DD) this run is for. Used to organize artifacts produced by nightly builds" type: string sha: + description: "Full git commit SHA to check out" type: string repo: + description: "Git repo to check out, in '{org}/{repo}' form, e.g. 'rapidsai/cudf'" type: string node_type: + description: | + Suffix, without leading '-', indicating the type of machine to run jobs on (e.g., 'cpu4' or 'gpu-l4-latest-1'). + Runner labels are of the form '{operating_system}-{arch}-{node_type}'. + See https://github.com/nv-gha-runners/enterprise-runner-configuration/blob/main/docs/runner-groups.md for a list + of valid values. type: string default: "cpu8" script: type: string - default: "ci/build_python.sh" + required: true + description: "Shell code to be executed in a step. Ideally this should just invoke a script managed in the repo the workflow runs from, like 'ci/build_python.sh'." + upload-artifacts: + type: boolean + default: true + required: false + description: "One of [true, false], true if artifacts should be uploaded to GitHub's artifact store" matrix_filter: + description: | + jq expression which modifies the matrix. + For example, 'map(select(.ARCH == "amd64"))' to achieve "only run amd64 jobs". type: string default: "." + alternative-gh-token-secret-name: + type: string + required: false + description: | + If provided, should contain the name of a secret in the repo which holds a GitHub API token. + When this is non-empty, that secret's value is used in place of the default repo-level token + anywhere that environment variable GH_TOKEN is set. This is especially useful for downloading + artifacts from other private repos, which repo tokens do not have access to. defaults: run: @@ -61,19 +89,15 @@ jobs: # export MATRIX=" # amd64 - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '11.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '12.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '11.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '12.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '11.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.8.0', LINUX_VER: 'rockylinux8' } + - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8' } + - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8' } + - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8' } + - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8' } # arm64 - - { ARCH: 'arm64', PY_VER: '3.10', CUDA_VER: '11.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'arm64', PY_VER: '3.10', CUDA_VER: '12.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'arm64', PY_VER: '3.11', CUDA_VER: '11.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'arm64', PY_VER: '3.11', CUDA_VER: '12.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '11.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '12.8.0', LINUX_VER: 'rockylinux8' } + - { ARCH: 'arm64', PY_VER: '3.10', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8' } + - { ARCH: 'arm64', PY_VER: '3.11', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8' } + - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8' } + - { ARCH: 'arm64', PY_VER: '3.13', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8' } " MATRIX="$( @@ -92,11 +116,11 @@ jobs: env: RAPIDS_ARTIFACTS_DIR: ${{ github.workspace }}/artifacts container: - image: rapidsai/ci-conda:cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }} + image: rapidsai/ci-conda:25.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }} env: RAPIDS_BUILD_TYPE: ${{ inputs.build_type }} steps: - - uses: aws-actions/configure-aws-credentials@v4 + - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: role-to-assume: ${{ vars.AWS_ROLE_ARN }} aws-region: ${{ vars.AWS_REGION }} @@ -106,41 +130,75 @@ jobs: repository: ${{ inputs.repo }} ref: ${{ inputs.sha }} fetch-depth: 0 + persist-credentials: true - name: Standardize repository information + env: + RAPIDS_REPOSITORY: ${{ inputs.repo || github.repository }} + RAPIDS_REF_NAME: ${{ inputs.branch || github.ref_name }} + RAPIDS_NIGHTLY_DATE: ${{ inputs.date }} run: | - echo "RAPIDS_REPOSITORY=${{ inputs.repo || github.repository }}" >> "${GITHUB_ENV}" - echo "RAPIDS_SHA=$(git rev-parse HEAD)" >> "${GITHUB_ENV}" - echo "RAPIDS_REF_NAME=${{ inputs.branch || github.ref_name }}" >> "${GITHUB_ENV}" - echo "RAPIDS_NIGHTLY_DATE=${{ inputs.date }}" >> "${GITHUB_ENV}" + { + echo "RAPIDS_REPOSITORY=${RAPIDS_REPOSITORY}" + echo "RAPIDS_SHA=$(git rev-parse HEAD)" + echo "RAPIDS_REF_NAME=${RAPIDS_REF_NAME}" + echo "RAPIDS_NIGHTLY_DATE=${RAPIDS_NIGHTLY_DATE}" + } >> "${GITHUB_ENV}" - name: Telemetry setup uses: rapidsai/shared-actions/telemetry-dispatch-setup@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + # DOES NOT NEED alternative-gh-token-secret-name - github.token is enough and more limited + GH_TOKEN: ${{ github.token }} with: extra_attributes: "rapids.PACKAGER=conda,rapids.CUDA_VER=${{ matrix.CUDA_VER }},rapids.PY_VER=${{ matrix.PY_VER }},rapids.ARCH=${{ matrix.ARCH }},rapids.LINUX_VER=${{ matrix.LINUX_VER }}" - name: Setup proxy cache uses: nv-gha-runners/setup-proxy-cache@main + # Per the docs at https://docs.github.com/en/rest/rate-limit/rate-limit?apiVersion=2022-11-28#get-rate-limit-status-for-the-authenticated-user, + # checking '/rate_limit | jq .' should not itself count against any rate limits. + # + # gh CLI is pre-installed on Github-hosted runners, but may not be on self-hosted runners. + - name: Check GitHub API rate limits + run: | + if ! type gh >/dev/null; then + echo "'gh' CLI is not installed... skipping rate-limits check" + else + gh api /rate_limit | jq . + fi + env: + # NEEDS alternative-gh-token-secret_name - API limits need to be for whatever token is used for upload/download. Repo token may be a different pool for rate limits. + GH_TOKEN: ${{ inputs.alternative-gh-token-secret-name && secrets[inputs.alternative-gh-token-secret-name] || github.token }} # zizmor: ignore[overprovisioned-secrets] - name: Python build - run: ${{ inputs.script }} + run: ${{ inputs.script }} # zizmor: ignore[template-injection] env: - GH_TOKEN: ${{ github.token }} + # NEEDS alternative-gh-token-secret-name - may require a token with more permissions + GH_TOKEN: ${{ inputs.alternative-gh-token-secret-name && secrets[inputs.alternative-gh-token-secret-name] || github.token }} # zizmor: ignore[overprovisioned-secrets] - name: Get Package Name and Location - run: | + if: ${{ inputs.upload-artifacts }} + run: | echo "RAPIDS_PACKAGE_NAME=$(RAPIDS_NO_PKG_EXTENSION=true rapids-package-name conda_python)" >> "${GITHUB_OUTPUT}" echo "CONDA_OUTPUT_DIR=${RAPIDS_CONDA_BLD_OUTPUT_DIR}" >> "${GITHUB_OUTPUT}" id: package-name - name: Show files to be uploaded + if: ${{ inputs.upload-artifacts }} + env: + CONDA_OUTPUT_DIR: ${{ steps.package-name.outputs.CONDA_OUTPUT_DIR }} run: | echo "Contents of directory to be uploaded:" - ls -R ${{ steps.package-name.outputs.CONDA_OUTPUT_DIR }} + ls -R "${CONDA_OUTPUT_DIR}" - uses: actions/upload-artifact@v4 + if: ${{ inputs.upload-artifacts }} with: + if-no-files-found: 'error' name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} path: ${{ steps.package-name.outputs.CONDA_OUTPUT_DIR }} - name: Upload additional artifacts if: "!cancelled()" - run: rapids-upload-artifacts-dir cuda${RAPIDS_CUDA_VERSION%%.*}_$(arch)_py${RAPIDS_PY_VERSION//.} + run: rapids-upload-artifacts-dir "cuda${RAPIDS_CUDA_VERSION%%.*}_$(arch)_py${RAPIDS_PY_VERSION//.}" - name: Telemetry upload attributes uses: rapidsai/shared-actions/telemetry-dispatch-stash-job-artifacts@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + # DOES NOT NEED alternative-gh-token-secret-name - github.token is enough and more limited + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/conda-python-tests.yaml b/.github/workflows/conda-python-tests.yaml index b5bca2a3..643c416a 100644 --- a/.github/workflows/conda-python-tests.yaml +++ b/.github/workflows/conda-python-tests.yaml @@ -11,26 +11,54 @@ on: type: string default: "auto" branch: + description: | + Git branch the workflow run targets. + This is required even when 'sha' is provided because it is also used for organizing artifacts. type: string date: + description: "Date (YYYY-MM-DD) this run is for. Used to organize artifacts produced by nightly builds" type: string sha: + description: "Full git commit SHA to check out" type: string repo: + description: "Git repo to check out, in '{org}/{repo}' form, e.g. 'rapidsai/cudf'" type: string script: type: string - default: "ci/test_python.sh" + required: true + description: "Shell code to be executed in a step. Ideally this should just invoke a script managed in the repo the workflow runs from, like 'ci/test_python.sh'." run_codecov: type: boolean default: true matrix_filter: + description: | + jq expression which modifies the matrix. + For example, 'map(select(.ARCH == "amd64"))' to achieve "only run amd64 jobs". type: string default: "." container-options: + description: | + Command-line arguments passed to 'docker run' when starting the container this workflow runs in. + This should be provided as a single string to be inlined into 'docker run', not an array. + For example, '--quiet --ulimit nofile=2048'. required: false type: string default: "-e _NOOP" + build_workflow_name: + description: | + Name of a workflow file that produced artifacts to be downloaded in this run. + If not set (the default), artifact-handling scripts use RAPIDS-conventional defaults (like "build.yaml" when "build_type == nightly"). + required: false + type: string + alternative-gh-token-secret-name: + type: string + required: false + description: | + If provided, should contain the name of a secret in the repo which holds a GitHub API token. + When this is non-empty, that secret's value is used in place of the default repo-level token + anywhere that environment variable GH_TOKEN is set. This is especially useful for downloading + artifacts from other private repos, which repo tokens do not have access to. defaults: run: @@ -84,21 +112,20 @@ jobs: export MATRICES=" pull-request: # amd64 - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '11.4.3', LINUX_VER: 'rockylinux8', GPU: 'l4', DRIVER: 'earliest', DEPENDENCIES: 'oldest' } - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '11.8.0', LINUX_VER: 'rockylinux8', GPU: 'l4', DRIVER: 'earliest', DEPENDENCIES: 'oldest' } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.8.0', LINUX_VER: 'ubuntu24.04', GPU: 'h100', DRIVER: 'latest', DEPENDENCIES: 'latest' } + - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '12.2.2', LINUX_VER: 'rockylinux8', GPU: 'l4', DRIVER: 'earliest', DEPENDENCIES: 'oldest' } + - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '12.9.1', LINUX_VER: 'ubuntu24.04', GPU: 'h100', DRIVER: 'latest', DEPENDENCIES: 'latest' } # arm64 - - { ARCH: 'arm64', PY_VER: '3.11', CUDA_VER: '12.0.1', LINUX_VER: 'ubuntu20.04', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'latest' } + - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '12.0.1', LINUX_VER: 'ubuntu22.04', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'latest' } nightly: # amd64 - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '11.4.3', LINUX_VER: 'rockylinux8', GPU: 'l4', DRIVER: 'earliest', DEPENDENCIES: 'oldest' } - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '11.8.0', LINUX_VER: 'ubuntu20.04', GPU: 'h100', DRIVER: 'latest', DEPENDENCIES: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '12.0.1', LINUX_VER: 'rockylinux8', GPU: 'l4', DRIVER: 'latest', DEPENDENCIES: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.8.0', LINUX_VER: 'ubuntu24.04', GPU: 'h100', DRIVER: 'latest', DEPENDENCIES: 'latest' } + - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '12.0.1', LINUX_VER: 'rockylinux8', GPU: 'l4', DRIVER: 'earliest', DEPENDENCIES: 'oldest' } + - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '12.2.2', LINUX_VER: 'ubuntu22.04', GPU: 'h100', DRIVER: 'latest', DEPENDENCIES: 'latest' } + - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.0.1', LINUX_VER: 'rockylinux8', GPU: 'l4', DRIVER: 'latest', DEPENDENCIES: 'latest' } + - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '12.9.1', LINUX_VER: 'ubuntu24.04', GPU: 'h100', DRIVER: 'latest', DEPENDENCIES: 'latest' } # arm64 - { ARCH: 'arm64', PY_VER: '3.10', CUDA_VER: '12.2.2', LINUX_VER: 'ubuntu22.04', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'oldest' } - - { ARCH: 'arm64', PY_VER: '3.11', CUDA_VER: '11.8.0', LINUX_VER: 'ubuntu20.04', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'latest' } - - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '12.8.0', LINUX_VER: 'rockylinux8', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'latest' } + - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '12.2.2', LINUX_VER: 'ubuntu22.04', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'latest' } + - { ARCH: 'arm64', PY_VER: '3.13', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'latest' } " # only overwrite MATRIX_TYPE if it was set to 'auto' @@ -133,13 +160,13 @@ jobs: RAPIDS_DEPENDENCIES: ${{ matrix.DEPENDENCIES }} RAPIDS_TESTS_DIR: ${{ github.workspace }}/test-results container: - image: rapidsai/ci-conda:cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }} + image: rapidsai/ci-conda:25.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }} options: ${{ inputs.container-options }} env: RAPIDS_BUILD_TYPE: ${{ inputs.build_type }} NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} steps: - - uses: aws-actions/configure-aws-credentials@v4 + - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: role-to-assume: ${{ vars.AWS_ROLE_ARN }} aws-region: ${{ vars.AWS_REGION }} @@ -150,35 +177,54 @@ jobs: repository: ${{ inputs.repo }} ref: ${{ inputs.sha }} fetch-depth: 0 + persist-credentials: true - name: Standardize repository information - run: | - echo "RAPIDS_REPOSITORY=${{ inputs.repo || github.repository }}" >> "${GITHUB_ENV}" - echo "RAPIDS_SHA=$(git rev-parse HEAD)" >> "${GITHUB_ENV}" - echo "RAPIDS_REF_NAME=${{ inputs.branch || github.ref_name }}" >> "${GITHUB_ENV}" - echo "RAPIDS_NIGHTLY_DATE=${{ inputs.date }}" >> "${GITHUB_ENV}" + uses: rapidsai/shared-actions/rapids-github-info@main + with: + repo: ${{ inputs.repo }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + build_workflow_name: ${{ inputs.build_workflow_name }} # This has to be AFTER the checkout step. It creates a telemetry-artifacts directory, # and the checkout step would destroy it. - name: Telemetry setup uses: rapidsai/shared-actions/telemetry-dispatch-setup@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + # DOES NOT NEED alternative-gh-token-secret-name - github.token is enough and more limited + GH_TOKEN: ${{ github.token }} with: extra_attributes: "rapids.PACKAGER=conda,rapids.CUDA_VER=${{ matrix.CUDA_VER }},rapids.PY_VER=${{ matrix.PY_VER }},rapids.ARCH=${{ matrix.ARCH }},rapids.LINUX_VER=${{ matrix.LINUX_VER }},rapids.GPU=${{ matrix.GPU }},rapids.DRIVER=${{ matrix.DRIVER }},rapids.DEPENDENCIES=${{ matrix.DEPENDENCIES }}" - name: Setup proxy cache uses: nv-gha-runners/setup-proxy-cache@main continue-on-error: true - # Skip the cache on RDS Lab nodes - if: ${{ matrix.GPU != 'v100' }} + # Per the docs at https://docs.github.com/en/rest/rate-limit/rate-limit?apiVersion=2022-11-28#get-rate-limit-status-for-the-authenticated-user, + # checking '/rate_limit | jq .' should not itself count against any rate limits. + # + # gh CLI is pre-installed on Github-hosted runners, but may not be on self-hosted runners. + - name: Check GitHub API rate limits + run: | + if ! type gh >/dev/null; then + echo "'gh' CLI is not installed... skipping rate-limits check" + else + gh api /rate_limit | jq . + fi + env: + # NEEDS alternative-gh-token-secret_name - API limits need to be for whatever token is used for upload/download. Repo token may be a different pool for rate limits. + GH_TOKEN: ${{ inputs.alternative-gh-token-secret-name && secrets[inputs.alternative-gh-token-secret-name] || github.token }} # zizmor: ignore[overprovisioned-secrets] - name: Python tests - run: ${{ inputs.script }} + run: ${{ inputs.script }} # zizmor: ignore[template-injection] env: - GH_TOKEN: ${{ github.token }} + # NEEDS alternative-gh-token-secret-name - may require a token with more permissions + GH_TOKEN: ${{ inputs.alternative-gh-token-secret-name && secrets[inputs.alternative-gh-token-secret-name] || github.token }} # zizmor: ignore[overprovisioned-secrets] - name: Generate test report - uses: test-summary/action@v2.4 + uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 with: paths: "${{ env.RAPIDS_TESTS_DIR }}/*.xml" if: always() @@ -195,8 +241,11 @@ jobs: --handle-no-reports-found - name: Upload additional artifacts if: "!cancelled()" - run: rapids-upload-artifacts-dir cuda${RAPIDS_CUDA_VERSION%%.*}_$(arch)_py${RAPIDS_PY_VERSION//.} + run: rapids-upload-artifacts-dir "cuda${RAPIDS_CUDA_VERSION%%.*}_$(arch)_py${RAPIDS_PY_VERSION//.}" - name: Telemetry upload attributes uses: rapidsai/shared-actions/telemetry-dispatch-stash-job-artifacts@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + # DOES NOT NEED alternative-gh-token-secret-name - github.token is enough and more limited + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/conda-upload-packages.yaml b/.github/workflows/conda-upload-packages.yaml index 3cbd4cda..74c2337d 100644 --- a/.github/workflows/conda-upload-packages.yaml +++ b/.github/workflows/conda-upload-packages.yaml @@ -6,19 +6,35 @@ on: required: true type: string branch: + description: | + Git branch the workflow run targets. + This is required even when 'sha' is provided because it is also used for organizing artifacts. type: string date: + description: "Date (YYYY-MM-DD) this run is for. Used to organize artifacts produced by nightly builds" type: string sha: + description: "Full git commit SHA to check out" type: string repo: + description: "Git repo to check out, in '{org}/{repo}' form, e.g. 'rapidsai/cudf'" type: string skip_upload_pkgs: + description: | + Space-delimited string of package names to skip uploading to anaconda.org. + When this is empty (the default), all conda packages found in CI artifacts for a given + run will be uploaded to anaconda.org type: string upload_to_label: description: The label that should be applied to packages uploaded to Anaconda.org type: string default: main + build_workflow_name: + description: | + Name of a workflow file that produced artifacts to be downloaded in this run. + If not set (the default), artifact-handling scripts use RAPIDS-conventional defaults (like "build.yaml" when "build_type == nightly"). + required: false + type: string defaults: run: @@ -43,32 +59,43 @@ jobs: upload: runs-on: linux-amd64-cpu4 container: - image: rapidsai/ci-conda:latest + image: rapidsai/ci-conda:25.10-latest # zizmor: ignore[unpinned-images] env: RAPIDS_BUILD_TYPE: ${{ inputs.build_type }} steps: - name: Telemetry setup uses: rapidsai/shared-actions/telemetry-dispatch-setup@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} - - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ vars.AWS_ROLE_ARN }} - aws-region: ${{ vars.AWS_REGION }} - role-duration-seconds: 43200 # 12h + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + GH_TOKEN: ${{ github.token }} - uses: actions/checkout@v4 with: repository: ${{ inputs.repo }} ref: ${{ inputs.sha }} fetch-depth: 0 + persist-credentials: true - name: Standardize repository information - run: | - echo "RAPIDS_REPOSITORY=${{ inputs.repo || github.repository }}" >> "${GITHUB_ENV}" - echo "RAPIDS_SHA=$(git rev-parse HEAD)" >> "${GITHUB_ENV}" - echo "RAPIDS_REF_NAME=${{ inputs.branch || github.ref_name }}" >> "${GITHUB_ENV}" - echo "RAPIDS_NIGHTLY_DATE=${{ inputs.date }}" >> "${GITHUB_ENV}" + uses: rapidsai/shared-actions/rapids-github-info@main + with: + repo: ${{ inputs.repo }} + branch: ${{ inputs.branch }} + build_workflow_name: ${{ inputs.build_workflow_name }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + # Per the docs at https://docs.github.com/en/rest/rate-limit/rate-limit?apiVersion=2022-11-28#get-rate-limit-status-for-the-authenticated-user, + # checking '/rate_limit | jq .' should not itself count against any rate limits. + - name: Check GitHub API rate limits + run: | + if ! type gh >/dev/null; then + echo "'gh' CLI is not installed... skipping rate-limits check" + else + gh api /rate_limit | jq . + fi + env: + GH_TOKEN: ${{ github.token }} - name: Set Proper Conda Upload Token run: | RAPIDS_CONDA_TOKEN=${{ secrets.CONDA_RAPIDSAI_NIGHTLY_TOKEN }} @@ -77,11 +104,14 @@ jobs: fi echo "RAPIDS_CONDA_TOKEN=${RAPIDS_CONDA_TOKEN}" >> "${GITHUB_ENV}" - name: Upload packages - run: rapids-upload-to-anaconda + run: rapids-upload-to-anaconda-github env: SKIP_UPLOAD_PKGS: ${{ inputs.skip_upload_pkgs }} RAPIDS_CONDA_UPLOAD_LABEL: ${{ inputs.upload_to_label }} + GH_TOKEN: ${{ github.token }} - name: Telemetry upload attributes uses: rapidsai/shared-actions/telemetry-dispatch-stash-job-artifacts@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/custom-job.yaml b/.github/workflows/custom-job.yaml index 09bbb436..0ddb4e90 100644 --- a/.github/workflows/custom-job.yaml +++ b/.github/workflows/custom-job.yaml @@ -6,32 +6,74 @@ on: required: true type: string branch: + description: | + Git branch the workflow run targets. + This is required even when 'sha' is provided because it is also used for organizing artifacts. type: string date: + description: "Date (YYYY-MM-DD) this run is for. Used to organize artifacts produced by nightly builds" type: string sha: + description: "Full git commit SHA to check out" type: string repo: + description: "Git repo to check out, in '{org}/{repo}' form, e.g. 'rapidsai/cudf'" type: string arch: + description: "One of [amd64, arm64]. CPU architecture to run on." type: string default: "amd64" node_type: + description: | + Suffix, without leading '-', indicating the type of machine to run jobs on (e.g., 'cpu4' or 'gpu-l4-latest-1'). + Runner labels are of the form '{operating_system}-{arch}-{node_type}'. + See https://github.com/nv-gha-runners/enterprise-runner-configuration/blob/main/docs/runner-groups.md for a list + of valid values. type: string default: "cpu8" container_image: + description: "Container image URI" type: string - default: "rapidsai/ci-conda:latest" - run_script: - required: true + default: "rapidsai/ci-conda:25.10-latest" + script: + required: false type: string + description: "Shell code to be executed in a step. Ideally this should just invoke a script managed in the repo the workflow runs from, like 'ci/test_java.sh'." file_to_upload: + description: | + Path to file(s) to be uploaded as a CI artifact. + If a directory is provided, all files in it will be bundled into a zip archive. + See 'path' input at https://github.com/actions/upload-artifact?tab=readme-ov-file#inputs type: string default: "gh-status.json" + artifact-name: + description: | + Identifier for the GitHub Actions artifact created by uploading ``file_to_upload``. + See 'name' input at https://github.com/actions/upload-artifact?tab=readme-ov-file#inputs + type: string + default: "result" continue-on-error: + description: | + If false (the default), treat job failures as workflow failures. + If true, job failures do not result in workflow failures (useful for implementing optional CI workflows). + See https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idcontinue-on-error type: boolean required: false default: false + build_workflow_name: + description: | + Name of a workflow file that produced artifacts to be downloaded in this run. + If not set (the default), artifact-handling scripts use RAPIDS-conventional defaults (like "build.yaml" when "build_type == nightly"). + required: false + type: string + alternative-gh-token-secret-name: + type: string + required: false + description: | + If provided, should contain the name of a secret in the repo which holds a GitHub API token. + When this is non-empty, that secret's value is used in place of the default repo-level token + anywhere that environment variable GH_TOKEN is set. This is especially useful for downloading + artifacts from other private repos, which repo tokens do not have access to. defaults: run: @@ -59,12 +101,12 @@ jobs: runs-on: "linux-${{ inputs.arch }}-${{ inputs.node_type }}" continue-on-error: ${{ inputs.continue-on-error }} container: - image: ${{ inputs.container_image }} + image: ${{ inputs.container_image }} # zizmor: ignore[unpinned-images] env: RAPIDS_BUILD_TYPE: ${{ inputs.build_type }} NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} steps: - - uses: aws-actions/configure-aws-credentials@v4 + - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: role-to-assume: ${{ vars.AWS_ROLE_ARN }} aws-region: ${{ vars.AWS_REGION }} @@ -74,36 +116,60 @@ jobs: repository: ${{ inputs.repo }} ref: ${{ inputs.sha }} fetch-depth: 0 + persist-credentials: true - name: Telemetry setup uses: rapidsai/shared-actions/telemetry-dispatch-setup@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + # DOES NOT NEED alternative-gh-token-secret-name - github.token is enough and more limited + GH_TOKEN: ${{ github.token }} - name: Get PR Info if: startsWith(github.ref_name, 'pull-request/') id: get-pr-info uses: nv-gha-runners/get-pr-info@main - name: Add PR Info if: startsWith(github.ref_name, 'pull-request/') - run: | + run: | # zizmor: ignore[template-injection] echo "RAPIDS_BASE_BRANCH=${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.ref }}" >> "${GITHUB_ENV}" - name: Standardize repository information + uses: rapidsai/shared-actions/rapids-github-info@main + with: + repo: ${{ inputs.repo }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + build_workflow_name: ${{ inputs.build_workflow_name }} + # Per the docs at https://docs.github.com/en/rest/rate-limit/rate-limit?apiVersion=2022-11-28#get-rate-limit-status-for-the-authenticated-user, + # checking '/rate_limit | jq .' should not itself count against any rate limits. + # + # gh CLI is pre-installed on Github-hosted runners, but may not be on self-hosted runners. + - name: Check GitHub API rate limits run: | - echo "RAPIDS_NIGHTLY_DATE=${{ inputs.date }}" >> "${GITHUB_ENV}" - echo "RAPIDS_REF_NAME=${{ inputs.branch || github.ref_name }}" >> "${GITHUB_ENV}" - echo "RAPIDS_REPOSITORY=${{ inputs.repo || github.repository }}" >> "${GITHUB_ENV}" - echo "RAPIDS_SHA=$(git rev-parse HEAD)" >> "${GITHUB_ENV}" - + if ! type gh >/dev/null; then + echo "'gh' CLI is not installed... skipping rate-limits check" + else + gh api /rate_limit | jq . + fi + env: + # NEEDS alternative-gh-token-secret_name - API limits need to be for whatever token is used for upload/download. Repo token may be a different pool for rate limits. + GH_TOKEN: ${{ inputs.alternative-gh-token-secret-name && secrets[inputs.alternative-gh-token-secret-name] || github.token }} # zizmor: ignore[overprovisioned-secrets] - name: Run script - run: ${{ inputs.run_script }} + run: ${INPUTS_SCRIPT} env: - GH_TOKEN: ${{ github.token }} + # NEEDS alternative-gh-token-secret-name - may require a token with more permissions + GH_TOKEN: ${{ inputs.alternative-gh-token-secret-name && secrets[inputs.alternative-gh-token-secret-name] || github.token }} # zizmor: ignore[overprovisioned-secrets] + INPUTS_SCRIPT: ${{ inputs.script }} - name: Upload file to GitHub Artifact uses: actions/upload-artifact@v4 with: - name: result + name: ${{ inputs.artifact-name }} path: ${{ inputs.file_to_upload }} if-no-files-found: ignore - name: Telemetry upload attributes uses: rapidsai/shared-actions/telemetry-dispatch-stash-job-artifacts@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + # DOES NOT NEED alternative-gh-token-secret-name - github.token is enough and more limited + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/pr-builder.yaml b/.github/workflows/pr-builder.yaml index b0b01721..80e77217 100644 --- a/.github/workflows/pr-builder.yaml +++ b/.github/workflows/pr-builder.yaml @@ -2,6 +2,9 @@ on: workflow_call: inputs: needs: + description: | + JSON string with the content of the 'needs' context for a GitHub Actions workflow. + For details, see https://docs.github.com/en/actions/reference/accessing-contextual-information-about-workflow-runs#example-contents-of-the-needs-context required: false type: string default: '{}' @@ -15,3 +18,20 @@ jobs: env: NEEDS: ${{ inputs.needs }} run: jq -en 'env.NEEDS | fromjson | all(.result as $result | ["success", "skipped"] | any($result == .))' + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 1 + persist-credentials: false + - name: Check for private token usage + env: + ERROR_MSG: "PR validation failed: Private token access is not allowed to be merged onto the development branch. Remove any uses of input 'alternative-gh-token-secret-name'." + # The 'grep' call below is intentionally not using '-q' or redirecting to /dev/null. + # + # Those choices + the use of '-n' mean that the exact file path and line numbers of the offending uses + # will show up in logs. + run: | + if grep -n -R -E 'alternative\-gh\-token\-secret\-name\:' ./.github; then + echo "::error::$ERROR_MSG" + exit 1 + fi diff --git a/.github/workflows/project-get-item-id.yaml b/.github/workflows/project-get-item-id.yaml index e64ac424..155a8b07 100644 --- a/.github/workflows/project-get-item-id.yaml +++ b/.github/workflows/project-get-item-id.yaml @@ -50,9 +50,10 @@ jobs: # Query up to 10 projects for the PR # There's no graphQL filter configured to query by a specific project # So we need to query all projects and filter the result ourselves - gh api graphql -F nodeId="$ENV_ITEM_NODE_ID" -f query=' + # shellcheck disable=SC2016 + gh api graphql -f nodeId="$ENV_ITEM_NODE_ID" -f query=' query($nodeId: ID!) { - node(id: $nodeId) { + node(id: $nodeId) { ... on PullRequest { projectItems(first: 10) { nodes { @@ -75,10 +76,10 @@ jobs: } } }' > project_data.json - + # Use jq to do the actual filtering, using --arg for safe variable passing - item_project_id=$(jq --arg project_id "$ENV_PROJECT_ID" -r '.data.node.projectItems.nodes[] | + item_project_id=$(jq --arg project_id "${ENV_PROJECT_ID}" -r '.data.node.projectItems.nodes[] | select(.project.id == $project_id) | .id' project_data.json) - echo "ITEM_PROJECT_ID=$item_project_id" >> $GITHUB_OUTPUT + echo "ITEM_PROJECT_ID=${item_project_id}" >> "${GITHUB_OUTPUT}" continue-on-error: true diff --git a/.github/workflows/project-get-set-iteration-field.yaml b/.github/workflows/project-get-set-iteration-field.yaml index 325fcd7c..7c854c43 100644 --- a/.github/workflows/project-get-set-iteration-field.yaml +++ b/.github/workflows/project-get-set-iteration-field.yaml @@ -19,7 +19,7 @@ on: description: "The graphQL node ID of the iteration field" type: string required: true - + ITEM_PROJECT_ID: description: "The issue or PR's graphQL project-specific ID" type: string @@ -69,7 +69,8 @@ jobs: run: | # Get current iteration iteration id # The current iteration is always the first element in the returned list - gh api graphql -F projectId="$ENV_PROJECT_ID" -F fieldName="$ENV_ITERATION_FIELD_NAME" -f query=' + # shellcheck disable=SC2016 + gh api graphql -f projectId="$ENV_PROJECT_ID" -f fieldName="$ENV_ITERATION_FIELD_NAME" -f query=' query($projectId: ID!, $fieldName: String!) { node(id: $projectId) { ... on ProjectV2 { @@ -88,10 +89,10 @@ jobs: } } }' > iteration_option_data.json - + # Use jq with --arg for safe variable handling current_iteration_option_id=$(jq --exit-status -r '.data.node.field.configuration.iterations[0].id' iteration_option_data.json) - echo "ITERATION_OPTION_ID=$current_iteration_option_id" >> "$GITHUB_OUTPUT" + echo "ITERATION_OPTION_ID=$current_iteration_option_id" >> "${GITHUB_OUTPUT}" continue-on-error: true - name: Update item iteration field @@ -104,10 +105,11 @@ jobs: ENV_ITERATION_FIELD_ID: ${{ inputs.ITERATION_FIELD_ID }} ENV_ITERATION_OPTION_ID: ${{ steps.get_iteration_option_id.outputs.ITERATION_OPTION_ID }} run: | - gh api graphql -F projectId="$ENV_PROJECT_ID" \ - -F itemId="$ENV_ITEM_PROJECT_ID" \ - -F fieldId="$ENV_ITERATION_FIELD_ID" \ - -F iterationId="$ENV_ITERATION_OPTION_ID" \ + # shellcheck disable=SC2016 + gh api graphql -f projectId="$ENV_PROJECT_ID" \ + -f itemId="$ENV_ITEM_PROJECT_ID" \ + -f fieldId="$ENV_ITERATION_FIELD_ID" \ + -f iterationId="$ENV_ITERATION_OPTION_ID" \ -f query=' mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $iterationId: String!) { updateProjectV2ItemFieldValue( @@ -140,4 +142,5 @@ jobs: UPDATE_FIELD_TYPE: "iteration" UPDATE_FIELD_ID: ${{ inputs.ITERATION_FIELD_ID }} UPDATE_FIELD_VALUE: ${{ needs.get_set_iteration_option_id.outputs.ITERATION_OPTION_ID }} - secrets: inherit + secrets: + ADD_TO_PROJECT_GITHUB_TOKEN: ${{ secrets.ADD_TO_PROJECT_GITHUB_TOKEN }} diff --git a/.github/workflows/project-get-set-single-select-field.yaml b/.github/workflows/project-get-set-single-select-field.yaml index a43c0083..20841a7f 100644 --- a/.github/workflows/project-get-set-single-select-field.yaml +++ b/.github/workflows/project-get-set-single-select-field.yaml @@ -19,7 +19,7 @@ on: description: "The graphQL node ID of the single-select field" type: string required: true - + ITEM_PROJECT_ID: description: "The issue or PR's graphQL project-specific ID" type: string @@ -77,8 +77,9 @@ jobs: # Get single_select option id if [ -z "$ENV_SINGLE_SELECT_OPTION_VALUE" ]; then # No option specified, get first option in list - gh api graphql -F projectId="$ENV_PROJECT_ID" \ - -F fieldName="$ENV_SINGLE_SELECT_FIELD_NAME" \ + # shellcheck disable=SC2016 + gh api graphql -f projectId="$ENV_PROJECT_ID" \ + -f fieldName="$ENV_SINGLE_SELECT_FIELD_NAME" \ -f query=' query($projectId: ID!, $fieldName: String!) { node(id: $projectId) { @@ -93,26 +94,27 @@ jobs: }' > single_select_option_data.json else # Get specific value - gh api graphql -F projectId="$ENV_PROJECT_ID" \ - -F fieldName="$ENV_SINGLE_SELECT_FIELD_NAME" \ - -F optionName="$ENV_SINGLE_SELECT_OPTION_VALUE" \ + # shellcheck disable=SC2016 + gh api graphql -f projectId="$ENV_PROJECT_ID" \ + -f fieldName="$ENV_SINGLE_SELECT_FIELD_NAME" \ + -f optionName="$ENV_SINGLE_SELECT_OPTION_VALUE" \ -f query=' query($projectId: ID!, $fieldName: String!, $optionName: String!) { node(id: $projectId) { ... on ProjectV2 { field(name: $fieldName) { ... on ProjectV2SingleSelectField { - options(names: $optionName) {id} + options(names: [$optionName]) {id} } } } } }' > single_select_option_data.json fi - + # Use jq with --exit-status to fail if no option is found current_single_select_option_id=$(jq --exit-status -r '.data.node.field.options[0].id' single_select_option_data.json) - echo "SINGLE_SELECT_OPTION_ID=$current_single_select_option_id" >> "$GITHUB_OUTPUT" + echo "SINGLE_SELECT_OPTION_ID=$current_single_select_option_id" >> "${GITHUB_OUTPUT}" continue-on-error: true - name: Update item single_select field @@ -125,10 +127,11 @@ jobs: ENV_SINGLE_SELECT_FIELD_ID: ${{ inputs.SINGLE_SELECT_FIELD_ID }} ENV_SINGLE_SELECT_OPTION_ID: ${{ steps.get_single_select_option_id.outputs.SINGLE_SELECT_OPTION_ID }} run: | - gh api graphql -F projectId="$ENV_PROJECT_ID" \ - -F itemId="$ENV_ITEM_PROJECT_ID" \ - -F fieldId="$ENV_SINGLE_SELECT_FIELD_ID" \ - -F optionId="$ENV_SINGLE_SELECT_OPTION_ID" \ + # shellcheck disable=SC2016 + gh api graphql -f projectId="$ENV_PROJECT_ID" \ + -f itemId="$ENV_ITEM_PROJECT_ID" \ + -f fieldId="$ENV_SINGLE_SELECT_FIELD_ID" \ + -f optionId="$ENV_SINGLE_SELECT_OPTION_ID" \ -f query=' mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $optionId: String!) { updateProjectV2ItemFieldValue( @@ -161,4 +164,5 @@ jobs: UPDATE_FIELD_TYPE: "single_select" UPDATE_FIELD_ID: ${{ inputs.SINGLE_SELECT_FIELD_ID }} UPDATE_FIELD_VALUE: ${{ needs.get_set_single_select_option_id.outputs.SINGLE_SELECT_OPTION_ID }} - secrets: inherit + secrets: + ADD_TO_PROJECT_GITHUB_TOKEN: ${{ secrets.ADD_TO_PROJECT_GITHUB_TOKEN }} diff --git a/.github/workflows/project-high-level-tracker.yaml b/.github/workflows/project-high-level-tracker.yaml new file mode 100644 index 00000000..617ad3b6 --- /dev/null +++ b/.github/workflows/project-high-level-tracker.yaml @@ -0,0 +1,268 @@ +name: RAPIDS High Level Tracking + +on: + workflow_call: + inputs: + PROJECT_ID: + description: "Project ID" + default: "PVT_kwDOAp2shc4ArK9w" + type: string + + # ------------- Field IDs ------------- + WEEK_FIELD_ID: + description: "Week Field ID" + default: "PVTIF_lADOAp2shc4ArK9wzgiTHg8" + type: string + RELEASE_FIELD_ID: + description: "Release Field ID" + default: "PVTF_lADOAp2shc4ArK9wzgpEYKw" + type: string + STAGE_FIELD_ID: + description: "Stage Field ID" + default: "PVTSSF_lADOAp2shc4ArK9wzgiTH8g" + type: string + START_DATE_FIELD_ID: + description: "Start Date Field ID" + default: "PVTF_lADOAp2shc4ArK9wzgiTHmQ" + type: string + END_DATE_FIELD_ID: + description: "End Date Field ID" + default: "PVTF_lADOAp2shc4ArK9wzgiTHmU" + type: string + + # ------------- Option IDs ------------- + # Stage options + STAGE_PLANNED_NAME: + description: "📓 Planned option ID" + default: "📓 Planned" + type: string + STAGE_DEVELOPMENT_NAME: + description: "💻 Development option ID" + default: "💻 Development" + type: string + STAGE_BURNDOWN_NAME: + description: "🔥 Burndown option ID" + default: "🔥 Burndown" + type: string + STAGE_CODE_FREEZE_NAME: + description: "❄ Code Freeze option ID" + default: "❄ Code Freeze" + type: string + STAGE_HOTFIX_NAME: + description: "🩹 Hotfix option ID" + default: "🩹 Hotfix" + type: string + + secrets: + ADD_TO_PROJECT_GITHUB_TOKEN: + description: "Project Access Token" + required: true + +jobs: + process-branch-name: + runs-on: ubuntu-latest + outputs: + release: ${{ steps.process-branch-name.outputs.branch-name }} + steps: + - name: Extract branch name + id: process-branch-name + run: | + branch=${{ github.event.pull_request.base.ref }} + release=${branch#branch-} + echo "release=$release" >> "$GITHUB_OUTPUT" + + get-stage-field: + runs-on: ubuntu-latest + outputs: + stage-id: ${{ steps.get-stage-field.outputs.stage-id }} + needs: + - process-branch-name + steps: + # Add cache step for releases.json + # Pinned to the 4.2 commit SHA + - name: Cache releases.json + id: cache-releases + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 + with: + path: /tmp/releases.json + # Cache key based on date - refresh daily to ensure we get updates + key: ${{ runner.os }}-releases-json-${{ github.run_id }}-${{ github.run_number }} + restore-keys: | + ${{ runner.os }}-releases-json- + + - name: Get stage field ID + id: get-stage-field + run: | + # Use the "nightly" info from this JSON: + # https://raw.githubusercontent.com/rapidsai/docs/refs/heads/main/_data/releases.json + # to determine the stage for the new release + + # Check if cache exists, download only if needed + if [ ! -f "/tmp/releases.json" ] || [ "$(find /tmp/releases.json -mtime +1 -print)" ]; then + echo "Cache miss or expired. Downloading releases.json..." + curl -s https://raw.githubusercontent.com/rapidsai/docs/main/_data/releases.json -o /tmp/releases.json + + # Validate the downloaded JSON + if ! jq empty /tmp/releases.json 2>/dev/null; then + echo "Error: Downloaded file is not valid JSON" + rm -f /tmp/releases.json + exit 1 + fi + else + echo "Using cached releases.json" + fi + + # Use the cached file + RELEASES_JSON=$(cat /tmp/releases.json) + + # Extract current repo name from GitHub context + REPO_NAME=$(echo "$GITHUB_REPOSITORY" | cut -d '/' -f 2) + echo "Repository: \"$REPO_NAME\"" + + # Check if repo is in cudf_dev group + CUDF_DEV_REPOS=("cuDF" "RMM" "rapids-cmake" "raft" "dask-cuda" "ucx-py") + IS_CUDF_DEV="false" + for repo in "${CUDF_DEV_REPOS[@]}"; do + if [[ "$REPO_NAME" == "$repo" ]]; then + IS_CUDF_DEV="true" + break + fi + done + echo "Is cudf_dev group: \"$IS_CUDF_DEV\"" + + # Get the release from branch name + RELEASE="${{ needs.process-branch-name.outputs.release }}" + echo "Release: $RELEASE" + + # Get current date in YYYY-MM-DD format + CURRENT_DATE=$(date +%Y-%m-%d) + echo "Current date: $CURRENT_DATE" + + # Extract dates from the releases.json for the current release + if [[ "$IS_CUDF_DEV" == "true" ]]; then + PLANNING_START=$(echo "$RELEASES_JSON" | jq -r --arg rel "$RELEASE" '.[$rel].cudf_dev.planning.start') + DEV_START=$(echo "$RELEASES_JSON" | jq -r --arg rel "$RELEASE" '.[$rel].cudf_dev.development.start') + BURNDOWN_START=$(echo "$RELEASES_JSON" | jq -r --arg rel "$RELEASE" '.[$rel].cudf_dev.burndown.start') + FREEZE_START=$(echo "$RELEASES_JSON" | jq -r --arg rel "$RELEASE" '.[$rel].cudf_dev.freeze.start') + RELEASE_DATE=$(echo "$RELEASES_JSON" | jq -r --arg rel "$RELEASE" '.[$rel].date') + else + PLANNING_START=$(echo "$RELEASES_JSON" | jq -r --arg rel "$RELEASE" '.[$rel].planning.start') + DEV_START=$(echo "$RELEASES_JSON" | jq -r --arg rel "$RELEASE" '.[$rel].development.start') + BURNDOWN_START=$(echo "$RELEASES_JSON" | jq -r --arg rel "$RELEASE" '.[$rel].burndown.start') + FREEZE_START=$(echo "$RELEASES_JSON" | jq -r --arg rel "$RELEASE" '.[$rel].freeze.start') + RELEASE_DATE=$(echo "$RELEASES_JSON" | jq -r --arg rel "$RELEASE" '.[$rel].date') + fi + + echo "Planning start: $PLANNING_START" + echo "Development start: $DEV_START" + echo "Burndown start: $BURNDOWN_START" + echo "Freeze start: $FREEZE_START" + echo "Release date: $RELEASE_DATE" + + # Determine stage based on current date + STAGE_ID="${{ inputs.STAGE_PLANNED_NAME }}" + + if [[ "$CURRENT_DATE" < "$DEV_START" ]]; then + STAGE_ID="${{ inputs.STAGE_PLANNED_NAME }}" + echo "Stage: Planned" + elif [[ "$CURRENT_DATE" < "$BURNDOWN_START" ]]; then + STAGE_ID="${{ inputs.STAGE_DEVELOPMENT_NAME }}" + echo "Stage: Development" + elif [[ "$CURRENT_DATE" < "$FREEZE_START" ]]; then + STAGE_ID="${{ inputs.STAGE_BURNDOWN_NAME }}" + echo "Stage: Burndown" + elif [[ "$CURRENT_DATE" < "$RELEASE_DATE" ]]; then + STAGE_ID="${{ inputs.STAGE_CODE_FREEZE_NAME }}" + echo "Stage: Code Freeze" + else + STAGE_ID="${{ inputs.STAGE_HOTFIX_NAME }}" + echo "Stage: Hotfix" + fi + + # Set output + echo "stage-id=$STAGE_ID" >> "$GITHUB_OUTPUT" + + get-project-id: + uses: ./.github/workflows/project-get-item-id.yaml + secrets: inherit + permissions: + contents: read + with: + PROJECT_ID: ${{ inputs.PROJECT_ID }} + ITEM_NODE_ID: "${{ github.event.pull_request.node_id }}" + + update-release-field: + uses: ./.github/workflows/project-set-text-date-numeric-field.yaml + needs: + - get-project-id + - process-branch-name + with: + PROJECT_ID: ${{ inputs.PROJECT_ID }} + FIELD_ID: ${{ inputs.RELEASE_FIELD_ID }} + FIELD_TYPE: text + SET_VALUE: ${{ needs.process-branch-name.outputs.release }} + ITEM_PROJECT_ID: "${{ needs.get-project-id.outputs.ITEM_PROJECT_ID }}" + ITEM_NODE_ID: "${{ github.event.pull_request.node_id }}" + UPDATE_LINKED_ISSUES: true + secrets: inherit + + update-stage-field: + uses: ./.github/workflows/project-get-set-single-select-field.yaml + needs: + - get-project-id + - process-branch-name + - get-stage-field + with: + PROJECT_ID: ${{ inputs.PROJECT_ID }} + SINGLE_SELECT_FIELD_ID: ${{ inputs.STAGE_FIELD_ID }} + SINGLE_SELECT_FIELD_NAME: "Stage" + SINGLE_SELECT_OPTION_VALUE: ${{ needs.get-stage-field.outputs.stage-id }} + ITEM_PROJECT_ID: "${{ needs.get-project-id.outputs.ITEM_PROJECT_ID }}" + ITEM_NODE_ID: "${{ github.event.pull_request.node_id }}" + UPDATE_ITEM: true + UPDATE_LINKED_ISSUES: true + secrets: inherit + + set-opened-date-field: + uses: ./.github/workflows/project-set-text-date-numeric-field.yaml + if: github.event.action == 'opened' || github.event.action == 'workflow_dispatch' + needs: + - get-project-id + with: + PROJECT_ID: ${{ inputs.PROJECT_ID }} + FIELD_ID: ${{ inputs.START_DATE_FIELD_ID }} + FIELD_TYPE: date + SET_VALUE: ${{ github.event.pull_request.created_at }} + ITEM_PROJECT_ID: "${{ needs.get-project-id.outputs.ITEM_PROJECT_ID }}" + ITEM_NODE_ID: "${{ github.event.pull_request.node_id }}" + UPDATE_LINKED_ISSUES: true + secrets: inherit + + set-closed-date-field: + uses: ./.github/workflows/project-set-text-date-numeric-field.yaml + if: github.event.action == 'closed' || github.event.action == 'workflow_dispatch' + needs: + - get-project-id + with: + PROJECT_ID: ${{ inputs.PROJECT_ID }} + FIELD_ID: ${{ inputs.END_DATE_FIELD_ID }} + FIELD_TYPE: date + SET_VALUE: ${{ github.event.pull_request.closed_at }} + ITEM_PROJECT_ID: "${{ needs.get-project-id.outputs.ITEM_PROJECT_ID }}" + ITEM_NODE_ID: "${{ github.event.pull_request.node_id }}" + UPDATE_LINKED_ISSUES: true + secrets: inherit + + update-week-field: + uses: ./.github/workflows/project-get-set-iteration-field.yaml + needs: + - get-project-id + with: + PROJECT_ID: ${{ inputs.PROJECT_ID }} + ITERATION_FIELD_ID: ${{ inputs.WEEK_FIELD_ID }} + ITERATION_FIELD_NAME: "Week" + ITEM_PROJECT_ID: "${{ needs.get-project-id.outputs.ITEM_PROJECT_ID }}" + ITEM_NODE_ID: "${{ github.event.pull_request.node_id }}" + UPDATE_ITEM: true + UPDATE_LINKED_ISSUES: true + secrets: inherit diff --git a/.github/workflows/project-set-text-date-numeric-field.yaml b/.github/workflows/project-set-text-date-numeric-field.yaml index 235a4bd1..2df3aa46 100644 --- a/.github/workflows/project-set-text-date-numeric-field.yaml +++ b/.github/workflows/project-set-text-date-numeric-field.yaml @@ -24,7 +24,7 @@ on: description: "The graphQL node ID of the field" type: string required: true - + ITEM_PROJECT_ID: description: "The issue or PR's graphQL project-specific ID" type: string @@ -32,7 +32,6 @@ on: ITEM_NODE_ID: description: "The issue or PR's graphQL node ID" - default: null type: string required: true @@ -66,11 +65,12 @@ jobs: run: | # Set the field based on the inputted desired value if [ "$ENV_FIELD_TYPE" == "date" ] || [ "$ENV_FIELD_TYPE" == "text" ]; then - gh api graphql -F projectId="$ENV_PROJECT_ID" \ - -F itemId="$ENV_ITEM_PROJECT_ID" \ - -F fieldId="$ENV_FIELD_ID" \ - -F value="$ENV_SET_VALUE" \ - -F fieldType="$ENV_FIELD_TYPE" \ + # shellcheck disable=SC2016 + gh api graphql -f projectId="$ENV_PROJECT_ID" \ + -f itemId="$ENV_ITEM_PROJECT_ID" \ + -f fieldId="$ENV_FIELD_ID" \ + -f value="$ENV_SET_VALUE" \ + -f fieldType="$ENV_FIELD_TYPE" \ -f query=' mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $value: String!, $fieldType: String!) { updateProjectV2ItemFieldValue( @@ -88,10 +88,11 @@ jobs: }' elif [ "$ENV_FIELD_TYPE" == "number" ]; then - gh api graphql -F projectId="$ENV_PROJECT_ID" \ - -F itemId="$ENV_ITEM_PROJECT_ID" \ - -F fieldId="$ENV_FIELD_ID" \ - -F value="$ENV_SET_VALUE" \ + # shellcheck disable=SC2016 + gh api graphql -f projectId="$ENV_PROJECT_ID" \ + -f itemId="$ENV_ITEM_PROJECT_ID" \ + -f fieldId="$ENV_FIELD_ID" \ + -f value="$ENV_SET_VALUE" \ -f query=' mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $value: Float!) { updateProjectV2ItemFieldValue( @@ -114,6 +115,7 @@ jobs: fi # Validate the response + # shellcheck disable=SC2181 if [ $? -ne 0 ]; then echo "Error: Failed to update field value" >&2 exit 1 @@ -132,4 +134,5 @@ jobs: UPDATE_FIELD_TYPE: ${{inputs.FIELD_TYPE}} UPDATE_FIELD_ID: ${{ inputs.FIELD_ID }} UPDATE_FIELD_VALUE: ${{ inputs.SET_VALUE }} - secrets: inherit + secrets: + ADD_TO_PROJECT_GITHUB_TOKEN: ${{ secrets.ADD_TO_PROJECT_GITHUB_TOKEN }} diff --git a/.github/workflows/project-update-linked-issues.yaml b/.github/workflows/project-update-linked-issues.yaml index b0e0f7b8..80979a95 100644 --- a/.github/workflows/project-update-linked-issues.yaml +++ b/.github/workflows/project-update-linked-issues.yaml @@ -61,7 +61,8 @@ jobs: ENV_UPDATE_FIELD_VALUE: ${{ inputs.UPDATE_FIELD_VALUE }} run: | # Find the linked issues to the PR - gh api graphql -F prNodeId="$ENV_PR_NODE_ID" -f query=' + # shellcheck disable=SC2016 + gh api graphql -f prNodeId="$ENV_PR_NODE_ID" -f query=' query($prNodeId: ID!) { node(id: $prNodeId) { ... on PullRequest { @@ -82,7 +83,7 @@ jobs: }' > linked_issues.json # Use jq with proper variable handling to get issue IDs - issue_ids=$(jq --exit-status --arg project_id "$ENV_PROJECT_ID" -r ' + issue_ids=$(jq --arg project_id "$ENV_PROJECT_ID" -r ' .data.node.closingIssuesReferences.nodes[].projectItems.nodes[] | select(.project.id == $project_id) | .id' linked_issues.json) @@ -101,10 +102,11 @@ jobs: case "$ENV_UPDATE_FIELD_TYPE" in "iteration") - gh api graphql -F projectId="$ENV_PROJECT_ID" \ - -F itemId="$issue_id" \ - -F fieldId="$ENV_UPDATE_FIELD_ID" \ - -F iterationId="$ENV_UPDATE_FIELD_VALUE" \ + # shellcheck disable=SC2016 + gh api graphql -f projectId="$ENV_PROJECT_ID" \ + -f itemId="$issue_id" \ + -f fieldId="$ENV_UPDATE_FIELD_ID" \ + -f iterationId="$ENV_UPDATE_FIELD_VALUE" \ -f query=' mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $iterationId: String!) { updateProjectV2ItemFieldValue( @@ -121,10 +123,11 @@ jobs: ;; "single_select") - gh api graphql -F projectId="$ENV_PROJECT_ID" \ - -F itemId="$issue_id" \ - -F fieldId="$ENV_UPDATE_FIELD_ID" \ - -F optionId="$ENV_UPDATE_FIELD_VALUE" \ + # shellcheck disable=SC2016 + gh api graphql -f projectId="$ENV_PROJECT_ID" \ + -f itemId="$issue_id" \ + -f fieldId="$ENV_UPDATE_FIELD_ID" \ + -f optionId="$ENV_UPDATE_FIELD_VALUE" \ -f query=' mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $optionId: String!) { updateProjectV2ItemFieldValue( @@ -141,11 +144,12 @@ jobs: ;; "date"|"text") - gh api graphql -F projectId="$ENV_PROJECT_ID" \ - -F itemId="$issue_id" \ - -F fieldId="$ENV_UPDATE_FIELD_ID" \ - -F value="$ENV_UPDATE_FIELD_VALUE" \ - -F fieldType="$ENV_UPDATE_FIELD_TYPE" \ + # shellcheck disable=SC2016 + gh api graphql -f projectId="$ENV_PROJECT_ID" \ + -f itemId="$issue_id" \ + -f fieldId="$ENV_UPDATE_FIELD_ID" \ + -f value="$ENV_UPDATE_FIELD_VALUE" \ + -f fieldType="$ENV_UPDATE_FIELD_TYPE" \ -f query=' mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $value: String!, $fieldType: String!) { updateProjectV2ItemFieldValue( @@ -162,10 +166,11 @@ jobs: ;; "number") - gh api graphql -F projectId="$ENV_PROJECT_ID" \ - -F itemId="$issue_id" \ - -F fieldId="$ENV_UPDATE_FIELD_ID" \ - -F value="$ENV_UPDATE_FIELD_VALUE" \ + # shellcheck disable=SC2016 + gh api graphql -f projectId="$ENV_PROJECT_ID" \ + -f itemId="$issue_id" \ + -f fieldId="$ENV_UPDATE_FIELD_ID" \ + -f value="$ENV_UPDATE_FIELD_VALUE" \ -f query=' mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $value: Float!) { updateProjectV2ItemFieldValue( @@ -187,6 +192,7 @@ jobs: ;; esac + # shellcheck disable=SC2181 if [ $? -ne 0 ]; then echo "Error: Failed to update field for issue $issue_id" >&2 exit 1 diff --git a/.github/workflows/wheels-build.yaml b/.github/workflows/wheels-build.yaml index 06359837..6a34fc1e 100644 --- a/.github/workflows/wheels-build.yaml +++ b/.github/workflows/wheels-build.yaml @@ -3,14 +3,19 @@ name: Build RAPIDS wheels on: workflow_call: inputs: - # repo and branch - repo: - type: string branch: + description: | + Git branch the workflow run targets. + This is required even when 'sha' is provided because it is also used for organizing artifacts. type: string date: + description: "Date (YYYY-MM-DD) this run is for. Used to organize artifacts produced by nightly builds" type: string sha: + description: "Full git commit SHA to check out" + type: string + repo: + description: "Git repo to check out, in '{org}/{repo}' form, e.g. 'rapidsai/cudf'" type: string build_type: description: "One of: [branch, nightly, pull-request]" @@ -19,42 +24,72 @@ on: script: required: true type: string - package-type: - required: false + description: "Shell code to be executed in a step. Ideally this should just invoke a script managed in the repo the workflow runs from, like 'ci/build_wheel.sh'." + package-name: + required: true type: string - wheel-name: - required: false + description: "Distribution name, without any other qualifiers (e.g. 'pylibcudf', not 'pylibcudf-cu12-cp311-manylinux_2_24_aarch64')" + package-type: + description: "One of: [cpp, python]" + required: true type: string pure-wheel: required: false type: boolean default: false + description: "One of [true, false], true if the wheel is not dependent on operating system, Python minor version, or CPU architecture" + append-cuda-suffix: + required: false + type: boolean + default: true + description: "One of [true, false] to indicate if CUDA version should be appended to the wheel name" # allow a bigger runner instance node_type: + description: | + Suffix, without leading '-', indicating the type of machine to run jobs on (e.g., 'cpu4' or 'gpu-l4-latest-1'). + Runner labels are of the form '{operating_system}-{arch}-{node_type}'. + See https://github.com/nv-gha-runners/enterprise-runner-configuration/blob/main/docs/runner-groups.md for a list + of valid values. required: false type: string default: "cpu16" # general settings matrix_filter: + description: | + jq expression which modifies the matrix. + For example, 'map(select(.ARCH == "amd64"))' to achieve "only run amd64 jobs". type: string default: "." - - # Extra repository that will be cloned into the project directory. + upload-artifacts: + type: boolean + default: true + required: false + description: "One of [true, false], true if artifacts should be uploaded to GitHub's artifact store" extra-repo: required: false type: string default: '' + description: "Extra repository that will be cloned into the project directory." extra-repo-sha: required: false type: string default: '' - # Note that this is the _name_ of a secret containing the key, not the key itself. + description: "Commit SHA in 'extra-repo' to clone." extra-repo-deploy-key: required: false type: string default: '' + description: "The _name_ of a secret containing a deploy key for 'extra-repo' (not the key itself)." + alternative-gh-token-secret-name: + type: string + required: false + description: | + If provided, should contain the name of a secret in the repo which holds a GitHub API token. + When this is non-empty, that secret's value is used in place of the default repo-level token + anywhere that environment variable GH_TOKEN is set. This is especially useful for downloading + artifacts from other private repos, which repo tokens do not have access to. defaults: run: @@ -94,19 +129,15 @@ jobs: # export MATRIX=" # amd64 - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '11.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '12.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '11.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '12.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '11.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.8.0', LINUX_VER: 'rockylinux8' } + - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8' } + - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8' } + - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8' } + - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8' } # arm64 - - { ARCH: 'arm64', PY_VER: '3.10', CUDA_VER: '11.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'arm64', PY_VER: '3.10', CUDA_VER: '12.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'arm64', PY_VER: '3.11', CUDA_VER: '11.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'arm64', PY_VER: '3.11', CUDA_VER: '12.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '11.8.0', LINUX_VER: 'rockylinux8' } - - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '12.8.0', LINUX_VER: 'rockylinux8' } + - { ARCH: 'arm64', PY_VER: '3.10', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8' } + - { ARCH: 'arm64', PY_VER: '3.11', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8' } + - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8' } + - { ARCH: 'arm64', PY_VER: '3.13', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8' } " MATRIX="$( @@ -124,12 +155,12 @@ jobs: env: RAPIDS_ARTIFACTS_DIR: ${{ github.workspace }}/artifacts container: - image: "rapidsai/ci-wheel:cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" + image: "rapidsai/ci-wheel:25.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" env: RAPIDS_BUILD_TYPE: ${{ inputs.build_type }} steps: - - uses: aws-actions/configure-aws-credentials@v4 + - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: role-to-assume: ${{ vars.AWS_ROLE_ARN }} aws-region: ${{ vars.AWS_REGION }} @@ -154,9 +185,11 @@ jobs: - name: Preprocess extra repos id: preprocess-extras if: ${{ inputs.extra-repo != '' }} + env: + EXTRA_REPO: ${{ inputs.extra-repo }} run: | - EXTRA_REPO_PATH=$(echo ${{ inputs.extra-repo }} | cut -d "/" -f 2) - echo "EXTRA_REPO_PATH=${EXTRA_REPO_PATH}" >> $GITHUB_OUTPUT + EXTRA_REPO_PATH=$(echo "$EXTRA_REPO" | cut -d "/" -f 2) + echo "EXTRA_REPO_PATH=${EXTRA_REPO_PATH}" >> "${GITHUB_OUTPUT}" - name: checkout extra repos uses: actions/checkout@v4 @@ -165,7 +198,7 @@ jobs: repository: ${{ inputs.extra-repo }} ref: ${{ inputs.extra-repo-sha }} path: "./${{ steps.preprocess-extras.outputs.EXTRA_REPO_PATH }}" - ssh-key: ${{ secrets[inputs.extra-repo-deploy-key] }} + ssh-key: ${{ secrets[inputs.extra-repo-deploy-key] }} # zizmor: ignore[overprovisioned-secrets] persist-credentials: false - name: Setup proxy cache @@ -175,53 +208,81 @@ jobs: - name: Telemetry setup uses: rapidsai/shared-actions/telemetry-dispatch-setup@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + # DOES NOT NEED alternative-gh-token-secret-name - github.token is enough and more limited + GH_TOKEN: ${{ github.token }} with: extra_attributes: "rapids.PACKAGER=wheel,rapids.CUDA_VER=${{ matrix.CUDA_VER }},rapids.PY_VER=${{ matrix.PY_VER }},rapids.ARCH=${{ matrix.ARCH }},rapids.LINUX_VER=${{ matrix.LINUX_VER }}" - - name: Build and repair the wheel + # Per the docs at https://docs.github.com/en/rest/rate-limit/rate-limit?apiVersion=2022-11-28#get-rate-limit-status-for-the-authenticated-user, + # checking '/rate_limit | jq .' should not itself count against any rate limits. + # + # gh CLI is pre-installed on Github-hosted runners, but may not be on self-hosted runners. + - name: Check GitHub API rate limits run: | - ${{ inputs.script }} + if ! type gh >/dev/null; then + echo "'gh' CLI is not installed... skipping rate-limits check" + else + gh api /rate_limit | jq . + fi env: - GH_TOKEN: ${{ github.token }} + # NEEDS alternative-gh-token-secret_name - API limits need to be for whatever token is used for upload/download. Repo token may be a different pool for rate limits. + GH_TOKEN: ${{ inputs.alternative-gh-token-secret-name && secrets[inputs.alternative-gh-token-secret-name] || github.token }} # zizmor: ignore[overprovisioned-secrets] + - name: Build and repair the wheel + run: ${{ inputs.script }} # zizmor: ignore[template-injection] + env: + # NEEDS alternative-gh-token-secret-name - may require a token with more permissions + GH_TOKEN: ${{ inputs.alternative-gh-token-secret-name && secrets[inputs.alternative-gh-token-secret-name] || github.token }} # zizmor: ignore[overprovisioned-secrets] # Use a shell that loads the rc file so that we get the compiler settings shell: bash -leo pipefail {0} - name: Get package name - if: inputs.package-type != '' + if: ${{ inputs.upload-artifacts }} env: + APPEND_CUDA_SUFFIX: ${{ inputs.append-cuda-suffix }} + PACKAGE_NAME: ${{ inputs.package-name }} PACKAGE_TYPE: ${{ inputs.package-type }} - WHEEL_NAME: ${{ inputs.wheel-name }} PURE_WHEEL: ${{ inputs.pure-wheel }} run: | - if [ -z "${WHEEL_NAME}" ]; then - WHEEL_NAME="${RAPIDS_REPOSITORY#*/}" + if [ -z "${PACKAGE_NAME}" ]; then + PACKAGE_NAME="${RAPIDS_REPOSITORY#*/}" + fi + export "RAPIDS_PY_CUDA_SUFFIX=$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" + if [ "${APPEND_CUDA_SUFFIX}" = "true" ]; then + export "RAPIDS_PY_WHEEL_NAME=${PACKAGE_NAME}_${RAPIDS_PY_CUDA_SUFFIX}" + else + export "RAPIDS_PY_WHEEL_NAME=${PACKAGE_NAME}" fi - export "RAPIDS_PY_CUDA_SUFFIX=$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - export "RAPIDS_PY_WHEEL_NAME=${WHEEL_NAME}_${RAPIDS_PY_CUDA_SUFFIX}" if [ "${PURE_WHEEL}" = "true" ]; then export "RAPIDS_PY_WHEEL_PURE=1" fi - echo "RAPIDS_PACKAGE_NAME=$(RAPIDS_NO_PKG_EXTENSION=true rapids-package-name wheel_${PACKAGE_TYPE})" >> "${GITHUB_OUTPUT}" + echo "RAPIDS_PACKAGE_NAME=$(RAPIDS_NO_PKG_EXTENSION=true rapids-package-name "wheel_${PACKAGE_TYPE}")" >> "${GITHUB_OUTPUT}" echo "WHEEL_OUTPUT_DIR=${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" >> "${GITHUB_OUTPUT}" id: package-name - name: Show files to be uploaded - if: inputs.package-type != '' + if: ${{ inputs.upload-artifacts }} + env: + WHEEL_OUTPUT_DIR: ${{ steps.package-name.outputs.WHEEL_OUTPUT_DIR }} run: | echo "Contents of directory to be uploaded:" - ls -R ${{ steps.package-name.outputs.WHEEL_OUTPUT_DIR }} - + ls -R "$WHEEL_OUTPUT_DIR" + - uses: actions/upload-artifact@v4 - if: inputs.package-type != '' + if: ${{ inputs.upload-artifacts }} with: + if-no-files-found: 'error' name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} path: ${{ steps.package-name.outputs.WHEEL_OUTPUT_DIR }} - + - name: Upload additional artifacts if: "!cancelled()" - run: rapids-upload-artifacts-dir cuda${RAPIDS_CUDA_VERSION%%.*}_$(arch)_py${RAPIDS_PY_VERSION//.} + run: rapids-upload-artifacts-dir "cuda${RAPIDS_CUDA_VERSION%%.*}_$(arch)_py${RAPIDS_PY_VERSION//.}" - name: Telemetry upload attributes - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + # DOES NOT NEED alternative-gh-token-secret-name - github.token is enough and more limited + GH_TOKEN: ${{ github.token }} uses: rapidsai/shared-actions/telemetry-dispatch-stash-job-artifacts@main continue-on-error: true diff --git a/.github/workflows/wheels-publish.yaml b/.github/workflows/wheels-publish.yaml index 96148846..59ec2258 100644 --- a/.github/workflows/wheels-publish.yaml +++ b/.github/workflows/wheels-publish.yaml @@ -3,14 +3,19 @@ name: Publish RAPIDS wheels on: workflow_call: inputs: - # repo and branch - repo: - type: string branch: + description: | + Git branch the workflow run targets. + This is required even when 'sha' is provided because it is also used for organizing artifacts. type: string date: + description: "Date (YYYY-MM-DD) this run is for. Used to organize artifacts produced by nightly builds" type: string sha: + description: "Full git commit SHA to check out" + type: string + repo: + description: "Git repo to check out, in '{org}/{repo}' form, e.g. 'rapidsai/cudf'" type: string build_type: description: "One of: [branch, nightly, pull-request]" @@ -19,6 +24,8 @@ on: # general settings package-name: + description: | + Distribution name, without any other qualifiers (e.g. 'pylibcudf', not 'pylibcudf-cu12-cp311-manylinux_2_24_aarch64') required: true type: string package-type: @@ -27,9 +34,15 @@ on: default: python type: string publish_to_pypi: - required: false + description: "If true, the wheel will be published to pypi.org" type: boolean default: false + build_workflow_name: + description: | + Name of a workflow file that produced artifacts to be downloaded in this run. + If not set (the default), artifact-handling scripts use RAPIDS-conventional defaults (like "build.yaml" when "build_type == nightly"). + required: false + type: string permissions: actions: read @@ -55,16 +68,10 @@ jobs: container: # CUDA toolkit version of the container is irrelevant in the publish step. # This just uploads already-built wheels to remote storage. - image: "rapidsai/ci-wheel:latest" + image: "rapidsai/ci-wheel:25.10-latest" # zizmor: ignore[unpinned-images] env: RAPIDS_BUILD_TYPE: ${{ inputs.build_type }} steps: - - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ vars.AWS_ROLE_ARN }} - aws-region: ${{ vars.AWS_REGION }} - role-duration-seconds: 43200 # 12h - - name: checkout code repo uses: actions/checkout@v4 with: @@ -76,20 +83,40 @@ jobs: - name: Telemetry setup uses: rapidsai/shared-actions/telemetry-dispatch-setup@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + GH_TOKEN: ${{ github.token }} - name: Standardize repository information uses: rapidsai/shared-actions/rapids-github-info@main with: repo: ${{ inputs.repo }} branch: ${{ inputs.branch }} + build_workflow_name: ${{ inputs.build_workflow_name }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} - - name: Download wheels from downloads.rapids.ai and publish to anaconda repository - run: rapids-wheels-anaconda "${{ inputs.package-name }}" "${{ inputs.package-type }}" + # Per the docs at https://docs.github.com/en/rest/rate-limit/rate-limit?apiVersion=2022-11-28#get-rate-limit-status-for-the-authenticated-user, + # checking '/rate_limit | jq .' should not itself count against any rate limits. + - name: Check GitHub API rate limits + run: | + if ! type gh >/dev/null; then + echo "'gh' CLI is not installed... skipping rate-limits check" + else + gh api /rate_limit | jq . + fi + env: + GH_TOKEN: ${{ github.token }} + + - name: Download wheels from artifact storage and publish to anaconda repository + run: rapids-wheels-anaconda-github "${INPUTS_PACKAGE_NAME}" "${INPUTS_PACKAGE_TYPE}" env: + GH_TOKEN: ${{ github.token }} RAPIDS_CONDA_TOKEN: ${{ secrets.CONDA_RAPIDSAI_WHEELS_NIGHTLY_TOKEN }} + PACKAGENAME: ${{ inputs.package-name }} + PACKAGETYPE: ${{ inputs.package-type }} + INPUTS_PACKAGE_NAME: ${{ inputs.package-name }} + INPUTS_PACKAGE_TYPE: ${{ inputs.package-type }} - name: Check if build is release id: check_if_release @@ -97,9 +124,9 @@ jobs: if: ${{ inputs.publish_to_pypi }} run: | if rapids-is-release-build; then - echo "is_release_build=true" | tee -a ${GITHUB_OUTPUT} + echo "is_release_build=true" | tee -a "${GITHUB_OUTPUT}" else - echo "is_release_build=false" | tee -a ${GITHUB_OUTPUT} + echo "is_release_build=false" | tee -a "${GITHUB_OUTPUT}" fi - name: Publish the downloaded wheels to PyPI @@ -112,4 +139,6 @@ jobs: - name: Telemetry upload attributes uses: rapidsai/shared-actions/telemetry-dispatch-stash-job-artifacts@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/wheels-test.yaml b/.github/workflows/wheels-test.yaml index cc97ed89..5a215655 100644 --- a/.github/workflows/wheels-test.yaml +++ b/.github/workflows/wheels-test.yaml @@ -3,14 +3,19 @@ name: Test RAPIDS wheels on: workflow_call: inputs: - # repo and branch - repo: - type: string branch: + description: | + Git branch the workflow run targets. + This is required even when 'sha' is provided because it is also used for organizing artifacts. type: string date: + description: "Date (YYYY-MM-DD) this run is for. Used to organize artifacts produced by nightly builds" type: string sha: + description: "Full git commit SHA to check out" + type: string + repo: + description: "Git repo to check out, in '{org}/{repo}' form, e.g. 'rapidsai/cudf'" type: string build_type: description: "One of: [branch, nightly, pull-request]" @@ -23,23 +28,53 @@ on: default: "auto" script: type: string - default: "ci/test_wheel.sh" + required: true + description: "Shell code to be executed in a step. Ideally this should just invoke a script managed in the repo the workflow runs from, like 'ci/test_wheel.sh'." matrix_filter: + description: | + jq expression which modifies the matrix. + For example, 'map(select(.ARCH == "amd64"))' to achieve "only run amd64 jobs". type: string default: "." container-options: + description: | + Command-line arguments passed to 'docker run' when starting the container this workflow runs in. + This should be provided as a single string to be inlined into 'docker run', not an array. + For example, '--quiet --ulimit nofile=2048'. required: false type: string default: "-e _NOOP" test_summary_show: + description: | + Sets the 'show:' input to the test-summary/action third-party action. + The default, 'fail', means "only show failing tests in the summary in the GitHub UI". + See https://github.com/test-summary/action?tab=readme-ov-file#options for a list of + available options. required: false type: string default: "fail" - # the use of secrets in shared-workflows is discouraged, especially for public repositories. - # these values were added for situations where the use of secrets is unavoidable. - secrets: - RAPIDS_AUX_SECRET_1: + rapids-aux-secret-1: required: false + type: string + default: '' + description: | + The NAME (not value) of a GitHub secret in the calling repo. + This allows callers of the workflow to make a single secret available in the job's + environment, via environment variable `RAPIDS_AUX_SECRET_1`. + build_workflow_name: + description: | + Name of a workflow file that produced artifacts to be downloaded in this run. + If not set (the default), artifact-handling scripts use RAPIDS-conventional defaults (like "build.yaml" when "build_type == nightly"). + required: false + type: string + alternative-gh-token-secret-name: + type: string + required: false + description: | + If provided, should contain the name of a secret in the repo which holds a GitHub API token. + When this is non-empty, that secret's value is used in place of the default repo-level token + anywhere that environment variable GH_TOKEN is set. This is especially useful for downloading + artifacts from other private repos, which repo tokens do not have access to. defaults: run: @@ -93,19 +128,19 @@ jobs: export MATRICES=" pull-request: # amd64 - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '12.8.0', LINUX_VER: 'ubuntu24.04', GPU: 'l4', DRIVER: 'latest', DEPENDENCIES: 'oldest' } + - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '12.9.1', LINUX_VER: 'ubuntu24.04', GPU: 'l4', DRIVER: 'latest', DEPENDENCIES: 'oldest' } # arm64 - - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '11.8.0', LINUX_VER: 'ubuntu20.04', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'latest' } + - { ARCH: 'arm64', PY_VER: '3.13', CUDA_VER: '12.9.1', LINUX_VER: 'ubuntu22.04', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'latest' } nightly: # amd64 - - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '11.8.0', LINUX_VER: 'rockylinux8', GPU: 'l4', DRIVER: 'latest', DEPENDENCIES: 'oldest' } - - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '12.0.1', LINUX_VER: 'ubuntu20.04', GPU: 'l4', DRIVER: 'latest', DEPENDENCIES: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.8.0', LINUX_VER: 'ubuntu22.04', GPU: 'l4', DRIVER: 'earliest', DEPENDENCIES: 'latest' } - - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.8.0', LINUX_VER: 'ubuntu24.04', GPU: 'h100', DRIVER: 'latest', DEPENDENCIES: 'latest' } + - { ARCH: 'amd64', PY_VER: '3.10', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8', GPU: 'l4', DRIVER: 'latest', DEPENDENCIES: 'oldest' } + - { ARCH: 'amd64', PY_VER: '3.11', CUDA_VER: '12.0.1', LINUX_VER: 'ubuntu22.04', GPU: 'l4', DRIVER: 'latest', DEPENDENCIES: 'latest' } + - { ARCH: 'amd64', PY_VER: '3.12', CUDA_VER: '12.9.1', LINUX_VER: 'ubuntu22.04', GPU: 'l4', DRIVER: 'earliest', DEPENDENCIES: 'latest' } + - { ARCH: 'amd64', PY_VER: '3.13', CUDA_VER: '12.9.1', LINUX_VER: 'ubuntu24.04', GPU: 'h100', DRIVER: 'latest', DEPENDENCIES: 'latest' } # arm64 - - { ARCH: 'arm64', PY_VER: '3.10', CUDA_VER: '11.8.0', LINUX_VER: 'rockylinux8', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'oldest' } - - { ARCH: 'arm64', PY_VER: '3.11', CUDA_VER: '12.2.2', LINUX_VER: 'ubuntu20.04', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'latest' } - - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '12.8.0', LINUX_VER: 'ubuntu24.04', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'latest' } + - { ARCH: 'arm64', PY_VER: '3.10', CUDA_VER: '12.9.1', LINUX_VER: 'rockylinux8', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'oldest' } + - { ARCH: 'arm64', PY_VER: '3.12', CUDA_VER: '12.2.2', LINUX_VER: 'ubuntu22.04', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'latest' } + - { ARCH: 'arm64', PY_VER: '3.13', CUDA_VER: '12.9.1', LINUX_VER: 'ubuntu24.04', GPU: 'a100', DRIVER: 'latest', DEPENDENCIES: 'latest' } " # only overwrite MATRIX_TYPE if it was set to 'auto' @@ -139,13 +174,13 @@ jobs: matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }} runs-on: "linux-${{ matrix.ARCH }}-gpu-${{ matrix.GPU }}-${{ matrix.DRIVER }}-1" container: - image: "rapidsai/citestwheel:cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" + image: "rapidsai/citestwheel:25.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" options: ${{ inputs.container-options }} env: NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} # GPU jobs must set this container env variable RAPIDS_BUILD_TYPE: ${{ inputs.build_type }} steps: - - uses: aws-actions/configure-aws-credentials@v4 + - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: role-to-assume: ${{ vars.AWS_ROLE_ARN }} aws-region: ${{ vars.AWS_REGION }} @@ -164,7 +199,10 @@ jobs: - name: Telemetry setup uses: rapidsai/shared-actions/telemetry-dispatch-setup@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + # DOES NOT NEED alternative-gh-token-secret-name - github.token is enough and more limited + GH_TOKEN: ${{ github.token }} with: extra_attributes: "rapids.PACKAGER=wheel,rapids.CUDA_VER=${{ matrix.CUDA_VER }},rapids.PY_VER=${{ matrix.PY_VER }},rapids.ARCH=${{ matrix.ARCH }},rapids.LINUX_VER=${{ matrix.LINUX_VER }},rapids.GPU=${{ matrix.GPU }},rapids.DRIVER=${{ matrix.DRIVER }},rapids.DEPENDENCIES=${{ matrix.DEPENDENCIES }}" @@ -175,21 +213,37 @@ jobs: branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} + build_workflow_name: ${{ inputs.build_workflow_name }} - name: Setup proxy cache uses: nv-gha-runners/setup-proxy-cache@main continue-on-error: true - # Skip the cache on RDS Lab nodes - if: ${{ matrix.GPU != 'v100' }} + + # Per the docs at https://docs.github.com/en/rest/rate-limit/rate-limit?apiVersion=2022-11-28#get-rate-limit-status-for-the-authenticated-user, + # checking '/rate_limit | jq .' should not itself count against any rate limits. + # + # gh CLI is pre-installed on Github-hosted runners, but may not be on self-hosted runners. + - name: Check GitHub API rate limits + run: | + if ! type gh >/dev/null; then + echo "'gh' CLI is not installed... skipping rate-limits check" + else + gh api /rate_limit | jq . + fi + env: + # NEEDS alternative-gh-token-secret_name - API limits need to be for whatever token is used for upload/download. Repo token may be a different pool for rate limits. + GH_TOKEN: ${{ inputs.alternative-gh-token-secret-name && secrets[inputs.alternative-gh-token-secret-name] || github.token }} # zizmor: ignore[overprovisioned-secrets] - name: Run tests - run: ${{ inputs.script }} + run: ${INPUTS_SCRIPT} env: - GH_TOKEN: ${{ github.token }} - RAPIDS_AUX_SECRET_1: ${{ secrets.RAPIDS_AUX_SECRET_1 }} + # NEEDS alternative-gh-token-secret-name - may require a token with more permissions + GH_TOKEN: ${{ inputs.alternative-gh-token-secret-name && secrets[inputs.alternative-gh-token-secret-name] || github.token }} # zizmor: ignore[overprovisioned-secrets] + RAPIDS_AUX_SECRET_1: ${{ inputs.rapids-aux-secret-1 != '' && secrets[inputs.rapids-aux-secret-1] || '' }} # zizmor: ignore[overprovisioned-secrets] + INPUTS_SCRIPT: ${{ inputs.script }} - name: Generate test report - uses: test-summary/action@v2.4 + uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 with: paths: "${{ env.RAPIDS_TESTS_DIR }}/*.xml" show: ${{ inputs.test_summary_show }} @@ -197,8 +251,11 @@ jobs: - name: Upload additional artifacts if: "!cancelled()" - run: rapids-upload-artifacts-dir cuda${RAPIDS_CUDA_VERSION%%.*}_$(arch)_py${RAPIDS_PY_VERSION//.} + run: rapids-upload-artifacts-dir "cuda${RAPIDS_CUDA_VERSION%%.*}_$(arch)_py${RAPIDS_PY_VERSION//.}" - name: Telemetry upload attributes uses: rapidsai/shared-actions/telemetry-dispatch-stash-job-artifacts@main continue-on-error: true - if: ${{ vars.TELEMETRY_ENABLED == 'true' && github.run_attempt == '1' }} + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + env: + # DOES NOT NEED alternative-gh-token-secret-name - github.token is enough and more limited + GH_TOKEN: ${{ github.token }} diff --git a/.github/zizmor.yml b/.github/zizmor.yml new file mode 100644 index 00000000..d7371b03 --- /dev/null +++ b/.github/zizmor.yml @@ -0,0 +1,7 @@ +rules: + unpinned-uses: + config: + policies: + rapidsai/shared-actions/*: ref-pin + nv-gha-runners/*: ref-pin + actions/*: ref-pin diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..3f70e99a --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,37 @@ +--- +# Copyright (c) 2025, NVIDIA CORPORATION. + +ci: + skip: [actionlint-docker] + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + - id: check-added-large-files + - id: check-yaml + - id: end-of-file-fixer + - repo: https://github.com/rhysd/actionlint + rev: v1.7.7 + hooks: + - id: actionlint-docker + - repo: https://github.com/codespell-project/codespell + rev: v2.4.1 + hooks: + - id: codespell + additional_dependencies: [tomli] + - repo: https://github.com/rapidsai/pre-commit-hooks + rev: v0.7.0 + hooks: + - id: verify-copyright + - repo: https://github.com/zizmorcore/zizmor-pre-commit + # Zizmor version. + rev: v1.12.1 + hooks: + # Run the linter. + - id: zizmor + exclude: | + (?x)^( + .github/workflows/build-in-devcontainer.yaml + )$ diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 4a4258b3..0fd20e86 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -24,4 +24,7 @@ function sed_runner() { for FILE in .github/workflows/*.yaml; do sed_runner "/rapidsai\/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}" + + # Update CI image tags + sed_runner "/rapidsai\/ci.*:[0-9\.]*-/ s/:[0-9\.]*-/:${NEXT_SHORT_TAG}-/g" "${FILE}" done diff --git a/docs/project-automation-readme.md b/docs/project-automation-readme.md new file mode 100644 index 00000000..99c5ecce --- /dev/null +++ b/docs/project-automation-readme.md @@ -0,0 +1,170 @@ +# RAPIDS Project Automation + +## Overview + +This collection of GitHub Actions workflows supports automation of the management of GitHub Projects, helping track development progress across RAPIDS repositories. + +The automations can also synchronize fields from PR to Issue such as the statuses, release information, and development stages into project fields, making project management effortless and consistent. + +## 🔍 The Challenge + +GitHub Projects are powerful, but automating them is not straightforward: + +- Each item has a **project-specific ID** that's different from its global node ID +- Different field types (text, date, single-select, iteration) require different GraphQL mutations +- Linking PRs to issues and keeping them in sync requires complex operations + - Work often begins in an Issue, but then is completed in a PR, requiring this synchronization + +### PRs can be linked to multiple issues, and issues can be linked to multiple PRs which can lead to challenges in automation + +The PR-to-Issue linkage exists within the PR's API, but not within the Issue's API. Additionally, in GitHub if many PRs are linked to an issue, closing _any_ of the PRs will close the issue. + +The shared workflows here follow a similar pattern in that if `update-linked-issues` is run, it will update all issues linked to the PR if they are in the same project as the target workflow (ie issues with multiple Projects will only have the matching Project fields updated). + +## 🧩 Workflow Architecture + +By centralizing this logic in reusable workflows, all RAPIDS repositories maintain consistent project tracking without duplicating code. + +Using the high-level tracker as an example, the automation is built as a set of reusable workflows that handle different aspects of project management (♻️ denotes a shared workflow): + +```mermaid +flowchart LR + A[PR Event] --> B[project-high-level-tracker ♻️] + B --> C[process-branch-name] + B --> D[get-stage-field] + B --> E[get-project-id ♻️] + D --> F[update-stage-field ♻️] + C --> G[update-release-field ♻️] + E --> G + E --> F + E --> H[update-week-field ♻️] + E --> I[set-opened-date-field ♻️] + E --> J[set-closed-date-field ♻️] + F --> K[update-linked-issues ♻️] + G --> K + H --> K + I --> K + J --> K +``` + +## 📁 Workflows + +### Core Workflows + +1. **project-get-item-id.yaml** + Gets the project-specific ID for an item (PR or issue) within a project - a critical first step for all operations. + +2. **project-set-text-date-numeric-field.yaml** + Updates text, date, or numeric fields in the project. + +3. **project-get-set-single-select-field.yaml** + Handles single-select (dropdown) fields like Status, Stage, etc. + +4. **project-get-set-iteration-field.yaml** + Manages iteration fields for sprint/week tracking. + +### Support Workflows + +5. **project-update-linked-issues.yaml** + Synchronizes linked issues with the PR's field values.
+ Unfortunately, we cannot go from Issue -> PR; the linkage exists only within the PR's API. + +6. **project-high-level-tracker.yaml** + The main entry point that orchestrates RAPIDS high-level project tracking, handling PR events and updating project fields based on release schedules. + +## 📋 Configuration + +### Project Field IDs + +The workflows require GraphQL node IDs for the project and its fields. These are provided as inputs: + +```yaml +inputs: + PROJECT_ID: + description: "Project ID" + default: "PVT_placeholder" # PVT = projectV2 + WEEK_FIELD_ID: + description: "Week Field ID" + default: "PVTIF_placeholder" + # ... other field IDs +``` + +### Getting Project and Field IDs + +To gather the required GraphQL node IDs for your project setup, use the included helper script: + +```bash +python docs/project-graphql-helper.py --token YOUR_GITHUB_TOKEN --org PROJECT_ORG_NAME --project PROJECT_NUMBER +``` + +Using the cuDF project https://github.com/orgs/rapidsai/projects/128: +```bash +python docs/project-graphql-helper.py -t ghp_placeholder -o rapidsai -p 128 +``` + +The script will output: +1. The Project ID (starts with `PVT_`) - use this for the `PROJECT_ID` input +2. A dictionary of all fields in the project with their IDs and metadata: + - Regular fields (text, date, etc.) `PVTF_...` + - Single-select fields include their options with option IDs `PVTSSF_...` + - Iteration fields include configuration details `PVTIF_...` + +Example output: +``` +PVT_kwDOAp2shc4AiNzl + +'Release': {'id': 'PVTSSF_lADOAp2shc4AiNzlzgg52UQ', + 'name': 'Release', + 'options': {'24.12': '582d2086', + '25.02': 'ee3d53a3', + '25.04': '0e757f49', + 'Backlog': 'be6006c4'}}, + 'Status': {'id': 'PVTSSF_lADOAp2shc4AiNzlzgaxNac', + 'name': 'Status', + 'options': {'Blocked': 'b0c2860f', + 'Done': '98236657', + 'In Progress': '47fc9ee4', + 'Todo': '1ba1e8b7'}}, +``` + +Use these IDs to populate the workflow input variables in your caller workflow. + +## 📊 Use Cases + +### Tracking PR Progress + +The workflow automatically tracks where a PR is in the development lifecycle: + +1. When a PR is opened against a release branch, it's tagged with the release +2. The stage is updated based on the current date relative to release milestones +3. Week iteration fields are updated to show current sprint +4. All linked issues inherit these values + +## 🚀 How The High-Level Tracker Works + +***Prerequisites*** +Set up a caller workflow that runs `project-high-level-tracker.yaml` on PR events in your repository. +Ensure your project is set up to auto-add PRs to the project when they are opened. + +1. When a PR is opened, updated, or closed, the workflow: + - Extracts the release from the branch name + - Determines the current development stage based on release timelines + - Updates fields in the project + - Synchronizes all linked issues + +2. The workflow uses a cached `releases.json` file to determine the current stage: + - Planning + - Development + - Burndown + - Code Freeze + - Hotfix + +## 📚 Related Resources + +- [GitHub GraphQL API Documentation](https://docs.github.com/en/graphql) +- [GitHub Projects API](https://docs.github.com/en/issues/planning-and-tracking-with-projects/automating-your-project/using-the-api-to-manage-projects) + +--- + +> [!Note] +> These workflows are designed for the RAPIDS ecosystem but can be adapted for any organization using GitHub Projects for development tracking. diff --git a/docs/project-graphql-helper.py b/docs/project-graphql-helper.py new file mode 100644 index 00000000..85221b33 --- /dev/null +++ b/docs/project-graphql-helper.py @@ -0,0 +1,110 @@ +# Setting up imports and Globals +import requests +import json +import argparse +from pprint import pprint + +# Building helper functions +def get_project_info(org, project_number, token): + headers = {"Authorization": f"Bearer {token}"} + + query = ''' + query($org: String!, $number: Int!) { + organization(login: $org){ + projectV2(number: $number) { + id + } + } + } + ''' + + variables = { + "org": org, + "number": int(project_number), + } + + data = { + "query": query, + "variables": variables, + } + + response = requests.post("https://api.github.com/graphql", headers=headers, json=data) + response_json = json.loads(response.text) + + project_id = response_json['data']['organization']['projectV2']['id'] + + query = ''' + query($node: ID!){ + node(id: $node) { + ... on ProjectV2 { + fields(first: 20) { + nodes { + ... on ProjectV2Field { + id + name + } + ... on ProjectV2IterationField { + id + name + configuration { + iterations { + startDate + id + } + } + } + ... on ProjectV2SingleSelectField { + id + name + options { + id + name + } + } + } + } + } + } + } + ''' + + variables = { + "node": project_id, + } + + data = { + "query": query, + "variables": variables, + } + + fields_response = requests.post("https://api.github.com/graphql", headers=headers, json=data) + fields_response_json = json.loads(fields_response.text) + + not_project_fields = ['Title', 'Assignees', 'Labels', 'Linked pull requests', 'Reviewers', 'Repository', 'Milestone', 'Tracks'] + field_names = [{'name': field['name'], 'id': field['id']} for field in fields_response_json['data']['node']['fields']['nodes'] if field['name'] not in not_project_fields] + fields = {field['name']:field for field in fields_response_json['data']['node']['fields']['nodes'] if field['name'] not in not_project_fields} + + for field in fields.values(): + if 'options' in field.keys(): + field['options'] = {option['name']: option['id'] for option in field['options']} + + return project_id, fields + +def main(): + # Set up argument parser + parser = argparse.ArgumentParser(description='GitHub Project GraphQL Helper') + parser.add_argument('--token', '-t', required=True, help='GitHub personal access token') + parser.add_argument('--org', '-o', required=True, help='GitHub organization name') + parser.add_argument('--project', '-p', required=True, type=int, help='GitHub project number') + + args = parser.parse_args() + + # Use the provided arguments + project_id, fields = get_project_info(org=args.org, project_number=args.project, token=args.token) + + pprint(project_id) + pprint(fields) + +if __name__ == "__main__": + main() + diff --git a/renovate.json b/renovate.json index 4bd832f5..c02dfddf 100644 --- a/renovate.json +++ b/renovate.json @@ -1,4 +1,10 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": ["config:base"] + "extends": ["config:base"], + "packageRules": [ + { + "matchPackagePatterns": ["^rapidsai/"], + "enabled": false + } + ] }