From ee341622418fce34bf3258ac937acab24ae711ac Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Fri, 29 Aug 2025 14:54:51 +1000 Subject: [PATCH 1/9] feature: update ado yamls to use pool-ubuntu-2404 --- .azure-pipelines/templates/variables.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/templates/variables.yml b/.azure-pipelines/templates/variables.yml index 5267cc2f51e..5ccf57a61fb 100644 --- a/.azure-pipelines/templates/variables.yml +++ b/.azure-pipelines/templates/variables.yml @@ -1,5 +1,5 @@ variables: - ubuntu_pool: 'pool-ubuntu-2204' + ubuntu_pool: 'pool-ubuntu-2404' ubuntu_multi_core_pool: 'pool-ubuntu-latest-multi-core' windows_pool: 'pool-windows-2019' ubuntu_arm64_pool: 'pool-ubuntu-latest-arm64' From 9a6d87c7ef9d3a01d66dc32377a802233a5888c4 Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Tue, 9 Sep 2025 09:54:34 +1000 Subject: [PATCH 2/9] temp: add temp step to echo vars --- azure-pipelines.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6e9d72a1c3c..9f78743b717 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -67,6 +67,18 @@ jobs: echo "Reject pull request directly to master branch" exit 1 +- job: ShowVariables + pool: + name: ubuntu-latest + steps: + - script: | + echo "Pipeline workspace: $PIPELINE_WORKSPACE" + echo "Listing repo root:" + ls -R $PIPELINE_WORKSPACE/.azure-pipelines/templates + echo "Contents of variables.yml:" + cat $PIPELINE_WORKSPACE/.azure-pipelines/templates/variables.yml + displayName: 'Inspect variables template' + - job: CredentialScanner displayName: "Credential Scanner" pool: From 65926d571bd0c6dc9ef65d299b8cae0dbec621ca Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Tue, 9 Sep 2025 10:02:21 +1000 Subject: [PATCH 3/9] temp: change pipeline to show variables --- azure-pipelines.yml | 1202 +------------------------------------------ 1 file changed, 5 insertions(+), 1197 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9f78743b717..9c77a81f3d8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,75 +1,14 @@ -resources: -- repo: self - -trigger: - batch: true - branches: - include: - - '*' - exclude: - - 'release*' - -pr: - branches: - include: - - '*' +trigger: none +pr: none variables: - template: ${{ variables.Pipeline.Workspace }}/.azure-pipelines/templates/variables.yml -- name: Codeql.Enabled - value: false -- name: ComponentDetection.ForceScan - value: eq(variables['Build.SourceBranch'], 'refs/heads/release') - -parameters: -- name: architectures - type: object - default: - - name: AMD64 - value: amd64 - pool: pool-ubuntu-latest-multi-core - - name: ARM64 - value: arm64 - pool: pool-ubuntu-latest-arm64 jobs: -- job: CheckPullRequest - displayName: "Check the Format of Pull Request Title and Content" - condition: and(succeeded(), in(variables['System.PullRequest.TargetBranch'], 'dev', 'release', 'main')) - - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - bash: | - echo "Check Title of Pull Request: #$(System.PullRequest.PullRequestNumber)" - title=$(curl https://api.github.com/repos/$(Build.Repository.Name)/pulls/$(System.PullRequest.PullRequestNumber) | jq -r '.title') - echo $title - body=$(curl https://api.github.com/repos/$(Build.Repository.Name)/pulls/$(System.PullRequest.PullRequestNumber) | jq -r '.body') - echo $body - if [ "$(System.PullRequest.TargetBranch)" != "release" ] && echo $title | grep -iqF hotfix:; then - echo "Hotfix PR should target release branch." - exit 1 - fi - if [ "$(System.PullRequest.TargetBranch)" == "main" ]; then - echo "Feature PR should target dev branch." - exit 1 - fi - python scripts/ci/check_pull_request.py "$title" "$body" - -- job: RejectPullRequestToMasterBranch - displayName: "Reject Pull Request To Master Branch" - condition: and(succeeded(), eq(variables['System.PullRequest.TargetBranch'], 'master')) - - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - bash: | - echo "Reject pull request directly to master branch" - exit 1 - - job: ShowVariables + displayName: 'Inspect Variables Template' pool: - name: ubuntu-latest + vmImage: ubuntu-latest steps: - script: | echo "Pipeline workspace: $PIPELINE_WORKSPACE" @@ -77,1135 +16,4 @@ jobs: ls -R $PIPELINE_WORKSPACE/.azure-pipelines/templates echo "Contents of variables.yml:" cat $PIPELINE_WORKSPACE/.azure-pipelines/templates/variables.yml - displayName: 'Inspect variables template' - -- job: CredentialScanner - displayName: "Credential Scanner" - pool: - name: ${{ variables.windows_pool }} - steps: - - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@3 - displayName: 'Run Credential Scanner' - inputs: - toolVersion: '2.1.17' - suppressionsFile: './scripts/ci/credscan/CredScanSuppressions.json' - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2 - displayName: 'Post Analysis' - inputs: - GdnBreakAllTools: false - GdnBreakGdnToolCredScan: true - GdnBreakGdnToolCredScanSeverity: Error - -- job: PolicyCheck - displayName: "Policy Check" - pool: - name: ${{ variables.windows_pool }} - steps: - - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@2 - displayName: 'Run Policy Check' - inputs: - targetType: F - result: PoliCheckResult.xml - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2 - displayName: 'Post Analysis' - inputs: - GdnBreakAllTools: false - GdnBreakGdnToolPoliCheck: true - GdnBreakGdnToolPoliCheckSeverity: Error - -- job: ExtractMetadata - displayName: Extract Metadata - - condition: succeeded() - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - task: Bash@3 - displayName: 'Extract Version' - inputs: - targetType: 'filePath' - filePath: scripts/release/get_version.sh - - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: metadata' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: metadata - -- job: VerifyLinuxRequirements - displayName: 'Verify src/azure-cli/requirements.*.Linux.txt' - condition: succeeded() - pool: - name: ${{ variables.ubuntu_pool }} - - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.13' - inputs: - versionSpec: 3.13 - - - bash: ./scripts/ci/dependency_check.sh - displayName: 'Verify src/azure-cli/requirements.py3.Linux.txt' - -- job: VerifyDarwinRequirements - displayName: 'Verify src/azure-cli/requirements.*.Darwin.txt' - condition: succeeded() - pool: - vmImage: ${{ variables.macos_pool }} - - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.13' - inputs: - versionSpec: 3.13 - - - bash: ./scripts/ci/dependency_check.sh - displayName: 'Verify src/azure-cli/requirements.py3.Darwin.txt' - -- job: VerifyWindowsRequirements - displayName: 'Verify src/azure-cli/requirements.*.Windows.txt' - condition: succeeded() - pool: - name: ${{ variables.windows_pool }} - - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.13' - inputs: - versionSpec: 3.13 - - - task: BatchScript@1 - inputs: - filename: ./scripts/ci/dependency_check.bat - displayName: 'Verify src/azure-cli/requirements.py3.Windows.txt' - -- job: VerifyVersions - displayName: Verify Command Module Versions - condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/release'), eq(variables['System.PullRequest.TargetBranch'], 'release'))) - - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.13' - inputs: - versionSpec: 3.13 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - azdev verify history - - displayName: 'Verify History' - -- job: BuildWindowsMSI - displayName: Build Windows MSI - strategy: - matrix: - x86: - Platform: x86 - x64: - Platform: x64 - - dependsOn: ExtractMetadata - condition: succeeded() - pool: - name: ${{ variables.windows_pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: PipAuthenticate@1 - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) - - - script: | - set ARCH=$(Platform) - set TARGET=msi - set /p CLI_VERSION=<$(System.ArtifactsDirectory)/metadata/version - set - - build_scripts/windows/scripts/build.cmd - displayName: 'Build Windows MSI' - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - inputs: - BuildDropPath: 'build_scripts/windows/out/' - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: MSI' - inputs: - TargetPath: 'build_scripts/windows/out/' - ArtifactName: msi-$(Platform) - -- job: BuildWindowsZIP - displayName: Build Windows ZIP - strategy: - matrix: - x64: - Platform: x64 - - dependsOn: ExtractMetadata - condition: succeeded() - pool: - name: ${{ variables.windows_pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: PipAuthenticate@1 - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) - - - script: | - set ARCH=$(Platform) - set TARGET=zip - set /p CLI_VERSION=<$(System.ArtifactsDirectory)/metadata/version - set - - build_scripts/windows/scripts/build.cmd - displayName: 'Build Windows ZIP' - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - inputs: - BuildDropPath: 'build_scripts/windows/out/' - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: ZIP' - inputs: - TargetPath: 'build_scripts/windows/out/' - ArtifactName: zip-$(Platform) - -- job: TestMsiInstallation - displayName: Test MSI Installation - strategy: - matrix: - x86: - Platform: x86 - x64: - Platform: x64 - - dependsOn: BuildWindowsMSI - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: ${{ variables.windows_pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/msi' - artifactName: msi-$(Platform) - - - task: PowerShell@2 - displayName: Install and Load CLI - inputs: - filePath: build_scripts\windows\scripts\test_msi_installation.ps1 - -- job: TestZipInstallation - displayName: Test ZIP Installation - dependsOn: BuildWindowsZIP - strategy: - matrix: - x64: - Platform: x64 - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: ${{ variables.windows_pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/zip' - artifactName: zip-$(Platform) - - - task: PowerShell@2 - displayName: Expand and Load CLI - inputs: - filePath: build_scripts\windows\scripts\test_zip_installation.ps1 - -- job: BuildDockerImageAzureLinux - displayName: Build Docker Image Azure Linux - dependsOn: BuildRpmPackagesAzureLinux - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: - pool: ${{ arch.pool }} - artifactName: docker-azurelinux3.0-${{ arch.value }} - dockerfile: azure-linux.dockerfile - packageArtifactName: rpm-azurelinux3.0-${{ arch.value }} - image: mcr.microsoft.com/azurelinux/base/core:3.0 - pool: - name: $(pool) - steps: - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/docker' - artifactName: $(packageArtifactName) - - bash: | - set -ex - mkdir docker-temp - mv $(Build.ArtifactStagingDirectory)/docker/*.rpm ./docker-temp/azure-cli.rpm - - bash scripts/release/docker/pipeline.sh - displayName: 'Build Docker' - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - DockerImagesToScan: 'clibuild$BUILD_BUILDNUMBER:latest' - - - task: PublishPipelineArtifact@0 - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(artifactName) - -- job: TestDockerImageAzureLinux - displayName: Test Docker Image Azure Linux - dependsOn: - - BuildDockerImageAzureLinux - - ExtractMetadata - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: - pool: ${{ arch.pool }} - artifactName: docker-azurelinux3.0-${{ arch.value }} - pool: - name: $(pool) - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Docker Image' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/docker' - artifactName: $(artifactName) - - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - - bash: | - set -exv - - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - IMAGE_NAME=clibuild$BUILD_BUILDNUMBER:latest - TAR_FILE=$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$CLI_VERSION.tar - - echo "== Test docker image ==" - - docker load < $TAR_FILE - docker run $IMAGE_NAME /bin/bash -c "time az self-test && time az --version && tdnf list --installed && sleep 5" - displayName: 'Bash Script' - -- job: BuildPythonWheel - displayName: Build Python Wheels - - dependsOn: ExtractMetadata - condition: succeeded() - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.13' - inputs: - versionSpec: 3.13 - - - task: PipAuthenticate@1 - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) - - - script: | - if [[ "$(Build.Reason)" == "PullRequest" ]]; then - branch=$(System.PullRequest.TargetBranch) - else - branch=$(Build.SourceBranchName) - fi - scripts/release/pypi/build.sh $branch - displayName: 'Run Wheel Build Script' - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: pypi' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: pypi - -- job: TestPythonWheel - displayName: Test Python Wheels - strategy: - matrix: - Python312: - python.version: '3.12' - Python313: - python.version: '3.13' - dependsOn: BuildPythonWheel - condition: succeeded() - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - task: DownloadPipelineArtifact@1 - displayName: 'Download PyPI Packages' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/pypi' - artifactName: pypi - - task: UsePythonVersion@0 - displayName: 'Use Python $(python.version)' - inputs: - versionSpec: '$(python.version)' - - bash: | - #!/usr/bin/env bash - set -ex - - CLI_VERSION=`cat $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata/version` - echo "== Testing pip install on $PYTHON_VERSION ==" - cd $BUILD_ARTIFACTSTAGINGDIRECTORY/pypi - pip install --find-links ./ azure_cli-$CLI_VERSION*whl && az self-test && az --version && sleep 5 - displayName: 'Test pip Install' - -- job: TestCore - displayName: Unit Test for Core - timeoutInMinutes: 10 - pool: - name: ${{ variables.ubuntu_pool }} - strategy: - matrix: - Python312: - python.version: '3.12' - Python313: - python.version: '3.13' - steps: - - template: .azure-pipelines/templates/automation_test.yml - parameters: - pythonVersion: '$(python.version)' - module: 'azure-cli-core' - -- job: TestTelemetry - displayName: Unit Test for Telemetry - timeoutInMinutes: 10 - pool: - name: ${{ variables.ubuntu_pool }} - strategy: - matrix: - Python312: - python.version: '3.12' - Python313: - python.version: '3.13' - steps: - - template: .azure-pipelines/templates/automation_test.yml - parameters: - pythonVersion: '$(python.version)' - module: 'azure-cli-telemetry' - -- job: IntegrationTestAgainstProfiles - displayName: Integration Test against Profiles - dependsOn: BuildPythonWheel - condition: succeeded() - timeoutInMinutes: 20 - - pool: - name: ${{ variables.ubuntu_pool }} - strategy: - matrix: - Python312: - python.version: '3.12' - Python313: - python.version: '3.13' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(python.version)' - inputs: - versionSpec: '$(python.version)' - - bash: pip install --upgrade pip wheel setuptools - displayName: 'Install pip and wheel' - - bash: ./scripts/ci/test_profile_integration.sh - displayName: 'Run Integration Test against Profiles' - -- job: TestExtensionsLoading - displayName: Test Extensions Loading - condition: succeeded() - timeoutInMinutes: 80 - - pool: - name: ${{ variables.ubuntu_pool }} - strategy: - matrix: - Python313: - python.version: '3.13' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(python.version)' - inputs: - versionSpec: '$(python.version)' - - bash: pip install --upgrade pip wheel setuptools - displayName: 'Install pip and wheel setuptools' - - bash: ./scripts/ci/test_extensions.sh - displayName: 'Load extensions' - -- job: BuildHomebrewFormula - displayName: Build Homebrew Formula - - dependsOn: BuildPythonWheel - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - bash: | - #!/bin/bash - - root=$(cd $(dirname $0); pwd) - - set -evx - - CLI_VERSION=`cat $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata/version` - HOMEBREW_UPSTREAM_URL=`curl -Ls -o /dev/null -w %{url_effective} https://api.github.com/repos/Azure/azure-cli/tarball/$BUILD_SOURCEVERSION` - - docker_files=$(cd $BUILD_SOURCESDIRECTORY/scripts/release/homebrew/docker; pwd) - src_files=$(cd $BUILD_SOURCESDIRECTORY/src; pwd) - - echo "Generating formula in docker container ... " - docker run -v $docker_files:/mnt/scripts \ - -v $src_files:/mnt/src \ - -e CLI_VERSION=$CLI_VERSION \ - -e HOMEBREW_UPSTREAM_URL=$HOMEBREW_UPSTREAM_URL \ - --name azurecli \ - mcr.microsoft.com/azurelinux/base/python:3 \ - /mnt/scripts/run.sh - - # clean up - rm -rf $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata - - docker cp azurecli:azure-cli.rb $BUILD_ARTIFACTSTAGINGDIRECTORY/azure-cli.rb - docker rm --force azurecli - displayName: 'Build homebrew formula' - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: homebrew' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: homebrew - -- job: TestHomebrewFormula - displayName: Test Homebrew Formula - - dependsOn: BuildHomebrewFormula - condition: succeeded() - pool: - vmImage: ${{ variables.macos_pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Homebrew' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/homebrew' - artifactName: homebrew - - - - bash: | - set -ev - # Force relink python@3.xx in Homebrew to resolve the conflict with pre-installed python 3.xx on macOS-12 image - # See: https://github.com/Azure/azure-cli/issues/29054 - python_version=3.13 - brew unlink python@$python_version && brew link --overwrite python@$python_version - - echo == Remove pre-installed azure-cli == - brew uninstall azure-cli - - echo == Install azure-cli.rb formula == - brew install --build-from-source $SYSTEM_ARTIFACTSDIRECTORY/homebrew/azure-cli.rb - - echo == Az Version == - az --version - - echo == Run Self-Test == - az self-test - - displayName: 'Bash Script' - -- job: TestHomebrewPackage - displayName: Test Homebrew Package - timeoutInMinutes: 180 - dependsOn: BuildHomebrewFormula - # condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - condition: false - pool: - vmImage: ${{ variables.macos_pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/homebrew' - artifactName: homebrew - - - - bash: ./scripts/release/homebrew/test_homebrew_package.sh - - - displayName: 'Test Homebrew Package' - - -- job: BuildRpmPackagesAzureLinux - displayName: Build Rpm Package - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: - image: mcr.microsoft.com/azurelinux/base/core:3.0 - artifact: rpm-azurelinux3.0-${{ arch.value }} - pool: ${{ arch.pool }} - pool: - name: $(pool) - steps: - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - - task: PipAuthenticate@1 - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) - - - task: Bash@3 - displayName: 'Build Rpm Package: Azure Linux' - inputs: - targetType: 'filePath' - filePath: scripts/release/rpm/pipeline_azurelinux.sh - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: rpm-azurelinux' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(artifact) - -- job: TestRpmPackagesAzureLinux - displayName: Test Rpm Package - timeoutInMinutes: 180 - dependsOn: - - BuildRpmPackagesAzureLinux - - ExtractMetadata - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: $(pool) - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: - image: mcr.microsoft.com/azurelinux/base/core:3.0 - artifact: rpm-azurelinux3.0-${{ arch.value }} - pool: ${{ arch.pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' - artifactName: $(artifact) - - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - - bash: | - set -ex - - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name "azure-cli-$CLI_VERSION-1.*.rpm" -printf '%f\n') - - echo "== Test rpm package on ${IMAGE} ==" - docker pull $IMAGE - docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh" - - displayName: 'Test Rpm Package Azure Linux' - -# TODO: rpmbuild on Red Hat UBI 8 is slow for unknown reason. Still working with Red Hat to investigate. -- job: BuildRpmPackages - displayName: Build Rpm Packages - # Do not run this job for Pull Requests due to the slowness - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: $(pool) - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - Red Hat Universal Base Image 8 ${{ arch.name }}: - dockerfile: ubi - image: registry.access.redhat.com/ubi8/ubi:8.4 - artifact: rpm-ubi8-${{ arch.value }} - python_package: python3.12 - pool: ${{ arch.pool }} - Red Hat Universal Base Image 9 ${{ arch.name }}: - dockerfile: ubi - image: registry.access.redhat.com/ubi9/ubi:9.0.0 - artifact: rpm-ubi9-${{ arch.value }} - python_package: python3.12 - pool: ${{ arch.pool }} - Red Hat Universal Base Image 10 ${{ arch.name }}: - dockerfile: ubi - image: registry.access.redhat.com/ubi10/ubi:10.0 - artifact: rpm-ubi10-${{ arch.value }} - python_package: python3.12 - pool: ${{ arch.pool }} - steps: - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - task: PipAuthenticate@1 - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) - - task: Bash@3 - displayName: 'Build Rpm Package' - inputs: - targetType: 'filePath' - filePath: scripts/release/rpm/pipeline.sh - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: rpm' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(artifact) - - -- job: TestRpmPackage - displayName: Test Rpm Package - timeoutInMinutes: 180 - dependsOn: - - BuildRpmPackages - - ExtractMetadata - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: $(pool) - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - Red Hat Universal Base Image 8 ${{ arch.name }}: - artifact: rpm-ubi8-${{ arch.value }} - distro: el8 - image: registry.access.redhat.com/ubi8/ubi:8.4 - python_package: python3.12 - python_cmd: python3.12 - pip_cmd: pip3.12 - pool: ${{ arch.pool }} - Red Hat Universal Base Image 9 ${{ arch.name }}: - artifact: rpm-ubi9-${{ arch.value }} - distro: el9 - image: registry.access.redhat.com/ubi9/ubi:9.0.0 - python_package: python3.12 - python_cmd: python3.12 - pip_cmd: pip3.12 - pool: ${{ arch.pool }} - Red Hat Universal Base Image 10 ${{ arch.name }}: - artifact: rpm-ubi10-${{ arch.value }} - distro: el10 - image: registry.access.redhat.com/ubi10/ubi:10.0 - python_package: python3.12 - python_cmd: python3.12 - pip_cmd: pip3.12 - pool: ${{ arch.pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' - artifactName: $(artifact) - - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - - bash: | - set -ex - - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name "azure-cli-$CLI_VERSION-1.${DISTRO}.*.rpm" -printf '%f\n') - - echo "== Test rpm package on ${IMAGE} ==" - docker pull $IMAGE - docker run --rm -e RPM_NAME=$RPM_NAME -e PYTHON_PACKAGE=${PYTHON_PACKAGE} -e PYTHON_CMD=${PYTHON_CMD} -e PIP_CMD=${PIP_CMD} -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh" - - displayName: 'Test Rpm Package' - -- job: BuildDebPackages - displayName: Build Deb Packages - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: $(pool) - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - # https://wiki.ubuntu.com/Releases - Jammy ${{ arch.name }}: - # 22.04 - deb_system: ubuntu - distro: jammy - arch: ${{ arch.value }} - pool: ${{ arch.pool }} - - Noble ${{ arch.name }}: - # 24.04 - deb_system: ubuntu - distro: noble - arch: ${{ arch.value }} - pool: ${{ arch.pool }} - - # https://wiki.debian.org/DebianReleases - Bullseye ${{ arch.name }}: - # 11 - deb_system: debian - distro: bullseye - arch: ${{ arch.value }} - pool: ${{ arch.pool }} - Bookworm ${{ arch.name }}: - # 12 - deb_system: debian - distro: bookworm - arch: ${{ arch.value }} - pool: ${{ arch.pool }} - steps: - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - task: PipAuthenticate@1 - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) - - task: Bash@3 - displayName: 'Build $(deb_system) $(distro) $(arch) Package' - inputs: - targetType: 'filePath' - filePath: scripts/release/debian/pipeline.sh - env: - DISTRO_BASE_IMAGE: mcr.microsoft.com/mirror/docker/library/$(deb_system):$(distro) - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: $(deb_system) $(distro) $(arch)' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(deb_system)-$(distro)-$(arch) - -- job: TestDebPackages - timeoutInMinutes: 180 - displayName: Test Deb Packages - dependsOn: - - BuildDebPackages - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - Jammy ${{ arch.name }}: - deb_system: ubuntu - distro: jammy - arch: ${{ arch.value }} - pool: ${{ arch.pool }} - Noble ${{ arch.name }}: - deb_system: ubuntu - distro: noble - arch: ${{ arch.value }} - pool: ${{ arch.pool }} - Bullseye ${{ arch.name }}: - deb_system: debian - distro: bullseye - arch: ${{ arch.value }} - pool: ${{ arch.pool }} - Bookworm ${{ arch.name }}: - deb_system: debian - distro: bookworm - arch: ${{ arch.value }} - pool: ${{ arch.pool }} - pool: - name: $(pool) - steps: - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download $(deb_system):$(distro) $(arch) Build' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/debian' - artifactName: $(deb_system)-$(distro)-$(arch) - - - task: Bash@3 - displayName: 'Test $(deb_system) $(distro) $(arch) Package' - env: - DISTRO_BASE_IMAGE: mcr.microsoft.com/mirror/docker/library/$(deb_system):$(distro) - inputs: - targetType: 'inline' - script: | - set -exv - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - - echo "== Test debian package on ${DISTRO} ==" - docker pull ${DISTRO_BASE_IMAGE} - docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" - -- job: CheckStyle - displayName: "Check CLI Style" - timeoutInMinutes: 120 - pool: - name: ${{ variables.ubuntu_multi_core_pool }} - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.13' - inputs: - versionSpec: 3.13 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - azdev style - -- job: CheckHeaders - displayName: "Check License, History, and DocMap" - condition: and(not(contains(variables['Build.SourceBranch'], 'lts')), not(contains(variables['System.PullRequest.TargetBranch'], 'lts'))) - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.13' - inputs: - versionSpec: 3.13 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - azdev verify license - azdev verify history - azdev verify document-map - -- job: PerformanceCheck - displayName: "PerformanceCheck" - strategy: - matrix: - Python312: - python.version: '3.12' - Python313: - python.version: '3.13' - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(python.version)' - inputs: - versionSpec: '$(python.version)' - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - azdev perf load-times - displayName: "Load Performance" - # - bash: | - # set -ev - # . env/bin/activate - - # azdev perf benchmark "version" "network vnet -h" "rest -h" "storage account" - # displayName: "Execution Performance" - -- job: CheckLinter - displayName: "Check CLI Linter" - - pool: - name: ${{ variables.ubuntu_multi_core_pool }} - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.13' - inputs: - versionSpec: 3.13 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - python scripts/ci/service_name.py - if [[ "$(System.PullRequest.TargetBranch)" != "" ]]; then - # If CI is set to shallow fetch, target branch should be expilictly fetched. - git fetch origin --depth=1 $(System.PullRequest.TargetBranch) - azdev linter --ci-exclusions --min-severity medium --repo=./ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch) - else - azdev linter --ci-exclusions --min-severity medium - fi - -- job: AzdevScan - condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) - displayName: "Secret Scan" - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.13' - inputs: - versionSpec: 3.13 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - git fetch origin --depth=1 $(System.PullRequest.TargetBranch) - declare -A secret_files - IFS_OLD=${IFS} - IFS=$'\n' - for FILE in `git diff --name-only --diff-filter=AM origin/$(System.PullRequest.TargetBranch)` ; do - echo $FILE - detected=$(azdev scan -f "$FILE" --continue-on-failure | python -c "import sys, json; print(json.load(sys.stdin)['secrets_detected'])") - if [ $detected == 'True' ]; then - printf "\033[0;31mDetected secrets from %s, You can run 'azdev mask' to remove secrets.\033[0m\n" "$FILE" - secret_files+=$FILE - fi - done - IFS=${IFS_OLD} - if [ "${#secret_files[@]}" -gt 0 ]; then - exit 1 - fi - -- job: CodegenCoverage - condition: in(variables['Build.Reason'], 'BatchedCI', 'IndividualCI') - timeoutInMinutes: 180 - displayName: "Codegen Coverage" - continueOnError: true - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' - inputs: - versionSpec: 3.11 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - # clone azure-cli-extensions - cd .. - git clone --depth 1 -b main https://github.com/Azure/azure-cli-extensions.git ./azure-cli-extensions - azdev extension repo add ./azure-cli-extensions - pip install setuptools==70.0.0 wheel==0.30.0 - azdev extension add "*" - pip install msrestazure markupsafe==2.0.1 - # Some extension will change the dependence, so run `azdev setup` again after all extensions installed. - azdev setup -c ./s -r ./azure-cli-extensions - - mkdir -p /tmp/module_stats - - find /mnt/vss/_work/1/s/src/azure-cli/azure/cli/command_modules/ -mindepth 1 -maxdepth 1 -type d -printf "%f\n" | grep -v '^__pycache__$' > /mnt/vss/_work/1/s/scripts/ci/core_modules.txt - echo "=== Core Modules ===" - cat /mnt/vss/_work/1/s/scripts/ci/core_modules.txt - - find /mnt/vss/_work/1/azure-cli-extensions/src/ -mindepth 1 -maxdepth 1 -type d -printf "%f\n" | grep -v '^__pycache__$' > /mnt/vss/_work/1/s/scripts/ci/extension_modules.txt - echo "=== Extension Modules ===" - cat /mnt/vss/_work/1/s/scripts/ci/extension_modules.txt - - for module in $(cat /mnt/vss/_work/1/s/scripts/ci/core_modules.txt); do - azdev statistics list-command-table $module --statistics-only > /tmp/module_stats/${module}.json || true - done - - for module in $(cat /mnt/vss/_work/1/s/scripts/ci/extension_modules.txt); do - azdev statistics list-command-table $module --statistics-only > /tmp/module_stats/${module}.json || true - done - azdev statistics list-command-table --statistics-only > /tmp/codegen_report.json || true - python /mnt/vss/_work/1/s/scripts/ci/codegen_report.py - env: - BUILD_ID: $(Build.BuildId) - BUILD_BRANCH: $(Build.SourceBranchName) - enabled: true + displayName: 'Show Variables' \ No newline at end of file From e0477450b2954d9bf2076f8e8385c5006a1c411b Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Tue, 9 Sep 2025 14:25:08 +1000 Subject: [PATCH 4/9] temp: change pool for temp pool print --- azure-pipelines.yml | 1202 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 1197 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9c77a81f3d8..83d7e195a11 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,14 +1,75 @@ -trigger: none -pr: none +resources: +- repo: self + +trigger: + batch: true + branches: + include: + - '*' + exclude: + - 'release*' + +pr: + branches: + include: + - '*' variables: - template: ${{ variables.Pipeline.Workspace }}/.azure-pipelines/templates/variables.yml +- name: Codeql.Enabled + value: false +- name: ComponentDetection.ForceScan + value: eq(variables['Build.SourceBranch'], 'refs/heads/release') + +parameters: +- name: architectures + type: object + default: + - name: AMD64 + value: amd64 + pool: pool-ubuntu-latest-multi-core + - name: ARM64 + value: arm64 + pool: pool-ubuntu-latest-arm64 jobs: +- job: CheckPullRequest + displayName: "Check the Format of Pull Request Title and Content" + condition: and(succeeded(), in(variables['System.PullRequest.TargetBranch'], 'dev', 'release', 'main')) + + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - bash: | + echo "Check Title of Pull Request: #$(System.PullRequest.PullRequestNumber)" + title=$(curl https://api.github.com/repos/$(Build.Repository.Name)/pulls/$(System.PullRequest.PullRequestNumber) | jq -r '.title') + echo $title + body=$(curl https://api.github.com/repos/$(Build.Repository.Name)/pulls/$(System.PullRequest.PullRequestNumber) | jq -r '.body') + echo $body + if [ "$(System.PullRequest.TargetBranch)" != "release" ] && echo $title | grep -iqF hotfix:; then + echo "Hotfix PR should target release branch." + exit 1 + fi + if [ "$(System.PullRequest.TargetBranch)" == "main" ]; then + echo "Feature PR should target dev branch." + exit 1 + fi + python scripts/ci/check_pull_request.py "$title" "$body" + +- job: RejectPullRequestToMasterBranch + displayName: "Reject Pull Request To Master Branch" + condition: and(succeeded(), eq(variables['System.PullRequest.TargetBranch'], 'master')) + + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - bash: | + echo "Reject pull request directly to master branch" + exit 1 + - job: ShowVariables - displayName: 'Inspect Variables Template' pool: - vmImage: ubuntu-latest + name: pool-ubuntu-latest-multi-core steps: - script: | echo "Pipeline workspace: $PIPELINE_WORKSPACE" @@ -16,4 +77,1135 @@ jobs: ls -R $PIPELINE_WORKSPACE/.azure-pipelines/templates echo "Contents of variables.yml:" cat $PIPELINE_WORKSPACE/.azure-pipelines/templates/variables.yml - displayName: 'Show Variables' \ No newline at end of file + displayName: 'Inspect variables template' + +- job: CredentialScanner + displayName: "Credential Scanner" + pool: + name: ${{ variables.windows_pool }} + steps: + - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@3 + displayName: 'Run Credential Scanner' + inputs: + toolVersion: '2.1.17' + suppressionsFile: './scripts/ci/credscan/CredScanSuppressions.json' + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2 + displayName: 'Post Analysis' + inputs: + GdnBreakAllTools: false + GdnBreakGdnToolCredScan: true + GdnBreakGdnToolCredScanSeverity: Error + +- job: PolicyCheck + displayName: "Policy Check" + pool: + name: ${{ variables.windows_pool }} + steps: + - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@2 + displayName: 'Run Policy Check' + inputs: + targetType: F + result: PoliCheckResult.xml + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2 + displayName: 'Post Analysis' + inputs: + GdnBreakAllTools: false + GdnBreakGdnToolPoliCheck: true + GdnBreakGdnToolPoliCheckSeverity: Error + +- job: ExtractMetadata + displayName: Extract Metadata + + condition: succeeded() + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - task: Bash@3 + displayName: 'Extract Version' + inputs: + targetType: 'filePath' + filePath: scripts/release/get_version.sh + + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: metadata' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: metadata + +- job: VerifyLinuxRequirements + displayName: 'Verify src/azure-cli/requirements.*.Linux.txt' + condition: succeeded() + pool: + name: ${{ variables.ubuntu_pool }} + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.13' + inputs: + versionSpec: 3.13 + + - bash: ./scripts/ci/dependency_check.sh + displayName: 'Verify src/azure-cli/requirements.py3.Linux.txt' + +- job: VerifyDarwinRequirements + displayName: 'Verify src/azure-cli/requirements.*.Darwin.txt' + condition: succeeded() + pool: + vmImage: ${{ variables.macos_pool }} + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.13' + inputs: + versionSpec: 3.13 + + - bash: ./scripts/ci/dependency_check.sh + displayName: 'Verify src/azure-cli/requirements.py3.Darwin.txt' + +- job: VerifyWindowsRequirements + displayName: 'Verify src/azure-cli/requirements.*.Windows.txt' + condition: succeeded() + pool: + name: ${{ variables.windows_pool }} + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.13' + inputs: + versionSpec: 3.13 + + - task: BatchScript@1 + inputs: + filename: ./scripts/ci/dependency_check.bat + displayName: 'Verify src/azure-cli/requirements.py3.Windows.txt' + +- job: VerifyVersions + displayName: Verify Command Module Versions + condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/release'), eq(variables['System.PullRequest.TargetBranch'], 'release'))) + + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.13' + inputs: + versionSpec: 3.13 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + azdev verify history + + displayName: 'Verify History' + +- job: BuildWindowsMSI + displayName: Build Windows MSI + strategy: + matrix: + x86: + Platform: x86 + x64: + Platform: x64 + + dependsOn: ExtractMetadata + condition: succeeded() + pool: + name: ${{ variables.windows_pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: PipAuthenticate@1 + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) + + - script: | + set ARCH=$(Platform) + set TARGET=msi + set /p CLI_VERSION=<$(System.ArtifactsDirectory)/metadata/version + set + + build_scripts/windows/scripts/build.cmd + displayName: 'Build Windows MSI' + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + inputs: + BuildDropPath: 'build_scripts/windows/out/' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: MSI' + inputs: + TargetPath: 'build_scripts/windows/out/' + ArtifactName: msi-$(Platform) + +- job: BuildWindowsZIP + displayName: Build Windows ZIP + strategy: + matrix: + x64: + Platform: x64 + + dependsOn: ExtractMetadata + condition: succeeded() + pool: + name: ${{ variables.windows_pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: PipAuthenticate@1 + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) + + - script: | + set ARCH=$(Platform) + set TARGET=zip + set /p CLI_VERSION=<$(System.ArtifactsDirectory)/metadata/version + set + + build_scripts/windows/scripts/build.cmd + displayName: 'Build Windows ZIP' + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + inputs: + BuildDropPath: 'build_scripts/windows/out/' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: ZIP' + inputs: + TargetPath: 'build_scripts/windows/out/' + ArtifactName: zip-$(Platform) + +- job: TestMsiInstallation + displayName: Test MSI Installation + strategy: + matrix: + x86: + Platform: x86 + x64: + Platform: x64 + + dependsOn: BuildWindowsMSI + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: ${{ variables.windows_pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/msi' + artifactName: msi-$(Platform) + + - task: PowerShell@2 + displayName: Install and Load CLI + inputs: + filePath: build_scripts\windows\scripts\test_msi_installation.ps1 + +- job: TestZipInstallation + displayName: Test ZIP Installation + dependsOn: BuildWindowsZIP + strategy: + matrix: + x64: + Platform: x64 + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: ${{ variables.windows_pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/zip' + artifactName: zip-$(Platform) + + - task: PowerShell@2 + displayName: Expand and Load CLI + inputs: + filePath: build_scripts\windows\scripts\test_zip_installation.ps1 + +- job: BuildDockerImageAzureLinux + displayName: Build Docker Image Azure Linux + dependsOn: BuildRpmPackagesAzureLinux + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + pool: ${{ arch.pool }} + artifactName: docker-azurelinux3.0-${{ arch.value }} + dockerfile: azure-linux.dockerfile + packageArtifactName: rpm-azurelinux3.0-${{ arch.value }} + image: mcr.microsoft.com/azurelinux/base/core:3.0 + pool: + name: $(pool) + steps: + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/docker' + artifactName: $(packageArtifactName) + - bash: | + set -ex + mkdir docker-temp + mv $(Build.ArtifactStagingDirectory)/docker/*.rpm ./docker-temp/azure-cli.rpm + + bash scripts/release/docker/pipeline.sh + displayName: 'Build Docker' + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + DockerImagesToScan: 'clibuild$BUILD_BUILDNUMBER:latest' + + - task: PublishPipelineArtifact@0 + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: $(artifactName) + +- job: TestDockerImageAzureLinux + displayName: Test Docker Image Azure Linux + dependsOn: + - BuildDockerImageAzureLinux + - ExtractMetadata + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + pool: ${{ arch.pool }} + artifactName: docker-azurelinux3.0-${{ arch.value }} + pool: + name: $(pool) + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Docker Image' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/docker' + artifactName: $(artifactName) + + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + + - bash: | + set -exv + + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + IMAGE_NAME=clibuild$BUILD_BUILDNUMBER:latest + TAR_FILE=$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$CLI_VERSION.tar + + echo "== Test docker image ==" + + docker load < $TAR_FILE + docker run $IMAGE_NAME /bin/bash -c "time az self-test && time az --version && tdnf list --installed && sleep 5" + displayName: 'Bash Script' + +- job: BuildPythonWheel + displayName: Build Python Wheels + + dependsOn: ExtractMetadata + condition: succeeded() + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.13' + inputs: + versionSpec: 3.13 + + - task: PipAuthenticate@1 + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) + + - script: | + if [[ "$(Build.Reason)" == "PullRequest" ]]; then + branch=$(System.PullRequest.TargetBranch) + else + branch=$(Build.SourceBranchName) + fi + scripts/release/pypi/build.sh $branch + displayName: 'Run Wheel Build Script' + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: pypi' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: pypi + +- job: TestPythonWheel + displayName: Test Python Wheels + strategy: + matrix: + Python312: + python.version: '3.12' + Python313: + python.version: '3.13' + dependsOn: BuildPythonWheel + condition: succeeded() + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + - task: DownloadPipelineArtifact@1 + displayName: 'Download PyPI Packages' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/pypi' + artifactName: pypi + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' + - bash: | + #!/usr/bin/env bash + set -ex + + CLI_VERSION=`cat $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata/version` + echo "== Testing pip install on $PYTHON_VERSION ==" + cd $BUILD_ARTIFACTSTAGINGDIRECTORY/pypi + pip install --find-links ./ azure_cli-$CLI_VERSION*whl && az self-test && az --version && sleep 5 + displayName: 'Test pip Install' + +- job: TestCore + displayName: Unit Test for Core + timeoutInMinutes: 10 + pool: + name: ${{ variables.ubuntu_pool }} + strategy: + matrix: + Python312: + python.version: '3.12' + Python313: + python.version: '3.13' + steps: + - template: .azure-pipelines/templates/automation_test.yml + parameters: + pythonVersion: '$(python.version)' + module: 'azure-cli-core' + +- job: TestTelemetry + displayName: Unit Test for Telemetry + timeoutInMinutes: 10 + pool: + name: ${{ variables.ubuntu_pool }} + strategy: + matrix: + Python312: + python.version: '3.12' + Python313: + python.version: '3.13' + steps: + - template: .azure-pipelines/templates/automation_test.yml + parameters: + pythonVersion: '$(python.version)' + module: 'azure-cli-telemetry' + +- job: IntegrationTestAgainstProfiles + displayName: Integration Test against Profiles + dependsOn: BuildPythonWheel + condition: succeeded() + timeoutInMinutes: 20 + + pool: + name: ${{ variables.ubuntu_pool }} + strategy: + matrix: + Python312: + python.version: '3.12' + Python313: + python.version: '3.13' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' + - bash: pip install --upgrade pip wheel setuptools + displayName: 'Install pip and wheel' + - bash: ./scripts/ci/test_profile_integration.sh + displayName: 'Run Integration Test against Profiles' + +- job: TestExtensionsLoading + displayName: Test Extensions Loading + condition: succeeded() + timeoutInMinutes: 80 + + pool: + name: ${{ variables.ubuntu_pool }} + strategy: + matrix: + Python313: + python.version: '3.13' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' + - bash: pip install --upgrade pip wheel setuptools + displayName: 'Install pip and wheel setuptools' + - bash: ./scripts/ci/test_extensions.sh + displayName: 'Load extensions' + +- job: BuildHomebrewFormula + displayName: Build Homebrew Formula + + dependsOn: BuildPythonWheel + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - bash: | + #!/bin/bash + + root=$(cd $(dirname $0); pwd) + + set -evx + + CLI_VERSION=`cat $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata/version` + HOMEBREW_UPSTREAM_URL=`curl -Ls -o /dev/null -w %{url_effective} https://api.github.com/repos/Azure/azure-cli/tarball/$BUILD_SOURCEVERSION` + + docker_files=$(cd $BUILD_SOURCESDIRECTORY/scripts/release/homebrew/docker; pwd) + src_files=$(cd $BUILD_SOURCESDIRECTORY/src; pwd) + + echo "Generating formula in docker container ... " + docker run -v $docker_files:/mnt/scripts \ + -v $src_files:/mnt/src \ + -e CLI_VERSION=$CLI_VERSION \ + -e HOMEBREW_UPSTREAM_URL=$HOMEBREW_UPSTREAM_URL \ + --name azurecli \ + mcr.microsoft.com/azurelinux/base/python:3 \ + /mnt/scripts/run.sh + + # clean up + rm -rf $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata + + docker cp azurecli:azure-cli.rb $BUILD_ARTIFACTSTAGINGDIRECTORY/azure-cli.rb + docker rm --force azurecli + displayName: 'Build homebrew formula' + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: homebrew' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: homebrew + +- job: TestHomebrewFormula + displayName: Test Homebrew Formula + + dependsOn: BuildHomebrewFormula + condition: succeeded() + pool: + vmImage: ${{ variables.macos_pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Homebrew' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/homebrew' + artifactName: homebrew + + + - bash: | + set -ev + # Force relink python@3.xx in Homebrew to resolve the conflict with pre-installed python 3.xx on macOS-12 image + # See: https://github.com/Azure/azure-cli/issues/29054 + python_version=3.13 + brew unlink python@$python_version && brew link --overwrite python@$python_version + + echo == Remove pre-installed azure-cli == + brew uninstall azure-cli + + echo == Install azure-cli.rb formula == + brew install --build-from-source $SYSTEM_ARTIFACTSDIRECTORY/homebrew/azure-cli.rb + + echo == Az Version == + az --version + + echo == Run Self-Test == + az self-test + + displayName: 'Bash Script' + +- job: TestHomebrewPackage + displayName: Test Homebrew Package + timeoutInMinutes: 180 + dependsOn: BuildHomebrewFormula + # condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + condition: false + pool: + vmImage: ${{ variables.macos_pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/homebrew' + artifactName: homebrew + + + - bash: ./scripts/release/homebrew/test_homebrew_package.sh + + + displayName: 'Test Homebrew Package' + + +- job: BuildRpmPackagesAzureLinux + displayName: Build Rpm Package + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + image: mcr.microsoft.com/azurelinux/base/core:3.0 + artifact: rpm-azurelinux3.0-${{ arch.value }} + pool: ${{ arch.pool }} + pool: + name: $(pool) + steps: + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + + - task: PipAuthenticate@1 + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) + + - task: Bash@3 + displayName: 'Build Rpm Package: Azure Linux' + inputs: + targetType: 'filePath' + filePath: scripts/release/rpm/pipeline_azurelinux.sh + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: rpm-azurelinux' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: $(artifact) + +- job: TestRpmPackagesAzureLinux + displayName: Test Rpm Package + timeoutInMinutes: 180 + dependsOn: + - BuildRpmPackagesAzureLinux + - ExtractMetadata + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: $(pool) + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + image: mcr.microsoft.com/azurelinux/base/core:3.0 + artifact: rpm-azurelinux3.0-${{ arch.value }} + pool: ${{ arch.pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' + artifactName: $(artifact) + + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + + - bash: | + set -ex + + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name "azure-cli-$CLI_VERSION-1.*.rpm" -printf '%f\n') + + echo "== Test rpm package on ${IMAGE} ==" + docker pull $IMAGE + docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh" + + displayName: 'Test Rpm Package Azure Linux' + +# TODO: rpmbuild on Red Hat UBI 8 is slow for unknown reason. Still working with Red Hat to investigate. +- job: BuildRpmPackages + displayName: Build Rpm Packages + # Do not run this job for Pull Requests due to the slowness + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: $(pool) + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Red Hat Universal Base Image 8 ${{ arch.name }}: + dockerfile: ubi + image: registry.access.redhat.com/ubi8/ubi:8.4 + artifact: rpm-ubi8-${{ arch.value }} + python_package: python3.12 + pool: ${{ arch.pool }} + Red Hat Universal Base Image 9 ${{ arch.name }}: + dockerfile: ubi + image: registry.access.redhat.com/ubi9/ubi:9.0.0 + artifact: rpm-ubi9-${{ arch.value }} + python_package: python3.12 + pool: ${{ arch.pool }} + Red Hat Universal Base Image 10 ${{ arch.name }}: + dockerfile: ubi + image: registry.access.redhat.com/ubi10/ubi:10.0 + artifact: rpm-ubi10-${{ arch.value }} + python_package: python3.12 + pool: ${{ arch.pool }} + steps: + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + - task: PipAuthenticate@1 + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) + - task: Bash@3 + displayName: 'Build Rpm Package' + inputs: + targetType: 'filePath' + filePath: scripts/release/rpm/pipeline.sh + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: rpm' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: $(artifact) + + +- job: TestRpmPackage + displayName: Test Rpm Package + timeoutInMinutes: 180 + dependsOn: + - BuildRpmPackages + - ExtractMetadata + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: $(pool) + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Red Hat Universal Base Image 8 ${{ arch.name }}: + artifact: rpm-ubi8-${{ arch.value }} + distro: el8 + image: registry.access.redhat.com/ubi8/ubi:8.4 + python_package: python3.12 + python_cmd: python3.12 + pip_cmd: pip3.12 + pool: ${{ arch.pool }} + Red Hat Universal Base Image 9 ${{ arch.name }}: + artifact: rpm-ubi9-${{ arch.value }} + distro: el9 + image: registry.access.redhat.com/ubi9/ubi:9.0.0 + python_package: python3.12 + python_cmd: python3.12 + pip_cmd: pip3.12 + pool: ${{ arch.pool }} + Red Hat Universal Base Image 10 ${{ arch.name }}: + artifact: rpm-ubi10-${{ arch.value }} + distro: el10 + image: registry.access.redhat.com/ubi10/ubi:10.0 + python_package: python3.12 + python_cmd: python3.12 + pip_cmd: pip3.12 + pool: ${{ arch.pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' + artifactName: $(artifact) + + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + + - bash: | + set -ex + + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name "azure-cli-$CLI_VERSION-1.${DISTRO}.*.rpm" -printf '%f\n') + + echo "== Test rpm package on ${IMAGE} ==" + docker pull $IMAGE + docker run --rm -e RPM_NAME=$RPM_NAME -e PYTHON_PACKAGE=${PYTHON_PACKAGE} -e PYTHON_CMD=${PYTHON_CMD} -e PIP_CMD=${PIP_CMD} -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh" + + displayName: 'Test Rpm Package' + +- job: BuildDebPackages + displayName: Build Deb Packages + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: $(pool) + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + # https://wiki.ubuntu.com/Releases + Jammy ${{ arch.name }}: + # 22.04 + deb_system: ubuntu + distro: jammy + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + + Noble ${{ arch.name }}: + # 24.04 + deb_system: ubuntu + distro: noble + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + + # https://wiki.debian.org/DebianReleases + Bullseye ${{ arch.name }}: + # 11 + deb_system: debian + distro: bullseye + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + Bookworm ${{ arch.name }}: + # 12 + deb_system: debian + distro: bookworm + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + steps: + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + - task: PipAuthenticate@1 + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) + - task: Bash@3 + displayName: 'Build $(deb_system) $(distro) $(arch) Package' + inputs: + targetType: 'filePath' + filePath: scripts/release/debian/pipeline.sh + env: + DISTRO_BASE_IMAGE: mcr.microsoft.com/mirror/docker/library/$(deb_system):$(distro) + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: $(deb_system) $(distro) $(arch)' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: $(deb_system)-$(distro)-$(arch) + +- job: TestDebPackages + timeoutInMinutes: 180 + displayName: Test Deb Packages + dependsOn: + - BuildDebPackages + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Jammy ${{ arch.name }}: + deb_system: ubuntu + distro: jammy + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + Noble ${{ arch.name }}: + deb_system: ubuntu + distro: noble + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + Bullseye ${{ arch.name }}: + deb_system: debian + distro: bullseye + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + Bookworm ${{ arch.name }}: + deb_system: debian + distro: bookworm + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + pool: + name: $(pool) + steps: + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download $(deb_system):$(distro) $(arch) Build' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/debian' + artifactName: $(deb_system)-$(distro)-$(arch) + + - task: Bash@3 + displayName: 'Test $(deb_system) $(distro) $(arch) Package' + env: + DISTRO_BASE_IMAGE: mcr.microsoft.com/mirror/docker/library/$(deb_system):$(distro) + inputs: + targetType: 'inline' + script: | + set -exv + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + + echo "== Test debian package on ${DISTRO} ==" + docker pull ${DISTRO_BASE_IMAGE} + docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" + +- job: CheckStyle + displayName: "Check CLI Style" + timeoutInMinutes: 120 + pool: + name: ${{ variables.ubuntu_multi_core_pool }} + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.13' + inputs: + versionSpec: 3.13 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + azdev style + +- job: CheckHeaders + displayName: "Check License, History, and DocMap" + condition: and(not(contains(variables['Build.SourceBranch'], 'lts')), not(contains(variables['System.PullRequest.TargetBranch'], 'lts'))) + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.13' + inputs: + versionSpec: 3.13 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + azdev verify license + azdev verify history + azdev verify document-map + +- job: PerformanceCheck + displayName: "PerformanceCheck" + strategy: + matrix: + Python312: + python.version: '3.12' + Python313: + python.version: '3.13' + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + azdev perf load-times + displayName: "Load Performance" + # - bash: | + # set -ev + # . env/bin/activate + + # azdev perf benchmark "version" "network vnet -h" "rest -h" "storage account" + # displayName: "Execution Performance" + +- job: CheckLinter + displayName: "Check CLI Linter" + + pool: + name: ${{ variables.ubuntu_multi_core_pool }} + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.13' + inputs: + versionSpec: 3.13 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + python scripts/ci/service_name.py + if [[ "$(System.PullRequest.TargetBranch)" != "" ]]; then + # If CI is set to shallow fetch, target branch should be expilictly fetched. + git fetch origin --depth=1 $(System.PullRequest.TargetBranch) + azdev linter --ci-exclusions --min-severity medium --repo=./ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch) + else + azdev linter --ci-exclusions --min-severity medium + fi + +- job: AzdevScan + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) + displayName: "Secret Scan" + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.13' + inputs: + versionSpec: 3.13 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + git fetch origin --depth=1 $(System.PullRequest.TargetBranch) + declare -A secret_files + IFS_OLD=${IFS} + IFS=$'\n' + for FILE in `git diff --name-only --diff-filter=AM origin/$(System.PullRequest.TargetBranch)` ; do + echo $FILE + detected=$(azdev scan -f "$FILE" --continue-on-failure | python -c "import sys, json; print(json.load(sys.stdin)['secrets_detected'])") + if [ $detected == 'True' ]; then + printf "\033[0;31mDetected secrets from %s, You can run 'azdev mask' to remove secrets.\033[0m\n" "$FILE" + secret_files+=$FILE + fi + done + IFS=${IFS_OLD} + if [ "${#secret_files[@]}" -gt 0 ]; then + exit 1 + fi + +- job: CodegenCoverage + condition: in(variables['Build.Reason'], 'BatchedCI', 'IndividualCI') + timeoutInMinutes: 180 + displayName: "Codegen Coverage" + continueOnError: true + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.11' + inputs: + versionSpec: 3.11 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + # clone azure-cli-extensions + cd .. + git clone --depth 1 -b main https://github.com/Azure/azure-cli-extensions.git ./azure-cli-extensions + azdev extension repo add ./azure-cli-extensions + pip install setuptools==70.0.0 wheel==0.30.0 + azdev extension add "*" + pip install msrestazure markupsafe==2.0.1 + # Some extension will change the dependence, so run `azdev setup` again after all extensions installed. + azdev setup -c ./s -r ./azure-cli-extensions + + mkdir -p /tmp/module_stats + + find /mnt/vss/_work/1/s/src/azure-cli/azure/cli/command_modules/ -mindepth 1 -maxdepth 1 -type d -printf "%f\n" | grep -v '^__pycache__$' > /mnt/vss/_work/1/s/scripts/ci/core_modules.txt + echo "=== Core Modules ===" + cat /mnt/vss/_work/1/s/scripts/ci/core_modules.txt + + find /mnt/vss/_work/1/azure-cli-extensions/src/ -mindepth 1 -maxdepth 1 -type d -printf "%f\n" | grep -v '^__pycache__$' > /mnt/vss/_work/1/s/scripts/ci/extension_modules.txt + echo "=== Extension Modules ===" + cat /mnt/vss/_work/1/s/scripts/ci/extension_modules.txt + + for module in $(cat /mnt/vss/_work/1/s/scripts/ci/core_modules.txt); do + azdev statistics list-command-table $module --statistics-only > /tmp/module_stats/${module}.json || true + done + + for module in $(cat /mnt/vss/_work/1/s/scripts/ci/extension_modules.txt); do + azdev statistics list-command-table $module --statistics-only > /tmp/module_stats/${module}.json || true + done + azdev statistics list-command-table --statistics-only > /tmp/codegen_report.json || true + python /mnt/vss/_work/1/s/scripts/ci/codegen_report.py + env: + BUILD_ID: $(Build.BuildId) + BUILD_BRANCH: $(Build.SourceBranchName) + enabled: true From 0eb1683b18e75f1a201e50f429a223add93fea96 Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Tue, 9 Sep 2025 14:35:45 +1000 Subject: [PATCH 5/9] temp: add job to find source of yaml --- azure-pipelines.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 83d7e195a11..7f66d4e2d75 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -54,7 +54,7 @@ jobs: echo "Feature PR should target dev branch." exit 1 fi - python scripts/ci/check_pull_request.py "$title" "$body" + python scripts/ci/check_pull_request.py "$title" "$body" - job: RejectPullRequestToMasterBranch displayName: "Reject Pull Request To Master Branch" @@ -79,6 +79,24 @@ jobs: cat $PIPELINE_WORKSPACE/.azure-pipelines/templates/variables.yml displayName: 'Inspect variables template' + - job: DebugTemplates + displayName: "Debug: List YAML templates + check ubuntu_pool" + pool: + vmImage: pool-ubuntu-latest-multi-core + steps: + - bash: | + echo "##[group]List all YAML files under workspace" + find "$(Pipeline.Workspace)" -type f \( -name "*.yml" -o -name "*.yaml" \) || true + echo "##[endgroup]" + + echo "##[group]Show relevant variables only" + echo "ubuntu_pool = $ubuntu_pool" + echo "Pipeline.Workspace = $PIPELINE_WORKSPACE" + echo "Build.SourcesDirectory = $BUILD_SOURCESDIRECTORY" + echo "System.DefaultWorkingDirectory = $SYSTEM_DEFAULTWORKINGDIRECTORY" + echo "##[endgroup]" + displayName: "Inspect workspace and pool variable" + - job: CredentialScanner displayName: "Credential Scanner" pool: From 78049277b318f4d1eb8b114f3a7db5e4d35c6819 Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Tue, 9 Sep 2025 14:37:00 +1000 Subject: [PATCH 6/9] temp: fix job indentation --- azure-pipelines.yml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7f66d4e2d75..18137641cdd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -79,23 +79,23 @@ jobs: cat $PIPELINE_WORKSPACE/.azure-pipelines/templates/variables.yml displayName: 'Inspect variables template' - - job: DebugTemplates - displayName: "Debug: List YAML templates + check ubuntu_pool" - pool: - vmImage: pool-ubuntu-latest-multi-core - steps: - - bash: | - echo "##[group]List all YAML files under workspace" - find "$(Pipeline.Workspace)" -type f \( -name "*.yml" -o -name "*.yaml" \) || true - echo "##[endgroup]" - - echo "##[group]Show relevant variables only" - echo "ubuntu_pool = $ubuntu_pool" - echo "Pipeline.Workspace = $PIPELINE_WORKSPACE" - echo "Build.SourcesDirectory = $BUILD_SOURCESDIRECTORY" - echo "System.DefaultWorkingDirectory = $SYSTEM_DEFAULTWORKINGDIRECTORY" - echo "##[endgroup]" - displayName: "Inspect workspace and pool variable" +- job: DebugTemplates + displayName: "Debug: List YAML templates + check ubuntu_pool" + pool: + vmImage: pool-ubuntu-latest-multi-core + steps: + - bash: | + echo "##[group]List all YAML files under workspace" + find "$(Pipeline.Workspace)" -type f \( -name "*.yml" -o -name "*.yaml" \) || true + echo "##[endgroup]" + + echo "##[group]Show relevant variables only" + echo "ubuntu_pool = $ubuntu_pool" + echo "Pipeline.Workspace = $PIPELINE_WORKSPACE" + echo "Build.SourcesDirectory = $BUILD_SOURCESDIRECTORY" + echo "System.DefaultWorkingDirectory = $SYSTEM_DEFAULTWORKINGDIRECTORY" + echo "##[endgroup]" + displayName: "Inspect workspace and pool variable" - job: CredentialScanner displayName: "Credential Scanner" From 7d4411318a109a16f034f13a0dea4e53b072c546 Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Tue, 9 Sep 2025 14:39:02 +1000 Subject: [PATCH 7/9] temp: change to name --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 18137641cdd..ca086382156 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -82,7 +82,7 @@ jobs: - job: DebugTemplates displayName: "Debug: List YAML templates + check ubuntu_pool" pool: - vmImage: pool-ubuntu-latest-multi-core + name: pool-ubuntu-latest-multi-core steps: - bash: | echo "##[group]List all YAML files under workspace" From c4e3584554589e217bd84375bf5b221aeba6ee21 Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Tue, 9 Sep 2025 14:49:07 +1000 Subject: [PATCH 8/9] temp: echo out vars --- azure-pipelines.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ca086382156..60ac5867e31 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -97,6 +97,12 @@ jobs: echo "##[endgroup]" displayName: "Inspect workspace and pool variable" + - bash: | + echo "UBUNTU_POOL = $UBUNTU_POOL" + echo "WINDOWS_POOL = $WINDOWS_POOL" + echo "UBUNTU_ARM64_POOL = $UBUNTU_ARM64_POOL" + displayName: "Check pool variables" + - job: CredentialScanner displayName: "Credential Scanner" pool: From bd6822112c8e16237876316826c57aa08933d5af Mon Sep 17 00:00:00 2001 From: Daniel Languiller Date: Thu, 11 Sep 2025 09:31:56 +1000 Subject: [PATCH 9/9] fix: revert ado debug statement --- azure-pipelines.yml | 38 +------------------------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 60ac5867e31..6e9d72a1c3c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -54,7 +54,7 @@ jobs: echo "Feature PR should target dev branch." exit 1 fi - python scripts/ci/check_pull_request.py "$title" "$body" + python scripts/ci/check_pull_request.py "$title" "$body" - job: RejectPullRequestToMasterBranch displayName: "Reject Pull Request To Master Branch" @@ -67,42 +67,6 @@ jobs: echo "Reject pull request directly to master branch" exit 1 -- job: ShowVariables - pool: - name: pool-ubuntu-latest-multi-core - steps: - - script: | - echo "Pipeline workspace: $PIPELINE_WORKSPACE" - echo "Listing repo root:" - ls -R $PIPELINE_WORKSPACE/.azure-pipelines/templates - echo "Contents of variables.yml:" - cat $PIPELINE_WORKSPACE/.azure-pipelines/templates/variables.yml - displayName: 'Inspect variables template' - -- job: DebugTemplates - displayName: "Debug: List YAML templates + check ubuntu_pool" - pool: - name: pool-ubuntu-latest-multi-core - steps: - - bash: | - echo "##[group]List all YAML files under workspace" - find "$(Pipeline.Workspace)" -type f \( -name "*.yml" -o -name "*.yaml" \) || true - echo "##[endgroup]" - - echo "##[group]Show relevant variables only" - echo "ubuntu_pool = $ubuntu_pool" - echo "Pipeline.Workspace = $PIPELINE_WORKSPACE" - echo "Build.SourcesDirectory = $BUILD_SOURCESDIRECTORY" - echo "System.DefaultWorkingDirectory = $SYSTEM_DEFAULTWORKINGDIRECTORY" - echo "##[endgroup]" - displayName: "Inspect workspace and pool variable" - - - bash: | - echo "UBUNTU_POOL = $UBUNTU_POOL" - echo "WINDOWS_POOL = $WINDOWS_POOL" - echo "UBUNTU_ARM64_POOL = $UBUNTU_ARM64_POOL" - displayName: "Check pool variables" - - job: CredentialScanner displayName: "Credential Scanner" pool: