Generate buildkite pipelines from Go code

.buildkite/integration.pipeline.yml

@@ -1,242 +1,15 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
+# Integration tests pipeline - dynamically generated from Go code.
+# See: dev-tools/buildkite/pipelines/integration_pipeline.go
 env:
-  VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
-  ASDF_MAGE_VERSION: 1.14.0
-
-  # Make Docker builds less verbose
-  BUILDKIT_PROGRESS: plain
-
-  # The following images are defined here and their values will be updated by updatecli
-  # Please do not change them manually.
-  IMAGE_UBUNTU_2204_X86_64: "platform-ingest-elastic-agent-ubuntu-2204-1762801856"
-  IMAGE_UBUNTU_2204_ARM_64: "platform-ingest-elastic-agent-ubuntu-2204-aarch64-1762801856"
-
-common:
-  - vault_docker_login: &vault_docker_login
-      elastic/vault-docker-login#v0.5.2:
-        secret_path: 'kv/ci-shared/platform-ingest/elastic_docker_registry'
+  IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1762801856"
 
 steps:
-  - group: "Integration tests: packaging"
-    key: "int-packaging"
-    notify:
-      - github_commit_status:
-          context: "buildkite/elastic-agent - Packaging"
-    steps:
-      # Build matrix is not used for packaging in favor to unique step keys
-      # Packaging amd64 non-containers
-      - label: ":package: amd64: zip,tar.gz,rpm,deb "
-        key: packaging-amd64
-        env:
-          PLATFORMS: "windows/amd64,linux/amd64"
-          PACKAGES: "zip,tar.gz,rpm,deb"
-        command: ".buildkite/scripts/steps/integration-package.sh"
-        artifact_paths:
-          - build/distributions/**
-        retry:
-          automatic:
-            limit: 1
-        agents:
-          provider: "gcp"
-          machineType: "n2-standard-8"
-          image: "${IMAGE_UBUNTU_2204_X86_64}"
-
-      # Temporary build step to verify building the elastic-agent works.
-      # This will be removed once OTEL_COMPONENT is removed, because it
-      # became the default.
-      - label: ":package: amd64: OTEL_COMPONENT zip,tar.gz,rpm,deb "
-        key: packaging-amd64-otel-component
-        env:
-          PLATFORMS: "windows/amd64,linux/amd64"
-          PACKAGES: "zip,tar.gz,rpm,deb"
-          OTEL_COMPONENT: "true"
-        command: ".buildkite/scripts/steps/integration-package.sh"
-        artifact_paths:
-          - build/distributions/**
-        retry:
-          automatic:
-            limit: 1
-        agents:
-          provider: "gcp"
-          machineType: "n2-standard-8"
-          image: "${IMAGE_UBUNTU_2204_X86_64}"
-
-      - label: ":package: amd64: FIPS tar.gz"
-        key: "packaging-amd64-fips"
-        env:
-          PACKAGES: "tar.gz"
-          PLATFORMS: "linux/amd64"
-          FIPS: "true"
-        command: ".buildkite/scripts/steps/integration-package.sh"
-        artifact_paths:
-          - build/distributions/**
-        retry:
-          automatic:
-            limit: 1
-        agents:
-          provider: "gcp"
-          machineType: "n2-standard-4"
-          image: "${IMAGE_UBUNTU_2204_X86_64}"
-
-      # Packaging linux and windows for arm64
-      - label: ":package: arm64: zip,tar.gz"
-        key: packaging-arm64
-        env:
-          PLATFORMS: "windows/arm64,linux/arm64"
-          PACKAGES: "tar.gz,zip"
-        command: ".buildkite/scripts/steps/integration-package.sh"
-        artifact_paths:
-          - build/distributions/**
-        retry:
-          automatic:
-            limit: 1
-        agents:
-          provider: "aws"
-          instanceType: "c6g.2xlarge"
-          image: "${IMAGE_UBUNTU_2204_ARM_64}"
-
-      # Temporary build step to verify building the elastic-agent works.
-      # This will be removed once OTEL_COMPONENT is removed, because it
-      # became the default.
-      - label: ":package: arm64: OTEL_COMPONENT zip,tar.gz"
-        key: packaging-arm64-otel-component
-        env:
-          PLATFORMS: "windows/arm64,linux/arm64"
-          PACKAGES: "tar.gz,zip"
-          OTEL_COMPONENT: "true"
-        command: ".buildkite/scripts/steps/integration-package.sh"
-        artifact_paths:
-          - build/distributions/**
-        retry:
-          automatic:
-            limit: 1
-        agents:
-          provider: "aws"
-          instanceType: "c6g.2xlarge"
-          image: "${IMAGE_UBUNTU_2204_ARM_64}"
-
-      - label: ":package: arm64: FIPS tar.gz"
-        key: "packaging-arm64-fips"
-        env:
-          PACKAGES: "tar.gz"
-          PLATFORMS: "linux/arm64"
-          FIPS: "true"
-        command: ".buildkite/scripts/steps/integration-package.sh"
-        artifact_paths:
-          - build/distributions/**
-        retry:
-          automatic:
-            limit: 1
-        agents:
-          provider: "aws"
-          instanceType: "c6g.2xlarge"
-          image: "${IMAGE_UBUNTU_2204_ARM_64}"
-
-      - label: ":package: amd64: Containers"
-        key: packaging-containers-amd64
-        env:
-          PACKAGES: "docker"
-          PLATFORMS: "linux/amd64"
-        command: |
-          .buildkite/scripts/steps/integration-package.sh
-          .buildkite/scripts/steps/integration-cloud-image-push.sh
-        artifact_paths:
-          - build/distributions/**
-        agents:
-          provider: "gcp"
-          machineType: "n2-standard-8"
-          diskSizeGb: 200
-          image: "${IMAGE_UBUNTU_2204_X86_64}"
-        plugins:
-          - *vault_docker_login
-
-      - label: ":package: arm64: Containers"
-        key: packaging-containers-arm64
-        env:
-          PACKAGES: "docker"
-          PLATFORMS: "linux/arm64"
-        command: |
-          .buildkite/scripts/steps/integration-package.sh
-        artifact_paths:
-          - build/distributions/**
-        agents:
-          provider: "aws"
-          instanceType: "c6g.4xlarge"
-          diskSizeGb: 200
-          image: "${IMAGE_UBUNTU_2204_ARM_64}"
-
-      - label: ":package: amd64: FIPS Containers"
-        key: packaging-containers-amd64-fips
-        env:
-          PACKAGES: "docker"
-          PLATFORMS: "linux/amd64"
-          FIPS: "true"
-        command: |
-          .buildkite/scripts/steps/integration-package.sh
-          .buildkite/scripts/steps/integration-cloud-image-push.sh
-        artifact_paths:
-          - build/distributions/**
-        agents:
-          provider: "gcp"
-          machineType: "n2-standard-8"
-          diskSizeGb: 200
-          image: "${IMAGE_UBUNTU_2204_X86_64}"
-        plugins:
-          - *vault_docker_login
-
-      - label: ":package: arm64: FIPS Containers"
-        key: packaging-containers-arm64-fips
-        env:
-          PACKAGES: "docker"
-          PLATFORMS: "linux/arm64"
-          FIPS: "true"
-        command: |
-          .buildkite/scripts/steps/integration-package.sh
-        artifact_paths:
-          - build/distributions/**
-        agents:
-          provider: "aws"
-          instanceType: "c6g.2xlarge"
-          diskSizeGb: 200
-          image: "${IMAGE_UBUNTU_2204_ARM_64}"
-
-      - label: ":package: amd64: OTEL_COMPONENT Containers"
-        key: packaging-containers-amd64-otel-component
-        env:
-          PACKAGES: "docker"
-          PLATFORMS: "linux/amd64"
-          OTEL_COMPONENT: "true"
-        command: |
-          .buildkite/scripts/steps/integration-package.sh
-        artifact_paths:
-          - build/distributions/**
-        agents:
-          provider: "gcp"
-          machineType: "n2-standard-8"
-          diskSizeGb: 200
-          image: "${IMAGE_UBUNTU_2204_X86_64}"
-        plugins:
-          - *vault_docker_login
-
-      - label: ":package: arm64: OTEL_COMPONENT Containers"
-        key: packaging-containers-arm64-otel-component
-        env:
-          PACKAGES: "docker"
-          PLATFORMS: "linux/arm64"
-          OTEL_COMPONENT: "true"
-        command: |
-          .buildkite/scripts/steps/integration-package.sh
-        artifact_paths:
-          - build/distributions/**
-        agents:
-          provider: "aws"
-          instanceType: "c6g.2xlarge"
-          diskSizeGb: 200
-          image: "${IMAGE_UBUNTU_2204_ARM_64}"
-
-  - label: "Triggering Integration tests"
-    command: "buildkite-agent pipeline upload .buildkite/bk.integration.pipeline.yml"
-
-  - label: "Triggering custom FIPS integration tests"
-    command: "buildkite-agent pipeline upload .buildkite/bk.integration-fips.pipeline.yml"
+  - label: ":pipeline: Upload pipeline"
+    command: |
+      source .buildkite/scripts/common.sh
+      mage buildkite:integration | buildkite-agent pipeline upload
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_2204_X86_64}"

.buildkite/pipeline.agentless-app-release.yaml

@@ -41,7 +41,7 @@ steps:
       diskSizeGb: 400
 
   # wait for packaging to be done
-  - wait: ~
+  - wait
 
   - label: "Publish to internal registry"
     key: "mirror-elastic-agent"
@@ -53,7 +53,7 @@ steps:
       - *docker_login_plugin
 
   # wait for metadata to be set
-  - wait: ~
+  - wait
 
   - label: ":docker: Validate docker image is built for all architectures"
     command: ".buildkite/scripts/steps/validate-agentless-docker-image.sh"
@@ -66,7 +66,7 @@ steps:
           secret_path: 'kv/ci-shared/platform-ingest/elastic_docker_registry'
 
   # wait for image architecture validation
-  - wait: ~
+  - wait
 
   - label: ":grey_question: Promote agentless app release if validation passes"
     # DRY_RUN will help to skip this step when set to true

.buildkite/pipeline.elastic-agent-gce-cleanup.yml

@@ -1,12 +1,12 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 # Removes stale GCE instances having matching labels, name prefixes and older than 24 hours
-# See gce-cleanup.sh and .buildkite/misc/gce-cleanup.yml
-env:
-  VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
+# This pipeline is dynamically generated from Go code.
+# See: dev-tools/buildkite/pipelines/gce_cleanup.go
 steps:
-  - label: "GCE Cleanup"
-    key: "gce-cleanup"
-    command: ".buildkite/scripts/steps/gce-cleanup.sh"
+  - label: ":pipeline: Upload pipeline"
+    command: |
+      source .buildkite/scripts/common.sh
+      mage buildkite:gceCleanup | buildkite-agent pipeline upload
     agents:
       provider: "gcp"