Improve kernelCTF auto releaser to handle Android releases (#284)

artmetla · web-flow · commit b706f60a9bbd · 2025-11-27T13:31:33.000+01:00
- The auto-releaser would wok in a following logic: 1. Access https://androidbuildinternal.googleapis.com/android/internal/build/v3/builds?branches=aosp-android-latest-release&buildAttemptStatus=complete&buildType=submitted&maxResults=1&successful=true&target=aosp_cf_x86_64_only_phone-userdebug to get an information about latest Android build available in JSON format. 2. We take the build number and using fetch_artifact (https://android.googlesource.com/tools/fetch_artifact/) attempt to download "kernel_version.txt" artefact from Android build to get all the necessary build details. 3. Using build details check if kernelCTF GCS bucket contains already the release. If not add it to releases that should be processed by kernelctf-release-build action. - The logic of kernelctf-release-build action updated to handle new naming style of Android releases.
diff --git a/.github/workflows/kernelctf-auto-releaser.yaml b/.github/workflows/kernelctf-auto-releaser.yaml
@@ -2,7 +2,7 @@ name: kernelCTF auto releaser
 on:
   workflow_dispatch:
   schedule:
-    - cron:  '0 12 * * *' # every day at 12:00 UTC
+    - cron: '0 12 * * *' # every day at 12:00 UTC
 permissions: {}
 defaults:
   run:
@@ -11,25 +11,57 @@ defaults:
 jobs:
   get_new_builds:
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     steps:
       - name: Test GCS SA key to fail quick before trying to make a build
         uses: google-github-actions/auth@v2
         with:
-          credentials_json: '${{secrets.KERNELCTF_GCS_SA_KEY}}'
+          credentials_json: '${{ secrets.KERNELCTF_GCS_SA_KEY }}'
           create_credentials_file: false
-
+      
       - name: Checkout repo
         uses: actions/checkout@v4
-
+      
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 'stable'
+      
       - name: Install prerequisites
-        run: sudo apt install -yq --no-install-recommends python3-lxml
+        run: sudo apt-get install -yq --no-install-recommends jq python3-lxml
+      
+      - name: Build fetch_artifact tool
+        run: |
+          git clone https://android.googlesource.com/tools/fetch_artifact
+          cd fetch_artifact
+          go build -o fetch_artifact .
+          sudo mv fetch_artifact /usr/local/bin/
+          cd ..
+          rm -rf fetch_artifact
+            
+      - name: Check latest Linux kernel versions
+        id: check_linux
+        run: python3 get_latest_lts_cos_versions.py
 
-      - id: check
-        name: Check latest kernel versions
-        run: ./get_latest_kernel_versions.py
+      - name: Check for new Android builds
+        id: check_android
+        run: python3 get_latest_android_versions.py
+            
+      - name: Merge all releases
+        id: merge
+        run: |
+          LINUX_RELEASES='${{ steps.check_linux.outputs.releases }}'
+          ANDROID_RELEASES='${{ steps.check_android.outputs.releases }}'
+          
+          # Merge both release arrays
+          MERGED=$(jq -sc 'add' <(echo "$LINUX_RELEASES") <(echo "$ANDROID_RELEASES"))
+          
+          echo "releases=$MERGED" >> $GITHUB_OUTPUT
+          echo "Merged releases: $MERGED"
+    
     outputs:
-      releases: ${{ steps.check.outputs.releases }}
-
+      releases: ${{ steps.merge.outputs.releases }}
+  
   build_release:
     needs: get_new_builds
     if: fromJSON(needs.get_new_builds.outputs.releases)[0] != null
diff --git a/.github/workflows/kernelctf-release-build.yaml b/.github/workflows/kernelctf-release-build.yaml
@@ -33,7 +33,7 @@ jobs:
         id: detect
         run: |
           # Detect Android builds by prefix, assume Linux for everything else
-          if [[ "${{inputs.releaseId}}" =~ ^android- ]]; then
+          if [[ "${{inputs.releaseId}}" =~ ^android ]]; then
             echo "type=android" >> $GITHUB_OUTPUT
             echo "Detected Android build from releaseId prefix"
           else
@@ -43,10 +43,23 @@ jobs:
 
       - name: Check release does not exist yet
         run: |
-          if curl --fail -I https://storage.googleapis.com/kernelctf-build/releases/${{inputs.releaseId}}/bzImage 2>/dev/null; then
-            echo "Error: Release ${{inputs.releaseId}} already exists"
+          RELEASE_ID="${{inputs.releaseId}}"
+          
+          # Detect build type
+          if [[ "$RELEASE_ID" =~ ^android ]]; then
+            # Check for Android release artifact
+            CHECK_FILE="cvd-host_package.tar.gz"
+          else
+            # Check for Linux release artifact
+            CHECK_FILE="bzImage"
+          fi
+          
+          if curl --fail -I "https://storage.googleapis.com/kernelctf-build/releases/${RELEASE_ID}/${CHECK_FILE}" 2>/dev/null; then
+            echo "Error: Release ${RELEASE_ID} already exists (found ${CHECK_FILE})"
             exit 1
           fi
+          
+          echo "Release ${RELEASE_ID} does not exist yet, proceeding with build"
       
       - name: Install Linux build prerequisites
         if: steps.detect.outputs.type == 'linux'
@@ -81,13 +94,24 @@ jobs:
         if: steps.detect.outputs.type == 'android'
         run: |
           RELEASE_ID="${{inputs.releaseId}}"
-          ARCH=$(echo $RELEASE_ID | cut -d'-' -f3)
-          BUILD_ID=$(echo $RELEASE_ID | cut -d'-' -f4)
+          
+          echo "Release ID: $RELEASE_ID"
+          
+          # Parse release ID format: android16-6.12.23-x86_64-14421689
+          # Use awk to split and extract fields
+          ARCH=$(echo "$RELEASE_ID" | awk -F'-' '{print $(NF-1)}')
+          BUILD_ID=$(echo "$RELEASE_ID" | awk -F'-' '{print $NF}')
           
           echo "Architecture: $ARCH"
           echo "Build ID: $BUILD_ID"
           
-          if [[ "$ARCH" == "x64" ]]; then
+          if [[ -z "$ARCH" || -z "$BUILD_ID" ]]; then
+            echo "Error: Failed to parse release ID"
+            echo "Expected format: androidXX-X.XX.XX-ARCH-BUILDID"
+            exit 1
+          fi
+          
+          if [[ "$ARCH" == "x86_64" ]]; then
             TARGET="aosp_cf_x86_64_only_phone-userdebug"
             TARGET_BASE="aosp_cf_x86_64_only_phone"
           elif [[ "$ARCH" == "arm64" ]]; then
@@ -118,7 +142,7 @@ jobs:
             -artifact=$IMG_ARTIFACT
           mv "$IMG_ARTIFACT" "$RELEASE_DIR/"
           
-          echo "Android artifacts downloaded to $RELEASE_DIR"        
+          echo "Android artifacts downloaded to $RELEASE_DIR"
 
       - name: Show releases
         run: find releases -type f|xargs ls -al
diff --git a/kernelctf/get_latest_android_versions.py b/kernelctf/get_latest_android_versions.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env -S python3 -u
+import json
+import subprocess
+import os
+import re
+from utils import *
+
+def validate_build_id(build_id):
+    if not re.match(r'^\d+$', str(build_id)):
+        fail(f"Invalid build_id format: {build_id}")
+    return build_id
+
+def validate_version_component(component, name):
+    if not re.match(r'^[a-z0-9.-]+$', component, re.IGNORECASE):
+        fail(f"Invalid {name} format: {component}")
+    return component
+
+def release_exists(release_id):
+    url = f"https://storage.googleapis.com/kernelctf-build/releases/{release_id}/cvd-host_package.tar.gz"
+    status_code = requests.head(url).status_code
+    
+    if status_code == 200:
+        return True
+    if status_code == 403:
+        return False
+    
+    fail(f"Unexpected HTTP status code for release check: {status_code} (url = {url})")
+
+def fetch_android_build():
+    API_URL = "https://androidbuildinternal.googleapis.com/android/internal/build/v3/builds"
+    TARGET = "aosp_cf_x86_64_only_phone-userdebug"
+    BRANCH = "aosp-android-latest-release"
+    
+    print(f"Fetching latest Android build for {TARGET}...")
+    
+    response = fetch(f"{API_URL}?branches={BRANCH}&buildAttemptStatus=complete&buildType=submitted&maxResults=1&successful=true&target={TARGET}")
+    
+    try:
+        data = json.loads(response)
+        builds = data.get('builds', [])
+        
+        if not builds:
+            fail("No builds found in API response")
+        
+        build_id = builds[0].get('buildId')
+        if not build_id:
+            fail("No buildId found in response")
+        
+        # Validate build_id is numeric only
+        build_id = validate_build_id(build_id)
+        
+        print(f"Latest build ID: {build_id}")
+        return {
+            'build_id': build_id,
+            'target': TARGET
+        }
+    
+    except (json.JSONDecodeError, KeyError) as e:
+        fail(f"Error parsing API response: {e}")
+
+def download_kernel_version(build_info):
+    build_id = build_info['build_id']
+    target = build_info['target']
+    
+    print(f"Fetching kernel_version.txt for build {build_id}...")
+    
+    try:
+        result = subprocess.run(
+            [
+                'fetch_artifact',
+                f'-target={target}',
+                f'-build_id={build_id}',
+                '-artifact=kernel_version.txt'
+            ],
+            capture_output=True,
+            text=True,
+            check=True
+        )
+        
+        if not os.path.exists('kernel_version.txt'):
+            fail("kernel_version.txt not found after download")
+        
+        with open('kernel_version.txt', 'r') as f:
+            kernel_version_string = f.read().strip()
+        
+        print(f"Kernel version string: {kernel_version_string}")
+        
+        os.remove('kernel_version.txt')
+        
+        return kernel_version_string
+    
+    except subprocess.CalledProcessError as e:
+        fail(f"fetch_artifact failed: {e.stderr}")
+
+def parse_kernel_version(kernel_version_string):
+    # Example: "6.12.23-android16-5-g2cc84bbe1269-ab13603476"
+    parts = kernel_version_string.split('-')
+    
+    if len(parts) < 2:
+        fail(f"Invalid kernel version format: {kernel_version_string}")
+    
+    kernel_version = validate_version_component(parts[0], "kernel_version")
+    android_version = validate_version_component(parts[1], "android_version")
+    
+    print(f"Kernel version: {kernel_version}")
+    print(f"Android version: {android_version}")
+    
+    return {
+        'kernel_version': kernel_version,
+        'android_version': android_version
+    }
+
+def get_latest_android_release():
+    # Fetch latest build info
+    build_info = fetch_android_build()
+    
+    # Download kernel version info
+    kernel_version_string = download_kernel_version(build_info)
+    
+    # Parse kernel version
+    version_info = parse_kernel_version(kernel_version_string)
+    
+    # Example build release ID: android16-6.12.23-x86_64-14421689
+    release_id = f"{version_info['android_version']}-{version_info['kernel_version']}-x86_64-{build_info['build_id']}"
+    
+    print(f"Release ID: {release_id}")
+    
+    # Check if release already exists
+    if release_exists(release_id):
+        print("Release already exists in GCS, skipping")
+        return []
+    
+    print("Release does not exist in GCS, will proceed with build")
+    return [{
+        "releaseId": release_id,
+        "branch": None
+    }]
+
+releases = get_latest_android_release()
+
+ghSet("OUTPUT", "releases=" + json.dumps(releases))
+print(f"Generated releases output: {json.dumps(releases)}")
diff --git a/kernelctf/get_latest_lts_cos_versions.py b/kernelctf/get_latest_lts_cos_versions.py
