From f2b4c43ded5e178fc4234d842e26aa6fbbb2671e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=98=BF=E7=94=B7?= <l.weinan@gmail.com>
Date: Tue, 20 Jan 2026 16:34:36 +0800
Subject: [PATCH 1/9] OCPBUGS-69923: ensure deterministic zone ordering for
 control plane machines #10188

---
 .../check/aws/zone-consistency/OWNERS         |   4 +
 .../check/aws/zone-consistency/cluster/OWNERS |   4 +
 ...k-aws-zone-consistency-cluster-commands.sh | 138 +++++++++++++++
 ...zone-consistency-cluster-ref.metadata.json |  10 ++
 ...heck-aws-zone-consistency-cluster-ref.yaml |  16 ++
 .../aws/zone-consistency/manifests/OWNERS     |   4 +
 ...aws-zone-consistency-manifests-commands.sh | 167 ++++++++++++++++++
 ...ne-consistency-manifests-ref.metadata.json |  10 ++
 ...ck-aws-zone-consistency-manifests-ref.yaml |  17 ++
 ...ller-rehearse-aws-ipi-provision-chain.yaml |   1 +
 .../ipi/install/ipi-install-chain.yaml        |   1 +
 11 files changed, 372 insertions(+)
 create mode 100644 ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/OWNERS
 create mode 100644 ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/OWNERS
 create mode 100755 ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-commands.sh
 create mode 100644 ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-ref.metadata.json
 create mode 100644 ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-ref.yaml
 create mode 100644 ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/OWNERS
 create mode 100755 ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-commands.sh
 create mode 100644 ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-ref.metadata.json
 create mode 100644 ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-ref.yaml

diff --git a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/OWNERS b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/OWNERS
new file mode 100644
index 0000000000000..88c1bfffd18a9
--- /dev/null
+++ b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/OWNERS
@@ -0,0 +1,4 @@
+approvers:
+- jianlinliu
+- gpei
+- yunjiang29
diff --git a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/OWNERS b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/OWNERS
new file mode 100644
index 0000000000000..88c1bfffd18a9
--- /dev/null
+++ b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/OWNERS
@@ -0,0 +1,4 @@
+approvers:
+- jianlinliu
+- gpei
+- yunjiang29
diff --git a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-commands.sh b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-commands.sh
new file mode 100755
index 0000000000000..d24650dddea96
--- /dev/null
+++ b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-commands.sh
@@ -0,0 +1,138 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+# save the exit code for junit xml file generated in step gather-must-gather
+# post check steps after cluster installation, exit code 101 if failed,
+# save to install-post-check-status.txt
+EXIT_CODE=101
+trap 'if [[ "$?" == 0 ]]; then EXIT_CODE=0; fi; echo "${EXIT_CODE}" > "${SHARED_DIR}/install-post-check-status.txt"' EXIT TERM
+
+echo "=========================================="
+echo "OCPBUGS-69923: Verify Cluster Zone Consistency"
+echo "=========================================="
+echo ""
+
+# Check required tools
+if ! command -v oc >/dev/null 2>&1; then
+    echo "Error: oc tool is required"
+    exit 1
+fi
+
+# Check if kubeconfig exists
+# kubeconfig is typically copied to SHARED_DIR after installation
+KUBECONFIG_PATH="${KUBECONFIG:-${SHARED_DIR}/kubeconfig}"
+if [ ! -f "$KUBECONFIG_PATH" ] && [ -f "/tmp/installer/auth/kubeconfig" ]; then
+    KUBECONFIG_PATH="/tmp/installer/auth/kubeconfig"
+fi
+
+if [ ! -f "$KUBECONFIG_PATH" ]; then
+    echo "Error: kubeconfig file does not exist"
+    echo "       Please ensure cluster installation is complete"
+    exit 1
+fi
+
+export KUBECONFIG="$KUBECONFIG_PATH"
+
+echo "Kubeconfig: $KUBECONFIG_PATH"
+echo ""
+
+# Check cluster connection
+if ! oc cluster-info >/dev/null 2>&1; then
+    echo "Error: Cannot connect to cluster"
+    echo "       Please check if kubeconfig file is correct and cluster is ready"
+    exit 1
+fi
+
+echo "✓ Successfully connected to cluster"
+echo ""
+
+# Get all master machines
+MASTER_MACHINES=$(oc get machine -n openshift-machine-api -l machine.openshift.io/cluster-api-machine-role=master -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || echo "")
+
+if [ -z "$MASTER_MACHINES" ]; then
+    echo "❌ Error: No master machines found"
+    echo "   Please ensure the cluster installation is complete"
+    exit 1
+fi
+
+echo "Found $(echo $MASTER_MACHINES | wc -w | tr -d ' ') master machine(s)"
+echo ""
+
+# Verify zone consistency for each machine
+echo "=========================================="
+echo "Check Zone Consistency for Each Machine"
+echo "=========================================="
+echo ""
+
+all_consistent=true
+machine_count=0
+ret=0
+
+for machine in $MASTER_MACHINES; do
+    machine_count=$((machine_count + 1))
+    echo "--- Machine: $machine ---"
+    
+    # Get zone label
+    zone_label=$(oc get machine "$machine" -n openshift-machine-api -o jsonpath='{.metadata.labels.machine\.openshift\.io/zone}' 2>/dev/null || echo "N/A")
+    
+    # Get zone from providerID
+    provider_id=$(oc get machine "$machine" -n openshift-machine-api -o jsonpath='{.spec.providerID}' 2>/dev/null || echo "")
+    provider_zone=$(echo "$provider_id" | grep -oP 'aws:///\K[^/]+' 2>/dev/null || echo "N/A")
+    
+    # Get availabilityZone from spec
+    spec_zone=$(oc get machine "$machine" -n openshift-machine-api -o jsonpath='{.spec.providerSpec.value.placement.availabilityZone}' 2>/dev/null || echo "N/A")
+    
+    echo "  Zone Label:        $zone_label"
+    echo "  ProviderID Zone:  $provider_zone"
+    echo "  Spec Zone:        $spec_zone"
+    
+    # Check consistency
+    if [ "$zone_label" != "N/A" ] && [ "$provider_zone" != "N/A" ] && [ "$spec_zone" != "N/A" ]; then
+        if [ "$zone_label" = "$provider_zone" ] && [ "$provider_zone" = "$spec_zone" ]; then
+            echo "  ✅ Zone consistent"
+        else
+            echo "  ❌ Zone inconsistent!"
+            all_consistent=false
+            ret=$((ret + 1))
+        fi
+    else
+        echo "  ⚠️  Warning: Some zone information is missing"
+        if [ "$zone_label" != "$provider_zone" ] || [ "$provider_zone" != "$spec_zone" ]; then
+            all_consistent=false
+            ret=$((ret + 1))
+        fi
+    fi
+    echo ""
+done
+
+# Summary
+echo "=========================================="
+echo "Verification Summary"
+echo "=========================================="
+echo ""
+echo "Checked $machine_count master machine(s)"
+echo ""
+
+if [ "$all_consistent" = true ]; then
+    echo "✅ Cluster verification PASSED: All machines have consistent zones!"
+    echo ""
+    echo "Cluster verification: PASS ✓"
+    echo ""
+    echo "Fix verification successful:"
+    echo "  - Zone label, ProviderID zone, and Spec zone are all consistent"
+    echo "  - Machines are created in the correct availability zones"
+    exit 0
+else
+    echo "❌ Cluster verification FAILED: Machines with inconsistent zones detected!"
+    echo ""
+    echo "Cluster verification: FAIL ✗"
+    echo ""
+    echo "Possible issues:"
+    echo "  1. Fix not effective"
+    echo "  2. Machines created in wrong availability zones"
+    echo "  3. Zone label does not match actual zone"
+    exit $ret
+fi
diff --git a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-ref.metadata.json b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-ref.metadata.json
new file mode 100644
index 0000000000000..12f2cc01f910f
--- /dev/null
+++ b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-ref.metadata.json
@@ -0,0 +1,10 @@
+{
+	"path": "cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-ref.yaml",
+	"owners": {
+		"approvers": [
+			"jianlinliu",
+			"gpei",
+			"yunjiang29"
+		]
+	}
+}
diff --git a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-ref.yaml b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-ref.yaml
new file mode 100644
index 0000000000000..97e59749e7f4c
--- /dev/null
+++ b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-ref.yaml
@@ -0,0 +1,16 @@
+ref:
+  as: cucushift-installer-check-aws-zone-consistency-cluster
+  from_image:
+    namespace: ocp
+    name: "4.16"
+    tag: upi-installer
+  grace_period: 10m
+  commands: cucushift-installer-check-aws-zone-consistency-cluster-commands.sh
+  resources:
+    requests:
+      cpu: 10m
+      memory: 100Mi
+  documentation: >-
+    Verify control plane machine zone consistency in installed cluster (OCPBUGS-69923).
+    This step verifies that actual cluster machines have consistent zone information
+    (zone label, providerID zone, and spec zone) after cluster installation is complete.
diff --git a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/OWNERS b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/OWNERS
new file mode 100644
index 0000000000000..88c1bfffd18a9
--- /dev/null
+++ b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/OWNERS
@@ -0,0 +1,4 @@
+approvers:
+- jianlinliu
+- gpei
+- yunjiang29
diff --git a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-commands.sh b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-commands.sh
new file mode 100755
index 0000000000000..f175a75c61591
--- /dev/null
+++ b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-commands.sh
@@ -0,0 +1,167 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+# save the exit code for junit xml file generated in step gather-must-gather
+# post check steps after manifest generation, exit code 100 if failed,
+# save to install-pre-config-status.txt
+EXIT_CODE=100
+trap 'if [[ "$?" == 0 ]]; then EXIT_CODE=0; fi; echo "${EXIT_CODE}" > "${SHARED_DIR}/install-pre-config-status.txt"' EXIT TERM
+
+echo "=========================================="
+echo "OCPBUGS-69923: Verify Manifest Zone Consistency"
+echo "=========================================="
+echo ""
+
+# Check required tools
+if ! command -v yq >/dev/null 2>&1; then
+    echo "Error: yq tool is required"
+    exit 1
+fi
+
+# Installation directory is typically /tmp/installer
+INSTALL_DIR="/tmp/installer"
+if [ ! -d "$INSTALL_DIR" ]; then
+    echo "Error: Installation directory does not exist: $INSTALL_DIR"
+    echo "       Manifests should be generated before this step"
+    exit 1
+fi
+
+echo "Installation directory: $INSTALL_DIR"
+echo ""
+
+# Check if manifest files exist
+CAPI_FILES=$(find "$INSTALL_DIR"/openshift -name "*cluster-api*master*.yaml" -type f 2>/dev/null | sort || true)
+MAPI_FILES=$(find "$INSTALL_DIR"/openshift -name "*machine-api*master*.yaml" -type f 2>/dev/null | sort || true)
+
+if [ -z "$CAPI_FILES" ]; then
+    echo "❌ Error: CAPI manifest files not found"
+    echo "   Please ensure manifests have been generated: openshift-install create manifests"
+    exit 1
+fi
+
+if [ -z "$MAPI_FILES" ]; then
+    echo "❌ Error: MAPI manifest files not found"
+    echo "   Please ensure manifests have been generated: openshift-install create manifests"
+    exit 1
+fi
+
+echo "Found CAPI and MAPI manifest files"
+echo ""
+
+# Get CAPI zones
+echo "=== CAPI Machine Zones ==="
+capi_zones=()
+capi_index=0
+for file in $CAPI_FILES; do
+    zone=$(yq eval '.spec.providerSpec.value.placement.availabilityZone' "$file" 2>/dev/null || echo "")
+    if [ -n "$zone" ] && [ "$zone" != "null" ] && [ "$zone" != "" ]; then
+        capi_zones+=("$zone")
+        echo "  master-$capi_index ($(basename "$file")): $zone"
+        capi_index=$((capi_index + 1))
+    fi
+done
+
+if [ ${#capi_zones[@]} -eq 0 ]; then
+    echo "  ⚠️  Warning: No CAPI zone information found"
+    echo "    Attempted path: .spec.providerSpec.value.placement.availabilityZone"
+    exit 1
+fi
+
+echo ""
+
+# Get MAPI zones
+echo "=== MAPI Machine Zones ==="
+mapi_zones=()
+
+# MAPI files are ControlPlaneMachineSet, need to extract zones from failureDomains
+for file in $MAPI_FILES; do
+    kind=$(yq eval '.kind' "$file" 2>/dev/null || echo "")
+    if [ "$kind" = "ControlPlaneMachineSet" ]; then
+        zones=$(yq eval '.spec.template.machines_v1beta1_machine_openshift_io.failureDomains.aws[].placement.availabilityZone' "$file" 2>/dev/null || echo "")
+        if [ -n "$zones" ]; then
+            master_count=${#capi_zones[@]}
+            if [ $master_count -eq 0 ]; then
+                master_count=3  # Default 3 masters
+            fi
+            
+            mapi_index=0
+            for zone in $zones; do
+                if [ "$zone" != "null" ] && [ -n "$zone" ] && [ "$zone" != "" ]; then
+                    if [ $mapi_index -lt $master_count ]; then
+                        mapi_zones+=("$zone")
+                        echo "  master-$mapi_index (from $(basename "$file")): $zone"
+                        mapi_index=$((mapi_index + 1))
+                    else
+                        break
+                    fi
+                fi
+            done
+        fi
+    else
+        # If it's a direct Machine object
+        zone=$(yq eval '.spec.providerSpec.value.placement.availabilityZone' "$file" 2>/dev/null || echo "")
+        if [ -n "$zone" ] && [ "$zone" != "null" ] && [ "$zone" != "" ]; then
+            mapi_zones+=("$zone")
+            echo "  master-${#mapi_zones[@]} ($(basename "$file")): $zone"
+        fi
+    fi
+done
+
+if [ ${#mapi_zones[@]} -eq 0 ]; then
+    echo "  ⚠️  Warning: No MAPI zone information found"
+    echo "    Attempted path: .spec.template.machines_v1beta1_machine_openshift_io.failureDomains.aws[].placement.availabilityZone"
+    exit 1
+fi
+
+echo ""
+
+# Compare
+echo "=========================================="
+echo "Consistency Check"
+echo "=========================================="
+
+if [ ${#capi_zones[@]} -ne ${#mapi_zones[@]} ]; then
+    echo "⚠️  Warning: CAPI and MAPI have different number of machines"
+    echo "   CAPI: ${#capi_zones[@]} machines"
+    echo "   MAPI: ${#mapi_zones[@]} machines"
+    echo ""
+fi
+
+all_match=true
+max_count=${#capi_zones[@]}
+if [ ${#mapi_zones[@]} -gt $max_count ]; then
+    max_count=${#mapi_zones[@]}
+fi
+
+ret=0
+for i in $(seq 0 $((max_count - 1))); do
+    capi_zone="${capi_zones[$i]:-N/A}"
+    mapi_zone="${mapi_zones[$i]:-N/A}"
+    
+    if [ "$capi_zone" = "$mapi_zone" ] && [ "$capi_zone" != "N/A" ]; then
+        echo "✓ Match: master-$i - Zone: $capi_zone"
+    else
+        echo "❌ Mismatch: master-$i - CAPI: $capi_zone, MAPI: $mapi_zone"
+        all_match=false
+        ret=$((ret + 1))
+    fi
+done
+
+echo ""
+
+if [ "$all_match" = true ]; then
+    echo "✅ Manifest verification PASSED: All machines have consistent zone allocation!"
+    echo ""
+    echo "Manifest verification: PASS ✓"
+    exit 0
+else
+    echo "❌ Manifest verification FAILED: Zone allocation inconsistency detected!"
+    echo ""
+    echo "Manifest verification: FAIL ✗"
+    echo ""
+    echo "This indicates that the fix for OCPBUGS-69923 may not be effective."
+    exit $ret
+fi
diff --git a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-ref.metadata.json b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-ref.metadata.json
new file mode 100644
index 0000000000000..3e302aba632f4
--- /dev/null
+++ b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-ref.metadata.json
@@ -0,0 +1,10 @@
+{
+	"path": "cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-ref.yaml",
+	"owners": {
+		"approvers": [
+			"jianlinliu",
+			"gpei",
+			"yunjiang29"
+		]
+	}
+}
diff --git a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-ref.yaml b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-ref.yaml
new file mode 100644
index 0000000000000..ed26d3183ffd0
--- /dev/null
+++ b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-ref.yaml
@@ -0,0 +1,17 @@
+ref:
+  as: cucushift-installer-check-aws-zone-consistency-manifests
+  from_image:
+    namespace: ocp
+    name: "4.16"
+    tag: upi-installer
+  grace_period: 10m
+  commands: cucushift-installer-check-aws-zone-consistency-manifests-commands.sh
+  resources:
+    requests:
+      cpu: 10m
+      memory: 100Mi
+  documentation: >-
+    Verify control plane machine zone allocation consistency in manifests (OCPBUGS-69923).
+    This step verifies that CAPI and MAPI manifests have consistent zone allocation
+    for control plane machines after manifests are generated.
+    This should run after 'create manifests' but before 'create cluster'.
diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/provision/cucushift-installer-rehearse-aws-ipi-provision-chain.yaml b/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/provision/cucushift-installer-rehearse-aws-ipi-provision-chain.yaml
index 80e6dc2ed82d4..95ccbd8378c04 100644
--- a/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/provision/cucushift-installer-rehearse-aws-ipi-provision-chain.yaml
+++ b/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/provision/cucushift-installer-rehearse-aws-ipi-provision-chain.yaml
@@ -7,5 +7,6 @@ chain:
   - chain: ipi-install
   - ref: enable-qe-catalogsource
   - chain: cucushift-installer-check
+  - ref: cucushift-installer-check-aws-zone-consistency-cluster
   documentation: |-
     Create an IPI cluster on AWS for QE e2e tests.
diff --git a/ci-operator/step-registry/ipi/install/ipi-install-chain.yaml b/ci-operator/step-registry/ipi/install/ipi-install-chain.yaml
index 7c772c2d6f15d..8c1302a13ddcb 100644
--- a/ci-operator/step-registry/ipi/install/ipi-install-chain.yaml
+++ b/ci-operator/step-registry/ipi/install/ipi-install-chain.yaml
@@ -5,6 +5,7 @@ chain:
   - ref: openshift-cluster-bot-rbac
   - ref: ipi-install-hosted-loki
   - ref: ipi-install-install
+  - ref: cucushift-installer-check-aws-zone-consistency-manifests
   - ref: ipi-install-times-collection
   - ref: nodes-readiness
   - ref: multiarch-validate-nodes

From 33e5f3a86ded2a5bb6acaeab3a99d38b9543ff14 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=98=BF=E7=94=B7?= <l.weinan@gmail.com>
Date: Tue, 20 Jan 2026 18:51:15 +0800
Subject: [PATCH 2/9] update

---
 ...k-aws-zone-consistency-cluster-commands.sh | 136 +++------------
 ...aws-zone-consistency-manifests-commands.sh | 161 +++++-------------
 2 files changed, 64 insertions(+), 233 deletions(-)

diff --git a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-commands.sh b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-commands.sh
index d24650dddea96..95c9da1262119 100755
--- a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-commands.sh
+++ b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-commands.sh
@@ -10,129 +10,41 @@ set -o pipefail
 EXIT_CODE=101
 trap 'if [[ "$?" == 0 ]]; then EXIT_CODE=0; fi; echo "${EXIT_CODE}" > "${SHARED_DIR}/install-post-check-status.txt"' EXIT TERM
 
-echo "=========================================="
-echo "OCPBUGS-69923: Verify Cluster Zone Consistency"
-echo "=========================================="
-echo ""
-
-# Check required tools
-if ! command -v oc >/dev/null 2>&1; then
-    echo "Error: oc tool is required"
-    exit 1
-fi
-
-# Check if kubeconfig exists
-# kubeconfig is typically copied to SHARED_DIR after installation
-KUBECONFIG_PATH="${KUBECONFIG:-${SHARED_DIR}/kubeconfig}"
-if [ ! -f "$KUBECONFIG_PATH" ] && [ -f "/tmp/installer/auth/kubeconfig" ]; then
-    KUBECONFIG_PATH="/tmp/installer/auth/kubeconfig"
-fi
-
-if [ ! -f "$KUBECONFIG_PATH" ]; then
-    echo "Error: kubeconfig file does not exist"
-    echo "       Please ensure cluster installation is complete"
-    exit 1
-fi
-
-export KUBECONFIG="$KUBECONFIG_PATH"
-
-echo "Kubeconfig: $KUBECONFIG_PATH"
-echo ""
-
-# Check cluster connection
-if ! oc cluster-info >/dev/null 2>&1; then
-    echo "Error: Cannot connect to cluster"
-    echo "       Please check if kubeconfig file is correct and cluster is ready"
-    exit 1
+if [ -f "${SHARED_DIR}/kubeconfig" ] ; then
+  export KUBECONFIG="${SHARED_DIR}/kubeconfig"
+else
+  echo "No KUBECONFIG found, exit now"
+  exit 1
 fi
 
-echo "✓ Successfully connected to cluster"
-echo ""
-
-# Get all master machines
 MASTER_MACHINES=$(oc get machine -n openshift-machine-api -l machine.openshift.io/cluster-api-machine-role=master -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || echo "")
 
 if [ -z "$MASTER_MACHINES" ]; then
-    echo "❌ Error: No master machines found"
-    echo "   Please ensure the cluster installation is complete"
-    exit 1
+  echo "No master machines found"
+  exit 1
 fi
 
-echo "Found $(echo $MASTER_MACHINES | wc -w | tr -d ' ') master machine(s)"
-echo ""
-
-# Verify zone consistency for each machine
-echo "=========================================="
-echo "Check Zone Consistency for Each Machine"
-echo "=========================================="
-echo ""
-
-all_consistent=true
-machine_count=0
 ret=0
 
 for machine in $MASTER_MACHINES; do
-    machine_count=$((machine_count + 1))
-    echo "--- Machine: $machine ---"
-    
-    # Get zone label
-    zone_label=$(oc get machine "$machine" -n openshift-machine-api -o jsonpath='{.metadata.labels.machine\.openshift\.io/zone}' 2>/dev/null || echo "N/A")
-    
-    # Get zone from providerID
-    provider_id=$(oc get machine "$machine" -n openshift-machine-api -o jsonpath='{.spec.providerID}' 2>/dev/null || echo "")
-    provider_zone=$(echo "$provider_id" | grep -oP 'aws:///\K[^/]+' 2>/dev/null || echo "N/A")
-    
-    # Get availabilityZone from spec
-    spec_zone=$(oc get machine "$machine" -n openshift-machine-api -o jsonpath='{.spec.providerSpec.value.placement.availabilityZone}' 2>/dev/null || echo "N/A")
-    
-    echo "  Zone Label:        $zone_label"
-    echo "  ProviderID Zone:  $provider_zone"
-    echo "  Spec Zone:        $spec_zone"
-    
-    # Check consistency
-    if [ "$zone_label" != "N/A" ] && [ "$provider_zone" != "N/A" ] && [ "$spec_zone" != "N/A" ]; then
-        if [ "$zone_label" = "$provider_zone" ] && [ "$provider_zone" = "$spec_zone" ]; then
-            echo "  ✅ Zone consistent"
-        else
-            echo "  ❌ Zone inconsistent!"
-            all_consistent=false
-            ret=$((ret + 1))
-        fi
-    else
-        echo "  ⚠️  Warning: Some zone information is missing"
-        if [ "$zone_label" != "$provider_zone" ] || [ "$provider_zone" != "$spec_zone" ]; then
-            all_consistent=false
-            ret=$((ret + 1))
-        fi
-    fi
-    echo ""
+  zone_label=$(oc get machine "$machine" -n openshift-machine-api -o jsonpath='{.metadata.labels.machine\.openshift\.io/zone}' 2>/dev/null || echo "")
+  provider_id=$(oc get machine "$machine" -n openshift-machine-api -o jsonpath='{.spec.providerID}' 2>/dev/null || echo "")
+  provider_zone=$(echo "$provider_id" | grep -oP 'aws:///\K[^/]+' 2>/dev/null || echo "")
+  spec_zone=$(oc get machine "$machine" -n openshift-machine-api -o jsonpath='{.spec.providerSpec.value.placement.availabilityZone}' 2>/dev/null || echo "")
+  
+  if [ -z "$zone_label" ] || [ -z "$provider_zone" ] || [ -z "$spec_zone" ]; then
+    echo "ERROR: $machine - missing zone information (label: $zone_label, providerID: $provider_zone, spec: $spec_zone)"
+    ret=$((ret + 1))
+  elif [ "$zone_label" != "$provider_zone" ] || [ "$provider_zone" != "$spec_zone" ]; then
+    echo "ERROR: $machine - zone inconsistent (label: $zone_label, providerID: $provider_zone, spec: $spec_zone)"
+    ret=$((ret + 1))
+  fi
 done
 
-# Summary
-echo "=========================================="
-echo "Verification Summary"
-echo "=========================================="
-echo ""
-echo "Checked $machine_count master machine(s)"
-echo ""
-
-if [ "$all_consistent" = true ]; then
-    echo "✅ Cluster verification PASSED: All machines have consistent zones!"
-    echo ""
-    echo "Cluster verification: PASS ✓"
-    echo ""
-    echo "Fix verification successful:"
-    echo "  - Zone label, ProviderID zone, and Spec zone are all consistent"
-    echo "  - Machines are created in the correct availability zones"
-    exit 0
+if [ $ret -eq 0 ]; then
+  echo "PASS: All machines have consistent zones"
 else
-    echo "❌ Cluster verification FAILED: Machines with inconsistent zones detected!"
-    echo ""
-    echo "Cluster verification: FAIL ✗"
-    echo ""
-    echo "Possible issues:"
-    echo "  1. Fix not effective"
-    echo "  2. Machines created in wrong availability zones"
-    echo "  3. Zone label does not match actual zone"
-    exit $ret
+  echo "FAIL: Machines with inconsistent zones detected"
 fi
+
+exit $ret
diff --git a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-commands.sh b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-commands.sh
index f175a75c61591..cd54ac096b36c 100755
--- a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-commands.sh
+++ b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-commands.sh
@@ -10,158 +10,77 @@ set -o pipefail
 EXIT_CODE=100
 trap 'if [[ "$?" == 0 ]]; then EXIT_CODE=0; fi; echo "${EXIT_CODE}" > "${SHARED_DIR}/install-pre-config-status.txt"' EXIT TERM
 
-echo "=========================================="
-echo "OCPBUGS-69923: Verify Manifest Zone Consistency"
-echo "=========================================="
-echo ""
-
-# Check required tools
-if ! command -v yq >/dev/null 2>&1; then
-    echo "Error: yq tool is required"
-    exit 1
-fi
-
-# Installation directory is typically /tmp/installer
 INSTALL_DIR="/tmp/installer"
+
 if [ ! -d "$INSTALL_DIR" ]; then
-    echo "Error: Installation directory does not exist: $INSTALL_DIR"
-    echo "       Manifests should be generated before this step"
-    exit 1
+  echo "Error: Installation directory does not exist: $INSTALL_DIR"
+  exit 1
 fi
 
-echo "Installation directory: $INSTALL_DIR"
-echo ""
-
-# Check if manifest files exist
 CAPI_FILES=$(find "$INSTALL_DIR"/openshift -name "*cluster-api*master*.yaml" -type f 2>/dev/null | sort || true)
 MAPI_FILES=$(find "$INSTALL_DIR"/openshift -name "*machine-api*master*.yaml" -type f 2>/dev/null | sort || true)
 
 if [ -z "$CAPI_FILES" ]; then
-    echo "❌ Error: CAPI manifest files not found"
-    echo "   Please ensure manifests have been generated: openshift-install create manifests"
-    exit 1
+  echo "Error: CAPI manifest files not found"
+  exit 1
 fi
 
 if [ -z "$MAPI_FILES" ]; then
-    echo "❌ Error: MAPI manifest files not found"
-    echo "   Please ensure manifests have been generated: openshift-install create manifests"
-    exit 1
+  echo "Error: MAPI manifest files not found"
+  exit 1
 fi
 
-echo "Found CAPI and MAPI manifest files"
-echo ""
-
 # Get CAPI zones
-echo "=== CAPI Machine Zones ==="
 capi_zones=()
-capi_index=0
 for file in $CAPI_FILES; do
-    zone=$(yq eval '.spec.providerSpec.value.placement.availabilityZone' "$file" 2>/dev/null || echo "")
-    if [ -n "$zone" ] && [ "$zone" != "null" ] && [ "$zone" != "" ]; then
-        capi_zones+=("$zone")
-        echo "  master-$capi_index ($(basename "$file")): $zone"
-        capi_index=$((capi_index + 1))
-    fi
+  zone=$(yq eval '.spec.providerSpec.value.placement.availabilityZone' "$file" 2>/dev/null || echo "")
+  if [ -n "$zone" ] && [ "$zone" != "null" ]; then
+    capi_zones+=("$zone")
+  fi
 done
 
 if [ ${#capi_zones[@]} -eq 0 ]; then
-    echo "  ⚠️  Warning: No CAPI zone information found"
-    echo "    Attempted path: .spec.providerSpec.value.placement.availabilityZone"
-    exit 1
+  echo "Error: No CAPI zone information found"
+  exit 1
 fi
 
-echo ""
-
 # Get MAPI zones
-echo "=== MAPI Machine Zones ==="
 mapi_zones=()
-
-# MAPI files are ControlPlaneMachineSet, need to extract zones from failureDomains
 for file in $MAPI_FILES; do
-    kind=$(yq eval '.kind' "$file" 2>/dev/null || echo "")
-    if [ "$kind" = "ControlPlaneMachineSet" ]; then
-        zones=$(yq eval '.spec.template.machines_v1beta1_machine_openshift_io.failureDomains.aws[].placement.availabilityZone' "$file" 2>/dev/null || echo "")
-        if [ -n "$zones" ]; then
-            master_count=${#capi_zones[@]}
-            if [ $master_count -eq 0 ]; then
-                master_count=3  # Default 3 masters
-            fi
-            
-            mapi_index=0
-            for zone in $zones; do
-                if [ "$zone" != "null" ] && [ -n "$zone" ] && [ "$zone" != "" ]; then
-                    if [ $mapi_index -lt $master_count ]; then
-                        mapi_zones+=("$zone")
-                        echo "  master-$mapi_index (from $(basename "$file")): $zone"
-                        mapi_index=$((mapi_index + 1))
-                    else
-                        break
-                    fi
-                fi
-            done
-        fi
-    else
-        # If it's a direct Machine object
-        zone=$(yq eval '.spec.providerSpec.value.placement.availabilityZone' "$file" 2>/dev/null || echo "")
-        if [ -n "$zone" ] && [ "$zone" != "null" ] && [ "$zone" != "" ]; then
-            mapi_zones+=("$zone")
-            echo "  master-${#mapi_zones[@]} ($(basename "$file")): $zone"
-        fi
-    fi
+  kind=$(yq eval '.kind' "$file" 2>/dev/null || echo "")
+  if [ "$kind" = "ControlPlaneMachineSet" ]; then
+    zones=$(yq eval '.spec.template.machines_v1beta1_machine_openshift_io.failureDomains.aws[].placement.availabilityZone' "$file" 2>/dev/null || echo "")
+    master_count=${#capi_zones[@]}
+    mapi_index=0
+    for zone in $zones; do
+      if [ "$zone" != "null" ] && [ -n "$zone" ] && [ $mapi_index -lt $master_count ]; then
+        mapi_zones+=("$zone")
+        mapi_index=$((mapi_index + 1))
+      fi
+    done
+  fi
 done
 
 if [ ${#mapi_zones[@]} -eq 0 ]; then
-    echo "  ⚠️  Warning: No MAPI zone information found"
-    echo "    Attempted path: .spec.template.machines_v1beta1_machine_openshift_io.failureDomains.aws[].placement.availabilityZone"
-    exit 1
-fi
-
-echo ""
-
-# Compare
-echo "=========================================="
-echo "Consistency Check"
-echo "=========================================="
-
-if [ ${#capi_zones[@]} -ne ${#mapi_zones[@]} ]; then
-    echo "⚠️  Warning: CAPI and MAPI have different number of machines"
-    echo "   CAPI: ${#capi_zones[@]} machines"
-    echo "   MAPI: ${#mapi_zones[@]} machines"
-    echo ""
-fi
-
-all_match=true
-max_count=${#capi_zones[@]}
-if [ ${#mapi_zones[@]} -gt $max_count ]; then
-    max_count=${#mapi_zones[@]}
+  echo "Error: No MAPI zone information found"
+  exit 1
 fi
 
+# Compare zones
 ret=0
-for i in $(seq 0 $((max_count - 1))); do
-    capi_zone="${capi_zones[$i]:-N/A}"
-    mapi_zone="${mapi_zones[$i]:-N/A}"
-    
-    if [ "$capi_zone" = "$mapi_zone" ] && [ "$capi_zone" != "N/A" ]; then
-        echo "✓ Match: master-$i - Zone: $capi_zone"
-    else
-        echo "❌ Mismatch: master-$i - CAPI: $capi_zone, MAPI: $mapi_zone"
-        all_match=false
-        ret=$((ret + 1))
-    fi
+for i in $(seq 0 $((${#capi_zones[@]} - 1))); do
+  capi_zone="${capi_zones[$i]}"
+  mapi_zone="${mapi_zones[$i]}"
+  if [ "$capi_zone" != "$mapi_zone" ]; then
+    echo "ERROR: master-$i zone mismatch - CAPI: $capi_zone, MAPI: $mapi_zone"
+    ret=$((ret + 1))
+  fi
 done
 
-echo ""
-
-if [ "$all_match" = true ]; then
-    echo "✅ Manifest verification PASSED: All machines have consistent zone allocation!"
-    echo ""
-    echo "Manifest verification: PASS ✓"
-    exit 0
+if [ $ret -eq 0 ]; then
+  echo "PASS: All machines have consistent zone allocation"
 else
-    echo "❌ Manifest verification FAILED: Zone allocation inconsistency detected!"
-    echo ""
-    echo "Manifest verification: FAIL ✗"
-    echo ""
-    echo "This indicates that the fix for OCPBUGS-69923 may not be effective."
-    exit $ret
+  echo "FAIL: Zone allocation inconsistency detected"
 fi
+
+exit $ret

From e009a48187bb9f15751e9dbf46d81efb74ecfeff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=98=BF=E7=94=B7?= <l.weinan@gmail.com>
Date: Tue, 20 Jan 2026 21:03:38 +0800
Subject: [PATCH 3/9] update

---
 ...nshift-tests-private-release-4.22__amd64-nightly.yaml | 9 +++++++++
 ...shift-installer-rehearse-aws-ipi-provision-chain.yaml | 1 -
 .../step-registry/ipi/install/ipi-install-chain.yaml     | 1 -
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.22__amd64-nightly.yaml b/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.22__amd64-nightly.yaml
index 15fed7bfab74d..228f324c114d7 100644
--- a/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.22__amd64-nightly.yaml
+++ b/ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.22__amd64-nightly.yaml
@@ -649,6 +649,15 @@ tests:
     test:
     - chain: openshift-e2e-test-qe-destructive
     workflow: cucushift-installer-rehearse-aws-ipi-edge-zone-cco-manual-security-token-service
+- as: aws-ipi-zone-consistency-f28
+  cron: 30 10 15 * *
+  steps:
+    cluster_profile: aws-qe
+    env:
+      BASE_DOMAIN: qe.devcluster.openshift.com
+    test:
+    - chain: openshift-e2e-test-qe
+    workflow: cucushift-installer-rehearse-aws-ipi-zone-consistency
 - as: aws-ipi-ovn-stress-f28
   cron: 19 17 7 * *
   steps:
diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/provision/cucushift-installer-rehearse-aws-ipi-provision-chain.yaml b/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/provision/cucushift-installer-rehearse-aws-ipi-provision-chain.yaml
index 95ccbd8378c04..80e6dc2ed82d4 100644
--- a/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/provision/cucushift-installer-rehearse-aws-ipi-provision-chain.yaml
+++ b/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/provision/cucushift-installer-rehearse-aws-ipi-provision-chain.yaml
@@ -7,6 +7,5 @@ chain:
   - chain: ipi-install
   - ref: enable-qe-catalogsource
   - chain: cucushift-installer-check
-  - ref: cucushift-installer-check-aws-zone-consistency-cluster
   documentation: |-
     Create an IPI cluster on AWS for QE e2e tests.
diff --git a/ci-operator/step-registry/ipi/install/ipi-install-chain.yaml b/ci-operator/step-registry/ipi/install/ipi-install-chain.yaml
index 8c1302a13ddcb..7c772c2d6f15d 100644
--- a/ci-operator/step-registry/ipi/install/ipi-install-chain.yaml
+++ b/ci-operator/step-registry/ipi/install/ipi-install-chain.yaml
@@ -5,7 +5,6 @@ chain:
   - ref: openshift-cluster-bot-rbac
   - ref: ipi-install-hosted-loki
   - ref: ipi-install-install
-  - ref: cucushift-installer-check-aws-zone-consistency-manifests
   - ref: ipi-install-times-collection
   - ref: nodes-readiness
   - ref: multiarch-validate-nodes

From 979d15d077e41ce6ff151a8716e99996e1c07dc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=98=BF=E7=94=B7?= <l.weinan@gmail.com>
Date: Tue, 20 Jan 2026 21:08:47 +0800
Subject: [PATCH 4/9] update

---
 .../rehearse/aws/ipi/zone-consistency/OWNERS  |  4 ++++
 ...pi-zone-consistency-workflow.metadata.json | 10 ++++++++++
 ...rse-aws-ipi-zone-consistency-workflow.yaml | 13 +++++++++++++
 ...-consistency-provision-chain.metadata.json | 10 ++++++++++
 ...-ipi-zone-consistency-provision-chain.yaml | 19 +++++++++++++++++++
 5 files changed, 56 insertions(+)
 create mode 100644 ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/OWNERS
 create mode 100644 ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/cucushift-installer-rehearse-aws-ipi-zone-consistency-workflow.metadata.json
 create mode 100644 ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/cucushift-installer-rehearse-aws-ipi-zone-consistency-workflow.yaml
 create mode 100644 ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/provision/cucushift-installer-rehearse-aws-ipi-zone-consistency-provision-chain.metadata.json
 create mode 100644 ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/provision/cucushift-installer-rehearse-aws-ipi-zone-consistency-provision-chain.yaml

diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/OWNERS b/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/OWNERS
new file mode 100644
index 0000000000000..88c1bfffd18a9
--- /dev/null
+++ b/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/OWNERS
@@ -0,0 +1,4 @@
+approvers:
+- jianlinliu
+- gpei
+- yunjiang29
diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/cucushift-installer-rehearse-aws-ipi-zone-consistency-workflow.metadata.json b/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/cucushift-installer-rehearse-aws-ipi-zone-consistency-workflow.metadata.json
new file mode 100644
index 0000000000000..dd8dffb14017b
--- /dev/null
+++ b/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/cucushift-installer-rehearse-aws-ipi-zone-consistency-workflow.metadata.json
@@ -0,0 +1,10 @@
+{
+	"path": "cucushift/installer/rehearse/aws/ipi/zone-consistency/cucushift-installer-rehearse-aws-ipi-zone-consistency-workflow.yaml",
+	"owners": {
+		"approvers": [
+			"jianlinliu",
+			"gpei",
+			"yunjiang29"
+		]
+	}
+}
diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/cucushift-installer-rehearse-aws-ipi-zone-consistency-workflow.yaml b/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/cucushift-installer-rehearse-aws-ipi-zone-consistency-workflow.yaml
new file mode 100644
index 0000000000000..f742f24288c2d
--- /dev/null
+++ b/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/cucushift-installer-rehearse-aws-ipi-zone-consistency-workflow.yaml
@@ -0,0 +1,13 @@
+workflow:
+  as: cucushift-installer-rehearse-aws-ipi-zone-consistency
+  steps:
+    pre:
+      - chain: cucushift-installer-rehearse-aws-ipi-zone-consistency-provision
+      - ref: cucushift-installer-reportportal-marker
+    post:
+      - chain: cucushift-installer-rehearse-aws-ipi-deprovision
+      - ref: send-results-to-reportportal
+  documentation: |-
+    This workflow tests OCPBUGS-69923: Control plane machine zone allocation consistency.
+    It verifies that CAPI and MAPI manifests have consistent zone allocation,
+    and that actual cluster machines have consistent zone information.
diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/provision/cucushift-installer-rehearse-aws-ipi-zone-consistency-provision-chain.metadata.json b/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/provision/cucushift-installer-rehearse-aws-ipi-zone-consistency-provision-chain.metadata.json
new file mode 100644
index 0000000000000..ec6d4c0082185
--- /dev/null
+++ b/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/provision/cucushift-installer-rehearse-aws-ipi-zone-consistency-provision-chain.metadata.json
@@ -0,0 +1,10 @@
+{
+	"path": "cucushift/installer/rehearse/aws/ipi/zone-consistency/provision/cucushift-installer-rehearse-aws-ipi-zone-consistency-provision-chain.metadata.json",
+	"owners": {
+		"approvers": [
+			"jianlinliu",
+			"gpei",
+			"yunjiang29"
+		]
+	}
+}
diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/provision/cucushift-installer-rehearse-aws-ipi-zone-consistency-provision-chain.yaml b/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/provision/cucushift-installer-rehearse-aws-ipi-zone-consistency-provision-chain.yaml
new file mode 100644
index 0000000000000..a4fbaa436d5a8
--- /dev/null
+++ b/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/provision/cucushift-installer-rehearse-aws-ipi-zone-consistency-provision-chain.yaml
@@ -0,0 +1,19 @@
+chain:
+  as: cucushift-installer-rehearse-aws-ipi-zone-consistency-provision
+  steps:
+  - chain: ipi-conf-aws
+  - ref: ipi-conf-aws-usage-info
+  - chain: aws-provision-iam-user-minimal-permission
+  - ref: ipi-install-rbac
+  - ref: openshift-cluster-bot-rbac
+  - ref: ipi-install-hosted-loki
+  - ref: ipi-install-install
+  - ref: cucushift-installer-check-aws-zone-consistency-manifests
+  - ref: ipi-install-times-collection
+  - ref: nodes-readiness
+  - ref: multiarch-validate-nodes
+  - ref: enable-qe-catalogsource
+  - chain: cucushift-installer-check
+  - ref: cucushift-installer-check-aws-zone-consistency-cluster
+  documentation: |-
+    Create an IPI cluster on AWS and verify zone consistency (OCPBUGS-69923).

From bd1fd2cc9f02b56aaa002a104275801181774652 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=98=BF=E7=94=B7?= <l.weinan@gmail.com>
Date: Tue, 20 Jan 2026 21:17:26 +0800
Subject: [PATCH 5/9] update

---
 ...-tests-private-release-4.22-periodics.yaml | 82 +++++++++++++++++++
 1 file changed, 82 insertions(+)

diff --git a/ci-operator/jobs/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.22-periodics.yaml b/ci-operator/jobs/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.22-periodics.yaml
index c9ccef8f64f4b..8191d6b0e66f9 100644
--- a/ci-operator/jobs/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.22-periodics.yaml
+++ b/ci-operator/jobs/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.22-periodics.yaml
@@ -22584,6 +22584,88 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build09
+  cron: 30 10 15 * *
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: release-4.22
+    org: openshift
+    repo: openshift-tests-private
+  labels:
+    ci-operator.openshift.io/cloud: aws
+    ci-operator.openshift.io/cloud-cluster-profile: aws-qe
+    ci-operator.openshift.io/variant: amd64-nightly
+    ci.openshift.io/generator: prowgen
+    job-release: "4.22"
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-openshift-openshift-tests-private-release-4.22-amd64-nightly-aws-ipi-zone-consistency-f28
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --oauth-token-path=/usr/local/github-credentials/oauth
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --target=aws-ipi-zone-consistency-f28
+      - --variant=amd64-nightly
+      command:
+      - ci-operator
+      image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
+      imagePullPolicy: Always
+      name: ""
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /usr/local/github-credentials
+        name: github-credentials-openshift-ci-robot-private-git-cloner
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: github-credentials-openshift-ci-robot-private-git-cloner
+      secret:
+        secretName: github-credentials-openshift-ci-robot-private-git-cloner
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build09
   cron: 26 15 5,12,19,26 * *

From 2d05cf9db4b167357c6c50b35b73c6e7fbc812e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=98=BF=E7=94=B7?= <l.weinan@gmail.com>
Date: Tue, 20 Jan 2026 21:18:13 +0800
Subject: [PATCH 6/9] update

---
 .../cucushift/installer/check/aws/zone-consistency/OWNERS      | 1 +
 .../installer/check/aws/zone-consistency/cluster/OWNERS        | 1 +
 ...taller-check-aws-zone-consistency-cluster-ref.metadata.json | 3 ++-
 .../installer/check/aws/zone-consistency/manifests/OWNERS      | 1 +
 ...ller-check-aws-zone-consistency-manifests-ref.metadata.json | 3 ++-
 5 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/OWNERS b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/OWNERS
index 88c1bfffd18a9..cfba660e13955 100644
--- a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/OWNERS
+++ b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/OWNERS
@@ -2,3 +2,4 @@ approvers:
 - jianlinliu
 - gpei
 - yunjiang29
+- liweinan
diff --git a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/OWNERS b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/OWNERS
index 88c1bfffd18a9..cfba660e13955 100644
--- a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/OWNERS
+++ b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/OWNERS
@@ -2,3 +2,4 @@ approvers:
 - jianlinliu
 - gpei
 - yunjiang29
+- liweinan
diff --git a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-ref.metadata.json b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-ref.metadata.json
index 12f2cc01f910f..8ec093c84fa9f 100644
--- a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-ref.metadata.json
+++ b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-ref.metadata.json
@@ -4,7 +4,8 @@
 		"approvers": [
 			"jianlinliu",
 			"gpei",
-			"yunjiang29"
+			"yunjiang29",
+			"liweinan"
 		]
 	}
 }
diff --git a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/OWNERS b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/OWNERS
index 88c1bfffd18a9..cfba660e13955 100644
--- a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/OWNERS
+++ b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/OWNERS
@@ -2,3 +2,4 @@ approvers:
 - jianlinliu
 - gpei
 - yunjiang29
+- liweinan
diff --git a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-ref.metadata.json b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-ref.metadata.json
index 3e302aba632f4..372f5c5eaa41a 100644
--- a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-ref.metadata.json
+++ b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-ref.metadata.json
@@ -4,7 +4,8 @@
 		"approvers": [
 			"jianlinliu",
 			"gpei",
-			"yunjiang29"
+			"yunjiang29",
+			"liweinan"
 		]
 	}
 }

From 89307ccf51c71a322d07adfa1fcc6ed45c30ba61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=98=BF=E7=94=B7?= <l.weinan@gmail.com>
Date: Tue, 20 Jan 2026 21:31:33 +0800
Subject: [PATCH 7/9] update

---
 .../rehearse/aws/ipi/zone-consistency/provision/OWNERS | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/provision/OWNERS

diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/provision/OWNERS b/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/provision/OWNERS
new file mode 100644
index 0000000000000..1dee593a8886f
--- /dev/null
+++ b/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/provision/OWNERS
@@ -0,0 +1,10 @@
+approvers:
+- jianlinliu
+- gpei
+- yunjiang29
+- liweinan
+reviewers:
+- jianlinliu
+- gpei
+- yunjiang29
+- liweinan

From 7cdb288426c0d666ed11d41388a503078f3be628 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=98=BF=E7=94=B7?= <l.weinan@gmail.com>
Date: Tue, 20 Jan 2026 21:38:19 +0800
Subject: [PATCH 8/9] update

---
 ...k-aws-zone-consistency-cluster-ref.metadata.json |  2 +-
 ...aws-zone-consistency-manifests-ref.metadata.json |  2 +-
 ...-aws-ipi-zone-consistency-workflow.metadata.json |  2 +-
 ...i-zone-consistency-provision-chain.metadata.json | 13 ++++++++++---
 4 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-ref.metadata.json b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-ref.metadata.json
index 8ec093c84fa9f..04b6cdf1510d0 100644
--- a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-ref.metadata.json
+++ b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/cluster/cucushift-installer-check-aws-zone-consistency-cluster-ref.metadata.json
@@ -8,4 +8,4 @@
 			"liweinan"
 		]
 	}
-}
+}
\ No newline at end of file
diff --git a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-ref.metadata.json b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-ref.metadata.json
index 372f5c5eaa41a..3d2b037b7a1bd 100644
--- a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-ref.metadata.json
+++ b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-ref.metadata.json
@@ -8,4 +8,4 @@
 			"liweinan"
 		]
 	}
-}
+}
\ No newline at end of file
diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/cucushift-installer-rehearse-aws-ipi-zone-consistency-workflow.metadata.json b/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/cucushift-installer-rehearse-aws-ipi-zone-consistency-workflow.metadata.json
index dd8dffb14017b..627460566115e 100644
--- a/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/cucushift-installer-rehearse-aws-ipi-zone-consistency-workflow.metadata.json
+++ b/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/cucushift-installer-rehearse-aws-ipi-zone-consistency-workflow.metadata.json
@@ -7,4 +7,4 @@
 			"yunjiang29"
 		]
 	}
-}
+}
\ No newline at end of file
diff --git a/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/provision/cucushift-installer-rehearse-aws-ipi-zone-consistency-provision-chain.metadata.json b/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/provision/cucushift-installer-rehearse-aws-ipi-zone-consistency-provision-chain.metadata.json
index ec6d4c0082185..b93c271e67102 100644
--- a/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/provision/cucushift-installer-rehearse-aws-ipi-zone-consistency-provision-chain.metadata.json
+++ b/ci-operator/step-registry/cucushift/installer/rehearse/aws/ipi/zone-consistency/provision/cucushift-installer-rehearse-aws-ipi-zone-consistency-provision-chain.metadata.json
@@ -1,10 +1,17 @@
 {
-	"path": "cucushift/installer/rehearse/aws/ipi/zone-consistency/provision/cucushift-installer-rehearse-aws-ipi-zone-consistency-provision-chain.metadata.json",
+	"path": "cucushift/installer/rehearse/aws/ipi/zone-consistency/provision/cucushift-installer-rehearse-aws-ipi-zone-consistency-provision-chain.yaml",
 	"owners": {
 		"approvers": [
 			"jianlinliu",
 			"gpei",
-			"yunjiang29"
+			"yunjiang29",
+			"liweinan"
+		],
+		"reviewers": [
+			"jianlinliu",
+			"gpei",
+			"yunjiang29",
+			"liweinan"
 		]
 	}
-}
+}
\ No newline at end of file

From c04ebd2c7a8bd9584511a9a385a0b80cd2d3c540 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=98=BF=E7=94=B7?= <l.weinan@gmail.com>
Date: Tue, 20 Jan 2026 23:52:40 +0800
Subject: [PATCH 9/9] Add manifests copy to SHARED_DIR and fix shellcheck
 issues

- Copy openshift manifests to SHARED_DIR/installer/openshift after create manifests
  in ipi-install-install step to allow post-install validation steps to access them
- Update zone-consistency-manifests script to check SHARED_DIR/installer first
- Fix shellcheck SC2086: quote variable in comparison
---
 ...eck-aws-zone-consistency-manifests-commands.sh | 15 ++++++++++-----
 .../install/ipi-install-install-commands.sh       |  7 +++++++
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-commands.sh b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-commands.sh
index cd54ac096b36c..2bc108178d9c2 100755
--- a/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-commands.sh
+++ b/ci-operator/step-registry/cucushift/installer/check/aws/zone-consistency/manifests/cucushift-installer-check-aws-zone-consistency-manifests-commands.sh
@@ -10,10 +10,15 @@ set -o pipefail
 EXIT_CODE=100
 trap 'if [[ "$?" == 0 ]]; then EXIT_CODE=0; fi; echo "${EXIT_CODE}" > "${SHARED_DIR}/install-pre-config-status.txt"' EXIT TERM
 
-INSTALL_DIR="/tmp/installer"
-
-if [ ! -d "$INSTALL_DIR" ]; then
-  echo "Error: Installation directory does not exist: $INSTALL_DIR"
+# Try to find installer directory
+# First check SHARED_DIR (manifests copied from ipi-install-install step)
+# Then check /tmp/installer (if running in the same pod)
+if [ -d "${SHARED_DIR}/installer" ]; then
+  INSTALL_DIR="${SHARED_DIR}/installer"
+elif [ -d "/tmp/installer" ]; then
+  INSTALL_DIR="/tmp/installer"
+else
+  echo "Error: Installation directory not found in ${SHARED_DIR}/installer or /tmp/installer"
   exit 1
 fi
 
@@ -53,7 +58,7 @@ for file in $MAPI_FILES; do
     master_count=${#capi_zones[@]}
     mapi_index=0
     for zone in $zones; do
-      if [ "$zone" != "null" ] && [ -n "$zone" ] && [ $mapi_index -lt $master_count ]; then
+      if [ "$zone" != "null" ] && [ -n "$zone" ] && [ $mapi_index -lt "$master_count" ]; then
         mapi_zones+=("$zone")
         mapi_index=$((mapi_index + 1))
       fi
diff --git a/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh b/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh
index 66d064ecdc6fd..51d4f3d4e2ef6 100755
--- a/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh
+++ b/ci-operator/step-registry/ipi/install/install/ipi-install-install-commands.sh
@@ -722,6 +722,13 @@ if test "${ret}" -ne 0 ; then
 fi
 set -o errexit
 
+# Copy manifests to SHARED_DIR for post-manifest validation steps
+# This allows steps that run after ipi-install-install to access manifests
+if [ -d "${dir}/openshift" ]; then
+  mkdir -p "${SHARED_DIR}/installer/openshift"
+  cp -r "${dir}/openshift"/* "${SHARED_DIR}/installer/openshift/" || true
+fi
+
 # Platform specific manifests adjustments
 case "${CLUSTER_TYPE}" in
 azure4|azure-arm64)