Add DocumentDB upgrade support with configurable PostgresImage and ImageVolume extensions

.github/actions/setup-test-environment/action.yml

@@ -54,6 +54,9 @@ inputs:
     description: 'Whether to use external images instead of loading from artifacts'
     required: false
     default: 'false'
+  documentdb-image:
+    description: 'DocumentDB image to use for the cluster'
+    required: true
   # GitHub configuration
   github-token:
     description: 'GitHub token for accessing packages'
@@ -197,10 +200,25 @@ runs:
         mongosh --version
         echo "✓ mongosh installed successfully for ${{ inputs.architecture }}"
 
+    - name: Create kind config file
+      shell: bash
+      run: |
+        cat > /tmp/kind-config.yaml <<EOF
+        kind: Cluster
+        apiVersion: kind.x-k8s.io/v1alpha4
+        featureGates:
+          ImageVolume: true
+        EOF
+        echo "Kind config file created:"
+        cat /tmp/kind-config.yaml
+
     - name: Create kind cluster
-      uses: helm/kind-action@v1.8.0
+      uses: helm/kind-action@v1.12.0
       with:
+        version: v0.31.0
+        node_image: kindest/node:v1.35.0
         cluster_name: documentdb-${{ inputs.test-type }}-${{ inputs.architecture }}-${{ inputs.test-scenario-name }}
+        config: /tmp/kind-config.yaml
 
     - name: Load Docker images into kind cluster (local build)
       if: inputs.use-external-images == 'false'
@@ -578,7 +596,7 @@ runs:
         spec:
           nodeCount: ${{ inputs.node-count }}
           instancesPerNode: ${{ inputs.instances-per-node }}
-          documentDBImage: ghcr.io/microsoft/documentdb/documentdb-local:16
+          documentDBImage: ${{ inputs.documentdb-image }}
           gatewayImage: ghcr.io/microsoft/documentdb/documentdb-local:16
           resource:
             storage:

.github/workflows/test-E2E.yml

@@ -57,6 +57,8 @@ env:
   DB_USERNAME: k8s_secret_user
   DB_PASSWORD: K8sSecret100
   DB_PORT: 10260
+  DOCUMENTDB_OLD_IMAGE: ghcr.io/guanzhousongmicrosoft/documentdb-pg18:0.109.0
+  DOCUMENTDB_IMAGE: ghcr.io/guanzhousongmicrosoft/documentdb-pg18:0.110.0
 
 jobs:
   # Conditional build workflow - only run if image_tag is not provided or on pull_request
@@ -143,10 +145,92 @@ jobs:
         db-port: ${{ env.DB_PORT }}
         image-tag: ${{ env.IMAGE_TAG }}
         chart-version: ${{ env.CHART_VERSION }}
+        documentdb-image: ${{ env.DOCUMENTDB_OLD_IMAGE }}
         use-external-images: ${{ github.event_name != 'pull_request' && inputs.image_tag != '' && inputs.image_tag != null }}
         github-token: ${{ secrets.GITHUB_TOKEN }}
         repository-owner: ${{ github.repository_owner }}
 
+    - name: Test DocumentDB Image Upgrade
+      run: |
+        echo "Testing DocumentDB extension image upgrade on ${{ matrix.architecture }}..."
+
+        OLD_IMAGE="${{ env.DOCUMENTDB_OLD_IMAGE }}"
+        NEW_IMAGE="${{ env.DOCUMENTDB_IMAGE }}"
+
+        # Verify current image is the old image
+        CURRENT_IMAGE=$(kubectl get documentdb $DB_NAME -n $DB_NS -o jsonpath='{.spec.documentDBImage}')
+        echo "Current DocumentDB image: $CURRENT_IMAGE"
+
+        if [[ "$CURRENT_IMAGE" != "$OLD_IMAGE" ]]; then
+          echo "❌ Expected old image $OLD_IMAGE but found $CURRENT_IMAGE"
+          exit 1
+        fi
+        echo "✓ Cluster deployed with old image"
+
+        # Check DocumentDB version in status before upgrade
+        DOCUMENTDB_VERSION_BEFORE=$(kubectl get documentdb $DB_NAME -n $DB_NS -o jsonpath='{.status.documentDBVersion}')
+        echo "DocumentDB version before upgrade: $DOCUMENTDB_VERSION_BEFORE"
+
+        echo ""
+        echo "Upgrading DocumentDB image to $NEW_IMAGE..."
+        kubectl patch documentdb $DB_NAME -n $DB_NS --type='merge' -p "{\"spec\":{\"documentDBImage\":\"$NEW_IMAGE\"}}"
+
+        echo "Waiting for cluster to be healthy with new image..."
+
+        # Extract expected version from image tag (e.g., "0.110.0" -> "0.110-0")
+        # NOTE: This parsing assumes semver format "X.Y.Z" where the last component becomes
+        # hyphen-separated in the extension version. Pre-release tags (e.g., "0.110.0-beta")
+        # are not currently supported and would not be converted correctly.
+        NEW_VERSION_TAG="${NEW_IMAGE##*:}"
+        EXPECTED_VERSION=$(echo "$NEW_VERSION_TAG" | sed 's/\.\([^.]*\)$/-\1/')
+        echo "Expected DocumentDB version after upgrade: $EXPECTED_VERSION"
+
+        timeout 600 bash -c '
+
+        while true; do
+          DB_STATUS=$(kubectl get documentdb '$DB_NAME' -n '$DB_NS' -o jsonpath="{.status.status}" 2>/dev/null)
+          CLUSTER_STATUS=$(kubectl get cluster '$DB_NAME' -n '$DB_NS' -o jsonpath="{.status.phase}" 2>/dev/null)
+
+          # Check DocumentDB version in status
+          DOCUMENTDB_VERSION=$(kubectl get documentdb '$DB_NAME' -n '$DB_NS' -o jsonpath="{.status.documentDBVersion}" 2>/dev/null || echo "N/A")
+
+          echo "DocumentDB status: $DB_STATUS, CNPG Cluster phase: $CLUSTER_STATUS, DocumentDB version: $DOCUMENTDB_VERSION"
+
+          if [[ "$DB_STATUS" == "Cluster in healthy state" && "$CLUSTER_STATUS" == "Cluster in healthy state" ]]; then
+            if [[ "$DOCUMENTDB_VERSION" == "'"$EXPECTED_VERSION"'" ]]; then
+              HEALTHY_PODS=$(kubectl get cluster '$DB_NAME' -n '$DB_NS' -o jsonpath="{.status.instancesStatus.healthy}" 2>/dev/null | jq length 2>/dev/null || echo "0")
+              if [[ "$HEALTHY_PODS" -ge "1" ]]; then
+                echo "✓ Cluster is healthy with new image and $HEALTHY_PODS healthy pods"
+                break
+              fi
+            fi
+          fi
+
+          sleep 10
+        done
+        '
+
+        echo "Verifying new image is applied..."
+        FINAL_IMAGE=$(kubectl get documentdb $DB_NAME -n $DB_NS -o jsonpath='{.spec.documentDBImage}')
+        echo "Final DocumentDB image in spec: $FINAL_IMAGE"
+
+        if [[ "$FINAL_IMAGE" != "$NEW_IMAGE" ]]; then
+          echo "❌ New image not applied to DocumentDB spec"
+          kubectl get documentdb $DB_NAME -n $DB_NS -o yaml
+          exit 1
+        fi
+
+        echo "✓ New image applied successfully"
+
+        # Check DocumentDB version in status after upgrade
+        DOCUMENTDB_VERSION_AFTER=$(kubectl get documentdb $DB_NAME -n $DB_NS -o jsonpath='{.status.documentDBVersion}')
+        echo "DocumentDB version before upgrade: $DOCUMENTDB_VERSION_BEFORE"
+        echo "DocumentDB version after upgrade: $DOCUMENTDB_VERSION_AFTER"
+
+        echo ""
+        echo "✅ DocumentDB image upgrade test completed successfully!"
+        echo "Upgraded from $OLD_IMAGE to $NEW_IMAGE"
+
     - name: Setup port forwarding for comprehensive tests
       uses: ./.github/actions/setup-port-forwarding
       with:

.github/workflows/test-backup-and-restore.yml

@@ -9,10 +9,6 @@ on:
     - cron: '0 2 * * *'
   workflow_dispatch:
     inputs:
-      documentdb_version:
-        description: 'DocumentDB image version to test'
-        required: false
-        default: '16'
       node_count:
         description: 'Number of DocumentDB nodes'
         required: false
@@ -27,11 +23,6 @@ on:
         description: 'Optional: Use existing image tag instead of building locally'
         required: false
         type: string
-      documentdb_version:
-        description: 'DocumentDB image version to test'
-        required: false
-        default: '16'
-        type: string
       node_count:
         description: 'Number of DocumentDB nodes'
         required: false
@@ -52,6 +43,7 @@ env:
   DB_USERNAME: k8s_secret_user
   DB_PASSWORD: K8sSecret100
   DB_PORT: 10260
+  DOCUMENTDB_IMAGE: ghcr.io/guanzhousongmicrosoft/documentdb-pg18:0.110.0
 
 jobs:
   # Conditional build workflow - only run if image_tag is not provided or on pull_request
@@ -131,6 +123,7 @@ jobs:
         db-port: ${{ env.DB_PORT }}
         image-tag: ${{ env.IMAGE_TAG }}
         chart-version: ${{ env.CHART_VERSION }}
+        documentdb-image: ${{ env.DOCUMENTDB_IMAGE }}
         use-external-images: ${{ github.event_name != 'pull_request' && inputs.image_tag != '' && inputs.image_tag != null }}
         github-token: ${{ secrets.GITHUB_TOKEN }}
         repository-owner: ${{ github.repository_owner }}
@@ -279,7 +272,7 @@ jobs:
         spec:
           nodeCount: ${{ matrix.node_count }}
           instancesPerNode: ${{ matrix.instances_per_node }}
-          documentDBImage: ghcr.io/microsoft/documentdb/documentdb-local:16
+          documentDBImage: ${{ env.DOCUMENTDB_IMAGE }}
           gatewayImage: ghcr.io/microsoft/documentdb/documentdb-local:16
           resource:
             storage:

.github/workflows/test-build-and-package.yml

@@ -118,52 +118,6 @@ jobs:
         path: sidecar-${{ matrix.arch }}-image.tar
         retention-days: 1
 
-  build-documentdb:
-    name: Build DocumentDB Images
-    timeout-minutes: 30
-    strategy:
-      matrix:
-        arch: [amd64, arm64]
-        include:
-          - arch: amd64
-            base_arch: AMD64
-            runner: ubuntu-22.04
-          - arch: arm64
-            base_arch: ARM64
-            runner: ubuntu-22.04-arm
-    runs-on: ${{ matrix.runner }}
-    steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
-
-    - name: Build documentdb Docker image for ${{ matrix.arch }}
-      run: |
-        echo "Building documentdb Docker image for ${{ matrix.arch }} architecture..."
-        docker buildx build \
-          --platform linux/${{ matrix.arch }} \
-          --build-arg ARCH=${{ matrix.base_arch }} \
-          --tag ghcr.io/${{ github.repository_owner }}/documentdb-kubernetes-operator/documentdb:${{ env.IMAGE_TAG }}-${{ matrix.arch }} \
-          --load \
-          -f .github/dockerfiles/Dockerfile_docdb .
-
-        echo "✓ DocumentDB Docker image built successfully for ${{ matrix.arch }}"
-
-    - name: Save documentdb Docker image as artifact
-      run: |
-        echo "Saving documentdb ${{ matrix.arch }} Docker image as tar file..."
-        docker save ghcr.io/${{ github.repository_owner }}/documentdb-kubernetes-operator/documentdb:${{ env.IMAGE_TAG }}-${{ matrix.arch }} > documentdb-${{ matrix.arch }}-image.tar
-        echo "✓ DocumentDB ${{ matrix.arch }} Docker image saved to tar file"
-
-    - name: Upload documentdb Docker image artifact
-      uses: actions/upload-artifact@v4
-      with:
-        name: build-docker-documentdb-${{ matrix.arch }}
-        path: documentdb-${{ matrix.arch }}-image.tar
-        retention-days: 1
-
   build-gateway:
     name: Build Gateway Images
     timeout-minutes: 30
@@ -214,7 +168,7 @@ jobs:
     name: Consolidate Platform Images
     runs-on: ubuntu-latest
     timeout-minutes: 15
-    needs: [build-operator, build-sidecar, build-documentdb, build-gateway]
+    needs: [build-operator, build-sidecar, build-gateway]
     outputs:
       image_tag: ${{ env.IMAGE_TAG }}
     steps:
@@ -273,29 +227,7 @@ jobs:
           ls -la ./artifacts/build-docker-sidecar-arm64/ || echo "Directory not found"
           exit 1
         fi
-
-        # Load documentdb images
-        echo "Loading documentdb AMD64 image..."
-        if [ -f ./artifacts/build-docker-documentdb-amd64/documentdb-amd64-image.tar ]; then
-          docker load < ./artifacts/build-docker-documentdb-amd64/documentdb-amd64-image.tar
-          echo "✓ DocumentDB AMD64 image loaded"
-        else
-          echo "❌ DocumentDB AMD64 image file not found"
-          ls -la ./artifacts/build-docker-documentdb-amd64/ || echo "Directory not found"
-          exit 1
-        fi
-
-        echo "Loading documentdb ARM64 image..."
-        if [ -f ./artifacts/build-docker-documentdb-arm64/documentdb-arm64-image.tar ]; then
-          docker load < ./artifacts/build-docker-documentdb-arm64/documentdb-arm64-image.tar
-          echo "✓ DocumentDB ARM64 image loaded"
-        else
-          echo "❌ DocumentDB ARM64 image file not found"
-          ls -la ./artifacts/build-docker-documentdb-arm64/ || echo "Directory not found"
-          exit 1
-        fi
-
-        # Load gateway images
+
         echo "Loading gateway AMD64 image..."
         if [ -f ./artifacts/build-docker-gateway-amd64/gateway-amd64-image.tar ]; then
           docker load < ./artifacts/build-docker-gateway-amd64/gateway-amd64-image.tar
@@ -324,14 +256,12 @@ jobs:
       run: |
         echo "Saving platform-specific images as artifacts..."
 
-        # Save all 8 platform-specific images (4 types x 2 architectures)
+        # Save all 6 platform-specific images (3 types x 2 architectures)
         docker save \
           ghcr.io/${{ github.repository_owner }}/documentdb-kubernetes-operator/operator:${{ env.IMAGE_TAG }}-amd64 \
           ghcr.io/${{ github.repository_owner }}/documentdb-kubernetes-operator/operator:${{ env.IMAGE_TAG }}-arm64 \
           ghcr.io/${{ github.repository_owner }}/documentdb-kubernetes-operator/sidecar:${{ env.IMAGE_TAG }}-amd64 \
           ghcr.io/${{ github.repository_owner }}/documentdb-kubernetes-operator/sidecar:${{ env.IMAGE_TAG }}-arm64 \
-          ghcr.io/${{ github.repository_owner }}/documentdb-kubernetes-operator/documentdb:${{ env.IMAGE_TAG }}-amd64 \
-          ghcr.io/${{ github.repository_owner }}/documentdb-kubernetes-operator/documentdb:${{ env.IMAGE_TAG }}-arm64 \
           ghcr.io/${{ github.repository_owner }}/documentdb-kubernetes-operator/gateway:${{ env.IMAGE_TAG }}-amd64 \
           ghcr.io/${{ github.repository_owner }}/documentdb-kubernetes-operator/gateway:${{ env.IMAGE_TAG }}-arm64 \
           > platform-specific-images.tar

.github/workflows/test-integration.yml

@@ -33,6 +33,7 @@ env:
   DB_USERNAME: default_user
   DB_PASSWORD: Admin100
   DB_PORT: 10260
+  DOCUMENTDB_IMAGE: ghcr.io/guanzhousongmicrosoft/documentdb-pg18:0.110.0
 
 jobs:
   # Use the reusable build workflow - only if no image tag is provided or on pull_request
@@ -119,6 +120,7 @@ jobs:
         db-port: ${{ env.DB_PORT }}
         image-tag: ${{ env.IMAGE_TAG }}
         chart-version: ${{ env.CHART_VERSION }}
+        documentdb-image: ${{ env.DOCUMENTDB_IMAGE }}
         use-external-images: ${{ github.event_name != 'pull_request' && github.event.inputs.image_tag != '' && github.event.inputs.image_tag != null }}
         github-token: ${{ secrets.GITHUB_TOKEN }}
         repository-owner: ${{ github.repository_owner }}

operator/documentdb-helm-chart/Chart.yaml

@@ -6,5 +6,5 @@ description: A Helm chart for deploying the DocumentDB operator
 appVersion: "0.1.3"
 dependencies:
   - name: cloudnative-pg
-    version: "0.26.1"
+    version: "0.27.0"
     repository: "https://cloudnative-pg.github.io/charts/"

operator/documentdb-helm-chart/crds/documentdb.io_dbs.yaml

@@ -192,6 +192,12 @@ spec:
                 maximum: 1
                 minimum: 1
                 type: integer
+              postgresImage:
+                default: ghcr.io/cloudnative-pg/postgresql:18-standard-bookworm
+                description: |-
+                  PostgresImage is the container image to use for the PostgreSQL server.
+                  If not specified, defaults to "ghcr.io/cloudnative-pg/postgresql:18-standard-bookworm".
+                type: string
               resource:
                 description: Resource specifies the storage resources for DocumentDB.
                 properties:
@@ -305,6 +311,10 @@ spec:
             properties:
               connectionString:
                 type: string
+              documentDBVersion:
+                description: DocumentDBVersion is the currently installed version
+                  of the DocumentDB extension.
+                type: string
               localPrimary:
                 type: string
               status:

operator/src/api/preview/documentdb_types.go

@@ -38,6 +38,12 @@ type DocumentDBSpec struct {
 	// If not specified, defaults to a version that matches the DocumentDB operator version.
 	GatewayImage string `json:"gatewayImage,omitempty"`
 
+	// PostgresImage is the container image to use for the PostgreSQL server.
+	// If not specified, defaults to "ghcr.io/cloudnative-pg/postgresql:18-standard-bookworm".
+	// +kubebuilder:default="ghcr.io/cloudnative-pg/postgresql:18-standard-bookworm"
+	// +optional
+	PostgresImage string `json:"postgresImage,omitempty"`
+
 	// DocumentDbCredentialSecret is the name of the Kubernetes Secret containing credentials
 	// for the DocumentDB gateway (expects keys `username` and `password`). If omitted,
 	// a default secret name `documentdb-credentials` is used.
@@ -214,6 +220,9 @@ type DocumentDBStatus struct {
 	TargetPrimary    string `json:"targetPrimary,omitempty"`
 	LocalPrimary     string `json:"localPrimary,omitempty"`
 
+	// DocumentDBVersion is the currently installed version of the DocumentDB extension.
+	DocumentDBVersion string `json:"documentDBVersion,omitempty"`
+
 	// TLS reports gateway TLS provisioning status (Phase 1).
 	TLS *TLSStatus `json:"tls,omitempty"`
 }

operator/src/config/crd/bases/documentdb.io_dbs.yaml

@@ -192,6 +192,12 @@ spec:
                 maximum: 1
                 minimum: 1
                 type: integer
+              postgresImage:
+                default: ghcr.io/cloudnative-pg/postgresql:18-standard-bookworm
+                description: |-
+                  PostgresImage is the container image to use for the PostgreSQL server.
+                  If not specified, defaults to "ghcr.io/cloudnative-pg/postgresql:18-standard-bookworm".
+                type: string
               resource:
                 description: Resource specifies the storage resources for DocumentDB.
                 properties:
@@ -305,6 +311,10 @@ spec:
             properties:
               connectionString:
                 type: string
+              documentDBVersion:
+                description: DocumentDBVersion is the currently installed version
+                  of the DocumentDB extension.
+                type: string
               localPrimary:
                 type: string
               status: