From de64860f1ca56d0a765a313a63c0032fabe983f2 Mon Sep 17 00:00:00 2001 From: Camila Macedo <7708031+camilamacedo86@users.noreply.github.com> Date: Thu, 22 Jan 2026 19:04:13 +0000 Subject: [PATCH] (e2e): Enhance e2e test to check workload resilience when catalogs goes away --- test/e2e/catalog_e2e_test.go | 245 ++++++++++++++--------------------- 1 file changed, 99 insertions(+), 146 deletions(-) diff --git a/test/e2e/catalog_e2e_test.go b/test/e2e/catalog_e2e_test.go index f003611e92..7a375d3e54 100644 --- a/test/e2e/catalog_e2e_test.go +++ b/test/e2e/catalog_e2e_test.go @@ -1864,28 +1864,23 @@ var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), fun By("Wait for operator deployment to be ready") var operatorDeployment *appsv1.Deployment - Eventually(func() error { + Eventually(func(g Gomega) { + var err error operatorDeployment, err = c.GetDeployment(generatedNamespace.GetName(), deploymentName) - if err != nil { - return err - } - if operatorDeployment.Spec.Replicas == nil || *operatorDeployment.Spec.Replicas == 0 { - return fmt.Errorf("deployment replicas is not set") - } - if operatorDeployment.Status.AvailableReplicas != *operatorDeployment.Spec.Replicas { - return fmt.Errorf("deployment %s not ready: %d/%d replicas available", - deploymentName, - operatorDeployment.Status.AvailableReplicas, - *operatorDeployment.Spec.Replicas) - } - if operatorDeployment.Status.ReadyReplicas != *operatorDeployment.Spec.Replicas { - return fmt.Errorf("deployment %s not ready: %d/%d replicas ready", - deploymentName, - operatorDeployment.Status.ReadyReplicas, - *operatorDeployment.Spec.Replicas) - } - return nil - }, pollDuration, pollInterval).Should(Succeed()) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(operatorDeployment.Spec.Replicas).NotTo(BeNil()) + g.Expect(*operatorDeployment.Spec.Replicas).NotTo(BeZero()) + g.Expect(operatorDeployment.Status.AvailableReplicas).To(Equal(*operatorDeployment.Spec.Replicas), + "deployment %s not ready: %d/%d replicas available", + deploymentName, + operatorDeployment.Status.AvailableReplicas, + *operatorDeployment.Spec.Replicas) + g.Expect(operatorDeployment.Status.ReadyReplicas).To(Equal(*operatorDeployment.Spec.Replicas), + "deployment %s not ready: %d/%d replicas ready", + deploymentName, + operatorDeployment.Status.ReadyReplicas, + *operatorDeployment.Spec.Replicas) + }).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed()) By("Record deployment state before catalog deletion") deploymentUID := operatorDeployment.UID @@ -1893,14 +1888,15 @@ var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), fun By("Verify ServiceAccount, Role, and RoleBinding created by OLM") var serviceAccount *corev1.ServiceAccount - Eventually(func() error { + Eventually(func(g Gomega) { + var err error serviceAccount, err = c.KubernetesInterface().CoreV1().ServiceAccounts(generatedNamespace.GetName()).Get( context.Background(), serviceAccountName, metav1.GetOptions{}, ) - return err - }, pollDuration, pollInterval).Should(Succeed()) + g.Expect(err).ShouldNot(HaveOccurred()) + }).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed()) serviceAccountUID := serviceAccount.UID // Roles and RoleBindings are owned by the CSV with generated names, so we list them by owner @@ -1911,35 +1907,27 @@ var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), fun }) var roleList *rbacv1.RoleList - Eventually(func() error { + Eventually(func(g Gomega) { + var err error roleList, err = c.KubernetesInterface().RbacV1().Roles(generatedNamespace.GetName()).List( context.Background(), metav1.ListOptions{LabelSelector: ownerSelector.String()}, ) - if err != nil { - return err - } - if len(roleList.Items) == 0 { - return fmt.Errorf("no roles found owned by CSV") - } - return nil - }, pollDuration, pollInterval).Should(Succeed()) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(roleList.Items).ToNot(BeEmpty(), "no roles found owned by CSV") + }).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed()) roleUID := roleList.Items[0].UID var roleBindingList *rbacv1.RoleBindingList - Eventually(func() error { + Eventually(func(g Gomega) { + var err error roleBindingList, err = c.KubernetesInterface().RbacV1().RoleBindings(generatedNamespace.GetName()).List( context.Background(), metav1.ListOptions{LabelSelector: ownerSelector.String()}, ) - if err != nil { - return err - } - if len(roleBindingList.Items) == 0 { - return fmt.Errorf("no rolebindings found owned by CSV") - } - return nil - }, pollDuration, pollInterval).Should(Succeed()) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(roleBindingList.Items).ToNot(BeEmpty(), "no rolebindings found owned by CSV") + }).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed()) roleBindingUID := roleBindingList.Items[0].UID By("Delete catalog source") @@ -1947,108 +1935,71 @@ var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), fun Expect(err).ShouldNot(HaveOccurred()) By("Wait for catalog source to be removed") - Eventually(func() error { + Eventually(func(g Gomega) { _, err := crc.OperatorsV1alpha1().CatalogSources(catalogSource.GetNamespace()).Get(context.Background(), catalogSource.GetName(), metav1.GetOptions{}) - if err == nil { - return fmt.Errorf("catalog source still exists") - } - if !k8serror.IsNotFound(err) { - return err - } - return nil - }, pollDuration, pollInterval).Should(Succeed()) + g.Expect(k8serror.IsNotFound(err)).To(BeTrue(), "catalog source should be deleted") + }).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed()) By("Wait for catalog source pod to be deleted") - Eventually(func() error { + Eventually(func(g Gomega) { listOpts := metav1.ListOptions{ LabelSelector: "olm.catalogSource=" + catalogSourceName, } pods, err := c.KubernetesInterface().CoreV1().Pods(catalogSource.GetNamespace()).List(context.Background(), listOpts) - if err != nil { - return err - } - if len(pods.Items) > 0 { - return fmt.Errorf("catalog source pod still exists: %d pods found", len(pods.Items)) - } - return nil - }, pollDuration, pollInterval).Should(Succeed()) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(pods.Items).To(BeEmpty(), "catalog source pod should be deleted") + }).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed()) By("Verify subscription behavior after catalog deletion") - Eventually(func() error { + Eventually(func(g Gomega) { sub, err := crc.OperatorsV1alpha1().Subscriptions(generatedNamespace.GetName()).Get( context.Background(), subscriptionName, metav1.GetOptions{}, ) - if err != nil { - return fmt.Errorf("failed to get subscription: %w", err) - } + g.Expect(err).ShouldNot(HaveOccurred(), "failed to get subscription") // Subscription should still track the installed CSV - if sub.Status.InstalledCSV != packageStable { - return fmt.Errorf("subscription InstalledCSV changed from %s to %s", packageStable, sub.Status.InstalledCSV) - } + g.Expect(sub.Status.InstalledCSV).To(Equal(packageStable), "subscription InstalledCSV should not change") // Verify catalog health behavior: if the deleted catalog is still in the health list, // it should be marked as unhealthy. If it's been removed from the list, that's also acceptable. for _, health := range sub.Status.CatalogHealth { if health.CatalogSourceRef != nil && health.CatalogSourceRef.Name == catalogSourceName { - if health.Healthy { - return fmt.Errorf("subscription still reports deleted catalog %s as healthy", catalogSourceName) - } + g.Expect(health.Healthy).To(BeFalse(), "subscription should not report deleted catalog %s as healthy", catalogSourceName) } } - - return nil - }, pollDuration, pollInterval).Should(Succeed()) + }).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed()) By("Verify CSV remains in succeeded state after catalog deletion") - Consistently(func() error { + Consistently(func(g Gomega) { fetchedCSV, err := crc.OperatorsV1alpha1().ClusterServiceVersions(generatedNamespace.GetName()).Get( context.Background(), installedCSV.GetName(), metav1.GetOptions{}, ) - if err != nil { - return fmt.Errorf("failed to get CSV: %w", err) - } - if fetchedCSV.Status.Phase != v1alpha1.CSVPhaseSucceeded { - return fmt.Errorf("CSV phase is %s, expected Succeeded", fetchedCSV.Status.Phase) - } - return nil - }, 3*time.Minute, pollInterval).Should(Succeed()) + g.Expect(err).ShouldNot(HaveOccurred(), "failed to get CSV") + g.Expect(fetchedCSV.Status.Phase).To(Equal(v1alpha1.CSVPhaseSucceeded), "CSV should remain in Succeeded state") + }).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed()) By("Verify deployment remains healthy and unchanged") - Consistently(func() error { + Consistently(func(g Gomega) { deployment, err := c.GetDeployment(generatedNamespace.GetName(), deploymentName) - if err != nil { - return fmt.Errorf("failed to get deployment: %w", err) - } - if deployment.UID != deploymentUID { - return fmt.Errorf("deployment was recreated") - } - if deployment.Spec.Replicas == nil { - return fmt.Errorf("deployment replicas is nil") - } - if deployment.Status.AvailableReplicas != expectedReplicas { - return fmt.Errorf("available replicas: got %d, want %d", deployment.Status.AvailableReplicas, expectedReplicas) - } - if deployment.Status.ReadyReplicas != expectedReplicas { - return fmt.Errorf("ready replicas: got %d, want %d", deployment.Status.ReadyReplicas, expectedReplicas) - } - return nil - }, 3*time.Minute, pollInterval).Should(Succeed()) + g.Expect(err).ShouldNot(HaveOccurred(), "failed to get deployment") + g.Expect(deployment.UID).To(Equal(deploymentUID), "deployment should not be recreated") + g.Expect(deployment.Spec.Replicas).NotTo(BeNil(), "deployment replicas should be set") + g.Expect(deployment.Status.AvailableReplicas).To(Equal(expectedReplicas), "available replicas should match expected") + g.Expect(deployment.Status.ReadyReplicas).To(Equal(expectedReplicas), "ready replicas should match expected") + }).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed()) By("Test OLM config management - add environment variable via subscription") - Eventually(func() error { + Eventually(func(g Gomega) { sub, err := crc.OperatorsV1alpha1().Subscriptions(generatedNamespace.GetName()).Get( context.Background(), subscriptionName, metav1.GetOptions{}, ) - if err != nil { - return err - } + g.Expect(err).ShouldNot(HaveOccurred()) if sub.Spec.Config == nil { sub.Spec.Config = &v1alpha1.SubscriptionConfig{} @@ -2062,26 +2013,25 @@ var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), fun sub, metav1.UpdateOptions{}, ) - return err - }, pollDuration, pollInterval).Should(Succeed()) + g.Expect(err).ShouldNot(HaveOccurred()) + }).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed()) By("Wait for deployment to have the environment variable") - Eventually(func() error { + Eventually(func(g Gomega) { deployment, err := c.GetDeployment(generatedNamespace.GetName(), deploymentName) - if err != nil { - return err - } - if len(deployment.Spec.Template.Spec.Containers) == 0 { - return fmt.Errorf("no containers in deployment") - } + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(deployment.Spec.Template.Spec.Containers).NotTo(BeEmpty(), "deployment should have containers") + container := deployment.Spec.Template.Spec.Containers[0] + envVarFound := false for _, env := range container.Env { if env.Name == "TEST_ENV_VAR" && env.Value == "test-value" { - return nil + envVarFound = true + break } } - return fmt.Errorf("TEST_ENV_VAR not found in deployment") - }, pollDuration, pollInterval).Should(Succeed()) + g.Expect(envVarFound).To(BeTrue(), "TEST_ENV_VAR should be found in deployment") + }).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed()) By("Delete the operator deployment to test OLM reconciliation") err = c.KubernetesInterface().AppsV1().Deployments(generatedNamespace.GetName()).Delete( @@ -2092,43 +2042,44 @@ var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), fun Expect(err).ShouldNot(HaveOccurred()) By("Wait for deployment to be deleted") - Eventually(func() error { + Eventually(func(g Gomega) { _, err := c.GetDeployment(generatedNamespace.GetName(), deploymentName) - if err == nil { - return fmt.Errorf("deployment still exists") - } - if !k8serror.IsNotFound(err) { - return err - } - return nil - }, pollDuration, pollInterval).Should(Succeed()) + g.Expect(k8serror.IsNotFound(err)).To(BeTrue(), "deployment should be deleted") + }).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed()) By("Wait for OLM to recreate the deployment") - Eventually(func() error { + // Use a longer timeout here since OLM needs to: + // 1. Detect the deployment deletion (via watch or reconciliation loop) + // 2. Recreate the deployment + // 3. Wait for the deployment to become ready (pull image, start pod, etc.) + // In slow/busy CI environments, this can take longer than the standard 5 minutes + Eventually(func(g Gomega) { deployment, err := c.GetDeployment(generatedNamespace.GetName(), deploymentName) - if err != nil { - return fmt.Errorf("deployment not recreated yet: %w", err) - } - if deployment.UID == deploymentUID { - return fmt.Errorf("deployment UID unchanged, not recreated") - } - if deployment.Spec.Replicas == nil { - return fmt.Errorf("deployment replicas is nil") - } - if deployment.Status.AvailableReplicas != expectedReplicas { - return fmt.Errorf("available replicas: got %d, want %d", deployment.Status.AvailableReplicas, expectedReplicas) - } - if deployment.Status.ReadyReplicas != expectedReplicas { - return fmt.Errorf("ready replicas: got %d, want %d", deployment.Status.ReadyReplicas, expectedReplicas) - } - return nil - }, pollDuration, pollInterval).Should(Succeed()) + g.Expect(err).ShouldNot(HaveOccurred(), "deployment should exist") + g.Expect(deployment.UID).NotTo(Equal(deploymentUID), "deployment should have new UID (recreated)") + g.Expect(deployment.Spec.Replicas).NotTo(BeNil(), "deployment replicas should be set") + g.Expect(*deployment.Spec.Replicas).NotTo(BeZero(), "deployment replicas should not be zero") + + // Check that pods are actually ready, not just that the deployment exists + g.Expect(deployment.Status.AvailableReplicas).To(Equal(expectedReplicas), + "deployment should have %d available replicas, got %d", expectedReplicas, deployment.Status.AvailableReplicas) + g.Expect(deployment.Status.ReadyReplicas).To(Equal(expectedReplicas), + "deployment should have %d ready replicas, got %d", expectedReplicas, deployment.Status.ReadyReplicas) + g.Expect(deployment.Status.UpdatedReplicas).To(Equal(expectedReplicas), + "deployment should have %d updated replicas, got %d", expectedReplicas, deployment.Status.UpdatedReplicas) + }).WithTimeout(8 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) By("Verify all resources were recreated by OLM with correct configuration") - recreatedDeployment, err := c.GetDeployment(generatedNamespace.GetName(), deploymentName) - Expect(err).ShouldNot(HaveOccurred()) - Expect(recreatedDeployment.UID).ToNot(Equal(deploymentUID), "deployment should have been recreated with new UID") + // Re-fetch the deployment to get the latest state after recreation + var recreatedDeployment *appsv1.Deployment + Eventually(func(g Gomega) { + var err error + recreatedDeployment, err = c.GetDeployment(generatedNamespace.GetName(), deploymentName) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(recreatedDeployment.UID).ToNot(Equal(deploymentUID), "deployment should have been recreated with new UID") + }).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed()) + // Verify ServiceAccount was NOT recreated (should have same UID) recreatedServiceAccount, err := c.KubernetesInterface().CoreV1().ServiceAccounts(generatedNamespace.GetName()).Get( context.Background(), serviceAccountName, @@ -2137,6 +2088,7 @@ var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), fun Expect(err).ShouldNot(HaveOccurred()) Expect(recreatedServiceAccount.UID).To(Equal(serviceAccountUID), "serviceaccount should not have been recreated (same UID)") + // Verify Role was NOT recreated (should have same UID) recreatedRoleList, err := c.KubernetesInterface().RbacV1().Roles(generatedNamespace.GetName()).List( context.Background(), metav1.ListOptions{LabelSelector: ownerSelector.String()}, @@ -2145,6 +2097,7 @@ var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), fun Expect(len(recreatedRoleList.Items)).To(BeNumerically(">", 0), "at least one role should exist") Expect(recreatedRoleList.Items[0].UID).To(Equal(roleUID), "role should not have been recreated (same UID)") + // Verify RoleBinding was NOT recreated (should have same UID) recreatedRoleBindingList, err := c.KubernetesInterface().RbacV1().RoleBindings(generatedNamespace.GetName()).List( context.Background(), metav1.ListOptions{LabelSelector: ownerSelector.String()},