Skip to content

🌱 (e2e): Enhance e2e test to check workiload resilience when catalogs goes away #3747

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-bot merged 1 commit into from
Jan 22, 2026
Merged

🌱 (e2e): Enhance e2e test to check workiload resilience when catalogs goes away #3747

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
245 changes: 99 additions & 146 deletions test/e2e/catalog_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1864,43 +1864,39 @@ var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), fun

By("Wait for operator deployment to be ready")
var operatorDeployment *appsv1.Deployment
Eventually(func() error {
Eventually(func(g Gomega) {
var err error
operatorDeployment, err = c.GetDeployment(generatedNamespace.GetName(), deploymentName)
if err != nil {
return err
}
if operatorDeployment.Spec.Replicas == nil || *operatorDeployment.Spec.Replicas == 0 {
return fmt.Errorf("deployment replicas is not set")
}
if operatorDeployment.Status.AvailableReplicas != *operatorDeployment.Spec.Replicas {
return fmt.Errorf("deployment %s not ready: %d/%d replicas available",
deploymentName,
operatorDeployment.Status.AvailableReplicas,
*operatorDeployment.Spec.Replicas)
}
if operatorDeployment.Status.ReadyReplicas != *operatorDeployment.Spec.Replicas {
return fmt.Errorf("deployment %s not ready: %d/%d replicas ready",
deploymentName,
operatorDeployment.Status.ReadyReplicas,
*operatorDeployment.Spec.Replicas)
}
return nil
}, pollDuration, pollInterval).Should(Succeed())
g.Expect(err).ShouldNot(HaveOccurred())
g.Expect(operatorDeployment.Spec.Replicas).NotTo(BeNil())
g.Expect(*operatorDeployment.Spec.Replicas).NotTo(BeZero())
g.Expect(operatorDeployment.Status.AvailableReplicas).To(Equal(*operatorDeployment.Spec.Replicas),
"deployment %s not ready: %d/%d replicas available",
deploymentName,
operatorDeployment.Status.AvailableReplicas,
*operatorDeployment.Spec.Replicas)
g.Expect(operatorDeployment.Status.ReadyReplicas).To(Equal(*operatorDeployment.Spec.Replicas),
"deployment %s not ready: %d/%d replicas ready",
deploymentName,
operatorDeployment.Status.ReadyReplicas,
*operatorDeployment.Spec.Replicas)
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())

By("Record deployment state before catalog deletion")
deploymentUID := operatorDeployment.UID
expectedReplicas := *operatorDeployment.Spec.Replicas

By("Verify ServiceAccount, Role, and RoleBinding created by OLM")
var serviceAccount *corev1.ServiceAccount
Eventually(func() error {
Eventually(func(g Gomega) {
var err error
serviceAccount, err = c.KubernetesInterface().CoreV1().ServiceAccounts(generatedNamespace.GetName()).Get(
context.Background(),
serviceAccountName,
metav1.GetOptions{},
)
return err
}, pollDuration, pollInterval).Should(Succeed())
g.Expect(err).ShouldNot(HaveOccurred())
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())
serviceAccountUID := serviceAccount.UID

// Roles and RoleBindings are owned by the CSV with generated names, so we list them by owner
Expand All @@ -1911,144 +1907,99 @@ var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), fun
})

var roleList *rbacv1.RoleList
Eventually(func() error {
Eventually(func(g Gomega) {
var err error
roleList, err = c.KubernetesInterface().RbacV1().Roles(generatedNamespace.GetName()).List(
context.Background(),
metav1.ListOptions{LabelSelector: ownerSelector.String()},
)
if err != nil {
return err
}
if len(roleList.Items) == 0 {
return fmt.Errorf("no roles found owned by CSV")
}
return nil
}, pollDuration, pollInterval).Should(Succeed())
g.Expect(err).ShouldNot(HaveOccurred())
g.Expect(roleList.Items).ToNot(BeEmpty(), "no roles found owned by CSV")
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())
roleUID := roleList.Items[0].UID

var roleBindingList *rbacv1.RoleBindingList
Eventually(func() error {
Eventually(func(g Gomega) {
var err error
roleBindingList, err = c.KubernetesInterface().RbacV1().RoleBindings(generatedNamespace.GetName()).List(
context.Background(),
metav1.ListOptions{LabelSelector: ownerSelector.String()},
)
if err != nil {
return err
}
if len(roleBindingList.Items) == 0 {
return fmt.Errorf("no rolebindings found owned by CSV")
}
return nil
}, pollDuration, pollInterval).Should(Succeed())
g.Expect(err).ShouldNot(HaveOccurred())
g.Expect(roleBindingList.Items).ToNot(BeEmpty(), "no rolebindings found owned by CSV")
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())
roleBindingUID := roleBindingList.Items[0].UID

By("Delete catalog source")
err = crc.OperatorsV1alpha1().CatalogSources(catalogSource.GetNamespace()).Delete(context.Background(), catalogSource.GetName(), metav1.DeleteOptions{})
Expect(err).ShouldNot(HaveOccurred())

By("Wait for catalog source to be removed")
Eventually(func() error {
Eventually(func(g Gomega) {
_, err := crc.OperatorsV1alpha1().CatalogSources(catalogSource.GetNamespace()).Get(context.Background(), catalogSource.GetName(), metav1.GetOptions{})
if err == nil {
return fmt.Errorf("catalog source still exists")
}
if !k8serror.IsNotFound(err) {
return err
}
return nil
}, pollDuration, pollInterval).Should(Succeed())
g.Expect(k8serror.IsNotFound(err)).To(BeTrue(), "catalog source should be deleted")
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())

By("Wait for catalog source pod to be deleted")
Eventually(func() error {
Eventually(func(g Gomega) {
listOpts := metav1.ListOptions{
LabelSelector: "olm.catalogSource=" + catalogSourceName,
}
pods, err := c.KubernetesInterface().CoreV1().Pods(catalogSource.GetNamespace()).List(context.Background(), listOpts)
if err != nil {
return err
}
if len(pods.Items) > 0 {
return fmt.Errorf("catalog source pod still exists: %d pods found", len(pods.Items))
}
return nil
}, pollDuration, pollInterval).Should(Succeed())
g.Expect(err).ShouldNot(HaveOccurred())
g.Expect(pods.Items).To(BeEmpty(), "catalog source pod should be deleted")
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())

By("Verify subscription behavior after catalog deletion")
Eventually(func() error {
Eventually(func(g Gomega) {
sub, err := crc.OperatorsV1alpha1().Subscriptions(generatedNamespace.GetName()).Get(
context.Background(),
subscriptionName,
metav1.GetOptions{},
)
if err != nil {
return fmt.Errorf("failed to get subscription: %w", err)
}
g.Expect(err).ShouldNot(HaveOccurred(), "failed to get subscription")

// Subscription should still track the installed CSV
if sub.Status.InstalledCSV != packageStable {
return fmt.Errorf("subscription InstalledCSV changed from %s to %s", packageStable, sub.Status.InstalledCSV)
}
g.Expect(sub.Status.InstalledCSV).To(Equal(packageStable), "subscription InstalledCSV should not change")

// Verify catalog health behavior: if the deleted catalog is still in the health list,
// it should be marked as unhealthy. If it's been removed from the list, that's also acceptable.
for _, health := range sub.Status.CatalogHealth {
if health.CatalogSourceRef != nil && health.CatalogSourceRef.Name == catalogSourceName {
if health.Healthy {
return fmt.Errorf("subscription still reports deleted catalog %s as healthy", catalogSourceName)
}
g.Expect(health.Healthy).To(BeFalse(), "subscription should not report deleted catalog %s as healthy", catalogSourceName)
}
}

return nil
}, pollDuration, pollInterval).Should(Succeed())
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())

By("Verify CSV remains in succeeded state after catalog deletion")
Consistently(func() error {
Consistently(func(g Gomega) {
fetchedCSV, err := crc.OperatorsV1alpha1().ClusterServiceVersions(generatedNamespace.GetName()).Get(
context.Background(),
installedCSV.GetName(),
metav1.GetOptions{},
)
if err != nil {
return fmt.Errorf("failed to get CSV: %w", err)
}
if fetchedCSV.Status.Phase != v1alpha1.CSVPhaseSucceeded {
return fmt.Errorf("CSV phase is %s, expected Succeeded", fetchedCSV.Status.Phase)
}
return nil
}, 3*time.Minute, pollInterval).Should(Succeed())
g.Expect(err).ShouldNot(HaveOccurred(), "failed to get CSV")
g.Expect(fetchedCSV.Status.Phase).To(Equal(v1alpha1.CSVPhaseSucceeded), "CSV should remain in Succeeded state")
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())

By("Verify deployment remains healthy and unchanged")
Consistently(func() error {
Consistently(func(g Gomega) {
deployment, err := c.GetDeployment(generatedNamespace.GetName(), deploymentName)
if err != nil {
return fmt.Errorf("failed to get deployment: %w", err)
}
if deployment.UID != deploymentUID {
return fmt.Errorf("deployment was recreated")
}
if deployment.Spec.Replicas == nil {
return fmt.Errorf("deployment replicas is nil")
}
if deployment.Status.AvailableReplicas != expectedReplicas {
return fmt.Errorf("available replicas: got %d, want %d", deployment.Status.AvailableReplicas, expectedReplicas)
}
if deployment.Status.ReadyReplicas != expectedReplicas {
return fmt.Errorf("ready replicas: got %d, want %d", deployment.Status.ReadyReplicas, expectedReplicas)
}
return nil
}, 3*time.Minute, pollInterval).Should(Succeed())
g.Expect(err).ShouldNot(HaveOccurred(), "failed to get deployment")
g.Expect(deployment.UID).To(Equal(deploymentUID), "deployment should not be recreated")
g.Expect(deployment.Spec.Replicas).NotTo(BeNil(), "deployment replicas should be set")
g.Expect(deployment.Status.AvailableReplicas).To(Equal(expectedReplicas), "available replicas should match expected")
g.Expect(deployment.Status.ReadyReplicas).To(Equal(expectedReplicas), "ready replicas should match expected")
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())

By("Test OLM config management - add environment variable via subscription")
Eventually(func() error {
Eventually(func(g Gomega) {
sub, err := crc.OperatorsV1alpha1().Subscriptions(generatedNamespace.GetName()).Get(
context.Background(),
subscriptionName,
metav1.GetOptions{},
)
if err != nil {
return err
}
g.Expect(err).ShouldNot(HaveOccurred())

if sub.Spec.Config == nil {
sub.Spec.Config = &v1alpha1.SubscriptionConfig{}
Expand All @@ -2062,26 +2013,25 @@ var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), fun
sub,
metav1.UpdateOptions{},
)
return err
}, pollDuration, pollInterval).Should(Succeed())
g.Expect(err).ShouldNot(HaveOccurred())
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())

By("Wait for deployment to have the environment variable")
Eventually(func() error {
Eventually(func(g Gomega) {
deployment, err := c.GetDeployment(generatedNamespace.GetName(), deploymentName)
if err != nil {
return err
}
if len(deployment.Spec.Template.Spec.Containers) == 0 {
return fmt.Errorf("no containers in deployment")
}
g.Expect(err).ShouldNot(HaveOccurred())
g.Expect(deployment.Spec.Template.Spec.Containers).NotTo(BeEmpty(), "deployment should have containers")

container := deployment.Spec.Template.Spec.Containers[0]
envVarFound := false
for _, env := range container.Env {
if env.Name == "TEST_ENV_VAR" && env.Value == "test-value" {
return nil
envVarFound = true
break
}
}
return fmt.Errorf("TEST_ENV_VAR not found in deployment")
}, pollDuration, pollInterval).Should(Succeed())
g.Expect(envVarFound).To(BeTrue(), "TEST_ENV_VAR should be found in deployment")
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())

By("Delete the operator deployment to test OLM reconciliation")
err = c.KubernetesInterface().AppsV1().Deployments(generatedNamespace.GetName()).Delete(
Expand All @@ -2092,43 +2042,44 @@ var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), fun
Expect(err).ShouldNot(HaveOccurred())

By("Wait for deployment to be deleted")
Eventually(func() error {
Eventually(func(g Gomega) {
_, err := c.GetDeployment(generatedNamespace.GetName(), deploymentName)
if err == nil {
return fmt.Errorf("deployment still exists")
}
if !k8serror.IsNotFound(err) {
return err
}
return nil
}, pollDuration, pollInterval).Should(Succeed())
g.Expect(k8serror.IsNotFound(err)).To(BeTrue(), "deployment should be deleted")
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())

By("Wait for OLM to recreate the deployment")
Eventually(func() error {
// Use a longer timeout here since OLM needs to:
// 1. Detect the deployment deletion (via watch or reconciliation loop)
// 2. Recreate the deployment
// 3. Wait for the deployment to become ready (pull image, start pod, etc.)
// In slow/busy CI environments, this can take longer than the standard 5 minutes
Eventually(func(g Gomega) {
deployment, err := c.GetDeployment(generatedNamespace.GetName(), deploymentName)
if err != nil {
return fmt.Errorf("deployment not recreated yet: %w", err)
}
if deployment.UID == deploymentUID {
return fmt.Errorf("deployment UID unchanged, not recreated")
}
if deployment.Spec.Replicas == nil {
return fmt.Errorf("deployment replicas is nil")
}
if deployment.Status.AvailableReplicas != expectedReplicas {
return fmt.Errorf("available replicas: got %d, want %d", deployment.Status.AvailableReplicas, expectedReplicas)
}
if deployment.Status.ReadyReplicas != expectedReplicas {
return fmt.Errorf("ready replicas: got %d, want %d", deployment.Status.ReadyReplicas, expectedReplicas)
}
return nil
}, pollDuration, pollInterval).Should(Succeed())
g.Expect(err).ShouldNot(HaveOccurred(), "deployment should exist")
g.Expect(deployment.UID).NotTo(Equal(deploymentUID), "deployment should have new UID (recreated)")
g.Expect(deployment.Spec.Replicas).NotTo(BeNil(), "deployment replicas should be set")
g.Expect(*deployment.Spec.Replicas).NotTo(BeZero(), "deployment replicas should not be zero")

// Check that pods are actually ready, not just that the deployment exists
g.Expect(deployment.Status.AvailableReplicas).To(Equal(expectedReplicas),
"deployment should have %d available replicas, got %d", expectedReplicas, deployment.Status.AvailableReplicas)
g.Expect(deployment.Status.ReadyReplicas).To(Equal(expectedReplicas),
"deployment should have %d ready replicas, got %d", expectedReplicas, deployment.Status.ReadyReplicas)
g.Expect(deployment.Status.UpdatedReplicas).To(Equal(expectedReplicas),
"deployment should have %d updated replicas, got %d", expectedReplicas, deployment.Status.UpdatedReplicas)
}).WithTimeout(8 * time.Minute).WithPolling(5 * time.Second).Should(Succeed())

By("Verify all resources were recreated by OLM with correct configuration")
recreatedDeployment, err := c.GetDeployment(generatedNamespace.GetName(), deploymentName)
Expect(err).ShouldNot(HaveOccurred())
Expect(recreatedDeployment.UID).ToNot(Equal(deploymentUID), "deployment should have been recreated with new UID")
// Re-fetch the deployment to get the latest state after recreation
var recreatedDeployment *appsv1.Deployment
Eventually(func(g Gomega) {
var err error
recreatedDeployment, err = c.GetDeployment(generatedNamespace.GetName(), deploymentName)
g.Expect(err).ShouldNot(HaveOccurred())
g.Expect(recreatedDeployment.UID).ToNot(Equal(deploymentUID), "deployment should have been recreated with new UID")
}).WithTimeout(pollDuration).WithPolling(pollInterval).Should(Succeed())

// Verify ServiceAccount was NOT recreated (should have same UID)
recreatedServiceAccount, err := c.KubernetesInterface().CoreV1().ServiceAccounts(generatedNamespace.GetName()).Get(
context.Background(),
serviceAccountName,
Expand All @@ -2137,6 +2088,7 @@ var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), fun
Expect(err).ShouldNot(HaveOccurred())
Expect(recreatedServiceAccount.UID).To(Equal(serviceAccountUID), "serviceaccount should not have been recreated (same UID)")

// Verify Role was NOT recreated (should have same UID)
recreatedRoleList, err := c.KubernetesInterface().RbacV1().Roles(generatedNamespace.GetName()).List(
context.Background(),
metav1.ListOptions{LabelSelector: ownerSelector.String()},
Expand All @@ -2145,6 +2097,7 @@ var _ = Describe("Starting CatalogSource e2e tests", Label("CatalogSource"), fun
Expect(len(recreatedRoleList.Items)).To(BeNumerically(">", 0), "at least one role should exist")
Expect(recreatedRoleList.Items[0].UID).To(Equal(roleUID), "role should not have been recreated (same UID)")

// Verify RoleBinding was NOT recreated (should have same UID)
recreatedRoleBindingList, err := c.KubernetesInterface().RbacV1().RoleBindings(generatedNamespace.GetName()).List(
context.Background(),
metav1.ListOptions{LabelSelector: ownerSelector.String()},
Expand Down