chore: fix intermittent E2E test fail in '1-040_validate_quoted_RBAC_group_names_test'

Makefile

@@ -174,7 +174,7 @@ e2e-tests-sequential-ginkgo: ginkgo ## Runs kuttl e2e sequential tests
 .PHONY: e2e-tests-parallel-ginkgo ## Runs kuttl e2e parallel tests, (Defaults to 5 runs at a time)
 e2e-tests-parallel-ginkgo: ginkgo
 	@echo "Running GitOps Operator parallel Ginkgo E2E tests..."
-	$(GINKGO_CLI) -p -v -procs=5 --trace --timeout 30m -r ./test/openshift/e2e/ginkgo/parallel
+	$(GINKGO_CLI) -p -v -procs=5 --trace --timeout 60m -r ./test/openshift/e2e/ginkgo/parallel
 
 .PHONY: e2e-tests-sequential
 e2e-tests-sequential:
@@ -285,7 +285,7 @@ kustomize: ## Download kustomize locally if necessary.
 GINKGO_CLI = $(shell pwd)/bin/ginkgo
 .PHONY: ginkgo
 ginkgo: ## Download ginkgo locally if necessary.
-	$(call go-get-tool,$(GINKGO_CLI),github.com/onsi/ginkgo/v2/ginkgo@v2.22.2)
+	$(call go-get-tool,$(GINKGO_CLI),github.com/onsi/ginkgo/v2/ginkgo@v2.25.3)
 
 
 # go-get-tool will 'go install' any package $2 and install it to $1.

test/openshift/e2e/ginkgo/fixture/argocd/fixture.go

@@ -261,7 +261,11 @@ func RunArgoCDCLI(args ...string) (string, error) {
 
 	cmdArgs := append([]string{"argocd"}, args...)
 
-	GinkgoWriter.Println("executing command", cmdArgs)
+	if args[0] == "login" {
+		GinkgoWriter.Println("executing command: argocd login (...)")
+	} else {
+		GinkgoWriter.Println("executing command", cmdArgs)
+	}
 
 	// #nosec G204
 	cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)

test/openshift/e2e/ginkgo/fixture/fixture.go

@@ -282,6 +282,8 @@ func CreateNamespace(name string) *corev1.Namespace {
 		Labels: NamespaceLabels,
 	}}
 
+	By("creating namespace '" + ns.Name + "'")
+
 	err := k8sClient.Create(context.Background(), ns)
 	Expect(err).ToNot(HaveOccurred())
 
@@ -902,6 +904,9 @@ func OutputDebugOnFail(namespaceParams ...any) {
 			continue
 		}
 
+		// Uncomment this to output the pod logs from app controller and repo server in ns
+		// outputAppControllerAndRepoLogsInNamespace(namespace)
+
 		GinkgoWriter.Println("")
 		GinkgoWriter.Println("----------------------------------------------------------------")
 		GinkgoWriter.Println("'kubectl get deployments -n " + namespace + " -o yaml")
@@ -932,6 +937,17 @@ func OutputDebugOnFail(namespaceParams ...any) {
 		GinkgoWriter.Println("----------------------------------------------------------------")
 	}
 
+	kubectlOutput, err = osFixture.ExecCommandWithOutputParam(false, true, "kubectl", "get", "applications", "-A", "-o", "yaml")
+	if err != nil {
+		GinkgoWriter.Println("unable to output all argo cd statuses", err, kubectlOutput)
+	} else {
+		GinkgoWriter.Println("")
+		GinkgoWriter.Println("----------------------------------------------------------------")
+		GinkgoWriter.Println("'kubectl get applications -A -o yaml':")
+		GinkgoWriter.Println(kubectlOutput)
+		GinkgoWriter.Println("----------------------------------------------------------------")
+	}
+
 	GinkgoWriter.Println("You can skip this debug output by setting 'SKIP_DEBUG_OUTPUT=true'")
 
 }
@@ -981,7 +997,7 @@ func outputPodLog(podSubstring string) {
 		return
 	}
 
-	// Look specifically for operator pod
+	// Look specifically for pod with name
 	matchingPods := []corev1.Pod{}
 	for idx := range podList.Items {
 		pod := podList.Items[idx]
@@ -992,19 +1008,19 @@ func outputPodLog(podSubstring string) {
 
 	if len(matchingPods) == 0 {
 		// This can happen when the operator is not running on the cluster
-		GinkgoWriter.Println("DebugOutputOperatorLogs was called, but no pods were found.")
+		GinkgoWriter.Println("outputPodLog was called looking for substring '" + podSubstring + "', but no pods were found.")
 		return
 	}
 
 	if len(matchingPods) != 1 {
-		GinkgoWriter.Println("unexpected number of operator pods", matchingPods)
+		GinkgoWriter.Println("unexpected number of pods", matchingPods)
 		return
 	}
 
 	// Extract operator logs
 	kubectlLogOutput, err := osFixture.ExecCommandWithOutputParam(false, true, "kubectl", "logs", "pod/"+matchingPods[0].Name, "manager", "-n", matchingPods[0].Namespace)
 	if err != nil {
-		GinkgoWriter.Println("unable to extract operator logs", err)
+		GinkgoWriter.Println("unable to extract logs for", matchingPods[0].Name, err)
 		return
 	}
 
@@ -1015,14 +1031,49 @@ func outputPodLog(podSubstring string) {
 
 	GinkgoWriter.Println("")
 	GinkgoWriter.Println("----------------------------------------------------------------")
-	GinkgoWriter.Println("Log output from operator pod:")
+	GinkgoWriter.Println("Log output from pod '" + matchingPods[0].Name + "' in " + matchingPods[0].Namespace + ":")
 	for _, line := range lines[startIndex:] {
 		GinkgoWriter.Println(">", line)
 	}
 	GinkgoWriter.Println("----------------------------------------------------------------")
 
 }
 
+//nolint:unused
+func outputAppControllerAndRepoLogsInNamespace(namespace string) {
+
+	var podList corev1.PodList
+	k8sClient, _ := utils.GetE2ETestKubeClient()
+	err := k8sClient.List(context.Background(), &podList, client.InNamespace(namespace))
+	Expect(err).ToNot(HaveOccurred())
+	for _, pod := range podList.Items {
+
+		want := strings.Contains(pod.Name, "repo-server") || strings.Contains(pod.Name, "application-controller")
+		if !want {
+			continue
+		}
+
+		kubectlLogOutput, err := osFixture.ExecCommandWithOutputParam(false, true, "kubectl", "logs", "pod/"+pod.Name, "-n", pod.Namespace)
+		if err != nil {
+			GinkgoWriter.Println("unable to extract logs for", pod.Name, err)
+			return
+		}
+
+		// Output only the last 500 lines
+		lines := strings.Split(kubectlLogOutput, "\n")
+
+		startIndex := max(len(lines)-500, 0)
+
+		GinkgoWriter.Println("")
+		GinkgoWriter.Println("----------------------------------------------------------------")
+		GinkgoWriter.Println("Log output from pod '" + pod.Name + "' in " + pod.Namespace + ":")
+		for _, line := range lines[startIndex:] {
+			GinkgoWriter.Println(">", line)
+		}
+		GinkgoWriter.Println("----------------------------------------------------------------")
+	}
+}
+
 func IsUpstreamOperatorTests() bool {
 	return false // This function should return true if running from argocd-operator repo, false if running from gitops-operator repo. This is to distinguish between tests in upstream argocd-operator and downstream gitops-operator repos.
 }

test/openshift/e2e/ginkgo/parallel/1-025-validate-managed-by-change_test.go

@@ -42,8 +42,14 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 	Context("1-025-validate-managed-by-change", func() {
 
 		var (
-			ctx       context.Context
-			k8sClient client.Client
+			ctx              context.Context
+			k8sClient        client.Client
+			test_1_25_argo1  *corev1.Namespace
+			test_1_25_argo2  *corev1.Namespace
+			test_1_25_target *corev1.Namespace
+			cleanup1         func()
+			cleanup2         func()
+			cleanup3         func()
 		)
 
 		BeforeEach(func() {
@@ -52,27 +58,39 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 			ctx = context.Background()
 		})
 
+		AfterEach(func() {
+
+			fixture.OutputDebugOnFail("test-1-25-argo1", "test-1-25-argo2", "test-1-25-target")
+
+			if cleanup3 != nil {
+				cleanup3()
+			}
+			if cleanup2 != nil {
+				cleanup2()
+			}
+			if cleanup1 != nil {
+				cleanup1()
+			}
+		})
+
 		It("ensuring that managed-by label can transition between two different Argo CD instances", func() {
 
 			By("creating 3 namespaces: 2 contain argo cd instances, and one will be managed by one of those namespaces")
 
-			test_1_25_argo1NS, cleanup1 := fixture.CreateNamespaceWithCleanupFunc("test-1-25-argo1")
-			defer cleanup1()
+			test_1_25_argo1, cleanup1 = fixture.CreateNamespaceWithCleanupFunc("test-1-25-argo1")
 
-			test_1_25_argo2NS, cleanup2 := fixture.CreateNamespaceWithCleanupFunc("test-1-25-argo2")
-			defer cleanup2()
+			test_1_25_argo2, cleanup2 = fixture.CreateNamespaceWithCleanupFunc("test-1-25-argo2")
 
-			test_1_25_targetNS, cleanup3 := fixture.CreateManagedNamespaceWithCleanupFunc("test-1-25-target", test_1_25_argo1NS.Name)
-			defer cleanup3()
+			test_1_25_target, cleanup3 = fixture.CreateManagedNamespaceWithCleanupFunc("test-1-25-target", test_1_25_argo1.Name)
 
 			argoCDtest_1_25_argo1 := &argov1beta1api.ArgoCD{
-				ObjectMeta: metav1.ObjectMeta{Name: "argocd", Namespace: test_1_25_argo1NS.Name},
+				ObjectMeta: metav1.ObjectMeta{Name: "argocd", Namespace: test_1_25_argo1.Name},
 				Spec:       argov1beta1api.ArgoCDSpec{},
 			}
 			Expect(k8sClient.Create(ctx, argoCDtest_1_25_argo1)).To(Succeed())
 
 			argoCDtest_1_25_argo2 := &argov1beta1api.ArgoCD{
-				ObjectMeta: metav1.ObjectMeta{Name: "argocd", Namespace: test_1_25_argo2NS.Name},
+				ObjectMeta: metav1.ObjectMeta{Name: "argocd", Namespace: test_1_25_argo2.Name},
 				Spec:       argov1beta1api.ArgoCDSpec{},
 			}
 			Expect(k8sClient.Create(ctx, argoCDtest_1_25_argo2)).To(Succeed())
@@ -91,7 +109,7 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 						TargetRevision: "HEAD",
 					},
 					Destination: argocdv1alpha1.ApplicationDestination{
-						Namespace: test_1_25_targetNS.Name,
+						Namespace: test_1_25_target.Name,
 						Server:    "https://kubernetes.default.svc",
 					},
 					Project: "default",
@@ -104,20 +122,20 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 			Expect(k8sClient.Create(ctx, app)).To(Succeed())
 
 			By("waiting for all pods to be ready in both Argo CD namespaces")
-			fixture.WaitForAllPodsInTheNamespaceToBeReady(test_1_25_argo1NS.Name, k8sClient)
-			fixture.WaitForAllPodsInTheNamespaceToBeReady(test_1_25_argo2NS.Name, k8sClient)
+			fixture.WaitForAllPodsInTheNamespaceToBeReady(test_1_25_argo1.Name, k8sClient)
+			fixture.WaitForAllPodsInTheNamespaceToBeReady(test_1_25_argo2.Name, k8sClient)
 
 			By("verifying Argo CD Application deployed as expected and is healthy and synced")
 			Eventually(app, "3m", "5s").Should(appFixture.HaveHealthStatusCode(health.HealthStatusHealthy))
 			Eventually(app, "60s", "5s").Should(appFixture.HaveSyncStatusCode(argocdv1alpha1.SyncStatusCodeSynced))
 
 			By("update 'test_1_25_target' NS to be managed by the second Argo CD instance, rather than the first")
-			namespaceFixture.Update(test_1_25_targetNS, func(n *corev1.Namespace) {
+			namespaceFixture.Update(test_1_25_target, func(n *corev1.Namespace) {
 				n.Labels["argocd.argoproj.io/managed-by"] = "test-1-25-argo2"
 			})
 
 			By("verifying that RoleBinding in 'test_1_25_target' is updated to the second namespace")
-			roleBindingIntest_1_25_targetNS := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "argocd-argocd-application-controller", Namespace: test_1_25_targetNS.Name}}
+			roleBindingIntest_1_25_targetNS := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "argocd-argocd-application-controller", Namespace: test_1_25_target.Name}}
 			Eventually(roleBindingIntest_1_25_targetNS).Should(rolebindingFixture.HaveSubject(rbacv1.Subject{
 				Kind:      "ServiceAccount",
 				Name:      "argocd-argocd-application-controller",
@@ -142,7 +160,7 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 						TargetRevision: "HEAD",
 					},
 					Destination: argocdv1alpha1.ApplicationDestination{
-						Namespace: test_1_25_targetNS.Name,
+						Namespace: test_1_25_target.Name,
 						Server:    "https://kubernetes.default.svc",
 					},
 					Project: "default",
@@ -155,12 +173,12 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 			Expect(k8sClient.Create(ctx, app_argo2)).To(Succeed())
 
 			By("First Argo CD instance Application should be unhealthy, because it is no longer managing the namespace")
-			Eventually(app, "4m", "1s").Should(appFixture.HaveHealthStatusCode(health.HealthStatusMissing))
-			Eventually(app, "4m", "1s").Should(appFixture.HaveSyncStatusCode(argocdv1alpha1.SyncStatusCodeUnknown))
+			Eventually(app, "4m", "5s").Should(appFixture.HaveHealthStatusCode(health.HealthStatusMissing))
+			Eventually(app, "4m", "5s").Should(appFixture.HaveSyncStatusCode(argocdv1alpha1.SyncStatusCodeUnknown))
 
 			By("Second Argo CD instance Application should be healthy, because it is now managing the namespace")
-			Eventually(app_argo2, "60s", "1s").Should(appFixture.HaveHealthStatusCode(health.HealthStatusHealthy))
-			Eventually(app_argo2, "60s", "1s").Should(appFixture.HaveSyncStatusCode(argocdv1alpha1.SyncStatusCodeSynced))
+			Eventually(app_argo2, "4m", "5s").Should(appFixture.HaveHealthStatusCode(health.HealthStatusHealthy))
+			Eventually(app_argo2, "4m", "5s").Should(appFixture.HaveSyncStatusCode(argocdv1alpha1.SyncStatusCodeSynced))
 		})
 
 	})

test/openshift/e2e/ginkgo/parallel/1-034_validate_webhook_notifications_test.go

@@ -380,7 +380,7 @@ UVwpFuaKz5vTCD36Gmmy/u8y
 			Eventually(notifConfigMap).Should(k8sFixture.ExistByName())
 			Eventually(notifConfigMap).Should(configmap.HaveStringDataKeyValueContainsSubstring("template.test-app-created", `{"created":"{{.app.metadata.name}}","type":"{{(call .repo.GetAppDetails).Type}}"}`))
 
-			By("creating an Argo CD Application that contains a notificatio annotation, which will trigger the notifications controller")
+			By("creating an Argo CD Application that contains a notification annotation, which will trigger the notifications controller")
 			app := &argocdv1alpha1.Application{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      "my-app-3",
@@ -419,7 +419,7 @@ UVwpFuaKz5vTCD36Gmmy/u8y
 
 				return strings.Contains(out, `{"created":"my-app-3","type":"Directory"}`)
 
-			}).Should(BeTrue())
+			}, "4m", "5s").Should(BeTrue())
 
 		})
 

test/openshift/e2e/ginkgo/parallel/1-116_annotation-based-tracking_test.go

@@ -41,8 +41,16 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 	Context("1-116_annotation-based-tracking_test", func() {
 
 		var (
-			k8sClient client.Client
-			ctx       context.Context
+			k8sClient        client.Client
+			ctx              context.Context
+			nsTestDemo1      *corev1.Namespace
+			nsTestDemo2      *corev1.Namespace
+			nsAppNS1         *corev1.Namespace
+			nsAppNS2         *corev1.Namespace
+			cleanupTestDemo1 func()
+			cleanupTestDemo2 func()
+			cleanupAppNS1    func()
+			cleanupAppNS2    func()
 		)
 
 		BeforeEach(func() {
@@ -53,16 +61,32 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 
 		})
 
+		AfterEach(func() {
+
+			fixture.OutputDebugOnFail(nsTestDemo1, nsTestDemo2, nsAppNS1, nsAppNS2)
+
+			if cleanupAppNS2 != nil {
+				cleanupAppNS2()
+			}
+			if cleanupAppNS1 != nil {
+				cleanupAppNS1()
+			}
+			if cleanupTestDemo2 != nil {
+				cleanupTestDemo2()
+			}
+			if cleanupTestDemo1 != nil {
+				cleanupTestDemo1()
+			}
+		})
+
 		It("verifies that when annotation tracking is enabled that Argo CD instance starts and has the annotation tracking value specified in ConfigMap", func() {
 
 			By("creating Argo CD instances with annotation+label in two different namespaces")
-			nsTestDemo1, cleanupFunc := fixture.CreateNamespaceWithCleanupFunc("argocd-test-demo-1")
-			defer cleanupFunc()
+			nsTestDemo1, cleanupTestDemo1 = fixture.CreateNamespaceWithCleanupFunc("argocd-test-demo-1")
 			Eventually(nsTestDemo1).Should(namespaceFixture.HavePhase(corev1.NamespaceActive))
 
-			nsTestDemo2, cleanupFunc := fixture.CreateNamespaceWithCleanupFunc("argocd-test-demo-2")
+			nsTestDemo2, cleanupTestDemo2 = fixture.CreateNamespaceWithCleanupFunc("argocd-test-demo-2")
 			Eventually(nsTestDemo2).Should(namespaceFixture.HavePhase(corev1.NamespaceActive))
-			defer cleanupFunc()
 
 			argoCDTestDemo1 := &argov1beta1api.ArgoCD{
 				ObjectMeta: metav1.ObjectMeta{Name: "argocd-instance-demo-1", Namespace: nsTestDemo1.Name},
@@ -87,12 +111,10 @@ var _ = Describe("GitOps Operator Parallel E2E Tests", func() {
 
 			By("creating 2 more namespaces, each managed by one of the above Argo CD instances")
 
-			nsAppNS1, cleanupFunc := fixture.CreateManagedNamespaceWithCleanupFunc("app-ns-1", "argocd-test-demo-1")
-			defer cleanupFunc()
+			nsAppNS1, cleanupAppNS1 = fixture.CreateManagedNamespaceWithCleanupFunc("app-ns-1", "argocd-test-demo-1")
 			Eventually(nsAppNS1).Should(namespaceFixture.HavePhase(corev1.NamespaceActive))
 
-			nsAppNS2, cleanupFunc := fixture.CreateManagedNamespaceWithCleanupFunc("app-ns-2", "argocd-test-demo-2")
-			defer cleanupFunc()
+			nsAppNS2, cleanupAppNS2 = fixture.CreateManagedNamespaceWithCleanupFunc("app-ns-2", "argocd-test-demo-2")
 			Eventually(nsAppNS2).Should(namespaceFixture.HavePhase(corev1.NamespaceActive))
 
 			By("creating an Application in each Argo CD instance, targeting one of the namespaces and verifying the deploy succeeds")