test e2e

Signed-off-by: zhujian <[email protected]>

Author: zhujian7

test/e2e/addonmanagement_test.go

@@ -74,7 +74,7 @@ var (
 	}
 )
 
-var _ = ginkgo.Describe("Enable addon management feature gate", ginkgo.Ordered, ginkgo.Label("addon-manager"), func() {
+var _ = ginkgo.FDescribe("Enable addon management feature gate", ginkgo.Ordered, ginkgo.Label("addon-manager"), func() {
 	addOnName := "hello-template"
 	addonInstallNamespace := "test-addon-template"
 
@@ -832,7 +832,7 @@ var _ = ginkgo.Describe("Enable addon management feature gate", ginkgo.Ordered,
 		}).ShouldNot(gomega.HaveOccurred())
 	})
 
-	ginkgo.It("ClusterManagementAddon deletion should wait for ManagedClusterAddons cleanup", func() {
+	ginkgo.FIt("ClusterManagementAddon deletion should wait for ManagedClusterAddons cleanup", func() {
 		ginkgo.By("Make sure addon is functioning before deletion")
 		configmap := &corev1.ConfigMap{
 			ObjectMeta: metav1.ObjectMeta{
@@ -867,18 +867,143 @@ var _ = ginkgo.Describe("Enable addon management feature gate", ginkgo.Ordered,
 			context.TODO(), addOnName, metav1.DeleteOptions{})
 		gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
+		ginkgo.By("Check that the ManagedClusterAddon is in deletion status")
+		gomega.Eventually(func() error {
+			addon, err := hub.AddonClient.AddonV1alpha1().ManagedClusterAddOns(universalClusterName).Get(
+				context.TODO(), addOnName, metav1.GetOptions{})
+			if err != nil {
+				if errors.IsNotFound(err) {
+					klog.Infof("ManagedClusterAddon %s is already deleted", addOnName)
+					return nil
+				}
+				klog.Errorf("Error getting ManagedClusterAddon: %v", err)
+				return err
+			}
+			if addon.DeletionTimestamp == nil {
+				klog.Infof("ManagedClusterAddon %s deletion timestamp not set yet", addOnName)
+				return fmt.Errorf("addon should have deletion timestamp set")
+			}
+			klog.Infof("ManagedClusterAddon %s has deletion timestamp set: %v", addOnName, addon.DeletionTimestamp)
+			klog.Infof("ManagedClusterAddon %s finalizers: %v", addOnName, addon.Finalizers)
+			klog.Infof("ManagedClusterAddon %s status conditions: %v", addOnName, addon.Status.Conditions)
+			return nil
+		}).ShouldNot(gomega.HaveOccurred())
+
 		ginkgo.By("The pre-delete job should clean up the configmap")
 		gomega.Eventually(func() error {
-			_, err := spoke.KubeClient.CoreV1().ConfigMaps(addonInstallNamespace).Get(
+			// Debug: Check all jobs in addon namespace first
+			allJobs, allJobsErr := spoke.KubeClient.BatchV1().Jobs(addonInstallNamespace).List(
+				context.Background(), metav1.ListOptions{})
+			if allJobsErr != nil {
+				klog.Errorf("Error listing all jobs in namespace %s: %v", addonInstallNamespace, allJobsErr)
+			} else {
+				klog.Infof("Found %d total jobs in namespace %s", len(allJobs.Items), addonInstallNamespace)
+				for _, j := range allJobs.Items {
+					klog.Infof("Job %s: Active=%d, Succeeded=%d, Failed=%d", j.Name, j.Status.Active, j.Status.Succeeded, j.Status.Failed)
+				}
+			}
+
+			// Debug: Check pre-delete job status
+			job, jobErr := spoke.KubeClient.BatchV1().Jobs(addonInstallNamespace).Get(
+				context.Background(), "hello-template-cleanup-configmap", metav1.GetOptions{})
+			if jobErr != nil {
+				if errors.IsNotFound(jobErr) {
+					klog.Infof("Pre-delete job not found yet")
+				} else {
+					klog.Errorf("Error getting pre-delete job: %v", jobErr)
+				}
+			} else {
+				klog.Infof("Pre-delete job status - Active: %d, Succeeded: %d, Failed: %d, Conditions: %v",
+					job.Status.Active, job.Status.Succeeded, job.Status.Failed, job.Status.Conditions)
+			}
+
+			// Debug: Check addon-manager logs to see if there are any issues
+			addonManagerPods, addonManagerErr := hub.KubeClient.CoreV1().Pods("open-cluster-management-hub").List(
+				context.Background(), metav1.ListOptions{
+					LabelSelector: "app=clustermanager-addon-manager-controller",
+				})
+			if addonManagerErr != nil {
+				klog.Errorf("Error listing addon-manager pods: %v", addonManagerErr)
+			} else {
+				klog.Infof("Found %d addon-manager pods", len(addonManagerPods.Items))
+				for _, pod := range addonManagerPods.Items {
+					klog.Infof("Addon-manager pod %s status: Phase=%s", pod.Name, pod.Status.Phase)
+
+					// Get recent addon-manager logs (last 50 lines)
+					logs, logErr := hub.KubeClient.CoreV1().Pods("open-cluster-management-hub").GetLogs(
+						pod.Name, &corev1.PodLogOptions{
+							TailLines: func(i int64) *int64 { return &i }(50),
+						}).DoRaw(context.Background())
+					if logErr != nil {
+						klog.Errorf("Error getting addon-manager logs for pod %s: %v", pod.Name, logErr)
+					} else {
+						klog.Infof("Recent addon-manager logs from pod %s: %s", pod.Name, string(logs))
+					}
+				}
+			}
+
+			// Debug: Check ManifestWorks for this addon
+			manifestWorks, mwErr := hub.WorkClient.WorkV1().ManifestWorks(universalClusterName).List(
+				context.Background(), metav1.ListOptions{
+					LabelSelector: fmt.Sprintf("addon.open-cluster-management.io/addon-name=%s", addOnName),
+				})
+			if mwErr != nil {
+				klog.Errorf("Error listing ManifestWorks: %v", mwErr)
+			} else {
+				klog.Infof("Found %d ManifestWorks for addon %s", len(manifestWorks.Items), addOnName)
+				for _, mw := range manifestWorks.Items {
+					klog.Infof("ManifestWork %s: DeletionTimestamp=%v, Status=%+v",
+						mw.Name, mw.DeletionTimestamp, mw.Status)
+					if len(mw.Status.ResourceStatus.Manifests) > 0 {
+						for _, manifest := range mw.Status.ResourceStatus.Manifests {
+							klog.Infof("  Resource %s/%s: Conditions=%v",
+								manifest.ResourceMeta.Kind, manifest.ResourceMeta.Name, manifest.Conditions)
+						}
+					}
+				}
+			}
+
+			if jobErr == nil {
+
+				// Debug: Get pods associated with this job to check their status
+				pods, podErr := spoke.KubeClient.CoreV1().Pods(addonInstallNamespace).List(
+					context.Background(), metav1.ListOptions{
+						LabelSelector: "job=hello-template-cleanup-configmap",
+					})
+				if podErr != nil {
+					klog.Errorf("Error listing job pods: %v", podErr)
+				} else {
+					for _, pod := range pods.Items {
+						klog.Infof("Job pod %s status: Phase=%s, Ready=%v, Conditions=%v",
+							pod.Name, pod.Status.Phase, pod.Status.ContainerStatuses, pod.Status.Conditions)
+
+						// If pod has failed, try to get logs
+						if pod.Status.Phase == "Failed" || pod.Status.Phase == "Succeeded" {
+							logs, logErr := spoke.KubeClient.CoreV1().Pods(addonInstallNamespace).GetLogs(
+								pod.Name, &corev1.PodLogOptions{}).DoRaw(context.Background())
+							if logErr != nil {
+								klog.Errorf("Error getting logs for pod %s: %v", pod.Name, logErr)
+							} else {
+								klog.Infof("Logs for job pod %s: %s", pod.Name, string(logs))
+							}
+						}
+					}
+				}
+			}
+
+			cms, err := spoke.KubeClient.CoreV1().ConfigMaps(addonInstallNamespace).Get(
 				context.Background(), configmap.Name, metav1.GetOptions{})
 			if err != nil {
 				if errors.IsNotFound(err) {
+					klog.Infof("Configmap %s successfully deleted", configmap.Name)
 					return nil
 				}
+				klog.Errorf("Error getting configmap: %v", err)
 				return err
 			}
-			return fmt.Errorf("the configmap should be deleted")
-		}, time.Second*180, time.Second*5).ShouldNot(gomega.HaveOccurred())
+			klog.Infof("Configmap %s still exists, waiting for deletion", configmap.Name)
+			return fmt.Errorf("the configmap should be deleted, %+v", cms)
+		}, time.Second*300, time.Second*3).ShouldNot(gomega.HaveOccurred())
 
 		ginkgo.By("ManagedClusterAddon should eventually be deleted after pre-delete job completes")
 		gomega.Eventually(func() error {