Add ipsec pod restart test

This commit adds required tests to ensure pod traffic
across nodes are not impacted upon multiple reboot of
ipsec dameonset pods.

Signed-off-by: Periyasamy Palanisamy <[email protected]>

Author: pperiyasamy

pkg/testsuites/standard_suites.go

@@ -260,7 +260,7 @@ var staticSuites = []ginkgo.TestSuite{
 			return strings.Contains(name, "[Suite:openshift/network/ipsec")
 		},
 		Parallelism: 1,
-		TestTimeout: 60 * time.Minute,
+		TestTimeout: 20 * time.Minute,
 	},
 	{
 		Name: "openshift/network/stress",

test/extended/networking/egressip_helpers.go

@@ -1391,15 +1391,6 @@ func (p *PortAllocator) allocatePort(port int) error {
 	return nil
 }
 
-// deleteDaemonSet deletes the Daemonset <namespace>/<dsName>.
-func deleteDaemonSet(clientset kubernetes.Interface, namespace, dsName string) error {
-	deleteOptions := metav1.DeleteOptions{}
-	if err := clientset.AppsV1().DaemonSets(namespace).Delete(context.TODO(), dsName, deleteOptions); err != nil {
-		return fmt.Errorf("Failed to delete DaemonSet %s/%s: %v", namespace, dsName, err)
-	}
-	return nil
-}
-
 // createHostNetworkedDaemonSetAndProbe creates a host networked pod in namespace <namespace> on
 // node <nodeName>. It will allocate a port to listen on and it will return
 // the DaemonSet or an error.

test/extended/networking/internal_ports.go

@@ -166,14 +166,14 @@ var _ = ginkgo.Describe("[sig-network] Internal connectivity", func() {
 				}
 			}
 		}
-		errs := parallelTest(6, testFns)
+		errs := ParallelTest(6, testFns)
 		o.Expect(errs).To(o.Equal([]error(nil)))
 	})
 })
 
-// parallelTest runs the provided fns in parallel with at most workers and returns an array of all
+// ParallelTest runs the provided fns in parallel with at most workers and returns an array of all
 // non nil errors.
-func parallelTest(workers int, fns []func() error) []error {
+func ParallelTest(workers int, fns []func() error) []error {
 	var wg sync.WaitGroup
 	work := make(chan func() error, workers)
 	results := make(chan error, workers)

test/extended/networking/ipsec.go

@@ -13,14 +13,17 @@ import (
 	mg "github.com/openshift/origin/test/extended/machine_config"
 	exutil "github.com/openshift/origin/test/extended/util"
 	"golang.org/x/sync/errgroup"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
 	e2eoutput "k8s.io/kubernetes/test/e2e/framework/pod/output"
+	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
 	"k8s.io/kubernetes/test/e2e/framework/statefulset"
 	admissionapi "k8s.io/pod-security-admission/api"
 
@@ -93,6 +96,8 @@ var (
 	rightServerCertName = "right_server"
 	// Expiration date for certificates.
 	certExpirationDate = time.Date(2034, time.April, 10, 0, 0, 0, 0, time.UTC)
+	// http endpoint port for the pod traffic test
+	port uint16 = 8080
 )
 
 type trafficType string
@@ -652,6 +657,130 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() {
 	})
 })
 
+var _ = g.Describe("[sig-network][Feature:IPsec] IPsec resilience", g.Ordered, func() {
+	oc := exutil.NewCLIWithPodSecurityLevel("ipsec", admissionapi.LevelPrivileged)
+	f := oc.KubeFramework()
+	var ipsecMode v1.IPsecMode
+
+	InOVNKubernetesContext(func() {
+		g.BeforeAll(func() {
+			var err error
+			ipsecConfig, err := getIPsecConfig(oc)
+			o.Expect(err).NotTo(o.HaveOccurred())
+			ipsecMode = ipsecConfig.mode
+		})
+
+		g.It("check pod traffic is working across nodes [apigroup:config.openshift.io] [Suite:openshift/network/ipsec]", func() {
+			g.By("creating test pods")
+			pods := createWebServerPods(oc, f.Namespace.Name)
+			g.By("checking crossing connectivity over the pods")
+			checkPodCrossConnectivity(pods)
+		})
+
+		g.It("check pod traffic is working across nodes after ipsec daemonset restart [apigroup:config.openshift.io] [Suite:openshift/network/ipsec]", func() {
+			// The IPsec daemonset manages IPsec connections between nodes for pod's east-west traffic.
+			// The IPsec daemonset exists only in IPsec full mode, so skip this test for other IPsec modes.
+			if ipsecMode != v1.IPsecModeFull {
+				e2eskipper.Skipf("cluster is configured with IPsec %s mode, so skipping the test", ipsecMode)
+			}
+			g.By("creating test pods")
+			pods := createWebServerPods(oc, f.Namespace.Name)
+			g.By("checking crossing connectivity over the pods")
+			checkPodCrossConnectivity(pods)
+			// Restart IPsec daemonset few times and check pod traffic is not impacted.
+			for i := 1; i <= 5; i++ {
+				g.By(fmt.Sprintf("attempt#%d restarting IPsec pods", i))
+				restartIPsecDaemonSet(oc)
+				g.By("checking crossing connectivity over the pods")
+				checkPodCrossConnectivity(pods)
+			}
+		})
+	})
+})
+
+func restartIPsecDaemonSet(oc *exutil.CLI) {
+	g.GinkgoHelper()
+	ds, err := getDaemonSet(oc, ovnNamespace, ovnIPsecDsName)
+	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(ds).NotTo(o.BeNil())
+	err = deleteDaemonSet(oc.AdminKubeClient(), ovnNamespace, ovnIPsecDsName)
+	o.Expect(err).NotTo(o.HaveOccurred())
+	// wait until CNO reconciles IPsec daemonset.
+	err = ensureIPsecFullEnabled(oc)
+	o.Expect(err).NotTo(o.HaveOccurred())
+}
+
+func createWebServerPods(oc *exutil.CLI, namespace string) []corev1.Pod {
+	g.GinkgoHelper()
+	immediate := int64(0)
+	ds := &appsv1.DaemonSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "ipsec-webserver",
+			Namespace: namespace,
+		},
+		Spec: appsv1.DaemonSetSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"apps": "ipsec-webserver",
+				},
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"apps": "ipsec-webserver",
+					},
+				},
+				Spec: corev1.PodSpec{
+					Tolerations: []corev1.Toleration{
+						{
+							Key:      "node-role.kubernetes.io/master",
+							Operator: corev1.TolerationOpExists,
+							Effect:   corev1.TaintEffectNoSchedule,
+						},
+					},
+					TerminationGracePeriodSeconds: &immediate,
+					Containers:                    []corev1.Container{e2epod.NewAgnhostContainer("agnhost-container", nil, nil, httpServerContainerCmd(port)...)},
+				},
+			},
+		},
+	}
+	ds, err := oc.AdminKubeClient().AppsV1().DaemonSets(namespace).Create(context.Background(), ds, metav1.CreateOptions{})
+	o.Expect(err).NotTo(o.HaveOccurred())
+	err = wait.PollUntilContextTimeout(context.Background(), 1*time.Second,
+		180*time.Second, true, func(ctx context.Context) (bool, error) {
+			return isDaemonSetRunning(oc, namespace, ds.Name)
+		})
+	o.Expect(err).NotTo(o.HaveOccurred())
+
+	pods, err := oc.AdminKubeClient().CoreV1().Pods(namespace).List(context.Background(),
+		metav1.ListOptions{LabelSelector: labels.Set(ds.Spec.Selector.MatchLabels).String()})
+	o.Expect(err).NotTo(o.HaveOccurred())
+	ds, err = getDaemonSet(oc, namespace, ds.Name)
+	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(len(pods.Items)).To(o.Equal(int(ds.Status.NumberAvailable)), fmt.Sprintf("%#v", pods.Items))
+	return pods.Items
+}
+
+func checkPodCrossConnectivity(pods []corev1.Pod) {
+	g.GinkgoHelper()
+	var testFns []func() error
+	for _, sourcePod := range pods {
+		for _, targetPod := range pods {
+			if sourcePod.Name == targetPod.Name {
+				// Skip if source and target pod are same, pod connectivity check is not required
+				// for this case.
+				continue
+			}
+			testFns = append(testFns, func() error {
+				framework.Logf("Checking pod connectivity from node %s to node %s", sourcePod.Spec.NodeName, targetPod.Spec.NodeName)
+				return connectToServer(podConfiguration{namespace: sourcePod.Namespace, name: sourcePod.Name}, targetPod.Status.PodIP, int(port))
+			})
+		}
+	}
+	errs := ParallelTest(6, testFns)
+	o.Expect(errs).To(o.Equal([]error(nil)))
+}
+
 func waitForIPsecConfigToComplete(oc *exutil.CLI, ipsecMode v1.IPsecMode) {
 	g.GinkgoHelper()
 	switch ipsecMode {

test/extended/networking/util.go

@@ -34,6 +34,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/storage/names"
 	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/kubernetes"
 	k8sclient "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/util/retry"
@@ -758,7 +759,7 @@ func isDaemonSetRunning(oc *exutil.CLI, namespace, name string) (bool, error) {
 	}
 	// Be sure that it has ds pod running in each node.
 	desired, scheduled, ready := ds.Status.DesiredNumberScheduled, ds.Status.CurrentNumberScheduled, ds.Status.NumberReady
-	return desired == scheduled && desired == ready, nil
+	return ds.Status.ObservedGeneration == ds.Generation && desired == scheduled && desired == ready, nil
 }
 
 func getDaemonSet(oc *exutil.CLI, namespace, name string) (*appsv1.DaemonSet, error) {
@@ -769,6 +770,15 @@ func getDaemonSet(oc *exutil.CLI, namespace, name string) (*appsv1.DaemonSet, er
 	return ds, err
 }
 
+// deleteDaemonSet deletes the Daemonset <namespace>/<dsName>.
+func deleteDaemonSet(clientset kubernetes.Interface, namespace, dsName string) error {
+	deleteOptions := metav1.DeleteOptions{}
+	if err := clientset.AppsV1().DaemonSets(namespace).Delete(context.TODO(), dsName, deleteOptions); err != nil {
+		return fmt.Errorf("failed to delete DaemonSet %s/%s: %v", namespace, dsName, err)
+	}
+	return nil
+}
+
 func createIPsecCertsMachineConfig(oc *exutil.CLI) (*mcfgv1.MachineConfig, error) {
 	nsCertMachineConfig, err := oc.MachineConfigurationClient().MachineconfigurationV1().MachineConfigs().Get(context.Background(),
 		nsCertMachineConfigName, metav1.GetOptions{})