Adds forbidParallelRuns configuration option (#31)

Author: bastjan

api/v1beta1/schedulercanary_types.go

@@ -42,6 +42,10 @@ type SchedulerCanarySpec struct {
 	// The default is 1 minute.
 	Interval metav1.Duration `json:"interval,omitempty"`
 
+	// ForbidParallelRuns will prevent the creation of a new canary pod if there is already a canary pod running.
+	// The default is false.
+	ForbidParallelRuns bool `json:"forbidParallelRuns,omitempty"`
+
 	// PodTemplate is the pod template to use for the canary pods.
 	PodTemplate corev1.PodTemplateSpec `json:"podTemplate,omitempty"`
 }

config/crd/bases/monitoring.appuio.io_schedulercanaries.yaml

@@ -43,6 +43,11 @@ spec:
           spec:
             description: SchedulerCanarySpec defines the desired state of SchedulerCanary
             properties:
+              forbidParallelRuns:
+                description: |-
+                  ForbidParallelRuns will prevent the creation of a new canary pod if there is already a canary pod running.
+                  The default is false.
+                type: boolean
               interval:
                 description: |-
                   Interval is the interval at which a canary pod will be created.

controllers/schedulercanary_controller.go

@@ -18,8 +18,10 @@ package controllers
 
 import (
 	"context"
+	"fmt"
 	"time"
 
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -68,6 +70,22 @@ func (r *SchedulerCanaryReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		return ctrl.Result{Requeue: true, RequeueAfter: rqi}, nil
 	}
 
+	if instance.Spec.ForbidParallelRuns {
+		// Check if there is already a canary pod running.
+		pods := &corev1.PodList{}
+		if err := r.Client.List(ctx, pods, client.InNamespace(instance.Namespace), client.MatchingLabels{instanceLabel: instance.Name}); err != nil {
+			return ctrl.Result{}, fmt.Errorf("forbidParallelRuns: error checking for already running pods, failed to list pods: %w", err)
+		}
+		if len(pods.Items) > 0 {
+			podNames := make([]string, len(pods.Items))
+			for i, pod := range pods.Items {
+				podNames[i] = pod.Name
+			}
+			l.Info("ForbidParallelRuns: already running pods found, skipping pod creation", "pods", podNames)
+			return ctrl.Result{}, nil
+		}
+	}
+
 	if err := r.createCanaryPod(ctx, instance); err != nil {
 		return ctrl.Result{}, err
 	}
@@ -85,6 +103,7 @@ func (r *SchedulerCanaryReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 func (r *SchedulerCanaryReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&monitoringv1beta1.SchedulerCanary{}).
+		Owns(&corev1.Pod{}).
 		Complete(r)
 }
 

controllers/schedulercanary_controller_test.go

@@ -70,6 +70,71 @@ var _ = Describe("SchedulerCanary controller", func() {
 
 	})
 
+	Context("ForbidParallelRuns", func() {
+		BeforeEach(func() {
+			ctx := context.Background()
+
+			schedulerCanary := &monitoringv1beta1.SchedulerCanary{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "my-canary",
+					Namespace: "default",
+				},
+				Spec: monitoringv1beta1.SchedulerCanarySpec{
+					Interval:                metav1.Duration{Duration: time.Millisecond},
+					MaxPodCompletionTimeout: metav1.Duration{Duration: time.Minute},
+					ForbidParallelRuns:      true,
+					PodTemplate: corev1.PodTemplateSpec{
+						Spec: corev1.PodSpec{
+							Containers: []corev1.Container{
+								{
+									Name:  "scheduler-canary",
+									Image: "busybox",
+								},
+							},
+						},
+					},
+				},
+			}
+			Expect(k8sClient.Create(ctx, schedulerCanary)).Should(Succeed())
+		})
+
+		It("ForbidParallelRuns: should not create more than one pod", func() {
+			ctx := context.Background()
+
+			Eventually(func() (int, error) {
+				pods := &corev1.PodList{}
+
+				err := k8sClient.List(ctx, pods, client.InNamespace("default"), client.MatchingLabels(map[string]string{
+					instanceLabel: "my-canary",
+				}))
+
+				return len(pods.Items), err
+			}, "10s", "250ms").Should(BeNumerically(">=", 1))
+
+			Consistently(func() (int, error) {
+				pods := &corev1.PodList{}
+
+				err := k8sClient.List(ctx, pods, client.InNamespace("default"), client.MatchingLabels(map[string]string{
+					instanceLabel: "my-canary",
+				}))
+
+				return len(pods.Items), err
+			}, "5s", "250ms").Should(Equal(1))
+
+			Expect(k8sClient.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace("default"))).Should(Succeed())
+
+			Eventually(func() (int, error) {
+				pods := &corev1.PodList{}
+
+				err := k8sClient.List(ctx, pods, client.InNamespace("default"), client.MatchingLabels(map[string]string{
+					instanceLabel: "my-canary",
+				}))
+
+				return len(pods.Items), err
+			}, "10s", "250ms").Should(BeNumerically(">=", 1))
+		})
+	})
+
 	AfterEach(func() {
 		ctx := context.Background()