Feature to delete cluster resources when TTL expires (#360)

* Feature to delete cluster resources when TTL expires

* Fix k8s version 1.25 cluster creation

Author: prateekgogia

operator/charts/kit-operator/crds/control-plane-crd.yaml

@@ -9634,6 +9634,8 @@ spec:
                           type: object
                       type: object
                   type: object
+                ttl:
+                  type: string
               type: object
             status:
               properties:

operator/pkg/apis/controlplane/v1alpha1/controlplane.go

@@ -46,10 +46,12 @@ type ControlPlaneList struct {
 // master and etcd are configured to run. By default, KIT uses all the default
 // values and ControlPlaneSpec can be empty.
 type ControlPlaneSpec struct {
-	KubernetesVersion         string     `json:"kubernetesVersion,omitempty"`
-	ColocateAPIServerWithEtcd bool       `json:"colocateAPIServerWithEtcd,omitempty"`
-	Master                    MasterSpec `json:"master,omitempty"`
-	Etcd                      Etcd       `json:"etcd,omitempty"`
+	KubernetesVersion         string `json:"kubernetesVersion,omitempty"`
+	ColocateAPIServerWithEtcd bool   `json:"colocateAPIServerWithEtcd,omitempty"`
+	// TTL is the duration for which control plane resources are active, once expired resource will be automatically deleted by the operator
+	TTL    string     `json:"ttl,omitempty"`
+	Master MasterSpec `json:"master,omitempty"`
+	Etcd   Etcd       `json:"etcd,omitempty"`
 }
 
 type Etcd struct {

operator/pkg/controllers/controller.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"time"
 
 	"github.com/awslabs/kubernetes-iteration-toolkit/operator/pkg/apis/controlplane/v1alpha1"
 	"github.com/awslabs/kubernetes-iteration-toolkit/operator/pkg/errors"
@@ -81,7 +82,26 @@ func (c *GenericController) reconcile(ctx context.Context, resource Object, pers
 	existingFinalizers := resource.GetFinalizers()
 	existingFinalizerSet := sets.NewString(existingFinalizers...)
 	finalizerStr := sets.NewString(fmt.Sprintf(FinalizerForAWSResources, c.Name()))
-	if resource.GetDeletionTimestamp() == nil {
+	if resource.GetDeletionTimestamp() == nil && ttlExpiredForControlPlane(resource) {
+		// when ttl expires for the cluster, finalize all sub resources
+		if result, err = c.finalizeSubResources(ctx, resource, existingFinalizerSet.Difference(finalizerStr).UnsortedList()); err != nil {
+			return *results.Failed, fmt.Errorf("finalizing resource controller %w", err)
+		}
+		if result.Requeue || result.RequeueAfter > 0 {
+			return *result, nil
+		}
+		// delete the CP object since it doesn't get auto deleted.
+		if err := c.Delete(ctx, resource); err != nil {
+			return *results.Failed, fmt.Errorf("deleting CP object, %w", err)
+		}
+		// once the object is deleted we don't need to merge patch anymore
+		return reconcile.Result{}, nil
+	}
+	if resource.GetDeletionTimestamp() != nil {
+		if result, err = c.finalizeSubResources(ctx, resource, existingFinalizerSet.Difference(finalizerStr).UnsortedList()); err != nil {
+			return *results.Failed, fmt.Errorf("finalizing resource controller %v, %w", c.Controller.Name(), err)
+		}
+	} else {
 		// Add finalizer for this controller
 		resource.SetFinalizers(existingFinalizerSet.Union(finalizerStr).UnsortedList())
 		result, err = c.Controller.Reconcile(ctx, resource)
@@ -90,13 +110,6 @@ func (c *GenericController) reconcile(ctx context.Context, resource Object, pers
 			return *results.Failed, fmt.Errorf("reconciling resource, %w", err)
 		}
 		resource.StatusConditions().MarkTrue(v1alpha1.Active)
-	} else {
-		if result, err = c.Controller.Finalize(ctx, resource); err != nil {
-			return *results.Failed, fmt.Errorf("finalizing resource controller %v, %w", c.Controller.Name(), err)
-		}
-		// Remove finalizer for this controller
-		resource.SetFinalizers(existingFinalizerSet.Difference(finalizerStr).UnsortedList())
-		zap.S().Infof("[%s] Successfully deleted", resource.GetName())
 	}
 	// If the finalizers have changed merge patch the object
 	if !reflect.DeepEqual(existingFinalizers, resource.GetFinalizers()) {
@@ -106,3 +119,34 @@ func (c *GenericController) reconcile(ctx context.Context, resource Object, pers
 	}
 	return *result, nil
 }
+
+func (c *GenericController) finalizeSubResources(ctx context.Context, resource Object, finalizers []string) (*reconcile.Result, error) {
+	result, err := c.Controller.Finalize(ctx, resource)
+	if err != nil {
+		return result, fmt.Errorf("finalizing resource controller %v, %w", c.Controller.Name(), err)
+	}
+	// Remove finalizer for this controller
+	resource.SetFinalizers(finalizers)
+	zap.S().Infof("[%s] Successfully deleted resources", resource.GetName())
+	return result, nil
+}
+
+func ttlExpiredForControlPlane(resource Object) bool {
+	cp, ok := resource.(*v1alpha1.ControlPlane)
+	if !ok {
+		return false
+	}
+	if cp.Spec.TTL != "" {
+		duration, err := time.ParseDuration(cp.Spec.TTL)
+		if err != nil {
+			zap.S().Errorf("parsing TTL duration, %w", err)
+			return false
+		}
+		deleteAfter := resource.GetCreationTimestamp().Add(duration)
+		if time.Now().After(deleteAfter) {
+			zap.S().Infof("[%v] control plane TTL expired, deleting cluster resources", cp.ClusterName())
+			return true
+		}
+	}
+	return false
+}

operator/pkg/controllers/master/kubeapiserver.go

@@ -17,6 +17,7 @@ package master
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/awslabs/kubernetes-iteration-toolkit/operator/pkg/apis/controlplane/v1alpha1"
@@ -79,7 +80,7 @@ func APIServerLabels(clustername string) map[string]string {
 func apiServerPodSpecFor(controlPlane *v1alpha1.ControlPlane) v1.PodSpec {
 	hostPathDirectoryOrCreate := v1.HostPathDirectoryOrCreate
 	hostPathDirectory := v1.HostPathDirectory
-	return v1.PodSpec{
+	return apiServerPodSpecForVersion(controlPlane.Spec.KubernetesVersion, &v1.PodSpec{
 		TerminationGracePeriodSeconds: aws.Int64(1),
 		HostNetwork:                   true,
 		DNSPolicy:                     v1.DNSClusterFirstWithHostNet,
@@ -425,7 +426,7 @@ func apiServerPodSpecFor(controlPlane *v1alpha1.ControlPlane) v1.PodSpec {
 				},
 			},
 		}},
-	}
+	})
 }
 
 func affinity(colocateAPIServerWithEtcd bool) *v1.Affinity {
@@ -449,3 +450,22 @@ func nodeSelector(clusterName string, colocateWithEtcd bool) map[string]string {
 	}
 	return selector
 }
+
+var (
+	disabledFlagsForAPIServer = map[string]struct{}{"--feature-gates": {}}
+)
+
+func apiServerPodSpecForVersion(version string, defaultSpec *v1.PodSpec) v1.PodSpec {
+	switch version {
+	case "1.25":
+		args := []string{}
+		for _, arg := range defaultSpec.Containers[0].Args {
+			if _, skip := disabledFlagsForAPIServer[strings.Split(arg, "=")[0]]; skip {
+				continue
+			}
+			args = append(args, arg)
+		}
+		defaultSpec.Containers[0].Args = args
+	}
+	return *defaultSpec
+}