fix: resolve secrets when generating eks userdata (backport #4285 to 2.1) (#4289)

* fix: resolve secrets when generating eks userdata

* fix: log line and resolve files error

Author: faiq

bootstrap/eks/controllers/eksconfig_controller.go

@@ -26,6 +26,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/pointer"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -145,6 +146,41 @@ func (r *EKSConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	return r.joinWorker(ctx, cluster, config, configOwner)
 }
 
+func (r *EKSConfigReconciler) resolveFiles(ctx context.Context, cfg *eksbootstrapv1.EKSConfig) ([]eksbootstrapv1.File, error) {
+	collected := make([]eksbootstrapv1.File, 0, len(cfg.Spec.Files))
+
+	for i := range cfg.Spec.Files {
+		in := cfg.Spec.Files[i]
+		if in.ContentFrom != nil {
+			data, err := r.resolveSecretFileContent(ctx, cfg.Namespace, in)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to resolve file source")
+			}
+			in.ContentFrom = nil
+			in.Content = string(data)
+		}
+		collected = append(collected, in)
+	}
+
+	return collected, nil
+}
+
+func (r *EKSConfigReconciler) resolveSecretFileContent(ctx context.Context, ns string, source eksbootstrapv1.File) ([]byte, error) {
+	secret := &corev1.Secret{}
+	key := types.NamespacedName{Namespace: ns, Name: source.ContentFrom.Secret.Name}
+	if err := r.Client.Get(ctx, key, secret); err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil, errors.Wrapf(err, "secret not found: %s", key)
+		}
+		return nil, errors.Wrapf(err, "failed to retrieve Secret %q", key)
+	}
+	data, ok := secret.Data[source.ContentFrom.Secret.Key]
+	if !ok {
+		return nil, errors.Errorf("secret references non-existent secret key: %q", source.ContentFrom.Secret.Key)
+	}
+	return data, nil
+}
+
 func (r *EKSConfigReconciler) joinWorker(ctx context.Context, cluster *clusterv1.Cluster, config *eksbootstrapv1.EKSConfig, configOwner *bsutil.ConfigOwner) (ctrl.Result, error) {
 	log := logger.FromContext(ctx)
 
@@ -191,6 +227,12 @@ func (r *EKSConfigReconciler) joinWorker(ctx context.Context, cluster *clusterv1
 	}
 
 	log.Info("Generating userdata")
+	files, err := r.resolveFiles(ctx, config)
+	if err != nil {
+		log.Info("Failed to resolve files for user data")
+		conditions.MarkFalse(config, eksbootstrapv1.DataSecretAvailableCondition, eksbootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
+		return ctrl.Result{}, err
+	}
 
 	nodeInput := &userdata.NodeInput{
 		// AWSManagedControlPlane webhooks default and validate EKSClusterName
@@ -208,7 +250,7 @@ func (r *EKSConfigReconciler) joinWorker(ctx context.Context, cluster *clusterv1
 		Users:                    config.Spec.Users,
 		DiskSetup:                config.Spec.DiskSetup,
 		Mounts:                   config.Spec.Mounts,
-		Files:                    config.Spec.Files,
+		Files:                    files,
 	}
 	if config.Spec.PauseContainer != nil {
 		nodeInput.PauseContainerAccount = &config.Spec.PauseContainer.AccountNumber

bootstrap/eks/controllers/eksconfig_controller_reconciler_test.go

@@ -203,6 +203,78 @@ func TestEKSConfigReconciler(t *testing.T) {
 			gomega.Expect(string(secret.Data["value"])).To(Not(Equal(string(expectedUserData))))
 		}).Should(Succeed())
 	})
+	t.Run("Should Reconcile an EKSConfig with a secret file reference", func(t *testing.T) {
+		g := NewWithT(t)
+		amcp := newAMCP("test-cluster")
+		//nolint: gosec // these are not credentials
+		secretPath := "/etc/secret.txt"
+		secretContent := "secretValue"
+		cluster := newCluster(amcp.Name)
+		machine := newMachine(cluster, "test-machine")
+		config := newEKSConfig(machine)
+		config.Spec.Files = append(config.Spec.Files, eksbootstrapv1.File{
+			ContentFrom: &eksbootstrapv1.FileSource{
+				Secret: eksbootstrapv1.SecretFileSource{
+					Name: "my-secret",
+					Key:  "secretKey",
+				},
+			},
+			Path: secretPath,
+		})
+		secret := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: "default",
+				Name:      "my-secret",
+			},
+			Data: map[string][]byte{
+				"secretKey": []byte(secretContent),
+			},
+		}
+		t.Logf(dump("amcp", amcp))
+		t.Logf(dump("config", config))
+		t.Logf(dump("machine", machine))
+		t.Logf(dump("cluster", cluster))
+		t.Logf(dump("secret", secret))
+		g.Expect(testEnv.Client.Create(ctx, secret)).To(Succeed())
+		g.Expect(testEnv.Client.Create(ctx, amcp)).To(Succeed())
+
+		// create a userData with the secret content and check if reconile.joinWorker
+		// resolves the userdata properly
+		expectedUserData, err := userdata.NewNode(&userdata.NodeInput{
+			ClusterName: amcp.Name,
+			Files: []eksbootstrapv1.File{
+				{
+					Content: secretContent,
+					Path:    secretPath,
+				},
+			},
+			KubeletExtraArgs: map[string]string{
+				"test-arg": "test-value",
+			},
+		})
+		g.Expect(err).To(BeNil())
+		reconciler := EKSConfigReconciler{
+			Client: testEnv.Client,
+		}
+		t.Logf(fmt.Sprintf("Calling reconcile on cluster '%s' and config '%s' should requeue", cluster.Name, config.Name))
+		g.Eventually(func(gomega Gomega) {
+			result, err := reconciler.joinWorker(ctx, cluster, config, configOwner("Machine"))
+			gomega.Expect(err).NotTo(HaveOccurred())
+			gomega.Expect(result.Requeue).To(BeFalse())
+		}).Should(Succeed())
+
+		secretList := &corev1.SecretList{}
+		testEnv.Client.List(ctx, secretList)
+		t.Logf(dump("secrets", secretList))
+		gotSecret := &corev1.Secret{}
+		g.Eventually(func(gomega Gomega) {
+			gomega.Expect(testEnv.Client.Get(ctx, client.ObjectKey{
+				Name:      config.Name,
+				Namespace: "default",
+			}, gotSecret)).To(Succeed())
+		}).Should(Succeed())
+		g.Expect(string(gotSecret.Data["value"])).To(Equal(string(expectedUserData)))
+	})
 }
 
 // newCluster return a CAPI cluster object.