refactor: switch from kubernetes-external-secrets to External Secrets Operator (#276)

Author: mglotov

terraform/layer2-k8s/README.md

@@ -31,7 +31,6 @@
 | <a name="module_aws_iam_cert_manager"></a> [aws\_iam\_cert\_manager](#module\_aws\_iam\_cert\_manager) | ../modules/aws-iam-eks-trusted | n/a |
 | <a name="module_aws_iam_elastic_stack"></a> [aws\_iam\_elastic\_stack](#module\_aws\_iam\_elastic\_stack) | ../modules/aws-iam-user-with-policy | n/a |
 | <a name="module_aws_iam_external_dns"></a> [aws\_iam\_external\_dns](#module\_aws\_iam\_external\_dns) | ../modules/aws-iam-eks-trusted | n/a |
-| <a name="module_aws_iam_external_secrets"></a> [aws\_iam\_external\_secrets](#module\_aws\_iam\_external\_secrets) | ../modules/aws-iam-eks-trusted | n/a |
 | <a name="module_aws_iam_gitlab_runner"></a> [aws\_iam\_gitlab\_runner](#module\_aws\_iam\_gitlab\_runner) | ../modules/aws-iam-eks-trusted | n/a |
 | <a name="module_aws_iam_kube_prometheus_stack_grafana"></a> [aws\_iam\_kube\_prometheus\_stack\_grafana](#module\_aws\_iam\_kube\_prometheus\_stack\_grafana) | ../modules/aws-iam-eks-trusted | n/a |
 | <a name="module_aws_iam_victoria_metrics_k8s_stack_grafana"></a> [aws\_iam\_victoria\_metrics\_k8s\_stack\_grafana](#module\_aws\_iam\_victoria\_metrics\_k8s\_stack\_grafana) | ../modules/aws-iam-eks-trusted | n/a |

terraform/layer2-k8s/eks-external-secrets.tf

@@ -8,23 +8,62 @@ locals {
     namespace     = local.helm_releases[index(local.helm_releases.*.id, "external-secrets")].namespace
   }
   external_secrets_values = <<VALUES
-# Environment variables to set on deployment pod
-env:
-  AWS_REGION: ${local.region}
-  AWS_DEFAULT_REGION: ${local.region}
-  POLLER_INTERVAL_MILLISECONDS: 30000
-  # trace, debug, info, warn, error, fatal
-  LOG_LEVEL: warn
-  LOG_MESSAGE_KEY: 'msg'
-  METRICS_PORT: 3001
+crds:
+  createClusterExternalSecret: false
+  createClusterSecretStore: true # without setting it to true, certcontroller couldn't start: {"level":"debug","ts":1651041439.6815717,"logger":"controller-runtime.healthz","msg":"healthz check failed","checker":"crd-inject","error":"resource not ready: clustersecretstores.external-secrets.io"}
 
-serviceAccount:
-  annotations:
-    "eks.amazonaws.com/role-arn": ${local.external_secrets.enabled ? module.aws_iam_external_secrets[0].role_arn : ""}
+processClusterExternalSecret: false
+processClusterStore: false
 
 securityContext:
-  # Required for use of IRSA, see https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts-technical-overview.html
-  fsGroup: 1000
+  capabilities:
+    drop:
+    - ALL
+  readOnlyRootFilesystem: true
+  runAsNonRoot: true
+  runAsUser: 1000
+
+resources:
+  requests:
+    cpu: 100m
+    memory: 128Mi
+  limits:
+    cpu: 200m
+    memory: 128Mi
+
+webhook:
+  securityContext:
+    capabilities:
+      drop:
+      - ALL
+    readOnlyRootFilesystem: true
+    runAsNonRoot: true
+    runAsUser: 1000
+
+  resources:
+    requests:
+      cpu: 50m
+      memory: 64Mi
+    limits:
+      cpu: 100m
+      memory: 64Mi
+
+certController:
+  securityContext:
+    capabilities:
+      drop:
+      - ALL
+    readOnlyRootFilesystem: true
+    runAsNonRoot: true
+    runAsUser: 1000
+
+  resources:
+    requests:
+      cpu: 50m
+      memory: 64Mi
+    limits:
+      cpu: 100m
+      memory: 64Mi
 VALUES
 }
 
@@ -56,6 +95,32 @@ module "external_secrets_namespace" {
         ]
       }
     },
+    {
+      name         = "allow-webhooks"
+      policy_types = ["Ingress"]
+      pod_selector = {
+        match_expressions = {
+          key      = "app.kubernetes.io/name"
+          operator = "In"
+          values   = ["${local.external_secrets.name}-webhook"]
+        }
+      }
+      ingress = {
+        ports = [
+          {
+            port     = "9443"
+            protocol = "TCP"
+          }
+        ]
+        from = [
+          {
+            ip_block = {
+              cidr = "0.0.0.0/0"
+            }
+          }
+        ]
+      }
+    },
     {
       name         = "allow-egress"
       policy_types = ["Egress"]
@@ -76,33 +141,6 @@ module "external_secrets_namespace" {
   ]
 }
 
-
-#tfsec:ignore:aws-iam-no-policy-wildcards
-module "aws_iam_external_secrets" {
-  count = local.external_secrets.enabled ? 1 : 0
-
-  source            = "../modules/aws-iam-eks-trusted"
-  name              = "${local.name}-${local.external_secrets.name}"
-  region            = local.region
-  oidc_provider_arn = local.eks_oidc_provider_arn
-  policy = jsonencode({
-    "Version" : "2012-10-17",
-    "Statement" : [
-      {
-        "Effect" : "Allow",
-        "Action" : [
-          "ssm:GetParameter",
-          "secretsmanager:GetResourcePolicy",
-          "secretsmanager:GetSecretValue",
-          "secretsmanager:DescribeSecret",
-          "secretsmanager:ListSecretVersionIds"
-        ],
-        "Resource" : "*"
-      }
-    ]
-  })
-}
-
 resource "helm_release" "external_secrets" {
   count = local.external_secrets.enabled ? 1 : 0
 

terraform/layer2-k8s/helm-releases.yaml

@@ -55,9 +55,9 @@ releases:
     namespace: external-dns
   - id: external-secrets
     enabled: true
-    chart: kubernetes-external-secrets
-    repository: https://external-secrets.github.io/kubernetes-external-secrets
-    chart_version: 8.5.5
+    chart: external-secrets
+    repository: https://charts.external-secrets.io
+    chart_version: 0.5.1
     namespace: external-secrets
   - id: gitlab-runner
     enabled: false