Merge pull request #57 from datafold/gerard-eng-941-test-that-workload-identity-can-backup-clickhouse-data

feat: Use IRSA for backups

Author: gtoonstra

main.tf

@@ -156,6 +156,19 @@ locals {
   )
 }
 
+module "clickhouse_backup" {
+  source = "./modules/clickhouse_backup"
+
+  deployment_name                = var.deployment_name
+  clickhouse_s3_bucket           = var.clickhouse_s3_bucket
+  s3_clickhouse_backup_tags      = var.s3_clickhouse_backup_tags
+  s3_backup_bucket_name_override = var.s3_backup_bucket_name_override
+}
+
+locals {
+  clickhouse_backup_bucket_arn  = module.clickhouse_backup.clickhouse_s3_bucket_arn
+}
+
 module "eks" {
   source = "./modules/eks"
 
@@ -181,11 +194,12 @@ module "eks" {
   k8s_public_access_cidrs             = var.k8s_public_access_cidrs
 
   k8s_access_bedrock                  = var.k8s_access_bedrock
+  clickhouse_backup_bucket_arn        = local.clickhouse_backup_bucket_arn
 }
 
 locals {
-  cluster_name        = module.eks.cluster_name
-  control_plane_sg_id = module.eks.control_plane_security_group_id
+cluster_name        = module.eks.cluster_name
+control_plane_sg_id = module.eks.control_plane_security_group_id
 }
 
 module "database" {
@@ -230,15 +244,6 @@ module "database" {
   rds_monitoring_interval                  = var.rds_monitoring_interval
 }
 
-module "clickhouse_backup" {
-  source = "./modules/clickhouse_backup"
-
-  deployment_name                = var.deployment_name
-  clickhouse_s3_bucket           = var.clickhouse_s3_bucket
-  s3_clickhouse_backup_tags      = var.s3_clickhouse_backup_tags
-  s3_backup_bucket_name_override = var.s3_backup_bucket_name_override
-}
-
 module "private_access" {
   count = var.deploy_private_access ? 1 : 0
   source = "./modules/private_access"
@@ -263,7 +268,7 @@ resource "aws_ebs_volume" "clickhouse_data" {
 
   tags = merge({
     Name = "${var.deployment_name}-clickhouse-data"
-  }, var.ebs_extra_tags)
+    }, var.ebs_extra_tags)
 }
 
 resource "aws_ebs_volume" "clickhouse_logs" {

modules/clickhouse_backup/iam.tf

@@ -2,14 +2,10 @@ output "clickhouse_s3_bucket" {
   value =  resource.aws_s3_bucket.clickhouse_backup.id
 }
 
-output "clickhouse_s3_region" {
-  value = resource.aws_s3_bucket.clickhouse_backup.region
-}
-
-output "clickhouse_access_key" {
-  value = resource.aws_iam_access_key.clickhouse_backup.id
+output "clickhouse_s3_bucket_arn" {
+  value =  resource.aws_s3_bucket.clickhouse_backup.arn
 }
 
-output "clickhouse_secret_key" {
-  value = resource.aws_iam_access_key.clickhouse_backup.secret
+output "clickhouse_s3_region" {
+  value = resource.aws_s3_bucket.clickhouse_backup.region
 }

modules/clickhouse_backup/outputs.tf

@@ -136,4 +136,10 @@ output "storage_worker_role_arn" {
 output "storage_worker_service_account_name" {
   value = var.storage_worker_service_account_name
   description = "The name of the service account for storage_worker"
-}
+}
+
+# Clickhouse backup
+output "clickhouse_backup_role_name" {
+  value = module.clickhouse_backup_role.iam_role_arn
+  description = "The name of the role for clickhouse backups"
+}

modules/eks/outputs.tf

@@ -28,7 +28,37 @@ resource "aws_iam_policy" "bedrock_access_policy" {
   tags = var.sg_tags
 }
 
-# 
+resource "aws_iam_policy" "clickhouse_backup_policy" {
+  name        = "${var.deployment_name}-clickhouse-backup-policy"
+  description = "Policy that allows clickhouse to make backups"
+
+  policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Effect = "Allow",
+        Action = [
+          "s3:ListBucket",
+        ],
+        Resource = [var.clickhouse_backup_bucket_arn]
+      },
+      {
+        Effect = "Allow",
+        Action = [
+          "s3:PutObject",
+          "s3:GetObject",
+          "s3:ListBucket",
+          "s3:DeleteObject"
+        ],
+        Resource = [
+          "${var.clickhouse_backup_bucket_arn}/*"
+        ]
+      }
+    ]
+  })
+}
+
+#
 # Roles
 # 
 
@@ -200,6 +230,18 @@ module "storage_worker_role" {
   }
 }
 
+module "clickhouse_backup_role" {
+  source             = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
+  role_name          = "${var.deployment_name}-${var.clickhouse_backup_service_account_name}"
+
+  oidc_providers = {
+    ex = {
+      provider_arn               = module.eks.oidc_provider_arn
+      namespace_service_accounts = ["${var.deployment_name}:${var.clickhouse_backup_service_account_name}"]
+    }
+  }
+}
+
 # Policy Attachments
 resource "aws_iam_role_policy_attachment" "bedrock_dfshell_attachment" {
   count      = var.k8s_access_bedrock ? 1 : 0
@@ -225,3 +267,8 @@ resource "aws_iam_role_policy_attachment" "bedrock_worker_interactive_attachment
   policy_arn = aws_iam_policy.bedrock_access_policy[0].arn
 }
 
+resource "aws_iam_role_policy_attachment" "clickhouse_backup_attachment" {
+  role       = module.clickhouse_backup_role.iam_role_name
+  policy_arn = aws_iam_policy.clickhouse_backup_policy.arn
+}
+

modules/eks/roles.tf

@@ -118,6 +118,17 @@ variable "sg_tags" {
   default     = {}
 }
 
+variable "clickhouse_backup_service_account_name" {
+  type        = string
+  default     = "datafold-clickhouse"
+  description = "Name of the service account for clickhouse backup"
+}
+
+variable "clickhouse_backup_bucket_arn" {
+  type        = string
+  description = "ARN of the backup bucket"
+}
+
 variable "dfshell_service_account_name" {
   type        = string
   default     = "datafold-dfshell"

modules/eks/variables.tf

@@ -133,14 +133,9 @@ output "clickhouse_s3_region" {
   description = "The region where the S3 bucket is created"
 }
 
-output "clickhouse_access_key" {
-  value = module.clickhouse_backup.clickhouse_access_key
-  description = "The access key of the IAM user doing the clickhouse backups."
-}
-
-output "clickhouse_secret_key" {
-  value = module.clickhouse_backup.clickhouse_secret_key
-  description = "The secret key of the IAM user doing the clickhouse backups."
+output "clickhouse_backup_role_name" {
+  value = module.eks.clickhouse_backup_role_name
+  description = "The name of the role for clickhouse backups"
 }
 
 output "private_access_vpces_name" {