[CONTP-921] Configure read-only root filesystem for Datadog Agent (#49)

* use init config and volume containers for rofs

* remove log volume to move logs to opt dir

* feat: remove init-config

* test: fix expected container and volume mount count

* feat: add rofs configuration option

* test: add rofs disabled to dd disabled test

* chore: merge dependson and test for rofs volume mounts

* feat: add precondition linux only for rofs

* feat: dd_readonly_root_filesystem false by default

Author: Mathew-Estafanous

modules/ecs_fargate/README.md

@@ -255,6 +255,7 @@ No modules.
 | <a name="input_dd_is_datadog_dependency_enabled"></a> [dd\_is\_datadog\_dependency\_enabled](#input\_dd\_is\_datadog\_dependency\_enabled) | Whether the Datadog Agent container is a dependency for other containers | `bool` | `false` | no |
 | <a name="input_dd_log_collection"></a> [dd\_log\_collection](#input\_dd\_log\_collection) | Configuration for Datadog Log Collection | <pre>object({<br/>    enabled = optional(bool, false)<br/>    fluentbit_config = optional(object({<br/>      registry                         = optional(string, "public.ecr.aws/aws-observability/aws-for-fluent-bit")<br/>      image_version                    = optional(string, "stable")<br/>      cpu                              = optional(number)<br/>      memory_limit_mib                 = optional(number)<br/>      is_log_router_essential          = optional(bool, false)<br/>      is_log_router_dependency_enabled = optional(bool, false)<br/>      environment = optional(list(object({<br/>        name  = string<br/>        value = string<br/>      })), [])<br/>      log_router_health_check = optional(object({<br/>        command      = optional(list(string))<br/>        interval     = optional(number)<br/>        retries      = optional(number)<br/>        start_period = optional(number)<br/>        timeout      = optional(number)<br/>        }),<br/>        {<br/>          command      = ["CMD-SHELL", "exit 0"]<br/>          interval     = 5<br/>          retries      = 3<br/>          start_period = 15<br/>          timeout      = 5<br/>        }<br/>      )<br/>      firelens_options = optional(object({<br/>        config_file_type  = optional(string)<br/>        config_file_value = optional(string)<br/>      }))<br/>      log_driver_configuration = optional(object({<br/>        host_endpoint = optional(string, "http-intake.logs.datadoghq.com")<br/>        tls           = optional(bool)<br/>        compress      = optional(string)<br/>        service_name  = optional(string)<br/>        source_name   = optional(string)<br/>        message_key   = optional(string)<br/>        }),<br/>        {<br/>          host_endpoint = "http-intake.logs.datadoghq.com"<br/>        }<br/>      )<br/>      mountPoints = optional(list(object({<br/>        sourceVolume : string,<br/>        containerPath : string,<br/>        readOnly : bool<br/>      })), [])<br/>      dependsOn = optional(list(object({<br/>        containerName : string,<br/>        condition : string<br/>      })), [])<br/>      }),<br/>      {<br/>        fluentbit_config = {<br/>          registry      = "public.ecr.aws/aws-observability/aws-for-fluent-bit"<br/>          image_version = "stable"<br/>          log_driver_configuration = {<br/>            host_endpoint = "http-intake.logs.datadoghq.com"<br/>          }<br/>        }<br/>      }<br/>    )<br/>  })</pre> | <pre>{<br/>  "enabled": false,<br/>  "fluentbit_config": {<br/>    "is_log_router_essential": false,<br/>    "log_driver_configuration": {<br/>      "host_endpoint": "http-intake.logs.datadoghq.com"<br/>    }<br/>  }<br/>}</pre> | no |
 | <a name="input_dd_memory_limit_mib"></a> [dd\_memory\_limit\_mib](#input\_dd\_memory\_limit\_mib) | Datadog Agent container memory limit in MiB | `number` | `null` | no |
+| <a name="input_dd_readonly_root_filesystem"></a> [dd\_readonly\_root\_filesystem](#input\_dd\_readonly\_root\_filesystem) | Datadog Agent container runs with read-only root filesystem enabled | `bool` | `false` | no |
 | <a name="input_dd_registry"></a> [dd\_registry](#input\_dd\_registry) | Datadog Agent image registry | `string` | `"public.ecr.aws/datadog/agent"` | no |
 | <a name="input_dd_service"></a> [dd\_service](#input\_dd\_service) | The task service name. Used for tagging (UST) | `string` | `null` | no |
 | <a name="input_dd_site"></a> [dd\_site](#input\_dd\_site) | Datadog Site | `string` | `"datadoghq.com"` | no |

modules/ecs_fargate/datadog.tf

@@ -77,6 +77,27 @@ locals {
     }
   ] : []
 
+  dd_agent_mount = concat(
+    local.apm_dsd_mount,
+    var.dd_readonly_root_filesystem ? [
+      {
+        containerPath = "/etc/datadog-agent"
+        sourceVolume  = "agent-config"
+        readOnly      = false
+      },
+      {
+        containerPath = "/tmp"
+        sourceVolume  = "agent-tmp"
+        readOnly      = false
+      },
+      {
+        containerPath = "/opt/datadog-agent/run"
+        sourceVolume  = "agent-run"
+        readOnly      = false
+      }
+    ] : []
+  )
+
   apm_socket_var = local.is_apm_socket_mount ? [
     {
       name  = "DD_TRACE_AGENT_URL"
@@ -223,6 +244,18 @@ locals {
     )
   ]
 
+  rofs_volumes = var.dd_readonly_root_filesystem ? [
+    {
+      name = "agent-config"
+    },
+    {
+      name = "agent-tmp"
+    },
+    {
+      name = "agent-run"
+    }
+  ] : []
+
   # Volume configuration for task
   apm_dsd_volume = local.is_apm_dsd_volume ? [
     {
@@ -238,6 +271,7 @@ locals {
 
   modified_volumes = concat(
     [for k, v in coalesce(var.volumes, []) : v],
+    local.rofs_volumes,
     local.apm_dsd_volume,
     local.cws_volume,
   )
@@ -264,6 +298,10 @@ locals {
       name  = "DD_INSTALL_INFO_INSTALLER_VERSION"
       value = local.install_info_installer_version
     },
+    {
+      name  = "DD_LOG_FILE"
+      value = "/opt/datadog-agent/run/logs"
+    }
   ]
 
   dynamic_env = [
@@ -308,52 +346,85 @@ locals {
     local.dd_environment,
   )
 
+  dd_agent_dependency = concat(
+    var.dd_readonly_root_filesystem ? [
+      {
+        condition     = "SUCCESS"
+        containerName = "init-volume"
+      }
+    ] : [],
+    try(var.dd_log_collection.fluentbit_config.is_log_router_dependency_enabled, false) && local.dd_firelens_log_configuration != null ? local.log_router_dependency : [],
+  )
+
   # Datadog Agent container definition
-  dd_agent_container = [
-    merge(
+  dd_agent_container = concat(
+    var.dd_readonly_root_filesystem ? [
       {
-        name         = "datadog-agent"
-        image        = "${var.dd_registry}:${var.dd_image_version}"
-        essential    = var.dd_essential
-        environment  = local.dd_agent_env
-        dockerLabels = var.dd_docker_labels
-        cpu          = var.dd_cpu
-        memory       = var.dd_memory_limit_mib
-        secrets = var.dd_api_key_secret != null ? [
+        cpu                    = 0
+        memory                 = 128
+        name                   = "init-volume"
+        image                  = "${var.dd_registry}:${var.dd_image_version}"
+        essential              = false
+        readOnlyRootFilesystem = true
+        command                = ["/bin/sh", "-c", "cp -vnR /etc/datadog-agent/* /agent-config/ && exit 0"]
+        mountPoints = [
           {
-            name      = "DD_API_KEY"
-            valueFrom = var.dd_api_key_secret.arn
+            sourceVolume  = "agent-config"
+            containerPath = "/agent-config"
+            readOnly      = false
           }
-        ] : []
-        portMappings = [
-          {
-            containerPort = 8125
-            hostPort      = 8125
-            protocol      = "udp"
-          },
-          {
-            containerPort = 8126
-            hostPort      = 8126
-            protocol      = "tcp"
+        ]
+      }
+    ] : [],
+    [
+      merge(
+        {
+          name         = "datadog-agent"
+          image        = "${var.dd_registry}:${var.dd_image_version}"
+          essential    = var.dd_essential
+          environment  = local.dd_agent_env
+          dockerLabels = var.dd_docker_labels
+          cpu          = var.dd_cpu
+          memory       = var.dd_memory_limit_mib
+
+          readonlyRootFilesystem = var.dd_readonly_root_filesystem
+          secrets = var.dd_api_key_secret != null ? [
+            {
+              name      = "DD_API_KEY"
+              valueFrom = var.dd_api_key_secret.arn
+            }
+          ] : []
+          portMappings = [
+            {
+              containerPort = 8125
+              hostPort      = 8125
+              protocol      = "udp"
+            },
+            {
+              containerPort = 8126
+              hostPort      = 8126
+              protocol      = "tcp"
+            }
+          ],
+
+          mountPoints      = local.dd_agent_mount,
+          logConfiguration = local.dd_firelens_log_configuration,
+          dependsOn        = local.dd_agent_dependency
+          systemControls   = []
+          volumesFrom      = []
+        },
+        try(var.dd_health_check.command == null, true) ? {} : {
+          healthCheck = {
+            command     = var.dd_health_check.command
+            interval    = var.dd_health_check.interval
+            timeout     = var.dd_health_check.timeout
+            retries     = var.dd_health_check.retries
+            startPeriod = var.dd_health_check.start_period
           }
-        ],
-        mountPoints      = local.apm_dsd_mount,
-        logConfiguration = local.dd_firelens_log_configuration,
-        dependsOn        = try(var.dd_log_collection.fluentbit_config.is_log_router_dependency_enabled, false) && local.dd_firelens_log_configuration != null ? local.log_router_dependency : [],
-        systemControls   = []
-        volumesFrom      = []
-      },
-      try(var.dd_health_check.command == null, true) ? {} : {
-        healthCheck = {
-          command     = var.dd_health_check.command
-          interval    = var.dd_health_check.interval
-          timeout     = var.dd_health_check.timeout
-          retries     = var.dd_health_check.retries
-          startPeriod = var.dd_health_check.start_period
         }
-      }
-    )
-  ]
+      )
+    ]
+  )
 
   dd_log_environment = var.dd_log_collection.fluentbit_config.environment != null ? var.dd_log_collection.fluentbit_config.environment : []
 

modules/ecs_fargate/main.tf

@@ -177,6 +177,10 @@ resource "aws_ecs_task_definition" "this" {
       condition     = var.dd_log_collection.enabled == false || (var.dd_log_collection.enabled == true && local.is_linux == true)
       error_message = "Log collection is not supported on Windows. Please set `dd_log_collection.enabled` to `false`."
     }
+    precondition {
+      condition     = var.dd_readonly_root_filesystem == false || (var.dd_readonly_root_filesystem == true && local.is_linux == true)
+      error_message = "Readonly root filesystem is only supported on Linux. Please set `dd_readonly_root_filesystem` to `false`."
+    }
     # Must provide only one of the two Datadog API key options
     precondition {
       condition     = (var.dd_api_key == null && var.dd_api_key_secret != null) || (var.dd_api_key != null && var.dd_api_key_secret == null)

modules/ecs_fargate/variables.tf

@@ -65,6 +65,13 @@ variable "dd_is_datadog_dependency_enabled" {
   nullable    = false
 }
 
+variable "dd_readonly_root_filesystem" {
+  description = "Datadog Agent container runs with read-only root filesystem enabled"
+  type        = bool
+  default     = false
+  nullable    = false
+}
+
 variable "dd_health_check" {
   description = "Datadog Agent health check configuration"
   type = object({

smoke_tests/ecs_fargate/all-dd-disabled.tf

@@ -17,6 +17,7 @@ module "dd_task_all_dd_disabled" {
   dd_tags                          = "team:cont-p, owner:container-monitoring"
   dd_essential                     = true
   dd_is_datadog_dependency_enabled = false
+  dd_readonly_root_filesystem      = false
 
   dd_environment = []
 

smoke_tests/ecs_fargate/all-dd-inputs.tf

@@ -17,6 +17,7 @@ module "dd_task_all_dd_inputs" {
   dd_tags                          = "team:cont-p, owner:container-monitoring"
   dd_essential                     = true
   dd_is_datadog_dependency_enabled = true
+  dd_readonly_root_filesystem      = true
 
   dd_environment = [
     {

smoke_tests/ecs_fargate/all-windows.tf

@@ -16,6 +16,8 @@ module "dd_task_all_windows" {
   dd_site    = var.dd_site
   dd_service = var.dd_service
 
+  dd_readonly_root_filesystem = false
+
   dd_apm = {
     enabled = true
   }

smoke_tests/ecs_fargate/apm-dsd-tcp-udp.tf

@@ -18,6 +18,8 @@ module "dd_task_apm_dsd_tcp_udp" {
   dd_tags      = "team:cont-p, owner:container-monitoring"
   dd_essential = true
 
+  dd_readonly_root_filesystem = true
+
   dd_dogstatsd = {
     enabled        = true,
     socket_enabled = false,

smoke_tests/ecs_fargate/logging-only.tf

@@ -15,6 +15,8 @@ module "dd_task_logging_only" {
   dd_service   = var.dd_service
   dd_essential = true
 
+  dd_readonly_root_filesystem = false
+
   dd_dogstatsd = {
     enabled = false,
   }

smoke_tests/ecs_fargate/ust-docker-labels.tf

@@ -20,6 +20,7 @@ module "dd_task_ust_docker_labels" {
   dd_essential = true
 
   dd_is_datadog_dependency_enabled = true
+  dd_readonly_root_filesystem      = true
 
   dd_log_collection = {
     enabled = true,