From 9ac07c926cbe26b6d893128f4a38881eeb482cc5 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Tue, 7 Oct 2025 17:35:44 +0200 Subject: [PATCH 1/6] Use additional domains from secret store instead local envs --- .env.template | 2 -- iac/provider-gcp/Makefile | 1 - iac/provider-gcp/init/main.tf | 22 +++++++++++++++++++++- iac/provider-gcp/init/outputs.tf | 4 ++++ iac/provider-gcp/main.tf | 11 +++++++++-- iac/provider-gcp/variables.tf | 6 ------ 6 files changed, 34 insertions(+), 12 deletions(-) diff --git a/.env.template b/.env.template index 36395fe933..47e4cc4811 100644 --- a/.env.template +++ b/.env.template @@ -58,8 +58,6 @@ FILESTORE_CACHE_CAPACITY_GB= # ------------------------------------------- Optional block ----------------------------------------------------------- # Managed Redis (default: false) REDIS_MANAGED=false -# Additional domains (separated by commas) -ADDITIONAL_DOMAINS= # Template bucket name (if you want to use a different bucket for templates then the default one) TEMPLATE_BUCKET_NAME= # Hash seed used for generating sandbox access tokens, not needed if you are not using them diff --git a/iac/provider-gcp/Makefile b/iac/provider-gcp/Makefile index d9c4ddc3d1..54579e72ac 100644 --- a/iac/provider-gcp/Makefile +++ b/iac/provider-gcp/Makefile @@ -36,7 +36,6 @@ tf_vars := TF_VAR_environment=$(TERRAFORM_ENVIRONMENT) \ $(call tfvar, GCP_REGION) \ $(call tfvar, GCP_ZONE) \ $(call tfvar, DOMAIN_NAME) \ - $(call tfvar, ADDITIONAL_DOMAINS) \ $(call tfvar, ADDITIONAL_API_SERVICES_JSON) \ $(call tfvar, PREFIX) \ $(call tfvar, ALLOW_SANDBOX_INTERNET) \ diff --git a/iac/provider-gcp/init/main.tf b/iac/provider-gcp/init/main.tf index db12efe1ca..b861b56f5b 100644 --- a/iac/provider-gcp/init/main.tf +++ b/iac/provider-gcp/init/main.tf @@ -208,6 +208,27 @@ resource "google_secret_manager_secret_version" "analytics_collector_api_token" depends_on = [time_sleep.secrets_api_wait_60_seconds] } +resource "google_secret_manager_secret" "routing_domains" { + secret_id = "${var.prefix}routing-domains" + + replication { + auto {} + } + + depends_on = [time_sleep.secrets_api_wait_60_seconds] +} + +resource "google_secret_manager_secret_version" "routing_domains" { + secret = google_secret_manager_secret.routing_domains.name + secret_data = jsonencode([]) + + lifecycle { + ignore_changes = [secret_data] + } + + depends_on = [time_sleep.secrets_api_wait_60_seconds] +} + resource "google_artifact_registry_repository" "orchestration_repository" { format = "DOCKER" repository_id = "e2b-orchestration" @@ -222,7 +243,6 @@ resource "time_sleep" "artifact_registry_api_wait_90_seconds" { create_duration = "90s" } - resource "google_artifact_registry_repository_iam_member" "orchestration_repository_member" { repository = google_artifact_registry_repository.orchestration_repository.name role = "roles/artifactregistry.reader" diff --git a/iac/provider-gcp/init/outputs.tf b/iac/provider-gcp/init/outputs.tf index be11b49b1c..2b2a72a546 100644 --- a/iac/provider-gcp/init/outputs.tf +++ b/iac/provider-gcp/init/outputs.tf @@ -30,6 +30,10 @@ output "analytics_collector_api_token_secret_name" { value = google_secret_manager_secret.analytics_collector_api_token.name } +output "routing_domains_secret_name" { + value = google_secret_manager_secret.routing_domains.name +} + output "orchestration_repository_name" { value = google_artifact_registry_repository.orchestration_repository.name } diff --git a/iac/provider-gcp/main.tf b/iac/provider-gcp/main.tf index 82eb375291..67c32cd10e 100644 --- a/iac/provider-gcp/main.tf +++ b/iac/provider-gcp/main.tf @@ -54,6 +54,14 @@ provider "google" { zone = var.gcp_zone } +data "google_secret_manager_secret_version" "routing_domains" { + secret = module.init.routing_domains_secret_name +} + +locals { + routing_domains = nonsensitive(jsondecode(data.google_secret_manager_secret_version.routing_domains.secret_data)) +} + module "init" { source = "./init" @@ -115,8 +123,7 @@ module "cluster" { nomad_port = var.nomad_port google_service_account_email = module.init.service_account_email domain_name = var.domain_name - additional_domains = (var.additional_domains != "" ? - [for item in split(",", var.additional_domains) : trimspace(item)] : []) + additional_domains = local.routing_domains additional_api_services = (var.additional_api_services_json != "" ? jsondecode(var.additional_api_services_json) : diff --git a/iac/provider-gcp/variables.tf b/iac/provider-gcp/variables.tf index d979c37dc6..30970e2963 100644 --- a/iac/provider-gcp/variables.tf +++ b/iac/provider-gcp/variables.tf @@ -330,12 +330,6 @@ variable "domain_name" { description = "The domain name where e2b will run" } -variable "additional_domains" { - type = string - description = "Additional domains which can be used to access the e2b cluster, separated by commas" - default = "" -} - variable "additional_api_services_json" { type = string description = < Date: Tue, 7 Oct 2025 17:36:22 +0200 Subject: [PATCH 2/6] Traefik ingress service --- iac/provider-gcp/main.tf | 4 + iac/provider-gcp/nomad-cluster/main.tf | 1 + .../nomad-cluster/network/main.tf | 5 + .../nomad-cluster/network/variables.tf | 8 ++ .../nomad-cluster/nodepool-api.tf | 5 + iac/provider-gcp/nomad-cluster/variables.tf | 8 ++ iac/provider-gcp/nomad/jobs/ingress.hcl | 97 +++++++++++++++++++ iac/provider-gcp/nomad/main.tf | 21 ++++ iac/provider-gcp/nomad/variables.tf | 8 ++ iac/provider-gcp/variables.tf | 13 +++ 10 files changed, 170 insertions(+) create mode 100644 iac/provider-gcp/nomad/jobs/ingress.hcl diff --git a/iac/provider-gcp/main.tf b/iac/provider-gcp/main.tf index 67c32cd10e..0ab7d1dda6 100644 --- a/iac/provider-gcp/main.tf +++ b/iac/provider-gcp/main.tf @@ -116,6 +116,7 @@ module "cluster" { api_use_nat = var.api_use_nat api_nat_ips = var.api_nat_ips + ingress_port = var.ingress_port edge_api_port = var.edge_api_port edge_proxy_port = var.edge_proxy_port api_port = var.api_port @@ -179,6 +180,9 @@ module "nomad" { clickhouse_job_constraint_prefix = var.clickhouse_job_constraint_prefix clickhouse_node_pool = var.clickhouse_node_pool + # Ingress + ingress_port = var.ingress_port + # API api_resources_cpu_count = var.api_resources_cpu_count api_resources_memory_mb = var.api_resources_memory_mb diff --git a/iac/provider-gcp/nomad-cluster/main.tf b/iac/provider-gcp/nomad-cluster/main.tf index c89f2f18da..3c9d5fb1d7 100644 --- a/iac/provider-gcp/nomad-cluster/main.tf +++ b/iac/provider-gcp/nomad-cluster/main.tf @@ -96,6 +96,7 @@ module "network" { api_use_nat = var.api_use_nat api_nat_ips = var.api_nat_ips + ingress_port = var.ingress_port api_port = var.api_port docker_reverse_proxy_port = var.docker_reverse_proxy_port network_name = var.network_name diff --git a/iac/provider-gcp/nomad-cluster/network/main.tf b/iac/provider-gcp/nomad-cluster/network/main.tf index 92927fe19a..c4a9508fa6 100644 --- a/iac/provider-gcp/nomad-cluster/network/main.tf +++ b/iac/provider-gcp/nomad-cluster/network/main.tf @@ -455,6 +455,11 @@ resource "google_compute_firewall" "default-hc" { } } + allow { + protocol = "tcp" + ports = [var.ingress_port.port] + } + dynamic "allow" { for_each = toset(var.additional_ports) diff --git a/iac/provider-gcp/nomad-cluster/network/variables.tf b/iac/provider-gcp/nomad-cluster/network/variables.tf index 640672814b..aa6db683f2 100644 --- a/iac/provider-gcp/nomad-cluster/network/variables.tf +++ b/iac/provider-gcp/nomad-cluster/network/variables.tf @@ -51,6 +51,14 @@ variable "api_port" { }) } +variable "ingress_port" { + type = object({ + name = string + port = number + health_path = string + }) +} + variable "docker_reverse_proxy_port" { type = object({ name = string diff --git a/iac/provider-gcp/nomad-cluster/nodepool-api.tf b/iac/provider-gcp/nomad-cluster/nodepool-api.tf index 9fbe6422ab..848fb6f92a 100644 --- a/iac/provider-gcp/nomad-cluster/nodepool-api.tf +++ b/iac/provider-gcp/nomad-cluster/nodepool-api.tf @@ -71,6 +71,11 @@ resource "google_compute_instance_group_manager" "api_pool" { port = var.docker_reverse_proxy_port.port } + named_port { + name = var.ingress_port.name + port = var.ingress_port.port + } + dynamic "named_port" { for_each = local.api_additional_ports content { diff --git a/iac/provider-gcp/nomad-cluster/variables.tf b/iac/provider-gcp/nomad-cluster/variables.tf index f18d7072a1..c6d449cbcb 100644 --- a/iac/provider-gcp/nomad-cluster/variables.tf +++ b/iac/provider-gcp/nomad-cluster/variables.tf @@ -100,6 +100,14 @@ variable "api_port" { }) } +variable "ingress_port" { + type = object({ + name = string + port = number + health_path = string + }) +} + variable "docker_reverse_proxy_port" { type = object({ name = string diff --git a/iac/provider-gcp/nomad/jobs/ingress.hcl b/iac/provider-gcp/nomad/jobs/ingress.hcl new file mode 100644 index 0000000000..b33450a2da --- /dev/null +++ b/iac/provider-gcp/nomad/jobs/ingress.hcl @@ -0,0 +1,97 @@ +job "ingress" { + datacenters = ["${gcp_zone}"] + node_pool = "${node_pool}" + priority = 90 + + group "ingress" { + count = ${count} + + constraint { + operator = "distinct_hosts" + value = "true" + } + + network { + port "ingress" { + static = "${ingress_port}" + } + + port "control" { + static = "${control_port}" + } + } + +# https://developer.hashicorp.com/nomad/docs/job-specification/update +%{ if update_stanza } + update { + max_parallel = 1 # Update only 1 node at a time + } +%{ endif } + + service { + port = "ingress" + name = "ingress" + task = "ingress" + + check { + type = "http" + name = "health" + path = "/ping" + interval = "3s" + timeout = "3s" + port = "${ingress_port}" + } + } + + task "ingress" { + driver = "docker" + + # If we need more than 30s we will need to update the max_kill_timeout in nomad + # https://developer.hashicorp.com/nomad/docs/configuration/client#max_kill_timeout + %{ if update_stanza } + kill_timeout = "24h" + %{ endif } + + kill_signal = "SIGTERM" + + config { + network_mode = "host" + image = "traefik:v3.5" + ports = ["control", "ingress"] + args = [ + # Entry-points that are set internally by Traefik + "--entrypoints.web.address=:${ingress_port}", + "--entrypoints.traefik.address=:${control_port}", + + # Traefik internals (logging, metrics, ...) + "--api.dashboard=true", + "--api.insecure=false", + + "--accesslog=true", + "--ping=true", + "--ping.entryPoint=web", + "--metrics=true", + "--metrics.prometheus=true", + "--metrics.prometheus.entryPoint=traefik", + + # Traefik Nomad provider + "--providers.nomad=true", + "--providers.nomad.endpoint.address=${nomad_endpoint}", + "--providers.nomad.endpoint.token=${nomad_token}", + + # Traefik Consul provider + "--providers.consulcatalog=true", + "--providers.consulcatalog.exposedByDefault=false", + "--providers.consulcatalog.endpoint.address=${consul_endpoint}", + "--providers.consulcatalog.endpoint.token=${consul_token}", + ] + } + + resources { + memory_max = ${memory_mb * 1.5} + memory = ${memory_mb} + cpu = ${cpu_count * 1000} + } + } + } +} \ No newline at end of file diff --git a/iac/provider-gcp/nomad/main.tf b/iac/provider-gcp/nomad/main.tf index eb93b32d23..076fdc0a43 100644 --- a/iac/provider-gcp/nomad/main.tf +++ b/iac/provider-gcp/nomad/main.tf @@ -68,6 +68,27 @@ resource "docker_image" "db_migrator_image" { platform = "linux/amd64/v8" } +resource "nomad_job" "ingress" { + jobspec = templatefile("${path.module}/jobs/ingress.hcl", + { + count = 1 + update_stanza = var.api_machine_count > 1 + cpu_count = 1 + memory_mb = 512 + node_pool = var.api_node_pool + gcp_zone = var.gcp_zone + + ingress_port = var.ingress_port.port + control_port = 8900 + + nomad_endpoint = "http://localhost:4646" + nomad_token = var.nomad_acl_token_secret + + consul_token = var.consul_acl_token_secret + consul_endpoint = "http://localhost:8500" + }) +} + resource "nomad_job" "api" { jobspec = templatefile("${path.module}/jobs/api.hcl", { update_stanza = var.api_machine_count > 1 diff --git a/iac/provider-gcp/nomad/variables.tf b/iac/provider-gcp/nomad/variables.tf index d946b7ca2a..7d68e8ea10 100644 --- a/iac/provider-gcp/nomad/variables.tf +++ b/iac/provider-gcp/nomad/variables.tf @@ -64,6 +64,14 @@ variable "api_port" { }) } +variable "ingress_port" { + type = object({ + name = string + port = number + health_path = string + }) +} + variable "api_resources_cpu_count" { type = number } diff --git a/iac/provider-gcp/variables.tf b/iac/provider-gcp/variables.tf index 30970e2963..00c59029c4 100644 --- a/iac/provider-gcp/variables.tf +++ b/iac/provider-gcp/variables.tf @@ -223,6 +223,19 @@ variable "api_port" { } } +variable "ingress_port" { + type = object({ + name = string + port = number + health_path = string + }) + default = { + name = "ingress" + port = 8800 + health_path = "/ping" + } +} + variable "docker_reverse_proxy_port" { type = object({ name = string From fde571e7a0a84fcd5618c4e22fd39a8f96a32132 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Tue, 7 Oct 2025 17:36:41 +0200 Subject: [PATCH 3/6] Load balancer for ingress backed by traefik --- .../nomad-cluster/network/ingress.tf | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 iac/provider-gcp/nomad-cluster/network/ingress.tf diff --git a/iac/provider-gcp/nomad-cluster/network/ingress.tf b/iac/provider-gcp/nomad-cluster/network/ingress.tf new file mode 100644 index 0000000000..eedb8ea79d --- /dev/null +++ b/iac/provider-gcp/nomad-cluster/network/ingress.tf @@ -0,0 +1,58 @@ +resource "google_compute_health_check" "ingress" { + name = "${var.prefix}ingress" + + timeout_sec = 3 + check_interval_sec = 5 + healthy_threshold = 2 + unhealthy_threshold = 2 + + http_health_check { + port = var.ingress_port.port + request_path = var.ingress_port.health_path + } +} + +resource "google_compute_backend_service" "ingress" { + name = "${var.prefix}ingress" + + protocol = "HTTP" + port_name = var.ingress_port.name + + session_affinity = null + health_checks = [google_compute_health_check.ingress.id] + + timeout_sec = 65 + + load_balancing_scheme = "EXTERNAL_MANAGED" + locality_lb_policy = "ROUND_ROBIN" + + backend { + group = var.api_instance_group + } +} + +resource "google_compute_url_map" "ingress" { + name = "${var.prefix}ingress" + default_service = google_compute_backend_service.ingress.self_link +} + +resource "google_compute_global_forwarding_rule" "ingress" { + name = "${var.prefix}ingress-forward-http" + ip_protocol = "TCP" + port_range = "443" + load_balancing_scheme = "EXTERNAL_MANAGED" + ip_address = google_compute_global_address.ingress_ipv4.address + target = google_compute_target_https_proxy.ingress.self_link +} + +resource "google_compute_global_address" "ingress_ipv4" { + name = "${var.prefix}ingress-ipv4" + ip_version = "IPV4" +} + +resource "google_compute_target_https_proxy" "ingress" { + name = "${var.prefix}ingress-https" + url_map = google_compute_url_map.ingress.self_link + + certificate_map = "//certificatemanager.googleapis.com/${google_certificate_manager_certificate_map.certificate_map.id}" +} From 3fa42ed8f53735fc5ab6e820629d28b79ec171e5 Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Thu, 9 Oct 2025 12:42:59 +0200 Subject: [PATCH 4/6] options to customize ingress job count --- iac/provider-gcp/Makefile | 1 + iac/provider-gcp/main.tf | 3 ++- iac/provider-gcp/nomad/main.tf | 2 +- iac/provider-gcp/nomad/variables.tf | 4 ++++ iac/provider-gcp/variables.tf | 5 +++++ 5 files changed, 13 insertions(+), 2 deletions(-) diff --git a/iac/provider-gcp/Makefile b/iac/provider-gcp/Makefile index 54579e72ac..790649c9b9 100644 --- a/iac/provider-gcp/Makefile +++ b/iac/provider-gcp/Makefile @@ -41,6 +41,7 @@ tf_vars := TF_VAR_environment=$(TERRAFORM_ENVIRONMENT) \ $(call tfvar, ALLOW_SANDBOX_INTERNET) \ $(call tfvar, API_RESOURCES_CPU_COUNT) \ $(call tfvar, API_RESOURCES_MEMORY_MB) \ + $(call tfvar, INGRESS_COUNT) \ $(call tfvar, CLIENT_PROXY_COUNT) \ $(call tfvar, CLIENT_PROXY_RESOURCES_CPU_COUNT) \ $(call tfvar, CLIENT_PROXY_RESOURCES_MEMORY_MB) \ diff --git a/iac/provider-gcp/main.tf b/iac/provider-gcp/main.tf index 0ab7d1dda6..a1aca19798 100644 --- a/iac/provider-gcp/main.tf +++ b/iac/provider-gcp/main.tf @@ -181,7 +181,8 @@ module "nomad" { clickhouse_node_pool = var.clickhouse_node_pool # Ingress - ingress_port = var.ingress_port + ingress_port = var.ingress_port + ingress_count = var.ingress_count # API api_resources_cpu_count = var.api_resources_cpu_count diff --git a/iac/provider-gcp/nomad/main.tf b/iac/provider-gcp/nomad/main.tf index 076fdc0a43..40a13cd6a2 100644 --- a/iac/provider-gcp/nomad/main.tf +++ b/iac/provider-gcp/nomad/main.tf @@ -71,7 +71,7 @@ resource "docker_image" "db_migrator_image" { resource "nomad_job" "ingress" { jobspec = templatefile("${path.module}/jobs/ingress.hcl", { - count = 1 + count = var.ingress_count update_stanza = var.api_machine_count > 1 cpu_count = 1 memory_mb = 512 diff --git a/iac/provider-gcp/nomad/variables.tf b/iac/provider-gcp/nomad/variables.tf index 7d68e8ea10..6e747e230f 100644 --- a/iac/provider-gcp/nomad/variables.tf +++ b/iac/provider-gcp/nomad/variables.tf @@ -72,6 +72,10 @@ variable "ingress_port" { }) } +variable "ingress_count" { + type = number +} + variable "api_resources_cpu_count" { type = number } diff --git a/iac/provider-gcp/variables.tf b/iac/provider-gcp/variables.tf index 00c59029c4..21dfbe07c7 100644 --- a/iac/provider-gcp/variables.tf +++ b/iac/provider-gcp/variables.tf @@ -155,6 +155,11 @@ variable "client_proxy_count" { default = 1 } +variable "ingress_count" { + type = number + default = 1 +} + variable "client_proxy_resources_memory_mb" { type = number default = 1024 From d13af647ff1c611926a017a784dbc0d9c228274a Mon Sep 17 00:00:00 2001 From: Jiri Sveceny Date: Thu, 9 Oct 2025 12:43:27 +0200 Subject: [PATCH 5/6] Additional domains can be provide from env and storage secret --- .env.template | 2 ++ iac/provider-gcp/Makefile | 1 + iac/provider-gcp/main.tf | 9 +++++++-- iac/provider-gcp/variables.tf | 6 ++++++ 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.env.template b/.env.template index 47e4cc4811..36395fe933 100644 --- a/.env.template +++ b/.env.template @@ -58,6 +58,8 @@ FILESTORE_CACHE_CAPACITY_GB= # ------------------------------------------- Optional block ----------------------------------------------------------- # Managed Redis (default: false) REDIS_MANAGED=false +# Additional domains (separated by commas) +ADDITIONAL_DOMAINS= # Template bucket name (if you want to use a different bucket for templates then the default one) TEMPLATE_BUCKET_NAME= # Hash seed used for generating sandbox access tokens, not needed if you are not using them diff --git a/iac/provider-gcp/Makefile b/iac/provider-gcp/Makefile index 790649c9b9..da545b52b6 100644 --- a/iac/provider-gcp/Makefile +++ b/iac/provider-gcp/Makefile @@ -36,6 +36,7 @@ tf_vars := TF_VAR_environment=$(TERRAFORM_ENVIRONMENT) \ $(call tfvar, GCP_REGION) \ $(call tfvar, GCP_ZONE) \ $(call tfvar, DOMAIN_NAME) \ + $(call tfvar, ADDITIONAL_DOMAINS) \ $(call tfvar, ADDITIONAL_API_SERVICES_JSON) \ $(call tfvar, PREFIX) \ $(call tfvar, ALLOW_SANDBOX_INTERNET) \ diff --git a/iac/provider-gcp/main.tf b/iac/provider-gcp/main.tf index a1aca19798..5a8118d961 100644 --- a/iac/provider-gcp/main.tf +++ b/iac/provider-gcp/main.tf @@ -59,7 +59,12 @@ data "google_secret_manager_secret_version" "routing_domains" { } locals { - routing_domains = nonsensitive(jsondecode(data.google_secret_manager_secret_version.routing_domains.secret_data)) + // Taking additional domains from local env is there just for backward compatibility + additional_domains_from_secret = nonsensitive(jsondecode(data.google_secret_manager_secret_version.routing_domains.secret_data)) + additional_domains_from_env = (var.additional_domains != "" ? + [for item in split(",", var.additional_domains) : trimspace(item)] : []) + + additional_domains = distinct(concat(local.additional_domains_from_env, local.additional_domains_from_secret)) } module "init" { @@ -124,8 +129,8 @@ module "cluster" { nomad_port = var.nomad_port google_service_account_email = module.init.service_account_email domain_name = var.domain_name - additional_domains = local.routing_domains + additional_domains = local.additional_domains additional_api_services = (var.additional_api_services_json != "" ? jsondecode(var.additional_api_services_json) : []) diff --git a/iac/provider-gcp/variables.tf b/iac/provider-gcp/variables.tf index 21dfbe07c7..97942b6e1a 100644 --- a/iac/provider-gcp/variables.tf +++ b/iac/provider-gcp/variables.tf @@ -348,6 +348,12 @@ variable "domain_name" { description = "The domain name where e2b will run" } +variable "additional_domains" { + type = string + description = "Additional domains which can be used to access the e2b cluster, separated by commas" + default = "" +} + variable "additional_api_services_json" { type = string description = < Date: Thu, 9 Oct 2025 12:46:47 +0200 Subject: [PATCH 6/6] Removed outdatec comment --- iac/provider-gcp/nomad/jobs/ingress.hcl | 2 -- 1 file changed, 2 deletions(-) diff --git a/iac/provider-gcp/nomad/jobs/ingress.hcl b/iac/provider-gcp/nomad/jobs/ingress.hcl index b33450a2da..378eff328e 100644 --- a/iac/provider-gcp/nomad/jobs/ingress.hcl +++ b/iac/provider-gcp/nomad/jobs/ingress.hcl @@ -46,8 +46,6 @@ job "ingress" { task "ingress" { driver = "docker" - # If we need more than 30s we will need to update the max_kill_timeout in nomad - # https://developer.hashicorp.com/nomad/docs/configuration/client#max_kill_timeout %{ if update_stanza } kill_timeout = "24h" %{ endif }