Implement ingress for easier services exposing #1314
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| resource "google_compute_health_check" "ingress" { | ||
| name = "${var.prefix}ingress" | ||
|
|
||
| timeout_sec = 3 | ||
| check_interval_sec = 5 | ||
| healthy_threshold = 2 | ||
| unhealthy_threshold = 2 | ||
|
|
||
| http_health_check { | ||
| port = var.ingress_port.port | ||
| request_path = var.ingress_port.health_path | ||
| } | ||
| } | ||
|
|
||
| resource "google_compute_backend_service" "ingress" { | ||
| name = "${var.prefix}ingress" | ||
|
|
||
| protocol = "HTTP" | ||
| port_name = var.ingress_port.name | ||
|
|
||
| session_affinity = null | ||
| health_checks = [google_compute_health_check.ingress.id] | ||
|
|
||
| timeout_sec = 65 | ||
|
|
||
| load_balancing_scheme = "EXTERNAL_MANAGED" | ||
| locality_lb_policy = "ROUND_ROBIN" | ||
|
|
||
| backend { | ||
| group = var.api_instance_group | ||
| } | ||
| } | ||
|
|
||
| resource "google_compute_url_map" "ingress" { | ||
|
There was a problem hiding this comment. Can we get a better name here? I think we now have two load balancers, "ingress" and "orch_map", neither of their names help understand what they do. Maybe "traefik" and "direct"? There was a problem hiding this comment. I like ingress as its common name and it describes what it is. Yes, "orch_map" is, in my opinion, a mistake, as it no longer makes sense. Ideally, I would like to transition away from the current load balancer once the migration is complete/rename it to something like "ingress-sandboxes" or a similar name to distinguish better. I don't like to call it Traefik, as we can switch the ingress backend at any time in the future, but I'm okay with you coming up with a better name. There was a problem hiding this comment. a note here, if we want to rename the There was a problem hiding this comment. I still don’t like that we would need two load balancers just because we cannot filter sandbox traffic. Will look into again tomorrow Yep, we can rename ingress to something else. Iam not sure about management/api as we can use it for something different in future. Ingress services sounds okay to me. There was a problem hiding this comment. I though it's actually quite nice to have separate LBs for user's sandbox traffic and our services traffic (different limitations, limits, HTTP support, etc), but maybe it's unnecessary There was a problem hiding this comment. Ideally, we should be able to match sandbox traffic to different rules (now it's catch-all fallback) so we can apply different limits/armor rules to them, then we don't need to have different LBs. For supporting newer versions of HTTP, etc, we can still relatively easily migrate everything, and I'm not sure if we would need some special LB that cannot handle both sandbox and services traffic. There was a problem hiding this comment. Okay, discovered that GCP Armor policy rule allows you to filter based on host regex, so we can use one shared backend and apply dynamic rules based on the domain there. This will solve our issue with needing two load balancers. I would stick with Example of a regexp that can catch sandbox traffic and apply rate limiting. In the same way, we can create rules for API limiting and other related restrictions. The good thing is that this only appends rules to already existing security policy, so we can push rules even from a private monorepo that will handle block/rate limit for services that are not open source. resource "google_compute_security_policy_rule" "sandbox-throttling-ip" {
security_policy = google_compute_security_policy.default["session"].name
action = "throttle"
priority = "500"
match {
expr {
expression = <<-EOT
request.headers["host"].matches("^(?i)[0-9]+-[a-z0-9-]+\\.e2b-jirka\\.dev$")
EOT
}
}
rate_limit_options {
conform_action = "allow"
exceed_action = "deny(429)"
enforce_on_key = ""
enforce_on_key_configs {
enforce_on_key_type = "IP"
}
rate_limit_threshold {
count = 40000
interval_sec = 60
}
}
description = "Requests to sandboxes from IP address"
}There was a problem hiding this comment. Internal docs that will contain all info related to migration and current state -> https://www.notion.so/e2bdev/Ingress-Migration-288b8c296873807a8264f1615602d11d |
||
| name = "${var.prefix}ingress" | ||
| default_service = google_compute_backend_service.ingress.self_link | ||
| } | ||
|
|
||
| resource "google_compute_global_forwarding_rule" "ingress" { | ||
| name = "${var.prefix}ingress-forward-http" | ||
| ip_protocol = "TCP" | ||
| port_range = "443" | ||
| load_balancing_scheme = "EXTERNAL_MANAGED" | ||
| ip_address = google_compute_global_address.ingress_ipv4.address | ||
| target = google_compute_target_https_proxy.ingress.self_link | ||
| } | ||
|
|
||
| resource "google_compute_global_address" "ingress_ipv4" { | ||
| name = "${var.prefix}ingress-ipv4" | ||
| ip_version = "IPV4" | ||
| } | ||
|
|
||
| resource "google_compute_target_https_proxy" "ingress" { | ||
| name = "${var.prefix}ingress-https" | ||
| url_map = google_compute_url_map.ingress.self_link | ||
|
|
||
| certificate_map = "//certificatemanager.googleapis.com/${google_certificate_manager_certificate_map.certificate_map.id}" | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| job "ingress" { | ||
| datacenters = ["${gcp_zone}"] | ||
| node_pool = "${node_pool}" | ||
| priority = 90 | ||
|
|
||
| group "ingress" { | ||
| count = ${count} | ||
|
|
||
| constraint { | ||
| operator = "distinct_hosts" | ||
| value = "true" | ||
| } | ||
|
|
||
| network { | ||
| port "ingress" { | ||
| static = "${ingress_port}" | ||
| } | ||
|
|
||
| port "control" { | ||
| static = "${control_port}" | ||
| } | ||
| } | ||
|
|
||
| # https://developer.hashicorp.com/nomad/docs/job-specification/update | ||
| %{ if update_stanza } | ||
| update { | ||
| max_parallel = 1 # Update only 1 node at a time | ||
| } | ||
| %{ endif } | ||
|
|
||
| service { | ||
| port = "ingress" | ||
| name = "ingress" | ||
| task = "ingress" | ||
|
|
||
| check { | ||
| type = "http" | ||
| name = "health" | ||
| path = "/ping" | ||
| interval = "3s" | ||
| timeout = "3s" | ||
| port = "${ingress_port}" | ||
| } | ||
| } | ||
|
|
||
| task "ingress" { | ||
| driver = "docker" | ||
|
|
||
| %{ if update_stanza } | ||
| kill_timeout = "24h" | ||
| %{ endif } | ||
|
|
||
| kill_signal = "SIGTERM" | ||
|
|
||
| config { | ||
| network_mode = "host" | ||
| image = "traefik:v3.5" | ||
| ports = ["control", "ingress"] | ||
| args = [ | ||
| # Entry-points that are set internally by Traefik | ||
| "--entrypoints.web.address=:${ingress_port}", | ||
| "--entrypoints.traefik.address=:${control_port}", | ||
|
|
||
| # Traefik internals (logging, metrics, ...) | ||
| "--api.dashboard=true", | ||
| "--api.insecure=false", | ||
|
|
||
| "--accesslog=true", | ||
| "--ping=true", | ||
| "--ping.entryPoint=web", | ||
| "--metrics=true", | ||
| "--metrics.prometheus=true", | ||
| "--metrics.prometheus.entryPoint=traefik", | ||
|
|
||
| # Traefik Nomad provider | ||
| "--providers.nomad=true", | ||
| "--providers.nomad.endpoint.address=${nomad_endpoint}", | ||
| "--providers.nomad.endpoint.token=${nomad_token}", | ||
|
|
||
| # Traefik Consul provider | ||
| "--providers.consulcatalog=true", | ||
| "--providers.consulcatalog.exposedByDefault=false", | ||
| "--providers.consulcatalog.endpoint.address=${consul_endpoint}", | ||
| "--providers.consulcatalog.endpoint.token=${consul_token}", | ||
sitole marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| ] | ||
| } | ||
|
|
||
| resources { | ||
| memory_max = ${memory_mb * 1.5} | ||
| memory = ${memory_mb} | ||
| cpu = ${cpu_count * 1000} | ||
| } | ||
| } | ||
| } | ||
| } | ||
Uh oh!
There was an error while loading. Please reload this page.