Merge branch 'uffd-extract' into use-uffd-wp-to-only-save-dirty-pages-written-to-eng-3093

Author: ValentaTomas

.golangci.yml

@@ -27,7 +27,6 @@ linters:
     - errchkjson
     - exhaustive
     - exhaustruct
-    - forbidigo  # enable this when we're committed to avoiding os.Getenv
     - funcorder
     - funlen
     - gochecknoglobals
@@ -70,11 +69,12 @@ linters:
     # disabled intentionally
 
   settings:
-    # enable this when we're committed to avoiding os.Getenv
-    #    forbidigo:
-    #      forbid:
-    #        - pattern: os\.Getenv
-    #          msg: Add your field to the configuration model instead.
+    forbidigo:
+      forbid:
+        - pattern: "^new$"
+          msg: "Use &Type{} instead."
+#        - pattern: os\.Getenv
+#          msg: "Add your field to the configuration model instead."
 
     gocritic:
       disabled-checks:
@@ -101,25 +101,21 @@ linters:
         - { disabled: true, name: function-length }
         - { disabled: true, name: function-result-limit }
         - { disabled: true, name: get-return }
-        - { disabled: true, name: if-return }
         - { disabled: true, name: import-alias-naming }
         - { disabled: true, name: import-shadowing }
         - { disabled: true, name: line-length-limit }
         - { disabled: true, name: max-control-nesting }
         - { disabled: true, name: max-public-structs }
         - { disabled: true, name: nested-structs }
         - { disabled: true, name: package-comments }
-        - { disabled: true, name: redefines-builtin-id }
         - { disabled: true, name: unchecked-type-assertion }
         - { disabled: true, name: unexported-naming }
         - { disabled: true, name: unexported-return }
         - { disabled: true, name: unhandled-error }  #  todo: enable this
         - { disabled: true, name: unnecessary-format }  # todo: enable this
         - { disabled: true, name: unnecessary-stmt }  # todo: enable this
-        - { disabled: true, name: unused-parameter }  # todo: enable this
         - { disabled: true, name: unused-receiver }
         - { disabled: true, name: use-errors-new }  # todo: enable this
-        - { disabled: true, name: useless-break }  # todo: enable this
         - { disabled: true, name: var-declaration }
         - { disabled: true, name: var-naming }
 
@@ -136,3 +132,7 @@ linters:
 
 run:
   go: 1.24.7
+
+issues:
+  max-issues-per-linter: 50
+  max-same-issues: 50

DEV-LOCAL.md

@@ -0,0 +1,14 @@
+# Develop the application locally
+
+1. `sudo modprobe nbd nbds_max=64`
+2. `make local-infra`: runs clickhouse, grafana, loki, memcached, mimir, otel, postgres, redis, tempo
+3. `cd packages/db && make migrate-local` initialize the database
+4. `cd packages/local-dev && go run seed-local-database.go` generate user, team, and token for local development 
+5. `cd packages/api && make run-local` run the api locally 
+6. `cd packages/orchestrator && make build-debug && sudo make run-local` run the orchestrator and template-manager locally.
+7. `cd packages/client-proxy && make run-local` run the client-proxy locally.
+
+# Services
+- grafana: http://localhost:53000)
+- postgres: postgres:postgres@127.0.0.1:5432
+- clickhouse: clickhouse:clickhouse@127.0.0.1:9000

Makefile

@@ -162,3 +162,7 @@ generate-mocks:
 .PHONY: tidy
 tidy:
 	scripts/golang-dependencies-integrity.sh
+
+.PHONY: local-infra
+local-infra:
+	docker compose --file ./packages/local-dev/docker-compose.yaml up --abort-on-container-failure

go.work

@@ -7,6 +7,7 @@ use (
 	./packages/db
 	./packages/docker-reverse-proxy
 	./packages/envd
+	./packages/local-dev
 	./packages/orchestrator
 	./packages/shared
 

iac/provider-gcp/Makefile

@@ -20,6 +20,8 @@ tf_vars := 	TF_VAR_environment=$(TERRAFORM_ENVIRONMENT) \
 	$(call tfvar, CLIENT_CLUSTER_CACHE_DISK_SIZE_GB) \
 	$(call tfvar, API_MACHINE_TYPE) \
 	$(call tfvar, API_CLUSTER_SIZE) \
+	$(call tfvar, API_USE_NAT) \
+	$(call tfvar, API_NAT_IPS) \
 	$(call tfvar, BUILD_MACHINE_TYPE) \
 	$(call tfvar, BUILD_CLUSTER_SIZE) \
 	$(call tfvar, BUILD_CLUSTER_ROOT_DISK_SIZE_GB) \

iac/provider-gcp/main.tf

@@ -105,8 +105,8 @@ module "cluster" {
   loki_node_pool         = var.loki_node_pool
   orchestrator_node_pool = var.orchestrator_node_pool
 
-  logs_health_proxy_port = var.logs_health_proxy_port
-  logs_proxy_port        = var.logs_proxy_port
+  api_use_nat = var.api_use_nat
+  api_nat_ips = var.api_nat_ips
 
   edge_api_port                = var.edge_api_port
   edge_proxy_port              = var.edge_proxy_port
@@ -203,10 +203,6 @@ module "nomad" {
 
   domain_name = var.domain_name
 
-  # Telemetry
-  logs_health_proxy_port = var.logs_health_proxy_port
-  logs_proxy_port        = var.logs_proxy_port
-
   # Logs
   loki_node_pool           = var.loki_node_pool
   loki_machine_count       = var.loki_cluster_size

iac/provider-gcp/nomad-cluster/main.tf

@@ -91,6 +91,10 @@ module "network" {
   cloudflare_api_token_secret_name = var.cloudflare_api_token_secret_name
 
   gcp_project_id = var.gcp_project_id
+  gcp_region     = var.gcp_region
+
+  api_use_nat = var.api_use_nat
+  api_nat_ips = var.api_nat_ips
 
   api_port                  = var.api_port
   docker_reverse_proxy_port = var.docker_reverse_proxy_port
@@ -106,9 +110,7 @@ module "network" {
   build_instance_group  = google_compute_instance_group_manager.build_pool.instance_group
   server_instance_group = google_compute_instance_group_manager.server_pool.instance_group
 
-  nomad_port             = var.nomad_port
-  logs_proxy_port        = var.logs_proxy_port
-  logs_health_proxy_port = var.logs_health_proxy_port
+  nomad_port = var.nomad_port
 
   cluster_tag_name = var.cluster_tag_name
 

iac/provider-gcp/nomad-cluster/network/main.tf

@@ -100,14 +100,6 @@ locals {
   health_checked_backends = { for backend_index, backend_value in local.backends : backend_index => backend_value }
 }
 
-# ======== IP ADDRESSES ====================
-
-// todo: (2025-09-22): this can be removed when all orchestrator will be rolled with internal logs collector server
-resource "google_compute_global_address" "orch_logs_ip" {
-  name = "${var.prefix}logs-ip"
-}
-
-
 # ======== CLOUDFLARE ====================
 
 data "cloudflare_zone" "domain" {
@@ -441,73 +433,6 @@ resource "google_compute_security_policy" "default" {
   }
 }
 
-module "gce_lb_http_logs" {
-  source            = "GoogleCloudPlatform/lb-http/google"
-  version           = "~> 12.1"
-  name              = "${var.prefix}external-logs-endpoint"
-  project           = var.gcp_project_id
-  address           = google_compute_global_address.orch_logs_ip.address
-  create_address    = false
-  target_tags       = [var.cluster_tag_name]
-  firewall_networks = [var.network_name]
-
-  labels = var.labels
-  backends = {
-    default = {
-      description                     = null
-      protocol                        = "HTTP"
-      port                            = var.logs_proxy_port.port
-      port_name                       = var.logs_proxy_port.name
-      timeout_sec                     = 20
-      connection_draining_timeout_sec = 1
-      enable_cdn                      = false
-      session_affinity                = null
-      affinity_cookie_ttl_sec         = null
-      custom_request_headers          = null
-      custom_response_headers         = null
-      security_policy                 = google_compute_security_policy.disable-bots-log-collector.self_link
-
-      health_check = {
-        check_interval_sec  = null
-        timeout_sec         = null
-        healthy_threshold   = null
-        unhealthy_threshold = null
-        request_path        = var.logs_health_proxy_port.health_path
-        port                = var.logs_health_proxy_port.port
-        host                = null
-        logging             = null
-      }
-
-      log_config = {
-        enable      = false
-        sample_rate = 0.0
-      }
-
-      groups = [
-        {
-          group                        = var.client_instance_group
-          balancing_mode               = null
-          capacity_scaler              = null
-          description                  = null
-          max_connections              = null
-          max_connections_per_instance = null
-          max_connections_per_endpoint = null
-          max_rate                     = null
-          max_rate_per_instance        = null
-          max_rate_per_endpoint        = null
-          max_utilization              = null
-        },
-      ]
-
-      iap_config = {
-        enable               = false
-        oauth2_client_id     = ""
-        oauth2_client_secret = ""
-      }
-    }
-  }
-}
-
 # Firewalls
 resource "google_compute_firewall" "default-hc" {
   name    = "${var.prefix}load-balancer-hc"
@@ -558,26 +483,6 @@ resource "google_compute_firewall" "client_proxy_firewall_ingress" {
   source_ranges = ["130.211.0.0/22", "35.191.0.0/16"]
 }
 
-resource "google_compute_firewall" "logs_collector_firewall_ingress" {
-  name    = "${var.prefix}${var.cluster_tag_name}-logs-collector-firewall-ingress"
-  network = var.network_name
-
-  allow {
-    protocol = "tcp"
-    # Health end point is already added by load balancer module automatically, but also adding it here just to make sure we don't remove it by accident
-    ports = [var.logs_proxy_port.port, var.logs_health_proxy_port.port]
-  }
-
-  priority = 999
-
-  direction   = "INGRESS"
-  target_tags = [var.cluster_tag_name]
-  # Load balancer health check IP ranges
-  # https://cloud.google.com/load-balancing/docs/health-check-concepts
-  source_ranges = ["130.211.0.0/22", "35.191.0.0/16"]
-}
-
-
 resource "google_compute_firewall" "internal_remote_connection_firewall_ingress" {
   name    = "${var.prefix}${var.cluster_tag_name}-internal-remote-connection-firewall-ingress"
   network = var.network_name
@@ -792,3 +697,37 @@ resource "google_compute_security_policy" "disable-bots-log-collector" {
     }
   }
 }
+
+# Cloud Router for NAT
+resource "google_compute_router" "nat_router" {
+  count   = var.api_use_nat ? 1 : 0
+  name    = "${var.prefix}nat-router"
+  network = var.network_name
+  region  = var.gcp_region
+}
+
+# Static IP addresses for NAT (only created if explicit IPs not provided)
+resource "google_compute_address" "nat_ips" {
+  count  = var.api_use_nat && length(var.api_nat_ips) == 0 ? 2 : 0
+  name   = "${var.prefix}nat-ip-${count.index + 1}"
+  region = var.gcp_region
+}
+
+# Cloud NAT for API nodes
+resource "google_compute_router_nat" "api_nat" {
+  count                              = var.api_use_nat ? 1 : 0
+  name                               = "${var.prefix}api-nat"
+  router                             = google_compute_router.nat_router[0].name
+  nat_ip_allocate_option             = "MANUAL_ONLY"
+  nat_ips                            = length(var.api_nat_ips) > 0 ? var.api_nat_ips : google_compute_address.nat_ips[*].self_link
+  source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
+
+  log_config {
+    enable = true
+    filter = "ERRORS_ONLY"
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}

iac/provider-gcp/nomad-cluster/network/outputs.tf

@@ -27,6 +27,17 @@ variable "gcp_project_id" {
   type = string
 }
 
+variable "gcp_region" {
+  type = string
+}
+
+variable "api_use_nat" {
+  type = bool
+}
+
+variable "api_nat_ips" {
+  type = list(string)
+}
 
 variable "cloudflare_api_token_secret_name" {
   type = string
@@ -63,21 +74,6 @@ variable "client_proxy_port" {
   })
 }
 
-variable "logs_proxy_port" {
-  type = object({
-    name = string
-    port = number
-  })
-}
-
-variable "logs_health_proxy_port" {
-  type = object({
-    name        = string
-    port        = number
-    health_path = string
-  })
-}
-
 variable "nomad_port" {
   type = number
 }