Skip to content

Remove load balancer infra for logs collector #1243

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 9, 2025
Merged

Remove load balancer infra for logs collector #1243

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions iac/provider-gcp/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,6 @@ module "cluster" {
api_use_nat = var.api_use_nat
api_nat_ips = var.api_nat_ips

logs_health_proxy_port = var.logs_health_proxy_port
logs_proxy_port = var.logs_proxy_port

edge_api_port = var.edge_api_port
edge_proxy_port = var.edge_proxy_port
api_port = var.api_port
Expand Down Expand Up @@ -206,10 +203,6 @@ module "nomad" {

domain_name = var.domain_name

# Telemetry
logs_health_proxy_port = var.logs_health_proxy_port
logs_proxy_port = var.logs_proxy_port

# Logs
loki_node_pool = var.loki_node_pool
loki_machine_count = var.loki_cluster_size
Expand Down
4 changes: 1 addition & 3 deletions iac/provider-gcp/nomad-cluster/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,9 +110,7 @@ module "network" {
build_instance_group = google_compute_instance_group_manager.build_pool.instance_group
server_instance_group = google_compute_instance_group_manager.server_pool.instance_group

nomad_port = var.nomad_port
logs_proxy_port = var.logs_proxy_port
logs_health_proxy_port = var.logs_health_proxy_port
nomad_port = var.nomad_port

cluster_tag_name = var.cluster_tag_name

Expand Down
95 changes: 0 additions & 95 deletions iac/provider-gcp/nomad-cluster/network/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,14 +100,6 @@ locals {
health_checked_backends = { for backend_index, backend_value in local.backends : backend_index => backend_value }
}

# ======== IP ADDRESSES ====================

// todo: (2025-09-22): this can be removed when all orchestrator will be rolled with internal logs collector server
resource "google_compute_global_address" "orch_logs_ip" {
name = "${var.prefix}logs-ip"
}


# ======== CLOUDFLARE ====================

data "cloudflare_zone" "domain" {
Expand Down Expand Up @@ -441,73 +433,6 @@ resource "google_compute_security_policy" "default" {
}
}

module "gce_lb_http_logs" {
source = "GoogleCloudPlatform/lb-http/google"
version = "~> 12.1"
name = "${var.prefix}external-logs-endpoint"
project = var.gcp_project_id
address = google_compute_global_address.orch_logs_ip.address
create_address = false
target_tags = [var.cluster_tag_name]
firewall_networks = [var.network_name]

labels = var.labels
backends = {
default = {
description = null
protocol = "HTTP"
port = var.logs_proxy_port.port
port_name = var.logs_proxy_port.name
timeout_sec = 20
connection_draining_timeout_sec = 1
enable_cdn = false
session_affinity = null
affinity_cookie_ttl_sec = null
custom_request_headers = null
custom_response_headers = null
security_policy = google_compute_security_policy.disable-bots-log-collector.self_link

health_check = {
check_interval_sec = null
timeout_sec = null
healthy_threshold = null
unhealthy_threshold = null
request_path = var.logs_health_proxy_port.health_path
port = var.logs_health_proxy_port.port
host = null
logging = null
}

log_config = {
enable = false
sample_rate = 0.0
}

groups = [
{
group = var.client_instance_group
balancing_mode = null
capacity_scaler = null
description = null
max_connections = null
max_connections_per_instance = null
max_connections_per_endpoint = null
max_rate = null
max_rate_per_instance = null
max_rate_per_endpoint = null
max_utilization = null
},
]

iap_config = {
enable = false
oauth2_client_id = ""
oauth2_client_secret = ""
}
}
}
}

# Firewalls
resource "google_compute_firewall" "default-hc" {
name = "${var.prefix}load-balancer-hc"
Expand Down Expand Up @@ -558,26 +483,6 @@ resource "google_compute_firewall" "client_proxy_firewall_ingress" {
source_ranges = ["130.211.0.0/22", "35.191.0.0/16"]
}

resource "google_compute_firewall" "logs_collector_firewall_ingress" {
name = "${var.prefix}${var.cluster_tag_name}-logs-collector-firewall-ingress"
network = var.network_name

allow {
protocol = "tcp"
# Health end point is already added by load balancer module automatically, but also adding it here just to make sure we don't remove it by accident
ports = [var.logs_proxy_port.port, var.logs_health_proxy_port.port]
}

priority = 999

direction = "INGRESS"
target_tags = [var.cluster_tag_name]
# Load balancer health check IP ranges
# https://cloud.google.com/load-balancing/docs/health-check-concepts
source_ranges = ["130.211.0.0/22", "35.191.0.0/16"]
}


resource "google_compute_firewall" "internal_remote_connection_firewall_ingress" {
name = "${var.prefix}${var.cluster_tag_name}-internal-remote-connection-firewall-ingress"
network = var.network_name
Expand Down
3 changes: 0 additions & 3 deletions iac/provider-gcp/nomad-cluster/network/outputs.tf
View file
Open in desktop

This file was deleted.

15 changes: 0 additions & 15 deletions iac/provider-gcp/nomad-cluster/network/variables.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,21 +74,6 @@ variable "client_proxy_port" {
})
}

variable "logs_proxy_port" {
type = object({
name = string
port = number
})
}

variable "logs_health_proxy_port" {
type = object({
name = string
port = number
health_path = string
})
}

variable "nomad_port" {
type = number
}
Expand Down
10 changes: 0 additions & 10 deletions iac/provider-gcp/nomad-cluster/nodepool-client.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,16 +74,6 @@ resource "google_compute_region_instance_group_manager" "client_pool" {
instance_template = google_compute_instance_template.client.id
}

named_port {
name = var.logs_health_proxy_port.name
port = var.logs_health_proxy_port.port
}

named_port {
name = var.logs_proxy_port.name
port = var.logs_proxy_port.port
}

auto_healing_policies {
health_check = google_compute_health_check.client_nomad_check.id
initial_delay_sec = 600
Expand Down
4 changes: 0 additions & 4 deletions iac/provider-gcp/nomad-cluster/outputs.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,3 @@
output "logs_proxy_ip" {
value = module.network.logs_proxy_ip
}

output "shared_chunk_cache_path" {
value = var.filestore_cache_enabled ? "${local.nfs_mount_path}/${local.nfs_mount_subdir}" : ""
}
16 changes: 0 additions & 16 deletions iac/provider-gcp/nomad-cluster/variables.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,22 +155,6 @@ variable "network_name" {
default = "default"
}

variable "logs_proxy_port" {
type = object({
name = string
port = number
})
}

variable "logs_health_proxy_port" {
type = object({
name = string
port = number
health_path = string
})
}


variable "google_service_account_email" {
type = string
}
Expand Down
9 changes: 9 additions & 0 deletions iac/provider-gcp/nomad/variables.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,10 @@ variable "logs_proxy_port" {
name = string
port = number
})
default = {
name = "logs"
port = 30006
}
}

variable "logs_health_proxy_port" {
Expand All @@ -191,6 +195,11 @@ variable "logs_health_proxy_port" {
port = number
health_path = string
})
default = {
name = "logs-health"
port = 44313
health_path = "/health"
}
}

variable "analytics_collector_host_secret_name" {
Expand Down
24 changes: 0 additions & 24 deletions iac/provider-gcp/variables.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,30 +195,6 @@ variable "edge_proxy_port" {
}
}

variable "logs_proxy_port" {
type = object({
name = string
port = number
})
default = {
name = "logs"
port = 30006
}
}

variable "logs_health_proxy_port" {
type = object({
name = string
port = number
health_path = string
})
default = {
name = "logs-health"
port = 44313
health_path = "/health"
}
}

variable "loki_cluster_size" {
type = number
default = 0
Expand Down