Skip to content

Commit ac88ba5

Browse files
upodroidupodroid
committed
enable nested virt & set single process oom kill
1 parent f3ac20e commit ac88ba5

File tree

8 files changed

+50
-71
lines changed

8 files changed

+50
-71
lines changed

‎infra/gcp/terraform/k8s-infra-prow-build/iam.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ module "iam" {
3232
"roles/secretmanager.secretAccessor" = [
3333
"serviceAccount:kubernetes-external-secrets@k8s-infra-prow-build.iam.gserviceaccount.com",
3434
"principal://iam.googleapis.com/projects/${module.project.project_number}/locations/global/workloadIdentityPools/${module.project.project_id}.svc.id.goog/subject/ns/external-secrets/sa/external-secrets",
35+
"principal://iam.googleapis.com/projects/180382678033/locations/global/workloadIdentityPools/k8s-infra-prow-build-trusted.svc.id.goog/subject/ns/external-secrets/sa/external-secrets",
3536
]
3637
}
3738
}

‎infra/gcp/terraform/k8s-infra-prow-build/main.tf‎

Lines changed: 26 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -90,14 +90,15 @@ module "prow_build_nodepool_c4_highmem_8_localssd" {
9090
"us-central1-c",
9191
"us-central1-f",
9292
]
93-
name = "pool6"
94-
initial_count = 1
95-
min_count = 1
96-
max_count = 80
97-
machine_type = "c4-highmem-8"
98-
disk_size_gb = 500
99-
disk_type = "hyperdisk-balanced"
100-
service_account = module.prow_build_cluster.cluster_node_sa.email
93+
name = "pool6"
94+
initial_count = 1
95+
min_count = 1
96+
max_count = 80
97+
machine_type = "c4-highmem-8-lssd"
98+
disk_size_gb = 100
99+
disk_type = "hyperdisk-balanced"
100+
enable_nested_virtualization = true
101+
service_account = module.prow_build_cluster.cluster_node_sa.email
101102
}
102103

103104
module "prow_build_nodepool_c4d_highmem_8_localssd" {
@@ -110,54 +111,15 @@ module "prow_build_nodepool_c4d_highmem_8_localssd" {
110111
"us-central1-b",
111112
"us-central1-c",
112113
]
113-
name = "pool7"
114-
initial_count = 1
115-
min_count = 10
116-
max_count = 80
117-
machine_type = "c4d-highmem-8-lssd" # has 2 local ssd disks attached
118-
disk_size_gb = 100
119-
disk_type = "hyperdisk-balanced"
120-
service_account = module.prow_build_cluster.cluster_node_sa.email
121-
}
122-
123-
124-
module "sig_node_node_pool_1_n4_highmem_8" {
125-
126-
source = "github.com/GoogleCloudPlatform/cloud-foundation-fabric//modules/gke-nodepool?ref=v39.0.0&depth=1"
127-
project_id = module.project.project_id
128-
name = "sig-node-pool1"
129-
location = module.prow_build_cluster.cluster.location
130-
cluster_name = module.prow_build_cluster.cluster.name
131-
132-
service_account = {
133-
email = module.prow_build_cluster.cluster_node_sa.email
134-
oauth_scopes = ["https://www.googleapis.com/auth/cloud-platform"]
135-
}
136-
137-
nodepool_config = {
138-
autoscaling = {
139-
max_node_count = 10
140-
min_node_count = 1 # 1 per zone
141-
}
142-
management = {
143-
auto_repair = true
144-
auto_upgrade = true
145-
}
146-
}
147-
148-
node_config = {
149-
machine_type = "n4-highmem-8"
150-
disk_type = "hyperdisk-balanced"
151-
image_type = "COS_CONTAINERD"
152-
gvnic = true
153-
workload_metadata_config_mode = "GKE_METADATA"
154-
shielded_instance_config = {
155-
enable_secure_boot = true
156-
}
157-
}
158-
159-
160-
taints = { dedicated = { value = "sig-node", effect = "NO_SCHEDULE" } }
114+
name = "pool7"
115+
initial_count = 1
116+
min_count = 10
117+
max_count = 80
118+
machine_type = "c4d-highmem-8-lssd" # has 1 local ssd disks attached
119+
disk_size_gb = 100
120+
disk_type = "hyperdisk-balanced"
121+
enable_nested_virtualization = true
122+
service_account = module.prow_build_cluster.cluster_node_sa.email
161123
}
162124

163125
module "prow_build_nodepool_c4a_highmem_8_localssd" {
@@ -170,13 +132,14 @@ module "prow_build_nodepool_c4a_highmem_8_localssd" {
170132
"us-central1-b",
171133
"us-central1-c",
172134
]
173-
name = "pool7-arm64"
174-
initial_count = 1
175-
min_count = 1
176-
max_count = 10
177-
machine_type = "c4a-highmem-8-lssd" # has 2 local ssd disks attached
178-
disk_size_gb = 100
179-
disk_type = "hyperdisk-balanced"
135+
name = "pool7-arm64"
136+
initial_count = 1
137+
min_count = 1
138+
max_count = 80
139+
machine_type = "c4a-highmem-8-lssd" # has 2 local ssd disks attached
140+
disk_size_gb = 100
141+
disk_type = "hyperdisk-balanced"
142+
enable_nested_virtualization = true
180143
// GKE automatically taints arm64 nodes
181144
// https://cloud.google.com/kubernetes-engine/docs/how-to/prepare-arm-workloads-for-deployment#overview
182145
service_account = module.prow_build_cluster.cluster_node_sa.email

‎infra/gcp/terraform/k8s-infra-prow-build/00-provider.tf‎ renamed to ‎infra/gcp/terraform/k8s-infra-prow-build/provider.tf‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,11 @@ terraform {
3030
required_providers {
3131
google = {
3232
source = "hashicorp/google"
33-
version = "~> 6.31.0"
33+
version = "~> 7.7.0"
3434
}
3535
google-beta = {
3636
source = "hashicorp/google-beta"
37-
version = "~> 6.31.0"
37+
version = "~> 7.7.0"
3838
}
3939
}
4040
}

‎infra/gcp/terraform/modules/gke-nodepool/main.tf‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@ resource "google_container_node_pool" "node_pool" {
4949

5050
service_account = var.service_account
5151
oauth_scopes = ["https://www.googleapis.com/auth/cloud-platform"]
52+
kubelet_config {
53+
single_process_oom_kill = false
54+
}
5255

5356
dynamic "ephemeral_storage_config" {
5457
for_each = var.ephemeral_local_ssd_count > 0 ? [var.ephemeral_local_ssd_count] : []
@@ -57,6 +60,11 @@ resource "google_container_node_pool" "node_pool" {
5760
}
5861
}
5962

63+
advanced_machine_features {
64+
enable_nested_virtualization = var.enable_nested_virtualization
65+
threads_per_core = 0
66+
}
67+
6068
// Needed for workload identity
6169
workload_metadata_config {
6270
mode = "GKE_METADATA"
@@ -72,6 +80,7 @@ resource "google_container_node_pool" "node_pool" {
7280
value = taint.value.value
7381
}
7482
}
83+
7584
}
7685

7786
// If we need to destroy the node pool, create the new one before destroying

‎infra/gcp/terraform/modules/gke-nodepool/variables.tf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,3 +107,9 @@ variable "service_account" {
107107
description = "The email address of the GCP Service Account to be associated with nodes in this node_pool"
108108
type = string
109109
}
110+
111+
variable "enable_nested_virtualization" {
112+
description = "Whether to enable nested virtualization on the node pool's VMs"
113+
type = bool
114+
default = false
115+
}

‎infra/gcp/terraform/modules/gke-nodepool/versions.tf‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,11 @@ terraform {
2020
required_providers {
2121
google = {
2222
source = "hashicorp/google"
23-
version = "~> 6.31.0"
23+
version = ">=6.31.0"
2424
}
2525
google-beta = {
2626
source = "hashicorp/google-beta"
27-
version = "~> 6.31.0"
27+
version = ">=6.31.0"
2828
}
2929
}
3030
}

‎infra/gcp/terraform/modules/gke-project/versions.tf‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,11 @@ terraform {
2020
required_providers {
2121
google = {
2222
source = "hashicorp/google"
23-
version = "~> 6.31.0"
23+
version = ">=6.31.0"
2424
}
2525
google-beta = {
2626
source = "hashicorp/google-beta"
27-
version = "~> 6.31.0"
27+
version = ">=6.31.0"
2828
}
2929
}
3030
}

‎infra/gcp/terraform/modules/workload-identity-service-account/versions.tf‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@ terraform {
1717
required_providers {
1818
google = {
1919
source = "hashicorp/google"
20-
version = "~> 6.31.0"
20+
version = ">=6.31.0"
2121
}
2222
google-beta = {
2323
source = "hashicorp/google-beta"
24-
version = "~> 6.31.0"
24+
version = ">=6.31.0"
2525
}
2626
}
2727
}

0 commit comments

Comments
 (0)