diff --git a/README.md b/README.md index 967aaf0..b6147fd 100644 --- a/README.md +++ b/README.md @@ -4,15 +4,21 @@ This repository contains the non-sensitive Kubernetes declarations powering the Secrets and credentials are managed separately in a Blackbox repository: [tfwiki/secrets](https://github.com/tfwiki/secrets) +> **Warning** +> We are migrating away from managing Kubernetes resources directly via manifest files (fiddly and error-prone) to managing them via Terraform. +> +> See the [terraform](./terraform) folder for progress on this migration + Rough notes: -### Prerequisites -* Kubernetes cluster running 1.8.x (to avoid hardcoding NFS Service IP in PersistantVolume declaration) -* Cloud SQL database `cloudsql-instance-credentials` https://cloud.google.com/sql/docs/mysql/connect-kubernetes-engine -* Persistant disk for mediawiki images (mounted via NFS) -* Global Static IP address +## Prerequisites + +- Kubernetes cluster running 1.8.x (to avoid hardcoding NFS Service IP in PersistantVolume declaration) +- Cloud SQL database `cloudsql-instance-credentials` +- Persistant disk for mediawiki images (mounted via NFS) +- Global Static IP address -### Task list +## Task list 1. Create cluster in Google Container Engine 2. Work on correct zone (`gcloud config set compute/zone [COMPUTE-ZONE]`) @@ -26,6 +32,6 @@ Rough notes: Syncing files from the Valve-hosted wiki is managed via the [`media-sync`](k8s/common/media-sync.yaml) job, but needs authorised SSH keys stored within a Kubernetes secret: -``` +```sh kubectl create secret generic media-sync-secret --from-file=ssh-privatekey=/path/to/.ssh/id_rsa --from-file=ssh-publickey=/path/to/.ssh/id_rsa.pub -``` \ No newline at end of file +``` diff --git a/terraform/.gitignore b/terraform/.gitignore new file mode 100644 index 0000000..35e05c9 --- /dev/null +++ b/terraform/.gitignore @@ -0,0 +1,34 @@ +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log + +# Exclude all .tfvars files, which are likely to contain sentitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +# +*.tfvars + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc \ No newline at end of file diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..6a7ced7 --- /dev/null +++ b/terraform/.terraform.lock.hcl @@ -0,0 +1,42 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "3.90.0" + constraints = "3.90.0" + hashes = [ + "h1:bhQD7Fb9BN+VdFjGSzoeCEfD9JrF7jTlnPX0Bhwtsc8=", + "h1:pbFJXXg4R/W6GeohQnpJp3qxFbAbVPZnLvL2BNNwVzc=", + "zh:177d6178fbe69a913915aee3a5cbe6d79e38539b0ad99d23df6f8dc9256ccce9", + "zh:3b614c8e6b263ead3cc33aabcc35a82312693b4f305e7a8ce8d9c8e5476bf163", + "zh:3da294f7f59266af1b0ffe8d2b482bd02adfa9aa63b87fdd349666cc759a969a", + "zh:5954c277868e5d57aafb79f7c97dd701ccc4b1c66828a6fc065fbdf96cbc1410", + "zh:6e1c52f99a97cc5fb467e131fac68785b3a4f1f519573256fad3656ff8557a05", + "zh:7fa3348306cab6aa2510cb10e69467abb0d204ede3640bf22e181c727c272c39", + "zh:8af593ad183128bbfdf5ea9534f332c3c88850884e88248786a21144f3cfe918", + "zh:90f4cd713ef8c361e34b4ee465c733f5ec1c6d395a022b41a3413240137550bd", + "zh:a71ea0ca26e96f2c62c8a69788daac30fbf093f7ff2eff4b4f9a0ce21cf860bd", + "zh:f9bad74242849708608cb26a8b95d25df5928d5a5e45e9aa6c43731b09dd6a8b", + "zh:fdc83cb240e4510a6a1c8f0269ac67b0096114ba5a05b43a6eca293b6f62e1c0", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.6.1" + constraints = ">= 2.0.3, 2.6.1" + hashes = [ + "h1:DWgawNO2C7IuXC2v9IjTSsqs1vZHSAbP4ilWQ0LdbwI=", + "h1:aw4mTyVx41Y/+lAGNJcRvylF2j0sRQD6up5/CXmj9ds=", + "zh:081fbaf9441ebb278753dcf05f318fa7d445e9599a600d7c525e9a18b871d4c8", + "zh:143bfbe871c628981d756ead47486e807fce876232d05607e0b8852ebee4eed8", + "zh:34f413a644eb952e3f041d67ef19200f4c286d374eae87b60fafdd8bf6bb5654", + "zh:370562be70233be730e1876d565710c3ef477e047f209cb3dff8a4a3217a6461", + "zh:443021df6d56e59e4d8dda8e57b506affff32b8a22de09661d21b98bc781fefb", + "zh:51a9501360b58adf9ee6e09fb81f555042ebc909ab36e06ccfc5e701e91f9923", + "zh:7d41d48b8291b98e0a4b7a1f79a9d1fe140a2e0d8df422c5b48cbae4c3fa615a", + "zh:881b3e44814d7d49a5820e2e4b13ee3d000b5baf7957df774a909f17472ece8a", + "zh:b860ff68a944de63fbe0a624c41f2e373711a2da4298c0f0cb151e00fb32a6b3", + "zh:c4ab48ea6e0f8d4a6db1abab1877addb2b21ecd126e505c74b8c85804bd92cbe", + "zh:e96589575dfd31eab48fcc85466dd49895925473c60c802b346cdb4037953350", + ] +} diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 0000000..448367f --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,39 @@ +# Terraform deployment stuffs + +All the Kubernetes stuff, but in Terraform so its easier to deal with. + +Resources to track/import: + +- Cluster itself + - [x] GKE cluster + - [x] GKE node pool +- Supporting infrastructure + - [ ] Blackfire + - [x] Ingress + - [ ] Cert manager ?? + - [ ] Filestore + - [ ] CloudSQL database + - [ ] (any other external resources?) +- Kubernetes deployments + - [x] Cloudsql-proxy daemonset + - [x] mcrouter daemonset + - [x] Mediawiki deployment + - [x] Mediawiki-update deployment + - [x] Memcached stateful set + - [x] Run-jobs deployment + - [x] Update special pages cron job + - [x] Varnish deployment +- Kubernetes services + - [x] all-varnish + - [x] cloudsql-proxy + - [x] mcrouter + - [x] mediawiki + - [x] memcached + - [x] nfs-server + - [x] nfs-varnish + +TODO + +- Extract appropriate variables from kubernetes configs +- Replace hardcoded resource references with usage of resource attributes +- Set up remote Terraform state diff --git a/terraform/config.tf b/terraform/config.tf new file mode 100644 index 0000000..b321d56 --- /dev/null +++ b/terraform/config.tf @@ -0,0 +1 @@ +# TODO: Config and secrets \ No newline at end of file diff --git a/terraform/gke-cluster/main.tf b/terraform/gke-cluster/main.tf new file mode 100644 index 0000000..5b848a4 --- /dev/null +++ b/terraform/gke-cluster/main.tf @@ -0,0 +1,59 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "3.90.0" + } + } +} + +data "google_container_engine_versions" "supported" { + location = var.google_zone + version_prefix = var.kubernetes_version +} + +resource "google_container_cluster" "default" { + name = var.cluster_name + location = var.google_zone + min_master_version = data.google_container_engine_versions.supported.latest_master_version + # node version must match master version + # https://www.terraform.io/docs/providers/google/r/container_cluster.html#node_version + node_version = data.google_container_engine_versions.supported.latest_master_version + initial_node_count = 0 + + resource_labels = { + "env" = var.env_label + } +} +resource "google_container_node_pool" "highcpu" { + name = "high-cpu-pool" + cluster = var.cluster_name + + node_locations = [ + var.google_zone + ] + + node_config { + machine_type = var.machine_type + + oauth_scopes = [ + "https://www.googleapis.com/auth/compute", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/service.management", + "https://www.googleapis.com/auth/servicecontrol", + ] + } + + autoscaling { + max_node_count = 9 + min_node_count = 3 + } + + depends_on = [ + # Can't directly reference this for cluster_name because it'll force a + # replacement due to diff name formats + google_container_cluster.default + ] +} diff --git a/terraform/gke-cluster/output.tf b/terraform/gke-cluster/output.tf new file mode 100644 index 0000000..f26dd5b --- /dev/null +++ b/terraform/gke-cluster/output.tf @@ -0,0 +1,7 @@ +output "node_version" { + value = google_container_cluster.default.node_version +} + +output "google_zone" { + value = var.google_zone +} diff --git a/terraform/gke-cluster/variables.tf b/terraform/gke-cluster/variables.tf new file mode 100644 index 0000000..7412fbf --- /dev/null +++ b/terraform/gke-cluster/variables.tf @@ -0,0 +1,20 @@ +variable "kubernetes_version" { + default = "1.18" +} + +variable "cluster_name" { + type = string +} + +variable "google_zone" { + type = string +} + +variable "machine_type" { + type = string + default = "n1-highcpu-32" +} + +variable "env_label" { + type = string +} diff --git a/terraform/kubernetes-config/cloudsql-proxy.tf b/terraform/kubernetes-config/cloudsql-proxy.tf new file mode 100644 index 0000000..19d8c3e --- /dev/null +++ b/terraform/kubernetes-config/cloudsql-proxy.tf @@ -0,0 +1,134 @@ +resource "kubernetes_service" "cloudsql_proxy" { + metadata { + name = "cloudsql-proxy" + + labels = { + app = "cloudsql-proxy" + } + } + + spec { + port { + name = "cloudsql-proxy" + protocol = "TCP" + port = 3306 + target_port = "cloudsql-proxy" + } + + selector = { + app = "cloudsql-proxy" + } + + type = "NodePort" + session_affinity = "None" + external_traffic_policy = "Cluster" + } +} + +resource "kubernetes_daemonset" "cloudsql_proxy" { + metadata { + name = "cloudsql-proxy" + + labels = { + app = "cloudsql-proxy" + } + } + + spec { + selector { + match_labels = { + app = "cloudsql-proxy" + } + } + + template { + metadata { + labels = { + app = "cloudsql-proxy" + } + } + + spec { + volume { + name = "cloudsql-instance-credentials" + + secret { + secret_name = "cloudsql-instance-credentials" + default_mode = "0644" + } + } + + volume { + name = "ssl-certs" + + host_path { + path = "/etc/ssl/certs" + } + } + + volume { + name = "cloudsql" + } + + container { + name = "cloudsql-proxy" + image = "gcr.io/cloudsql-docker/gce-proxy:1.11" + + # TODO: Extract variables + command = ["/cloud_sql_proxy", "--dir=/cloudsql", "-instances=tfwiki-182108:us-west1:tfwiki-production=tcp:0.0.0.0:3306", "-credential_file=/secrets/cloudsql/credentials.json"] + + port { + name = "cloudsql-proxy" + container_port = 3306 + protocol = "TCP" + } + + resources { + limits = { + cpu = "1024m" + + memory = "512Mi" + } + + requests = { + cpu = "512m" + + memory = "128Mi" + } + } + + volume_mount { + name = "cloudsql-instance-credentials" + read_only = true + mount_path = "/secrets/cloudsql" + } + + volume_mount { + name = "ssl-certs" + mount_path = "/etc/ssl/certs" + } + + volume_mount { + name = "cloudsql" + mount_path = "/cloudsql" + } + + termination_message_path = "/dev/termination-log" + termination_message_policy = "File" + image_pull_policy = "IfNotPresent" + } + + restart_policy = "Always" + termination_grace_period_seconds = 30 + dns_policy = "ClusterFirst" + } + } + + strategy { + type = "RollingUpdate" + } + + revision_history_limit = 10 + } +} + diff --git a/terraform/kubernetes-config/filestore.tf b/terraform/kubernetes-config/filestore.tf new file mode 100644 index 0000000..2ef5938 --- /dev/null +++ b/terraform/kubernetes-config/filestore.tf @@ -0,0 +1,41 @@ +resource "kubernetes_persistent_volume" "tfwiki_media_prod" { + metadata { + name = "tfwiki-media-prod" + } + + spec { + capacity = { + storage = "1T" + } + + access_modes = ["ReadWriteMany"] + + persistent_volume_source { + nfs { + # TODO: extract variable(s) + path = "/tfwiki" + server = "10.155.167.82" + } + } + } +} + +resource "kubernetes_persistent_volume_claim" "tfwiki_media_prod_claim" { + metadata { + name = "tfwiki-media-prod-claim" + namespace = "default" + } + + spec { + access_modes = ["ReadWriteMany"] + + resources { + requests = { + storage = "1T" + } + } + + volume_name = "tfwiki-media-prod" + } +} + diff --git a/terraform/kubernetes-config/ingress.tf b/terraform/kubernetes-config/ingress.tf new file mode 100644 index 0000000..fc26524 --- /dev/null +++ b/terraform/kubernetes-config/ingress.tf @@ -0,0 +1,29 @@ +resource "kubernetes_ingress" "prod_ingress" { + metadata { + name = "prod-ingress" + + annotations = { + "acme.cert-manager.io/http01-edit-in-place" = "true" + "cert-manager.io/cluster-issuer" = "letsencrypt-prod" + "kubernetes.io/ingress.class" = "gce" + + // TODO: Extract variable(s) + "kubernetes.io/ingress.global-static-ip-name" = "tfwiki-production-static-ip" + } + } + + spec { + backend { + // TODO: Pull from service resource + service_name = "varnish" + service_port = "varnish" + } + + tls { + // TODO: Extract variable(s) + hosts = ["wiki.teamfortress.com", "wiki.tf2.com", "prod.wiki.tf"] + secret_name = "prod-tls" + } + } +} + diff --git a/terraform/kubernetes-config/main.tf b/terraform/kubernetes-config/main.tf new file mode 100644 index 0000000..6783726 --- /dev/null +++ b/terraform/kubernetes-config/main.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.0.3" + } + } +} diff --git a/terraform/kubernetes-config/mcrouter.tf b/terraform/kubernetes-config/mcrouter.tf new file mode 100644 index 0000000..c09c2b3 --- /dev/null +++ b/terraform/kubernetes-config/mcrouter.tf @@ -0,0 +1,131 @@ +resource "kubernetes_service" "mcrouter" { + metadata { + name = "mcrouter" + + labels = { + app = "mcrouter" + } + } + + spec { + port { + name = "mcrouter-port" + port = 5000 + target_port = "mcrouter-port" + } + + selector = { + app = "mcrouter" + } + + cluster_ip = "None" + } +} + +resource "kubernetes_config_map" "mcrouter" { + metadata { + name = "mcrouter" + + labels = { + app = "mcrouter" + } + } + + data = { + // TODO: Pull any of these from service definitions ??? + "config.json" = "{\n \"pools\": {\n \"A\": {\n \"servers\": [\n \"memcached-0.memcached.default.svc.cluster.local:11211\",\n \"memcached-1.memcached.default.svc.cluster.local:11211\",\n \"memcached-2.memcached.default.svc.cluster.local:11211\",\n \"memcached-3.memcached.default.svc.cluster.local:11211\",\n \"memcached-4.memcached.default.svc.cluster.local:11211\",\n \"memcached-5.memcached.default.svc.cluster.local:11211\",\n \"memcached-6.memcached.default.svc.cluster.local:11211\",\n \"memcached-7.memcached.default.svc.cluster.local:11211\",\n \"memcached-8.memcached.default.svc.cluster.local:11211\",\n \"memcached-9.memcached.default.svc.cluster.local:11211\",\n ]\n }\n },\n \"route\": \"PoolRoute|A\"\n}" + } +} + +resource "kubernetes_daemonset" "mcrouter" { + metadata { + name = "mcrouter" + + labels = { + app = "mcrouter" + } + } + + spec { + selector { + match_labels = { + app = "mcrouter" + } + } + + template { + metadata { + labels = { + app = "mcrouter" + } + } + + spec { + volume { + name = "config" + + config_map { + name = "mcrouter" + } + } + + container { + name = "mcrouter" + image = "jphalip/mcrouter:0.36.0" + command = ["mcrouter"] + args = ["-p 5000", "--config-file=/etc/mcrouter/config.json"] + + port { + name = "mcrouter-port" + host_port = 5000 + container_port = 5000 + } + + resources { + limits = { + cpu = "2" + + memory = "512Mi" + } + + requests = { + cpu = "1" + + memory = "128Mi" + } + } + + volume_mount { + name = "config" + mount_path = "/etc/mcrouter" + } + + liveness_probe { + tcp_socket { + port = "mcrouter-port" + } + + initial_delay_seconds = 30 + timeout_seconds = 5 + } + + readiness_probe { + tcp_socket { + port = "mcrouter-port" + } + + initial_delay_seconds = 5 + timeout_seconds = 1 + } + + image_pull_policy = "Always" + } + } + } + + strategy { + type = "RollingUpdate" + } + } +} + diff --git a/terraform/kubernetes-config/mediawiki-update.tf b/terraform/kubernetes-config/mediawiki-update.tf new file mode 100644 index 0000000..b5d2bb6 --- /dev/null +++ b/terraform/kubernetes-config/mediawiki-update.tf @@ -0,0 +1,342 @@ +resource "kubernetes_job" "mediawiki_update" { + # TODO: Create these per update, organised another way? Perhaps a `jobs/[whatever].tf`? + # How to avoid re-run if setting infra up from scratch, where jobs aren't relevant? + count = 0 + + metadata { + name = "mediawiki-update" + + labels = { + app = "mediawiki-update" + } + } + + spec { + backoff_limit = 4 + + template { + metadata { + labels = { + app = "mediawiki-update" + } + } + + spec { + volume { + name = "mediawiki-images" + + persistent_volume_claim { + // TODO: Pull from relevant resource? + claim_name = "tfwiki-media-prod-claim" + } + } + + // TODO: Can extract common env blocks across all mediawiki definitions? + container { + name = "mediawiki-update" + image = "tfwiki/mediawiki:1.31-tfwiki4" + command = ["php", "/var/www/html/w/maintenance/update.php", "--skip-external-dependencies"] + + env { + name = "NODE_NAME" + + value_from { + field_ref { + field_path = "spec.nodeName" + } + } + } + + env { + name = "MEMCACHED_HOST" + value = "$(NODE_NAME):5001" + } + + env { + name = "SERVER_URL" + + value_from { + config_map_key_ref { + name = "mediawiki-config" + key = "server_url" + } + } + } + + env { + name = "SITENAME" + + value_from { + config_map_key_ref { + name = "mediawiki-config" + key = "sitename" + optional = true + } + } + } + + env { + name = "VARNISH_HOST" + + value_from { + config_map_key_ref { + name = "mediawiki-config" + key = "varnish_host" + optional = true + } + } + } + + env { + name = "TRUSTED_PROXIES" + + value_from { + config_map_key_ref { + name = "mediawiki-config" + key = "trusted_proxies" + optional = true + } + } + } + + env { + name = "BLACKFIRE_SOCKET" + + value_from { + config_map_key_ref { + name = "mediawiki-config" + key = "blackfire_socket" + optional = true + } + } + } + + env { + name = "RECAPTCHA_KEY" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "recaptcha.key" + } + } + } + + env { + name = "RECAPTCHA_SECRET" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "recaptcha.secret" + } + } + } + + env { + name = "EMAIL_EMERGENCY_CONTACT" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "email.emergency_contact" + optional = true + } + } + } + + env { + name = "EMAIL_PASSWORD_SENDER" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "email.password_sender" + optional = true + } + } + } + + env { + name = "SENTRY_DSN" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "sentry.dsn" + optional = true + } + } + } + + env { + name = "SMTP_HOST" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.host" + optional = true + } + } + } + + env { + name = "SMTP_IDHOST" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.idhost" + optional = true + } + } + } + + env { + name = "SMTP_PORT" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.port" + optional = true + } + } + } + + env { + name = "SMTP_AUTH" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.auth" + optional = true + } + } + } + + env { + name = "SMTP_USERNAME" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.username" + optional = true + } + } + } + + env { + name = "SMTP_PASSWORD" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.password" + optional = true + } + } + } + + env { + name = "SECRET_KEY" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "secret_key" + } + } + } + + env { + name = "STEAM_API_KEY" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "steam.api.key" + optional = true + } + } + } + + env { + name = "DB_PASSWORD" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "db.password" + optional = true + } + } + } + + env { + name = "DB_DATABASE" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "db.database" + optional = true + } + } + } + + env { + name = "DB_HOST" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "db.host" + optional = true + } + } + } + + env { + name = "DB_TYPE" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "db.type" + optional = true + } + } + } + + env { + name = "DB_USER" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "db.user" + optional = true + } + } + } + + volume_mount { + // TODO: Is this a another resource we can reference?? + name = "mediawiki-images" + mount_path = "/var/www/html/w/images" + sub_path = "prod" + } + + image_pull_policy = "Always" + } + + restart_policy = "Never" + } + } + } +} + diff --git a/terraform/kubernetes-config/mediawiki.tf b/terraform/kubernetes-config/mediawiki.tf new file mode 100644 index 0000000..6d43f94 --- /dev/null +++ b/terraform/kubernetes-config/mediawiki.tf @@ -0,0 +1,453 @@ +resource "kubernetes_service" "mediawiki" { + metadata { + name = "mediawiki" + + labels = { + app = "mediawiki" + } + } + + spec { + port { + name = "mediawiki" + protocol = "TCP" + port = 80 + target_port = "mediawiki" + } + + selector = { + app = "mediawiki" + + tier = "frontend" + } + + type = "NodePort" + session_affinity = "None" + external_traffic_policy = "Cluster" + } +} + +resource "kubernetes_horizontal_pod_autoscaler" "mediawiki" { + metadata { + name = "mediawiki" + namespace = "default" + } + + spec { + scale_target_ref { + // TODO: Pull from relevant resource? + kind = "Deployment" + name = "mediawiki" + api_version = "apps/v1beta1" + } + + min_replicas = 6 + max_replicas = 18 + target_cpu_utilization_percentage = 80 + } +} + +resource "kubernetes_deployment" "mediawiki" { + metadata { + name = "mediawiki" + + labels = { + app = "mediawiki" + } + } + + spec { + selector { + match_labels = { + app = "mediawiki" + + tier = "frontend" + } + } + + template { + metadata { + labels = { + app = "mediawiki" + + tier = "frontend" + } + } + + spec { + volume { + name = "mediawiki-images" + + persistent_volume_claim { + // TODO: Pull from relevant resource? + claim_name = "tfwiki-media-prod-claim" + } + } + + // TODO: Can extract common env blocks across all mediawiki definitions? + container { + name = "mediawiki" + image = "tfwiki/mediawiki:1.31-tfwiki6" + + port { + name = "mediawiki" + container_port = 80 + protocol = "TCP" + } + + env { + name = "NODE_NAME" + + value_from { + field_ref { + field_path = "spec.nodeName" + } + } + } + + env { + name = "MEMCACHED_HOST" + value = "$(NODE_NAME):5000" + } + + env { + name = "SERVER_URL" + + value_from { + config_map_key_ref { + name = "mediawiki-config" + key = "server_url" + } + } + } + + env { + name = "SITENAME" + + value_from { + config_map_key_ref { + name = "mediawiki-config" + key = "sitename" + optional = true + } + } + } + + env { + name = "VARNISH_HOST" + + value_from { + config_map_key_ref { + name = "mediawiki-config" + key = "varnish_host" + optional = true + } + } + } + + env { + name = "TRUSTED_PROXIES" + + value_from { + config_map_key_ref { + name = "mediawiki-config" + key = "trusted_proxies" + optional = true + } + } + } + + env { + name = "BLACKFIRE_SOCKET" + + value_from { + config_map_key_ref { + name = "mediawiki-config" + key = "blackfire_socket" + optional = true + } + } + } + + env { + name = "RECAPTCHA_KEY" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "recaptcha.key" + } + } + } + + env { + name = "RECAPTCHA_SECRET" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "recaptcha.secret" + } + } + } + + env { + name = "EMAIL_EMERGENCY_CONTACT" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "email.emergency_contact" + optional = true + } + } + } + + env { + name = "EMAIL_PASSWORD_SENDER" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "email.password_sender" + optional = true + } + } + } + + env { + name = "SENTRY_DSN" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "sentry.dsn" + optional = true + } + } + } + + env { + name = "SMTP_HOST" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.host" + optional = true + } + } + } + + env { + name = "SMTP_IDHOST" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.idhost" + optional = true + } + } + } + + env { + name = "SMTP_PORT" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.port" + optional = true + } + } + } + + env { + name = "SMTP_AUTH" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.auth" + optional = true + } + } + } + + env { + name = "SMTP_USERNAME" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.username" + optional = true + } + } + } + + env { + name = "SMTP_PASSWORD" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.password" + optional = true + } + } + } + + env { + name = "SECRET_KEY" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "secret_key" + } + } + } + + env { + name = "STEAM_API_KEY" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "steam.api.key" + optional = true + } + } + } + + env { + name = "DB_PASSWORD" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "db.password" + optional = true + } + } + } + + env { + name = "DB_DATABASE" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "db.database" + optional = true + } + } + } + + env { + name = "DB_HOST" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "db.host" + optional = true + } + } + } + + env { + name = "DB_TYPE" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "db.type" + optional = true + } + } + } + + env { + name = "DB_USER" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "db.user" + optional = true + } + } + } + + resources { + limits = { + cpu = "10" + + memory = "6Gi" + } + + requests = { + cpu = "8" + + memory = "5Gi" + } + } + + volume_mount { + // TODO: Is this a another resource we can reference?? + name = "mediawiki-images" + mount_path = "/var/www/html/w/images" + sub_path = "prod" + } + + liveness_probe { + http_get { + path = "/wiki/Main_Page" + port = "80" + scheme = "HTTP" + } + + timeout_seconds = 5 + period_seconds = 10 + success_threshold = 1 + failure_threshold = 6 + } + + readiness_probe { + http_get { + path = "/wiki/Main_Page" + port = "80" + scheme = "HTTP" + } + + timeout_seconds = 5 + period_seconds = 10 + success_threshold = 1 + failure_threshold = 3 + } + + termination_message_path = "/dev/termination-log" + termination_message_policy = "File" + image_pull_policy = "Always" + } + + restart_policy = "Always" + termination_grace_period_seconds = 30 + dns_policy = "ClusterFirst" + } + } + + strategy { + type = "RollingUpdate" + + rolling_update { + max_unavailable = "1" + max_surge = "1" + } + } + } +} + diff --git a/terraform/kubernetes-config/memcache.tf b/terraform/kubernetes-config/memcache.tf new file mode 100644 index 0000000..a848ca1 --- /dev/null +++ b/terraform/kubernetes-config/memcache.tf @@ -0,0 +1,118 @@ +resource "kubernetes_service" "memcached" { + metadata { + name = "memcached" + + labels = { + app = "memcached" + } + } + + spec { + port { + name = "memcached" + protocol = "TCP" + port = 11211 + target_port = "memcached" + } + + selector = { + app = "memcached" + } + + cluster_ip = "None" + type = "ClusterIP" + session_affinity = "None" + } +} + +resource "kubernetes_stateful_set" "memcached" { + metadata { + name = "memcached" + + labels = { + app = "memcached" + } + } + + spec { + replicas = 10 + + selector { + match_labels = { + app = "memcached" + } + } + + template { + metadata { + labels = { + app = "memcached" + } + } + + spec { + container { + name = "memcached" + image = "memcached:latest" + command = ["memcached", "-m 1024", "-c 10000"] + + port { + name = "memcached" + container_port = 11211 + protocol = "TCP" + } + + resources { + requests = { + cpu = "100m" + + memory = "1Gi" + } + } + + liveness_probe { + tcp_socket { + port = "memcached" + } + + initial_delay_seconds = 30 + timeout_seconds = 5 + period_seconds = 10 + success_threshold = 1 + failure_threshold = 3 + } + + readiness_probe { + tcp_socket { + port = "memcached" + } + + initial_delay_seconds = 5 + timeout_seconds = 1 + period_seconds = 10 + success_threshold = 1 + failure_threshold = 3 + } + + termination_message_path = "/dev/termination-log" + termination_message_policy = "File" + image_pull_policy = "Always" + } + + restart_policy = "Always" + termination_grace_period_seconds = 30 + dns_policy = "ClusterFirst" + } + } + + service_name = "memcached" + pod_management_policy = "OrderedReady" + + update_strategy { + type = "OnDelete" + } + + revision_history_limit = 10 + } +} + diff --git a/terraform/kubernetes-config/run-jobs.tf b/terraform/kubernetes-config/run-jobs.tf new file mode 100644 index 0000000..7fbf5b2 --- /dev/null +++ b/terraform/kubernetes-config/run-jobs.tf @@ -0,0 +1,370 @@ +resource "kubernetes_deployment" "run_jobs" { + metadata { + name = "run-jobs" + + labels = { + app = "run-jobs" + } + } + + spec { + replicas = 4 + + selector { + match_labels = { + app = "run-jobs" + } + } + + template { + metadata { + labels = { + app = "run-jobs" + } + } + + spec { + volume { + name = "mediawiki-images" + + persistent_volume_claim { + # TODO: Pull from relevant resource? + claim_name = "tfwiki-media-prod-claim" + } + } + + # Reuse env definitions across all mediawiki resources? + container { + name = "run-jobs" + image = "tfwiki/mediawiki:1.31-tfwiki4" + command = ["php", "/var/www/html/w/maintenance/runJobs.php", "--wait", "--procs 1", "--maxjobs 100"] + + env { + name = "NODE_NAME" + + value_from { + field_ref { + field_path = "spec.nodeName" + } + } + } + + env { + name = "MEMCACHED_HOST" + value = "$(NODE_NAME):5001" + } + + env { + name = "SERVER_URL" + + value_from { + config_map_key_ref { + name = "mediawiki-config" + key = "server_url" + } + } + } + + env { + name = "SITENAME" + + value_from { + config_map_key_ref { + name = "mediawiki-config" + key = "sitename" + optional = true + } + } + } + + env { + name = "VARNISH_HOST" + + value_from { + config_map_key_ref { + name = "mediawiki-config" + key = "varnish_host" + optional = true + } + } + } + + env { + name = "TRUSTED_PROXIES" + + value_from { + config_map_key_ref { + name = "mediawiki-config" + key = "trusted_proxies" + optional = true + } + } + } + + env { + name = "BLACKFIRE_SOCKET" + + value_from { + config_map_key_ref { + name = "mediawiki-config" + key = "blackfire_socket" + optional = true + } + } + } + + env { + name = "RECAPTCHA_KEY" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "recaptcha.key" + } + } + } + + env { + name = "RECAPTCHA_SECRET" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "recaptcha.secret" + } + } + } + + env { + name = "EMAIL_EMERGENCY_CONTACT" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "email.emergency_contact" + optional = true + } + } + } + + env { + name = "EMAIL_PASSWORD_SENDER" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "email.password_sender" + optional = true + } + } + } + + env { + name = "SENTRY_DSN" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "sentry.dsn" + optional = true + } + } + } + + env { + name = "SMTP_HOST" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.host" + optional = true + } + } + } + + env { + name = "SMTP_IDHOST" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.idhost" + optional = true + } + } + } + + env { + name = "SMTP_PORT" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.port" + optional = true + } + } + } + + env { + name = "SMTP_AUTH" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.auth" + optional = true + } + } + } + + env { + name = "SMTP_USERNAME" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.username" + optional = true + } + } + } + + env { + name = "SMTP_PASSWORD" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.password" + optional = true + } + } + } + + env { + name = "SECRET_KEY" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "secret_key" + } + } + } + + env { + name = "STEAM_API_KEY" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "steam.api.key" + optional = true + } + } + } + + env { + name = "DB_PASSWORD" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "db.password" + optional = true + } + } + } + + env { + name = "DB_DATABASE" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "db.database" + optional = true + } + } + } + + env { + name = "DB_HOST" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "db.host" + optional = true + } + } + } + + env { + name = "DB_TYPE" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "db.type" + optional = true + } + } + } + + env { + name = "DB_USER" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "db.user" + optional = true + } + } + } + + resources { + limits = { + cpu = "2" + + memory = "1Gi" + } + + requests = { + cpu = "1" + + memory = "512Mi" + } + } + + volume_mount { + name = "mediawiki-images" + mount_path = "/var/www/html/w/images" + sub_path = "prod" + } + + termination_message_path = "/dev/termination-log" + termination_message_policy = "File" + image_pull_policy = "Always" + } + + restart_policy = "Always" + termination_grace_period_seconds = 30 + dns_policy = "ClusterFirst" + } + } + + strategy { + type = "RollingUpdate" + + rolling_update { + max_unavailable = "1" + max_surge = "1" + } + } + } +} + diff --git a/terraform/kubernetes-config/update-special-pages.tf b/terraform/kubernetes-config/update-special-pages.tf new file mode 100644 index 0000000..50822e9 --- /dev/null +++ b/terraform/kubernetes-config/update-special-pages.tf @@ -0,0 +1,351 @@ +resource "kubernetes_cron_job" "update_special_pages" { + metadata { + name = "update-special-pages" + + labels = { + app = "update-special-pages" + } + } + + spec { + schedule = "0 * * * *" + concurrency_policy = "Allow" + + job_template { + metadata {} + + spec { + template { + metadata { + labels = { + app = "update-special-pages" + } + } + + spec { + volume { + name = "mediawiki-images" + + # TODO: is this a resource we can ref? + persistent_volume_claim { + claim_name = "tfwiki-media-prod-claim" + } + } + + # Reuse env defs across all mediawikis resources? + container { + name = "update-special-pages" + image = "tfwiki/mediawiki:1.31-tfwiki4" + command = ["php", "/var/www/html/w/maintenance/updateSpecialPages.php"] + + env { + name = "NODE_NAME" + + value_from { + field_ref { + field_path = "spec.nodeName" + } + } + } + + env { + name = "MEMCACHED_HOST" + value = "$(NODE_NAME):5001" + } + + env { + name = "SERVER_URL" + + value_from { + config_map_key_ref { + name = "mediawiki-config" + key = "server_url" + } + } + } + + env { + name = "SITENAME" + + value_from { + config_map_key_ref { + name = "mediawiki-config" + key = "sitename" + optional = true + } + } + } + + env { + name = "VARNISH_HOST" + + value_from { + config_map_key_ref { + name = "mediawiki-config" + key = "varnish_host" + optional = true + } + } + } + + env { + name = "TRUSTED_PROXIES" + + value_from { + config_map_key_ref { + name = "mediawiki-config" + key = "trusted_proxies" + optional = true + } + } + } + + env { + name = "BLACKFIRE_SOCKET" + + value_from { + config_map_key_ref { + name = "mediawiki-config" + key = "blackfire_socket" + optional = true + } + } + } + + env { + name = "RECAPTCHA_KEY" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "recaptcha.key" + } + } + } + + env { + name = "RECAPTCHA_SECRET" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "recaptcha.secret" + } + } + } + + env { + name = "EMAIL_EMERGENCY_CONTACT" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "email.emergency_contact" + optional = true + } + } + } + + env { + name = "EMAIL_PASSWORD_SENDER" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "email.password_sender" + optional = true + } + } + } + + env { + name = "SENTRY_DSN" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "sentry.dsn" + optional = true + } + } + } + + env { + name = "SMTP_HOST" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.host" + optional = true + } + } + } + + env { + name = "SMTP_IDHOST" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.idhost" + optional = true + } + } + } + + env { + name = "SMTP_PORT" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.port" + optional = true + } + } + } + + env { + name = "SMTP_AUTH" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.auth" + optional = true + } + } + } + + env { + name = "SMTP_USERNAME" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.username" + optional = true + } + } + } + + env { + name = "SMTP_PASSWORD" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "smtp.password" + optional = true + } + } + } + + env { + name = "SECRET_KEY" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "secret_key" + } + } + } + + env { + name = "STEAM_API_KEY" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "steam.api.key" + optional = true + } + } + } + + env { + name = "DB_PASSWORD" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "db.password" + optional = true + } + } + } + + env { + name = "DB_DATABASE" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "db.database" + optional = true + } + } + } + + env { + name = "DB_HOST" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "db.host" + optional = true + } + } + } + + env { + name = "DB_TYPE" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "db.type" + optional = true + } + } + } + + env { + name = "DB_USER" + + value_from { + secret_key_ref { + name = "mediawiki-secret" + key = "db.user" + optional = true + } + } + } + + volume_mount { + name = "mediawiki-images" + mount_path = "/var/www/html/w/images" + sub_path = "prod" + } + + termination_message_path = "/dev/termination-log" + termination_message_policy = "File" + image_pull_policy = "Always" + } + + restart_policy = "OnFailure" + termination_grace_period_seconds = 30 + dns_policy = "ClusterFirst" + } + } + } + } + + successful_jobs_history_limit = 3 + failed_jobs_history_limit = 1 + } +} + diff --git a/terraform/kubernetes-config/variables.tf b/terraform/kubernetes-config/variables.tf new file mode 100644 index 0000000..51fc6ba --- /dev/null +++ b/terraform/kubernetes-config/variables.tf @@ -0,0 +1,3 @@ +variable "cluster_name" { + type = string +} \ No newline at end of file diff --git a/terraform/kubernetes-config/varnish.tf b/terraform/kubernetes-config/varnish.tf new file mode 100644 index 0000000..9c6deba --- /dev/null +++ b/terraform/kubernetes-config/varnish.tf @@ -0,0 +1,139 @@ +resource "kubernetes_service" "varnish" { + metadata { + name = "varnish" + } + + spec { + port { + name = "varnish" + protocol = "TCP" + port = 80 + target_port = "varnish" + } + + selector = { + app = "varnish" + } + + type = "NodePort" + session_affinity = "None" + external_traffic_policy = "Cluster" + } +} + +resource "kubernetes_service" "all_varnish" { + metadata { + name = "all-varnish" + } + + spec { + port { + name = "varnish" + protocol = "TCP" + port = 80 + target_port = "varnish" + } + + selector = { + app = "varnish" + } + + cluster_ip = "None" + type = "ClusterIP" + session_affinity = "None" + } +} + +resource "kubernetes_deployment" "varnish" { + metadata { + name = "varnish" + + labels = { + app = "varnish" + } + } + + spec { + replicas = 4 + + selector { + match_labels = { + app = "varnish" + } + } + + template { + metadata { + name = "varnish" + + labels = { + app = "varnish" + } + } + + spec { + container { + name = "varnish" + image = "tfwiki/varnish:1.0.3" + + port { + name = "varnish" + container_port = 80 + protocol = "TCP" + } + + env { + // TODO: Is this a another resource we can reference?? + name = "BACKEND_HOST" + value = "mediawiki" + } + + resources { + limits = { + cpu = "1024m" + + memory = "3Gi" + } + + requests = { + cpu = "512m" + + memory = "2Gi" + } + } + + readiness_probe { + http_get { + path = "/wiki/Main_Page" + port = "80" + scheme = "HTTP" + } + + timeout_seconds = 1 + period_seconds = 10 + success_threshold = 1 + failure_threshold = 3 + } + + termination_message_path = "/dev/termination-log" + termination_message_policy = "File" + image_pull_policy = "IfNotPresent" + } + + restart_policy = "Always" + termination_grace_period_seconds = 30 + dns_policy = "ClusterFirst" + } + } + + strategy { + type = "RollingUpdate" + + rolling_update { + max_unavailable = "1" + max_surge = "1" + } + } + } +} + diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..9aec3d5 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,25 @@ +# Configure kubernetes provider with Oauth2 access token. +# https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config +# This fetches a new token, which will expire in 1 hour. +data "google_client_config" "default" { + depends_on = [module.gke-cluster] +} + +# Defer reading the cluster data until the GKE cluster exists. +data "google_container_cluster" "default" { + name = var.cluster_name + depends_on = [module.gke-cluster] +} + +module "gke-cluster" { + source = "./gke-cluster" + cluster_name = var.cluster_name + google_zone = var.google_zone + env_label = var.env_label +} + +module "kubernetes-config" { + depends_on = [module.gke-cluster] + source = "./kubernetes-config" + cluster_name = var.cluster_name +} diff --git a/terraform/providers.tf b/terraform/providers.tf new file mode 100644 index 0000000..1b5f5ac --- /dev/null +++ b/terraform/providers.tf @@ -0,0 +1,13 @@ +provider "kubernetes" { + host = "https://${data.google_container_cluster.default.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode( + data.google_container_cluster.default.master_auth[0].cluster_ca_certificate, + ) +} + +provider "google" { + project = var.google_project + region = var.google_region + zone = var.google_zone +} diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000..9475dcb --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,19 @@ +variable "cluster_name" { + type = string +} + +variable "google_project" { + type = string +} + +variable "google_region" { + type = string +} + +variable "google_zone" { + type = string +} + +variable "env_label" { + type = string +} \ No newline at end of file diff --git a/terraform/versions.tf b/terraform/versions.tf new file mode 100644 index 0000000..3455f55 --- /dev/null +++ b/terraform/versions.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.6.1" + } + + google = { + source = "hashicorp/google" + version = "3.90.0" + } + } + + required_version = ">= 1.0.10" +}