Add Secrets CRUD

.env.template

@@ -65,6 +65,12 @@ TEMPLATE_BUCKET_NAME=
 # Hash seed used for generating sandbox access tokens, not needed if you are not using them
 SANDBOX_ACCESS_TOKEN_HASH_SEED=abcdefghijklmnopqrstuvwxyz
 
+# If you have a compatible vault instance running (pkg/vault) and want to use secrets egress proxy
+VAULT_ADDR=
+VAULT_APPROLE_ROLE_ID=
+VAULT_APPROLE_SECRET_ID=
+VAULT_TLS_CA=
+
 # Integration tests variables (only for running integration tests locally)
 # your domain name, e.g. https://api.great-innovations.dev
 TESTS_API_SERVER_URL=

.github/actions/start-services/action.yml

@@ -75,6 +75,10 @@ runs:
         make -C packages/clickhouse migrate-without-build
       shell: bash
 
+    - name: Run Vault
+      run: bash .github/actions/start-services/start-vault.sh
+      shell: bash
+
     - name: Start Services
       env:
         ENVD_TIMEOUT: "60s"

.github/actions/start-services/start-vault.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+set -e
+
+echo "Starting Vault in dev mode..."
+docker run -d --name vault \
+  --cap-add=IPC_LOCK \
+  -e 'VAULT_DEV_ROOT_TOKEN_ID=myroot' \
+  -e 'VAULT_DEV_LISTEN_ADDRESS=0.0.0.0:8200' \
+  -p 8200:8200 \
+  hashicorp/vault:1.20.3
+
+echo "Waiting for Vault to be ready..."
+for i in {1..30}; do
+  if curl -s http://localhost:8200/v1/sys/health | grep -q "initialized"; then
+    echo "Vault is ready!"
+    break
+  fi
+  echo "Waiting for Vault... ($i/30)"
+  sleep 1
+done
+
+# Configure Vault
+export VAULT_ADDR='http://localhost:8200'
+export VAULT_TOKEN='myroot'
+
+echo "Enabling AppRole auth..."
+docker exec -e VAULT_ADDR=$VAULT_ADDR -e VAULT_TOKEN=$VAULT_TOKEN vault \
+  vault auth enable approle || true
+
+echo "Creating Vault policy..."
+cat > /tmp/vault-policy.hcl <<'POLICY'
+path "secret/data/*" {
+  capabilities = ["create", "read", "update", "delete"]
+}
+path "secret/metadata/*" {
+  capabilities = ["create", "read", "update", "delete"]
+}
+POLICY
+
+docker cp /tmp/vault-policy.hcl vault:/tmp/vault-policy.hcl
+docker exec -e VAULT_ADDR=$VAULT_ADDR -e VAULT_TOKEN=$VAULT_TOKEN vault \
+  vault policy write test-policy /tmp/vault-policy.hcl
+
+echo "Creating AppRole..."
+docker exec -e VAULT_ADDR=$VAULT_ADDR -e VAULT_TOKEN=$VAULT_TOKEN vault \
+  vault write auth/approle/role/test-role \
+  token_policies="test-policy" \
+  token_ttl=1h \
+  token_max_ttl=4h
+
+echo "Getting role-id and secret-id..."
+ROLE_ID=$(docker exec -e VAULT_ADDR=$VAULT_ADDR -e VAULT_TOKEN=$VAULT_TOKEN vault \
+  vault read -field=role_id auth/approle/role/test-role/role-id)
+SECRET_ID=$(docker exec -e VAULT_ADDR=$VAULT_ADDR -e VAULT_TOKEN=$VAULT_TOKEN vault \
+  vault write -field=secret_id -f auth/approle/role/test-role/secret-id)
+
+echo "Exporting Vault configuration..."
+if [ -n "$GITHUB_ENV" ]; then
+  echo "VAULT_ADDR=http://localhost:8200" >> $GITHUB_ENV
+  echo "VAULT_APPROLE_ROLE_ID=${ROLE_ID}" >> $GITHUB_ENV
+  echo "VAULT_APPROLE_SECRET_ID=${SECRET_ID}" >> $GITHUB_ENV
+else
+  echo "VAULT_ADDR=http://localhost:8200"
+  echo "VAULT_APPROLE_ROLE_ID=${ROLE_ID}"
+  echo "VAULT_APPROLE_SECRET_ID=${SECRET_ID}"
+fi
+
+echo "Vault setup complete!"
+

iac/provider-gcp/main.tf

@@ -190,6 +190,8 @@ module "nomad" {
   api_admin_token                           = random_password.api_admin_secret.result
   redis_url_secret_version                  = google_secret_manager_secret_version.redis_url
   sandbox_access_token_hash_seed            = random_password.sandbox_access_token_hash_seed.result
+  vault_api_approle_secret_name             = "${var.prefix}vault-api-approle"
+  vault_tls_ca_secret_name                  = "${var.prefix}vault-tls-ca"
 
   # Click Proxy
   client_proxy_count               = var.client_proxy_count

iac/provider-gcp/nomad/jobs/api.hcl

@@ -96,6 +96,10 @@ job "api" {
         REDIS_CLUSTER_URL              = "${redis_cluster_url}"
         DNS_PORT                       = "${dns_port_number}"
         SANDBOX_ACCESS_TOKEN_HASH_SEED = "${sandbox_access_token_hash_seed}"
+        VAULT_ADDR                     = "${vault_addr}"
+        VAULT_APPROLE_ROLE_ID          = "${vault_api_approle_creds != "" ? jsondecode(vault_api_approle_creds).role_id : ""}"
+        VAULT_APPROLE_SECRET_ID        = "${vault_api_approle_creds != "" ? jsondecode(vault_api_approle_creds).secret_id : ""}"
+        VAULT_TLS_CA                   = ${jsonencode(vault_tls_ca)}
 
         LOCAL_CLUSTER_ENDPOINT = "${local_cluster_endpoint}"
         LOCAL_CLUSTER_TOKEN    = "${local_cluster_token}"

iac/provider-gcp/nomad/main.tf

@@ -47,6 +47,14 @@ data "google_secret_manager_secret_version" "redis_url" {
   secret = var.redis_url_secret_version.secret
 }
 
+data "google_secret_manager_secret_version" "vault_api_approle" {
+  secret = var.vault_api_approle_secret_name
+}
+
+data "google_secret_manager_secret_version" "vault_tls_ca" {
+  secret = var.vault_tls_ca_secret_name
+}
+
 
 data "docker_registry_image" "api_image" {
   name = "${var.gcp_region}-docker.pkg.dev/${var.gcp_project_id}/${var.orchestration_repository_name}/api:latest"
@@ -101,6 +109,9 @@ resource "nomad_job" "api" {
     redis_cluster_url              = data.google_secret_manager_secret_version.redis_url.secret_data != "redis.service.consul" ? "${data.google_secret_manager_secret_version.redis_url.secret_data}:${var.redis_port.port}" : ""
     dns_port_number                = var.api_dns_port_number
     clickhouse_connection_string   = local.clickhouse_connection_string
+    vault_addr                     = "https://vault-leader.service.consul:8200"
+    vault_api_approle_creds        = data.google_secret_manager_secret_version.vault_api_approle.secret_data
+    vault_tls_ca                   = data.google_secret_manager_secret_version.vault_tls_ca.secret_data
     sandbox_access_token_hash_seed = var.sandbox_access_token_hash_seed
     db_migrator_docker_image       = docker_image.db_migrator_image.repo_digest
     launch_darkly_api_key          = trimspace(data.google_secret_manager_secret_version.launch_darkly_api_key.secret_data)

iac/provider-gcp/nomad/variables.tf

@@ -368,3 +368,13 @@ variable "filestore_cache_cleanup_files_per_loop" {
 variable "dockerhub_remote_repository_url" {
   type = string
 }
+
+variable "vault_api_approle_secret_name" {
+  type        = string
+  description = "The name of the Google Secret Manager secret containing Vault API AppRole credentials"
+}
+
+variable "vault_tls_ca_secret_name" {
+  type        = string
+  description = "The name of the Google Secret Manager secret containing Vault TLS CA certificate"
+}

packages/api/go.mod

@@ -167,12 +167,15 @@ require (
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-msgpack v1.1.5 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
+	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/hashicorp/hcl/v2 v2.19.1 // indirect
 	github.com/hashicorp/memberlist v0.5.0 // indirect
 	github.com/hashicorp/serf v0.10.1 // indirect
+	github.com/hashicorp/vault-client-go v0.4.3 // indirect
 	github.com/huandu/xstrings v1.4.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
@@ -232,6 +235,7 @@ require (
 	github.com/prometheus/exporter-toolkit v0.10.1-0.20230714054209-2f4150c63f97 // indirect
 	github.com/prometheus/procfs v0.16.0 // indirect
 	github.com/prometheus/prometheus v1.8.2-0.20200727090838-6f296594a852 // indirect
+	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/sercand/kuberesolver/v5 v5.1.1 // indirect