test(postgresql): e2e for TLS options

Adds e2e tests for the functionality of providing the TLS options by
testing with TLS turned on on the server side using the
`tls.enabled=true` and `tls.autoGenerated=true` PostgreSQL Chart
parameters.

To make the tests as robust as possible: port forwarding is not used,
but rather exposing the server port via NodePort; timeouts are increased
on kubectl wait's, from 2 min to 3 min.

The test iself reuses parts of the existing tests, and adds one
additional check that asserts if TLS is used.

Author: zregvart

cluster/local/integration_tests.sh

@@ -96,6 +96,15 @@ nodes:
   extraMounts:
   - hostPath: "${cache_path}/"
     containerPath: /cache
+  extraPortMappings:
+  - containerPort: 5432
+    hostPort: 5432
+  kubeadmConfigPatches:
+  - |
+    kind: ClusterConfiguration
+    apiServer:
+      extraArgs:
+        "service-node-port-range": "1-65535"
 EOF
   )"
   echo "${config}" | "${KIND}" create cluster --name="${K8S_CLUSTER}" --wait=5m --image="${node_image}" --config=-

cluster/local/postgresdb_functions.sh

@@ -21,6 +21,54 @@ setup_postgresdb_no_tls() {
   PORT_FORWARD_PID=$!
 }
 
+setup_postgresdb_tls() {
+  echo_step "Installing PostgresDB Helm chart into default namespace"
+  postgres_root_pw=$(LC_ALL=C tr -cd "A-Za-z0-9" </dev/urandom | head -c 32)
+
+  "${HELM}" repo update
+  "${HELM}" install postgresdb bitnami/postgresql \
+      --version 11.9.1 \
+      --set global.postgresql.auth.postgresPassword="${postgres_root_pw}" \
+      --set volumePermissions.enabled=true \
+      --set tls.enabled=true \
+      --set tls.autoGenerated=true \
+      --set primary.extraEnvVars[0].name=POSTGRESQL_TLS_CA_FILE \
+      --set primary.extraEnvVars[0].value=/opt/bitnami/postgresql/certs/ca.crt \
+      --set primary.pgHbaConfiguration='local all all trust
+hostssl all all 0.0.0.0/0 md5
+hostssl all all ::/0 md5' \
+      --wait
+
+  # Access via NodePort, the port-forward gets a connection reset after first use and will be no longer available
+  "${KUBECTL}" expose pod postgresdb-postgresql-0 \
+      --name=postgres-nodeport \
+      --type=NodePort \
+      --selector='statefulset.kubernetes.io/pod-name=postgresdb-postgresql-0' \
+      --overrides '{"apiVersion":"v1","spec":{"ports":[{"port":5432,"protocol":"TCP","targetPort":5432,"nodePort":5432}]}}'
+  "${KUBECTL}" wait service/postgres-nodeport --timeout 2m --for=jsonpath='{.status.loadBalancer}'
+
+  # double check that PostgreSQL is accessible via the NodePort
+  echo_step "Waiting for PostgreSQL to become accessible"
+  while ! pg_isready -h localhost -p 5432; do
+    echo -n .
+    sleep 0.5
+  done
+  echo_step_completed
+
+  echo_step "Setting up client connection secrets"
+  "${KUBECTL}" create secret generic postgresdb-creds \
+      --from-literal username="postgres" \
+      --from-literal password="${postgres_root_pw}" \
+      --from-literal endpoint="postgresdb-postgresql.default.svc.cluster.local" \
+      --from-literal port="5432"
+
+    # copy the TLS secret to crossplane-system namespace for the provider to access
+    "${KUBECTL}" get secret postgresdb-postgresql-crt -o yaml \
+      | "${KUBECTL}" patch --patch='{"metadata":{"namespace":"crossplane-system"}}' -o yaml --dry-run=client -f - \
+      | "${KUBECTL}" apply -f -
+  echo_step_completed
+}
+
 setup_provider_config_postgres_no_tls() {
   echo_step "creating ProviderConfig for PostgresDb with no TLS"
   local yaml="$( cat <<EOF
@@ -40,6 +88,55 @@ EOF
   echo "${yaml}" | "${KUBECTL}" apply -f -
 }
 
+setup_provider_config_postgres_tls() {
+  echo_step "creating ProviderConfig for PostgresDb with TLS"
+  local yaml="$(cat <<EOF
+apiVersion: pkg.crossplane.io/v1beta1
+kind: DeploymentRuntimeConfig
+metadata:
+  name: postgres-tls
+spec:
+  deploymentTemplate:
+    spec:
+      selector: {}
+      template:
+        spec:
+          containers:
+            - name: package-runtime
+              args:
+                - --debug
+              volumeMounts:
+                - mountPath: /certs/postgres
+                  name: postgresql-tls
+                  readOnly: true
+          volumes:
+          - name: postgresql-tls
+            secret:
+              secretName: postgresdb-postgresql-crt
+              defaultMode: 420
+              items:
+              - key: ca.crt
+                path: ca.crt
+---
+apiVersion: postgresql.sql.crossplane.io/v1alpha1
+kind: ProviderConfig
+metadata:
+  name: default
+spec:
+  sslRootCert: /certs/postgres/ca.crt
+  credentials:
+    source: PostgreSQLConnectionSecret
+    connectionSecretRef:
+      namespace: default
+      name: postgresdb-creds
+EOF
+  )"
+  echo "${yaml}" | "${KUBECTL}" apply -f -
+
+  # make use of the postgres-tls DeploymentRuntimeConfig
+  "${KUBECTL}" patch providers.pkg.crossplane.io/provider-sql --type=json --patch='[{"op":"add","path":"/spec/runtimeConfigRef","value":{"name":"postgres-tls"}}]'
+}
+
 setup_postgresdb_tests(){
 # install provider resources
 echo_step "creating PostgresDB Database resource"
@@ -59,19 +156,19 @@ echo_step "creating PostgresDB Schema resources"
 "${KUBECTL}" apply -f ${projectdir}/examples/postgresql/schema.yaml
 
 echo_step "check if Role is ready"
-"${KUBECTL}" wait --timeout 2m --for condition=Ready -f ${projectdir}/examples/postgresql/role.yaml
+"${KUBECTL}" wait --timeout 3m --for condition=Ready -f ${projectdir}/examples/postgresql/role.yaml
 echo_step_completed
 
 echo_step "check if database is ready"
-"${KUBECTL}" wait --timeout 2m --for condition=Ready -f ${projectdir}/examples/postgresql/database.yaml
+"${KUBECTL}" wait --timeout 3m --for condition=Ready -f ${projectdir}/examples/postgresql/database.yaml
 echo_step_completed
 
 echo_step "check if grant is ready"
-"${KUBECTL}" wait --timeout 2m --for condition=Ready -f ${projectdir}/examples/postgresql/grant.yaml
+"${KUBECTL}" wait --timeout 3m --for condition=Ready -f ${projectdir}/examples/postgresql/grant.yaml
 echo_step_completed
 
 echo_step "check if schema is ready"
-"${KUBECTL}" wait --timeout 2m --for condition=Ready -f ${projectdir}/examples/postgresql/schema.yaml
+"${KUBECTL}" wait --timeout 3m --for condition=Ready -f ${projectdir}/examples/postgresql/schema.yaml
 echo_step_completed
 }
 
@@ -168,6 +265,22 @@ check_observe_only_database(){
   echo_step_completed
 }
 
+check_tls_used() {
+  echo_step "check if TLS is used to connect"
+  local tls
+  tls="$(PGPASSWORD="${postgres_root_pw}" psql -h localhost -p 5432 -U postgres -wtAc "select pg_ssl.ssl FROM pg_stat_ssl pg_ssl JOIN pg_stat_activity pg_sa ON pg_ssl.pid = pg_sa.pid;")"
+
+  if [[ "${tls}" == t ]]; then
+      echo "Connected using TLS"
+      echo_info "OK"
+  else
+      echo "Did not connect using TLS"
+      echo_error "Not OK"
+  fi
+
+  echo_step_completed
+}
+
 delete_postgresdb_resources(){
   # uninstall
   echo_step "uninstalling ${PROJECT_NAME}"
@@ -179,14 +292,20 @@ delete_postgresdb_resources(){
 
   # ----------- cleaning postgres related resources
 
-  echo_step "kill port-forwarding"
-  kill $PORT_FORWARD_PID
+  if [[ -n "${PORT_FORWARD_PID}" ]]; then
+    echo_step "kill port-forwarding"
+    kill $PORT_FORWARD_PID
+    unset PORT_FORWARD_PID
+  fi
 
   echo_step "uninstalling secret and provider config for postgres"
   "${KUBECTL}" delete secret postgresdb-creds
 
   echo_step "Uninstalling PostgresDB Helm chart from default namespace"
   "${HELM}" uninstall postgresdb
+
+  # make sure to delete the PVC, otherwise the password will be reused from the first installation
+  "${KUBECTL}" delete --ignore-not-found=true pvc data-postgresdb-postgresql-0
 }
 
 integration_tests_postgres() {
@@ -198,4 +317,16 @@ integration_tests_postgres() {
   check_all_roles_privileges
   check_schema_privileges
   delete_postgresdb_resources
-}
+
+  tls_tests() {
+    local PGSSLMODE=require
+    setup_postgresdb_tls
+    setup_provider_config_postgres_tls
+    check_tls_used
+    setup_observe_only_database
+    setup_postgresdb_tests
+    check_all_roles_privileges
+    delete_postgresdb_resources
+  }
+  tls_tests
+}