feat: delegated-init support for Fly projects (#997)

* feat: delegated-init support for Fly projects

* chore: add a salt minion config

* chore: always extract the archive to /

* chore: extract newer archive only based on touch time in GHA

* chore: track salt version in aio, not always latest

* chore: bump image version

Author: tomashley

common.vars.pkr.hcl

@@ -1 +1 @@
-postgres-version = "15.1.1.61"
+postgres-version = "15.1.1.62"

docker/all-in-one/Dockerfile

@@ -154,6 +154,22 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     && mkdir -p /dist \
     && mkdir -p /data/opt && chmod go+rwx /data/opt
 
+####################
+# Install salt
+####################
+ENV DEBIAN_FRONTEND noninteractive
+ENV SALT_VERSION 3006
+
+# Install one-dir salt
+RUN mkdir /etc/apt/keyrings \
+    && curl -fsSL -o /etc/apt/keyrings/salt-archive-keyring-2023.gpg https://repo.saltproject.io/salt/py3/ubuntu/20.04/arm64/SALT-PROJECT-GPG-PUBKEY-2023.gpg \
+    && echo "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.gpg arch=arm64] https://repo.saltproject.io/salt/py3/ubuntu/20.04/arm64/$SALT_VERSION focal main" | tee /etc/apt/sources.list.d/salt.list \
+    && apt-get clean && apt-get update \
+    && apt-get install -y salt-minion
+
+ADD docker/all-in-one/etc/salt/minion /etc/salt/minion
+
+
 # Copy single binary dependencies
 COPY --from=pgrst /bin/postgrest /dist/
 COPY --from=gotrue /usr/local/bin/auth /dist/gotrue

docker/all-in-one/entrypoint.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -eou pipefail
 
+START_TIME=$(date +%s%N)
+
 PG_CONF=/etc/postgresql/postgresql.conf
 SUPERVISOR_CONF=/etc/supervisor/supervisord.conf
 
@@ -9,6 +11,16 @@ export CONFIGURED_FLAG_PATH=${CONFIGURED_FLAG_PATH:-$DATA_VOLUME_MOUNTPOINT/mach
 
 export MAX_IDLE_TIME_MINUTES=${MAX_IDLE_TIME_MINUTES:-5}
 
+function calculate_duration {
+  local start_time=$1
+  local end_time=$2
+
+  local duration=$((end_time - start_time))
+  local milliseconds=$((duration / 1000000))
+
+  echo "$milliseconds"
+}
+
 # Ref: https://gist.github.com/sj26/88e1c6584397bb7c13bd11108a579746
 function retry {
   # Pass 0 for unlimited retries
@@ -53,12 +65,12 @@ function enable_swap {
 }
 
 function push_lsn_checkpoint_file {
-    if [ "${PLATFORM_DEPLOYMENT:-}" != "true" ]; then
-      echo "Skipping push of LSN checkpoint file"
-      return
-    fi
+  if [ "${PLATFORM_DEPLOYMENT:-}" != "true" ]; then
+    echo "Skipping push of LSN checkpoint file"
+    return
+  fi
 
-    /usr/bin/admin-mgr lsn-checkpoint-push --immediately || echo "Failed to push LSN checkpoint"
+  /usr/bin/admin-mgr lsn-checkpoint-push --immediately || echo "Failed to push LSN checkpoint"
 }
 
 function graceful_shutdown {
@@ -71,46 +83,33 @@ function graceful_shutdown {
 }
 
 function enable_autoshutdown {
-    sed -i "s/autostart=.*/autostart=true/" /etc/supervisor/base-services/supa-shutdown.conf
+  sed -i "s/autostart=.*/autostart=true/" /etc/supervisor/base-services/supa-shutdown.conf
 }
 
 function enable_lsn_checkpoint_push {
-    sed -i "s/autostart=.*/autostart=true/" /etc/supervisor/base-services/lsn-checkpoint-push.conf
-    sed -i "s/autorestart=.*/autorestart=true/" /etc/supervisor/base-services/lsn-checkpoint-push.conf
+  sed -i "s/autostart=.*/autostart=true/" /etc/supervisor/base-services/lsn-checkpoint-push.conf
+  sed -i "s/autorestart=.*/autorestart=true/" /etc/supervisor/base-services/lsn-checkpoint-push.conf
 }
 
 function disable_fail2ban {
-  sed -i "s/command=.*/command=sleep 5/" /etc/supervisor/services/fail2ban.conf
   sed -i "s/autostart=.*/autostart=false/" /etc/supervisor/services/fail2ban.conf
   sed -i "s/autorestart=.*/autorestart=false/" /etc/supervisor/services/fail2ban.conf
 }
 
-function disable_gotrue {
-  sed -i "s/command=.*/command=sleep 5/" /etc/supervisor/services/gotrue.conf
-  sed -i "s/autostart=.*/autostart=false/" /etc/supervisor/services/gotrue.conf
-  sed -i "s/autorestart=.*/autorestart=false/" /etc/supervisor/services/gotrue.conf
-}
-
-function disable_pgbouncer {
-  sed -i "s/command=.*/command=sleep 5/" /etc/supervisor/services/pgbouncer.conf
-  sed -i "s/autostart=.*/autostart=false/" /etc/supervisor/services/pgbouncer.conf
-  sed -i "s/autorestart=.*/autorestart=false/" /etc/supervisor/services/pgbouncer.conf
-}
-
 function setup_postgres {
   tar -xzvf "$INIT_PAYLOAD_PATH" -C / ./etc/postgresql.schema.sql
   mv /etc/postgresql.schema.sql /docker-entrypoint-initdb.d/migrations/99-schema.sql
 
   tar -xzvf "$INIT_PAYLOAD_PATH" -C / ./etc/postgresql-custom/pgsodium_root.key
-  echo "include = '/etc/postgresql-custom/postgresql-platform-defaults.conf'" >> $PG_CONF
+  echo "include = '/etc/postgresql-custom/postgresql-platform-defaults.conf'" >>$PG_CONF
 
   # TODO (darora): walg enablement is temporarily performed here until changes from https://github.com/supabase/postgres/pull/639 get picked up
   # other things will still be needed in the future (auth_delay config)
   sed -i \
-      -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" \
-      -e "s|shared_preload_libraries = '\(.*\)'|shared_preload_libraries = '\1, auth_delay'|" \
-      -e "/# Automatically generated optimizations/i auth_delay.milliseconds = '3000'" \
-      "${PG_CONF}"
+    -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" \
+    -e "s|shared_preload_libraries = '\(.*\)'|shared_preload_libraries = '\1, auth_delay'|" \
+    -e "/# Automatically generated optimizations/i auth_delay.milliseconds = '3000'" \
+    "${PG_CONF}"
 
   # Setup ssl certs
   mkdir -p /etc/ssl/certs/postgres
@@ -169,6 +168,8 @@ function setup_postgres {
     chmod g+rx "/etc/wal-g"
     chmod g+rx "${WALG_CONF_DIR}"
   fi
+  DURATION=$(calculate_duration "$START_TIME" "$(date +%s%N)")
+  echo "E: Execution time to setting up postgresql: $DURATION milliseconds"
 }
 
 function setup_credentials {
@@ -178,6 +179,8 @@ function setup_credentials {
   export SERVICE_ROLE_KEY=${SERVICE_ROLE_KEY:-$(jq -r '.["service_key"]' /tmp/init.json)}
   export ADMIN_API_KEY=${ADMIN_API_KEY:-$(jq -r '.["supabase_admin_key"]' /tmp/init.json)}
   export JWT_SECRET=${JWT_SECRET:-$(jq -r '.["jwt_secret"]' /tmp/init.json)}
+  DURATION=$(calculate_duration "$START_TIME" "$(date +%s%N)")
+  echo "E: Execution time to setting up credentials: $DURATION milliseconds"
 }
 
 function report_health {
@@ -193,9 +196,9 @@ function report_health {
 }
 
 function run_prelaunch_hooks {
-    if [ -f "/etc/postgresql-custom/supautils.conf" ]; then
-      sed -i -e 's/dblink, //' "/etc/postgresql-custom/supautils.conf"
-    fi
+  if [ -f "/etc/postgresql-custom/supautils.conf" ]; then
+    sed -i -e 's/dblink, //' "/etc/postgresql-custom/supautils.conf"
+  fi
 }
 
 function start_supervisor {
@@ -206,19 +209,57 @@ function start_supervisor {
   /usr/bin/supervisord -c $SUPERVISOR_CONF
 }
 
+DELEGATED_ARCHIVE_PATH=/data/delegated-init.tar.gz
+DELEGATED_ENTRY_PATH=/data/delegated-entry.sh
+
+function fetch_and_execute_delegated_payload {
+  curl -s --time-cond $DELEGATED_ARCHIVE_PATH -o $DELEGATED_ARCHIVE_PATH "$DELEGATED_INIT_LOCATION"
+
+  if [ ! -f $DELEGATED_ARCHIVE_PATH ]; then
+    echo "No delegated payload found, bailing"
+    return
+  fi
+
+  # only extract a valid archive
+  if tar -tzf "$DELEGATED_ARCHIVE_PATH" &>/dev/null; then
+    TAR_MTIME_EPOCH=$(tar -tvzf "$DELEGATED_ARCHIVE_PATH" data/delegated-entry.sh | awk '{print $4, $5}' | xargs -I {} date -d {} +%s)
+
+    if [ -f $DELEGATED_ENTRY_PATH ]; then
+      FILE_MTIME_EPOCH=$(stat -c %Y "$DELEGATED_ENTRY_PATH")
+
+      if [ "$TAR_MTIME_EPOCH" -gt "$FILE_MTIME_EPOCH" ]; then
+        tar -xvzf "$DELEGATED_ARCHIVE_PATH" -C /
+      else
+        echo "TAR archive is not newer, skipping extraction"
+      fi
+    else
+      tar -xvzf "$DELEGATED_ARCHIVE_PATH" -C /
+    fi
+  else
+    echo "Invalid TAR archive"
+    return
+  fi
+
+  # Run our delegated entry script here
+  if [ -f "$DELEGATED_ENTRY_PATH" ]; then
+    chmod +x $DELEGATED_ENTRY_PATH
+    bash -c "$DELEGATED_ENTRY_PATH $START_TIME"
+  fi
+}
+
 # Increase max number of open connections
 ulimit -n 65536
 
 # Update pgsodium root key
 if [ "${PGSODIUM_ROOT_KEY:-}" ]; then
-  echo "${PGSODIUM_ROOT_KEY}" > /etc/postgresql-custom/pgsodium_root.key
+  echo "${PGSODIUM_ROOT_KEY}" >/etc/postgresql-custom/pgsodium_root.key
 fi
 
 # Update pgdata directory
 if [ "${PGDATA_REAL:-}" ]; then
-    mkdir -p "${PGDATA_REAL}"
-    chown -R postgres:postgres "${PGDATA_REAL}"
-    chmod -R g+rx "${PGDATA_REAL}"
+  mkdir -p "${PGDATA_REAL}"
+  chown -R postgres:postgres "${PGDATA_REAL}"
+  chmod -R g+rx "${PGDATA_REAL}"
 fi
 
 if [ "${PGDATA:-}" ]; then
@@ -239,7 +280,7 @@ export INIT_PAYLOAD_PATH=${INIT_PAYLOAD_PATH:-/tmp/payload.tar.gz}
 
 if [ "${INIT_PAYLOAD_PRESIGNED_URL:-}" ]; then
   curl -fsSL "$INIT_PAYLOAD_PRESIGNED_URL" -o "/tmp/payload.tar.gz" || true
-  if [ -f "/tmp/payload.tar.gz" ] && [ "/tmp/payload.tar.gz" != "$INIT_PAYLOAD_PATH" ] ; then
+  if [ -f "/tmp/payload.tar.gz" ] && [ "/tmp/payload.tar.gz" != "$INIT_PAYLOAD_PATH" ]; then
     mv "/tmp/payload.tar.gz" "$INIT_PAYLOAD_PATH"
   fi
 fi
@@ -298,11 +339,8 @@ if [ "${FAIL2BAN_DISABLED:-}" == "true" ]; then
 fi
 
 if [ "${GOTRUE_DISABLED:-}" == "true" ]; then
-  disable_gotrue
-fi
-
-if [ "${PGBOUNCER_DISABLED:-}" == "true" ]; then
-  disable_pgbouncer
+  sed -i "s/autostart=.*/autostart=false/" /etc/supervisor/services/gotrue.conf
+  sed -i "s/autorestart=.*/autorestart=false/" /etc/supervisor/services/gotrue.conf
 fi
 
 if [ "${PLATFORM_DEPLOYMENT:-}" == "true" ]; then
@@ -316,5 +354,12 @@ fi
 
 touch "$CONFIGURED_FLAG_PATH"
 run_prelaunch_hooks
-start_supervisor
-push_lsn_checkpoint_file
+
+if [ -n "${DELEGATED_INIT_LOCATION:-}" ]; then
+  fetch_and_execute_delegated_payload
+else
+  DURATION=$(calculate_duration "$START_TIME" "$(date +%s%N)")
+  echo "E: Execution time to starting supervisor: $DURATION milliseconds"
+  start_supervisor
+  push_lsn_checkpoint_file
+fi

docker/all-in-one/etc/salt/minion

@@ -0,0 +1,71 @@
+# Minions can connect to multiple masters simultaneously (all masters
+# are "hot"), or can be configured to failover if a master becomes
+# unavailable.  Multiple hot masters are configured by setting this
+# value to "str".  Failover masters can be requested by setting
+# to "failover".  MAKE SURE TO SET master_alive_interval if you are
+# using failover.
+# Setting master_type to 'disable' lets you have a running minion (with engines and
+# beacons) without a master connection
+master_type: disable
+
+# The minion can locally cache the return data from jobs sent to it, this
+# can be a good way to keep track of jobs the minion has executed
+# (on the minion side). By default this feature is disabled, to enable, set
+# cache_jobs to True.
+cache_jobs: True
+
+# The minion can take a while to start up when lspci and/or dmidecode is used
+# to populate the grains for the minion. Set this to False if you do not need
+# GPU hardware grains for your minion.
+enable_gpu_grains: False
+
+# Backup files that are replaced by file.managed and file.recurse under
+# 'cachedir'/file_backup relative to their original location and appended
+# with a timestamp. The only valid setting is "minion". Disabled by default.
+#
+# Alternatively this can be specified for each file in state files:
+# /etc/ssh/sshd_config:
+#   file.managed:
+#     - source: salt://ssh/sshd_config
+#     - backup: minion
+#
+backup_mode: minion
+
+#####     File Directory Settings    #####
+##########################################
+# The Salt Minion can redirect all file server operations to a local directory,
+# this allows for the same state tree that is on the master to be used if
+# copied completely onto the minion. This is a literal copy of the settings on
+# the master but used to reference a local directory on the minion.
+
+# Set the file client. The client defaults to looking on the master server for
+# files, but can be directed to look at the local file directory setting
+# defined below by setting it to "local". Setting a local file_client runs the
+# minion in masterless mode.
+file_client: local
+
+# The file directory works on environments passed to the minion, each environment
+# can have multiple root directories, the subdirectories in the multiple file
+# roots cannot match, otherwise the downloaded files will not be able to be
+# reliably ensured. A base environment is required to house the top file.
+# Example:
+# file_roots:
+#   base:
+#     - /srv/salt/
+#   dev:
+#     - /srv/salt/dev/services
+#     - /srv/salt/dev/states
+#   prod:
+#     - /srv/salt/prod/services
+#     - /srv/salt/prod/states
+#
+file_roots:
+  base:
+    - /opt/salt/state
+
+# The Salt pillar is searched for locally if file_client is set to local. If
+# this is the case, and pillar data is defined, then the pillar_roots need to
+# also be configured on the minion:
+pillar_roots:
+  base:
+    - /opt/salt/pillar