[Dry run automation] archive hardfork toolbox

Makefile

@@ -221,6 +221,7 @@ build-archive-utils: ocaml_checks reformat-diff ## Build archive node and relate
 		src/app/archive_blocks/archive_blocks.exe \
 		src/app/extract_blocks/extract_blocks.exe \
 		src/app/missing_blocks_auditor/missing_blocks_auditor.exe \
+		src/app/archive_hardfork_toolbox/archive_hardfork_toolbox.exe \
 		--profile=$(DUNE_PROFILE)  \
 		&& echo "✅ Build complete"
 

buildkite/src/Command/ArchiveNodeTest.dhall

@@ -19,7 +19,9 @@ in  { step =
                     [ "ARCHIVE_TEST_APP=mina-archive-node-test"
                     , "MINA_TEST_NETWORK_DATA=/etc/mina/test/archive/sample_db"
                     ]
-                    "src/test/archive/sample_db/archive_db.sql"
+                    ( RunWithPostgres.ScriptOrArchive.Script
+                        "src/test/archive/sample_db/archive_db.sql"
+                    )
                     ( Artifacts.fullDockerTag
                         Artifacts.Tag::{
                         , artifact = Artifacts.Type.FunctionalTestSuite

buildkite/src/Command/PatchArchiveTest.dhall

@@ -19,7 +19,9 @@ in  { step =
                     [ "PATCH_ARCHIVE_TEST_APP=mina-patch-archive-test"
                     , "NETWORK_DATA_FOLDER=/etc/mina/test/archive/sample_db"
                     ]
-                    "./src/test/archive/sample_db/archive_db.sql"
+                    ( RunWithPostgres.ScriptOrArchive.Script
+                        "./src/test/archive/sample_db/archive_db.sql"
+                    )
                     ( Artifacts.fullDockerTag
                         Artifacts.Tag::{
                         , artifact = Artifacts.Type.FunctionalTestSuite

buildkite/src/Command/ReplayerTest.dhall

@@ -17,7 +17,9 @@ in  { step =
               , commands =
                 [ RunWithPostgres.runInDockerWithPostgresConn
                     ([] : List Text)
-                    "./src/test/archive/sample_db/archive_db.sql"
+                    ( RunWithPostgres.ScriptOrArchive.Script
+                        "./src/test/archive/sample_db/archive_db.sql"
+                    )
                     ( Artifacts.fullDockerTag
                         Artifacts.Tag::{
                         , artifact = Artifacts.Type.FunctionalTestSuite

buildkite/src/Command/RunWithPostgres.dhall

@@ -57,10 +57,14 @@ let Cmd = ../Lib/Cmds.dhall
 
 let ContainerImages = ../Constants/ContainerImages.dhall
 
+let ScriptOrArchive
+    : Type
+    = < Script : Text | Archive : { Script : Text, Archive : Text } >
+
 let runInDockerWithPostgresConn
-    : List Text -> Text -> Text -> Text -> Cmd.Type
+    : List Text -> ScriptOrArchive -> Text -> Text -> Cmd.Type
     =     \(environment : List Text)
-      ->  \(initScript : Text)
+      ->  \(initScript : ScriptOrArchive)
       ->  \(docker : Text)
       ->  \(innerScript : Text)
       ->  let port = "5432"
@@ -97,13 +101,31 @@ let runInDockerWithPostgresConn
               : Text
               = "\\\$BUILDKITE_BUILD_CHECKOUT_PATH"
 
+          let runInitScript =
+                merge
+                  { Script =
+                          \(script : Text)
+                      ->  [ "docker exec ${postgresDockerName} psql ${pg_conn} -f /workdir/${script}"
+                          ]
+                  , Archive =
+                          \(archive : { Script : Text, Archive : Text })
+                      ->  [ "tar -xzf ${archive.Archive}"
+                          , "docker exec ${postgresDockerName} find /workdir -name \"${archive.Script}\" -exec psql ${pg_conn} -f {} \\;"
+                          ]
+                  }
+                  initScript
+
           in  Cmd.chain
-                [ "( docker stop ${postgresDockerName} && docker rm ${postgresDockerName} ) || true"
-                , "source buildkite/scripts/export-git-env-vars.sh"
-                , "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir --name ${postgresDockerName} -d -e POSTGRES_USER=${user} -e POSTGRES_PASSWORD=${password} -e POSTGRES_PASSWORD=${password} -e POSTGRES_DB=${dbName} ${dockerVersion}"
-                , "sleep 5"
-                , "docker exec ${postgresDockerName} psql ${pg_conn} -f /workdir/${initScript}"
-                , "docker run --pid=container:postgres --network host --volume ${outerDir}:/workdir --workdir /workdir --entrypoint bash ${envVars} ${docker} ${innerScript}"
-                ]
-
-in  { runInDockerWithPostgresConn = runInDockerWithPostgresConn }
+                (   [ "( docker stop ${postgresDockerName} && docker rm ${postgresDockerName} ) || true"
+                    , "source buildkite/scripts/export-git-env-vars.sh"
+                    , "docker run --network host --volume ${outerDir}:/workdir --workdir /workdir --name ${postgresDockerName} -d -e POSTGRES_USER=${user} -e POSTGRES_PASSWORD=${password} -e POSTGRES_DB=${dbName} ${dockerVersion}"
+                    , "sleep 5"
+                    ]
+                  # runInitScript
+                  # [ "docker run --pid=container:postgres --network host --volume ${outerDir}:/workdir --workdir /workdir --entrypoint bash ${envVars} ${docker} ${innerScript}"
+                    ]
+                )
+
+in  { runInDockerWithPostgresConn = runInDockerWithPostgresConn
+    , ScriptOrArchive = ScriptOrArchive
+    }

buildkite/src/Jobs/Test/ArchiveHardforkToolboxTest.dhall

@@ -0,0 +1,77 @@
+let S = ../../Lib/SelectFiles.dhall
+
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+let PipelineTag = ../../Pipeline/Tag.dhall
+
+let JobSpec = ../../Pipeline/JobSpec.dhall
+
+let Artifacts = ../../Constants/Artifacts.dhall
+
+let Dockers = ../../Constants/DockerVersions.dhall
+
+let BuildFlags = ../../Constants/BuildFlags.dhall
+
+let Command = ../../Command/Base.dhall
+
+let Size = ../../Command/Size.dhall
+
+let RunWithPostgres = ../../Command/RunWithPostgres.dhall
+
+let dependsOn =
+      Dockers.dependsOn
+        Dockers.DepsSpec::{
+        , buildFlags = BuildFlags.Type.Instrumented
+        , artifact = Artifacts.Type.FunctionalTestSuite
+        }
+
+let key = "archive-hardfork-toolbox-test"
+
+in  Pipeline.build
+      Pipeline.Config::{
+      , spec = JobSpec::{
+        , dirtyWhen =
+          [ S.strictlyStart (S.contains "src/app/archive_hardfork_toolbox")
+          , S.exactly
+              "buildkite/src/Jobs/Test/ArchiveHardforkToolboxTest"
+              "dhall"
+          , S.exactly
+              "scripts/tests/archive-hardfork-toolbox/hf_archive"
+              "tar.gz"
+          , S.exactly "scripts/tests/archive-hardfork-toolbox/runner" "sh"
+          ]
+        , path = "Test"
+        , name = "ArchiveHardforkToolboxTest"
+        , tags =
+          [ PipelineTag.Type.Long
+          , PipelineTag.Type.Test
+          , PipelineTag.Type.Stable
+          ]
+        }
+      , steps =
+        [ Command.build
+            Command.Config::{
+            , commands =
+              [ RunWithPostgres.runInDockerWithPostgresConn
+                  ([] : List Text)
+                  ( RunWithPostgres.ScriptOrArchive.Archive
+                      { Script = "hf_archive.sql"
+                      , Archive =
+                          "scripts/tests/archive-hardfork-toolbox/hf_archive.tar.gz"
+                      }
+                  )
+                  ( Artifacts.fullDockerTag
+                      Artifacts.Tag::{
+                      , artifact = Artifacts.Type.FunctionalTestSuite
+                      , buildFlags = BuildFlags.Type.Instrumented
+                      }
+                  )
+                  "scripts/tests/archive-hardfork-toolbox/runner.sh && buildkite/scripts/upload-partial-coverage-data.sh ${key} "
+              ]
+            , label = "Archive: Hardfork Toolbox Test"
+            , key = key
+            , target = Size.Large
+            , depends_on = dependsOn
+            }
+        ]
+      }

buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall

@@ -54,7 +54,9 @@ in  Pipeline.build
                   "export MINA_DEB_CODENAME=bullseye && source ./buildkite/scripts/export-git-env-vars.sh && echo \\\${MINA_DOCKER_TAG}"
               , RunWithPostgres.runInDockerWithPostgresConn
                   ([] : List Text)
-                  "./src/test/archive/sample_db/archive_db.sql"
+                  ( RunWithPostgres.ScriptOrArchive.Script
+                      "./src/test/archive/sample_db/archive_db.sql"
+                  )
                   rosettaDocker
                   "./buildkite/scripts/rosetta-indexer-test.sh"
               , Cmd.runInDocker

changes/17955.md

@@ -0,0 +1,3 @@
+Archive Hardfork Toolbox
+
+Utility tool for various hardfork verifications or confirmations regarding hardfork runbook automation

scripts/debian/builder-helpers.sh

@@ -536,6 +536,8 @@ copy_common_archive_configs() {
     "${BUILDDIR}/usr/local/bin/mina-archive-blocks"
   cp ./default/src/app/extract_blocks/extract_blocks.exe \
     "${BUILDDIR}/usr/local/bin/mina-extract-blocks"
+  cp ./default/src/app/archive_hardfork_toolbox/archive_hardfork_toolbox.exe \
+    "${BUILDDIR}/usr/local/bin/mina-archive-hardfork-toolbox"
 
   mkdir -p "${BUILDDIR}/etc/mina/archive"
   cp ../scripts/archive/missing-blocks-guardian.sh \

scripts/tests/archive-hardfork-toolbox/runner.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# This script runs the archive hardfork toolbox tests.
+# It assumes that the archive hardfork toolbox has already been built.
+# It requires the archive database URI to be provided via --archive-uri argument.
+
+# Parse command line arguments
+ARCHIVE_URI="${PG_CONN:-}"
+TOOLBOX_PATH="mina-archive-hardfork-toolbox"
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --archive-uri)
+            ARCHIVE_URI="$2"
+            shift 2
+            ;;
+        --toolbox-path)
+            TOOLBOX_PATH="$2"
+            shift 2
+            ;;
+        *)
+            echo "Unknown option: $1"
+            echo "Usage: $0 --archive-uri <database_uri> [--toolbox-path <path_to_binary>]"
+            exit 1
+            ;;
+    esac
+done
+
+# Validate required arguments
+if [[ -z "$ARCHIVE_URI" ]]; then
+    echo "Error: --archive-uri argument is required"
+    echo "Usage: $0 --archive-uri <database_uri> [--toolbox-path <path_to_binary>]"
+    exit 1
+fi
+
+echo "Using archive database URI: $ARCHIVE_URI"
+echo "Using toolbox binary: $TOOLBOX_PATH"
+
+# Create temporary directory for extracting the archive
+TEMP_DIR=$(mktemp -d)
+echo "Created temporary directory: $TEMP_DIR"
+
+# Cleanup function to remove temporary directory on exit
+cleanup() {
+    echo "Cleaning up temporary directory: $TEMP_DIR"
+    rm -rf "$TEMP_DIR"
+}
+trap cleanup EXIT
+
+# Get the directory of this script
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Extract the archive to temporary directory
+echo "Extracting hf_archive.tar.gz to $TEMP_DIR"
+tar -xzf "$SCRIPT_DIR/hf_archive.tar.gz" -C "$TEMP_DIR"
+
+
+# Run the archive hardfork toolbox tests
+echo "Running archive hardfork toolbox tests..."
+
+"$TOOLBOX_PATH" fork-candidate is-in-best-chain --archive-uri "$ARCHIVE_URI" --fork-state-hash 3NK3okhoURkke6dEd2SuQabU9VqvUKnKm4HxZbo5yoU525uGh4Zj --fork-height 296892  --fork-slot 446779
+
+"$TOOLBOX_PATH" fork-candidate confirmations --archive-uri "$ARCHIVE_URI" --fork-state-hash 3NK3okhoURkke6dEd2SuQabU9VqvUKnKm4HxZbo5yoU525uGh4Zj  --fork-slot 446779 --required-confirmations 1
+
+"$TOOLBOX_PATH" fork-candidate no-commands-after --archive-uri "$ARCHIVE_URI" --fork-state-hash 3NK3okhoURkke6dEd2SuQabU9VqvUKnKm4HxZbo5yoU525uGh4Zj  --fork-slot 446779
+
+"$TOOLBOX_PATH" verify-upgrade --archive-uri "$ARCHIVE_URI" --version 3.3.0
+
+"$TOOLBOX_PATH" validate-fork --archive-uri "$ARCHIVE_URI" --fork-slot 1067 --fork-state-hash 3NK38gNjWR6sE2MTKV8AqogjY6WaboPjSDq3zfpfVtiUgLMze1Wm

src/app/archive_hardfork_toolbox/Readme.md

@@ -0,0 +1,121 @@
+# Archive Hardfork Toolbox
+
+The Archive Hardfork Toolbox is a utility for verifying the integrity of archive database migrations and validating hardfork operations in the Mina protocol. This tool helps ensure that database schema upgrades and fork transitions maintain data consistency.
+
+## Overview
+
+This toolbox provides commands to:
+- Verify fork block candidates before migration
+- Validate database schema upgrades
+- Ensure fork block integrity and ancestry
+
+## Commands
+
+### fork-candidate
+
+A group of commands for pre-fork verifications to validate that a candidate block is suitable for forking.
+
+#### is-in-best-chain
+
+Verifies that the fork block is in the best chain of the blockchain.
+
+**Usage:**
+```bash
+archive_hardfork_toolbox fork-candidate is-in-best-chain \
+  --archive-uri "postgresql://user:pass@host:port/db" \
+  --fork-state-hash "3NKx..." \
+  --fork-height 12345 \
+  --fork-slot 67890
+```
+
+**Parameters:**
+- `--archive-uri`: URI for connecting to the mainnet archive database
+- `--fork-state-hash`: Hash of the fork state
+- `--fork-height`: Height of the fork block
+- `--fork-slot`: Global slot since genesis of the fork block
+
+#### confirmations
+
+Verifies that the fork block has the required number of confirmations.
+
+**Usage:**
+```bash
+archive_hardfork_toolbox fork-candidate confirmations \
+  --archive-uri "postgresql://user:pass@host:port/db" \
+  --fork-state-hash "3NKx..." \
+  --fork-slot 67890 \
+  --required-confirmations 290
+```
+
+**Parameters:**
+- `--archive-uri`: URI for connecting to the mainnet archive database
+- `--fork-state-hash`: Hash of the fork state
+- `--fork-slot`: Global slot since genesis of the fork block
+- `--required-confirmations`: Number of confirmations required for the fork block
+
+#### no-commands-after
+
+Verifies that no commands were executed after the fork block, ensuring a clean fork point.
+
+**Usage:**
+```bash
+archive_hardfork_toolbox fork-candidate no-commands-after \
+  --archive-uri "postgresql://user:pass@host:port/db" \
+  --fork-state-hash "3NKx..." \
+  --fork-slot 67890
+```
+
+**Parameters:**
+- `--archive-uri`: URI for connecting to the mainnet archive database
+- `--fork-state-hash`: Hash of the fork state
+- `--fork-slot`: Global slot since genesis of the fork block
+
+### verify-upgrade
+
+Verifies the upgrade from pre-fork to post-fork database schema.
+
+**Usage:**
+```bash
+archive_hardfork_toolbox verify-upgrade \
+  --archive-uri "postgresql://user:pass@host:port/db" \
+  --version "3.2.0"
+```
+
+**Parameters:**
+- `--archive-uri`: URI for connecting to the pre-fork mainnet archive database
+- `--version`: Version to upgrade to (e.g., "3.2.0")
+
+### validate-fork
+
+Validates the fork block and its ancestors to ensure blockchain integrity.
+
+**Usage:**
+```bash
+archive_hardfork_toolbox validate-fork \
+  --archive-uri "postgresql://user:pass@host:port/db" \
+  --fork-state-hash "3NKx..." \
+  --fork-slot 67890
+```
+
+**Parameters:**
+- `--archive-uri`: URI for connecting to the mainnet archive database
+- `--fork-state-hash`: Hash of the fork state
+- `--fork-slot`: Global slot since genesis of the fork block
+
+## Typical Workflow
+
+1. **Pre-fork validation**: Use the `fork-candidate` commands to ensure the chosen fork point is valid:
+   - Check if the block is in the best chain
+   - Verify sufficient confirmations
+   - Ensure no commands after the fork point
+
+2. **Schema upgrade**: Use `verify-upgrade` to validate the database migration process
+
+3. **Post-fork validation**: Use `validate-fork` to ensure the fork block and its ancestry remain intact
+
+## Database Connection
+
+All commands require an `--archive-uri` parameter that should be a PostgreSQL connection string in the format:
+```
+postgresql://username:password@hostname:port/database_name
+```