Implement release process for rustup

.github/workflows/ci.yml

@@ -28,7 +28,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-  local:
+  release:
     name: Local release
     runs-on: ubuntu-latest
 
@@ -48,7 +48,7 @@ jobs:
         run: docker compose up -d
 
       - name: Run the local release process for channel ${{ matrix.channel }}
-        run: ./run.sh ${{ matrix.channel }}
+        run: ./run.sh release ${{ matrix.channel }}
 
       - name: Validate the generated signatures
         run: docker compose exec -T local /src/local/check-signature.sh ${{ matrix.channel }}
@@ -61,6 +61,42 @@ jobs:
         env:
           RUSTUP_DIST_SERVER: http://localhost:9000/static
 
+  rustup:
+    name: Local rustup
+    runs-on: ubuntu-latest
+
+    env:
+      RUSTUP_OVERRIDE_VERSION: 99.0.0
+
+    strategy:
+      fail-fast: false
+      matrix:
+        channel: [stable, beta]
+
+    steps:
+      - name: Clone the source code
+        uses: actions/checkout@v3
+
+      - name: Ensure Rust Stable is up to date
+        run: rustup self update && rustup update stable
+
+      - name: Start the local environment
+        run: docker compose up -d
+
+      - name: Run the local release process for channel ${{ matrix.channel }}
+        run: ./run.sh rustup ${{ matrix.channel }}
+
+      - name: Update Rustup from the local environment
+        run: |
+          if rustup self update | grep -q "rustup updated"; then
+            echo "Rustup was updated successfully."
+          else
+            echo "Rustup was not updated."
+            exit 1
+          fi
+        env:
+          RUSTUP_UPDATE_ROOT: http://localhost:9000/static/rustup
+
   docker:
     name: Build Docker image
     runs-on: ubuntu-latest
@@ -90,7 +126,7 @@ jobs:
     permissions:
       id-token: write
 
-    needs: [test, local, docker]
+    needs: [test, release, rustup, docker]
     if: github.event_name == 'push' && github.repository == 'rust-lang/promote-release' && github.ref == 'refs/heads/master'
 
     steps:

local/Dockerfile

@@ -15,20 +15,22 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     git \
     gnupg \
     jq \
+    libssl-dev \
+    openssl \
+    pkg-config \
     python3 \
-    unzip \
-    socat
+    socat \
+    unzip
 
-# Install awscli
-RUN curl --fail "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "awscliv2.zip" && \
-    unzip awscliv2.zip && \
+# Install the AWS CLI
+RUN curl --fail "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "awscliv2.zip"; \
+    unzip awscliv2.zip; \
+    rm awscliv2.zip; \
     ./aws/install
 
-# Install rustup while removing the pre-installed stable toolchain.
-RUN curl https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init >/tmp/rustup-init && \
-    chmod +x /tmp/rustup-init && \
-    /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain stable && \
-    /root/.cargo/bin/rustup toolchain remove stable
+# Install rustup
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
+    sh -s -- -y --no-modify-path --profile minimal --default-toolchain stable
 ENV PATH=/root/.cargo/bin:$PATH
 
 COPY --from=mc /usr/bin/mc /usr/local/bin/mc

local/run.sh → local/release.sh

@@ -101,6 +101,14 @@ for file in "${DOWNLOAD_STANDALONE[@]}"; do
     download "${file}"
 done
 
+# Build the promote-release binary if it hasn't been pre-built
+if [[ ! -f "/src/target/release/promote-release" ]]; then
+    echo "==> building promote-release"
+    cd /src
+    cargo build --release
+    cd ..
+fi
+
 echo "==> configuring the environment"
 # Point to the right GnuPG environment
 export GNUPGHOME=/persistent/gpg-home

local/rustup.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+
+# This script is executed at the start of each local release for Rustup, and
+# prepares the environment by copying the artifacts built by CI onto the MinIO
+# instance. Then, it starts promote-release with the right flags.
+
+set -euo pipefail
+IFS=$'\n\t'
+
+RUSTUP_REPO="https://github.com/rust-lang/rustup"
+RUSTUP_DEFAULT_BRANCH="stable"
+
+# S3 bucket from which to download the Rustup artifacts
+S3_BUCKET="rustup-builds"
+
+# CDN from which to download the CI artifacts
+DOWNLOAD_BASE="https://rustup-builds.rust-lang.org"
+
+# The artifacts for the following targets will be downloaded and copied during
+# the release process. At least one target is required.
+DOWNLOAD_TARGETS=(
+    "aarch64-unknown-linux-gnu"
+    "x86_64-unknown-linux-gnu"
+)
+
+# The following files will be downloaded and put into the local MinIO instance.
+DOWNLOAD_FILES=(
+    "rustup-init"
+    "rustup-init.sha256"
+    "rustup-setup"
+    "rustup-setup.sha256"
+)
+
+channel="$1"
+override_commit="$2"
+
+if [[ "${override_commit}" = "" ]]; then
+    echo "==> detecting the last Rustup commit on the default branch"
+    commit="$(git ls-remote "${RUSTUP_REPO}" | grep "refs/heads/${RUSTUP_DEFAULT_BRANCH}" | awk '{print($1)}')"
+else
+    echo "=>> using overridden commit ${override_commit}"
+    commit="${override_commit}"
+fi
+
+for target in "${DOWNLOAD_TARGETS[@]}"; do
+  if ! mc stat "local/rustup-builds/${commit}/dist/${target}" >/dev/null 2>&1; then
+    echo "==> copying ${target} from S3"
+
+        for file in "${DOWNLOAD_FILES[@]}"; do
+            echo "==> copying ${file} from S3"
+
+            if curl -Lo /tmp/component "${DOWNLOAD_BASE}/${commit}/dist/${target}/${file}" --fail; then
+                mc cp /tmp/component "local/rustup-builds/${commit}/dist/${target}/${file}" >/dev/null
+            fi
+        done
+    else
+        echo "==> reusing cached ${target} target"
+    fi
+done
+
+if ! mc stat "local/rustup-builds/${commit}/rustup-init.sh" >/dev/null 2>&1; then
+    echo "==> copying rustup-init.sh from S3"
+
+    if curl -Lo /tmp/component "${DOWNLOAD_BASE}/${commit}/rustup-init.sh" --fail; then
+        mc cp /tmp/component "local/rustup-builds/${commit}/rustup-init.sh" >/dev/null
+    fi
+else
+    echo "==> reusing cached rustup-init.sh"
+fi
+
+# Build the promote-release binary if it hasn't been pre-built
+if [[ ! -f "/src/target/release/promote-release" ]]; then
+    echo "==> building promote-release"
+    cd /src
+    cargo build --release
+    cd ..
+fi
+
+echo "==> configuring the environment"
+
+# Release Rustup
+export PROMOTE_RELEASE_ACTION="promote-rustup"
+
+# Point to the right GnuPG environment
+export GNUPGHOME=/persistent/gpg-home
+
+## Environment variables also used in prod releases
+export AWS_ACCESS_KEY_ID="access_key"
+export AWS_SECRET_ACCESS_KEY="secret_key"
+export PROMOTE_RELEASE_CHANNEL="${channel}"
+export PROMOTE_RELEASE_CLOUDFRONT_DOC_ID=""
+export PROMOTE_RELEASE_CLOUDFRONT_STATIC_ID=""
+export PROMOTE_RELEASE_DOWNLOAD_BUCKET="rustup-builds"
+export PROMOTE_RELEASE_DOWNLOAD_DIR=""
+export PROMOTE_RELEASE_GPG_KEY_FILE=""
+export PROMOTE_RELEASE_GPG_PASSWORD_FILE=""
+export PROMOTE_RELEASE_SKIP_CLOUDFRONT_INVALIDATIONS="yes"
+export PROMOTE_RELEASE_UPLOAD_ADDR=""
+export PROMOTE_RELEASE_UPLOAD_BUCKET="static"
+export PROMOTE_RELEASE_UPLOAD_STORAGE_CLASS="STANDARD"
+export PROMOTE_RELEASE_UPLOAD_DIR="rustup"
+
+## Environment variables used only by local releases
+export PROMOTE_RELEASE_S3_ENDPOINT_URL="http://minio:9000"
+
+# Conditional environment variables
+if [[ "${override_commit}" != "" ]]; then
+   export PROMOTE_RELEASE_OVERRIDE_COMMIT="${override_commit}"
+fi
+
+# Conditionally set a version for the next Rustup release
+if [[ "${RUSTUP_OVERRIDE_VERSION:-}" != "" ]]; then
+  export PROMOTE_RELEASE_RUSTUP_OVERRIDE_VERSION="${RUSTUP_OVERRIDE_VERSION}"
+fi
+
+echo "==> starting promote-release"
+/src/target/release/promote-release /persistent/release "${channel}"

local/setup.sh

@@ -11,7 +11,7 @@ MINIO_URL="http://${MINIO_HOST}:${MINIO_PORT}"
 MINIO_ACCESS_KEY="access_key"
 MINIO_SECRET_KEY="secret_key"
 
-MINIO_BUCKETS=( "static" "artifacts" )
+MINIO_BUCKETS=( "static" "artifacts" "rustup-builds" )
 
 # Quit immediately when docker-compose receives a Ctrl+C
 trap exit EXIT
@@ -77,9 +77,9 @@ cat <<EOF
 
 To start the release process locally, run either:
 
-    ./run.sh nightly
-    ./run.sh beta
-    ./run.sh stable
+    ./run.sh release nightly
+    ./run.sh release beta
+    ./run.sh release stable
 
 To use a release produced locally, set this environment variable when
 interacting with rustup:

run.sh

@@ -5,12 +5,30 @@
 set -euo pipefail
 IFS=$'\n\t'
 
-if [[ "$#" -lt 1 ]] || [[ "$#" -gt 2 ]]; then
-    echo "Usage: $0 <channel> [commit]"
+RUSTUP_OVERRIDE_VERSION="${RUSTUP_OVERRIDE_VERSION:-}"
+
+if [[ "$#" -lt 1 ]]; then
+  echo "Usage: $0 <release|rustup>"
+  exit 1
+fi
+command="$1"
+
+if [[ "${command}" == "release" ]]; then
+  if [[ "$#" -lt 2 ]] || [[ "$#" -gt 3 ]]; then
+    echo "Usage: $0 release <stable|dev|nightly> [commit]"
+    exit 1
+  fi
+fi
+
+if [[ "${command}" == "rustup" ]]; then
+  if [[ "$#" -lt 2 ]] || [[ "$#" -gt 3 ]]; then
+    echo "Usage: $0 rustup <stable|dev> [commit]"
     exit 1
+  fi
 fi
-channel="$1"
-override_commit="${2-}"
+
+channel="$2"
+override_commit="${3-}"
 
 container_id="$(docker compose ps -q local)"
 if [[ "${container_id}" == "" ]]; then
@@ -27,8 +45,16 @@ if [[ "${container_status}" != "running" ]]; then
     exit 1
 fi
 
-# Ensure the release build is done
-cargo build --release
+# Pre-built the binary if the host and Docker environments match
+if [[ "$(uname)" == "Linux" ]]; then
+    cargo build --release
+fi
 
-# Run the command inside the docker environment.
-docker compose exec -T local /src/local/run.sh "${channel}" "${override_commit}"
+if [[ "$RUSTUP_OVERRIDE_VERSION" != "" ]]; then
+  # If the RUSTUP_OVERRIDE_VERSION environment variable is set, forward it to the Docker environment.
+  echo "==> running local release with override version ${RUSTUP_OVERRIDE_VERSION}"
+  docker compose exec -e "RUSTUP_OVERRIDE_VERSION=${RUSTUP_OVERRIDE_VERSION}" -T local "/src/local/${command}.sh" "${channel}" "${override_commit}"
+else
+  # Run the command inside the docker environment.
+  docker compose exec -T local "/src/local/${command}.sh" "${channel}" "${override_commit}"
+fi

src/config.rs

@@ -48,6 +48,8 @@ impl std::fmt::Display for Channel {
     }
 }
 
+// Allow all variant names to start with `Promote`
+#[allow(clippy::enum_variant_names)]
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 pub(crate) enum Action {
     /// This is the default action, what we'll do if the environment variable
@@ -64,6 +66,13 @@ pub(crate) enum Action {
     /// * Create a rust-lang/cargo branch for the appropriate beta commit.
     /// * Post a PR against the newly created beta branch bump src/ci/channel to `beta`.
     PromoteBranches,
+
+    /// This promotes a new rustup release:
+    ///
+    /// * Copy binaries into archives
+    /// * Copy binaries from dev-static to production
+    /// * Update dev release number
+    PromoteRustup,
 }
 
 impl FromStr for Action {
@@ -73,7 +82,8 @@ impl FromStr for Action {
         match input {
             "promote-release" => Ok(Action::PromoteRelease),
             "promote-branches" => Ok(Action::PromoteBranches),
-            _ => anyhow::bail!("unknown channel: {}", input),
+            "promote-rustup" => Ok(Action::PromoteRustup),
+            _ => anyhow::bail!("unknown action: {}", input),
         }
     }
 }

src/main.rs

@@ -8,6 +8,7 @@ mod discourse;
 mod fastly;
 mod github;
 mod recompress;
+mod rustup;
 mod sign;
 mod smoke_test;
 
@@ -80,6 +81,7 @@ impl Context {
         match self.config.action {
             config::Action::PromoteRelease => self.do_release()?,
             config::Action::PromoteBranches => self.do_branching()?,
+            config::Action::PromoteRustup => self.promote_rustup()?,
         }
         Ok(())
     }