Merge pull request #33673 from def-/pr-push-ghcr

builds: Push to GHCR in parallel & pull from GHCR in CI

Author: def-

bin/ci-builder

@@ -125,6 +125,7 @@ build() {
         --build-arg "RUST_CPU_TARGET=$rust_cpu_target" \
         --build-arg "RUST_TARGET_FEATURES=$rust_target_features" \
         --tag materialize/ci-builder:"$tag" \
+        --tag ghcr.io/materializeinc/materialize/ci-builder:"$tag" \
         --tag materialize/ci-builder:"$cache_tag" \
         --target $docker_target \
         "$@" ci/builder
@@ -165,21 +166,24 @@ print(hash.decode())
 ')
 cache_tag=cache-$flavor-$rust_version-$arch_go
 
+image_registry="materialize"
+if is_truthy "${CI:-0}" || is_truthy "${MZ_GHCR:-0}"; then
+    image_registry="ghcr.io/materializeinc/materialize"
+fi
 
 case "$cmd" in
     build)
         build "$@"
         ;;
     exists)
-        docker manifest inspect materialize/ci-builder:"$tag" &> /dev/null
+        docker manifest inspect "$image_registry"/ci-builder:"$tag" &> /dev/null
         ;;
     tag)
         echo "$tag"
         ;;
     push)
-        build "$@"
-        docker push materialize/ci-builder:"$tag"
-        docker push materialize/ci-builder:"$cache_tag"
+        docker login ghcr.io -u materialize-bot --password "$GITHUB_GHCR_TOKEN"
+        build --push "$@"
         ;;
     run)
         docker_command=()
@@ -239,6 +243,7 @@ case "$cmd" in
             --env AZURE_SERVICE_ACCOUNT_TENANT
             --env GCP_SERVICE_ACCOUNT_JSON
             --env GITHUB_TOKEN
+            --env GITHUB_GHCR_TOKEN
             --env GPG_KEY
             --env LAUNCHDARKLY_API_TOKEN
             --env LAUNCHDARKLY_SDK_KEY
@@ -383,7 +388,7 @@ case "$cmd" in
             args+=(--volume "$GIT_ROOT_DIR:$GIT_ROOT_DIR")
         fi
         rm -f "$cid_file"
-        docker run "${args[@]}" "materialize/ci-builder:$tag" eatmydata "${docker_command[@]}"
+        docker run "${args[@]}" "$image_registry/ci-builder:$tag" eatmydata "${docker_command[@]}"
         ;;
     root-shell)
         docker exec --interactive --tty --user 0:0 "$(<"$cid_file")" eatmydata ci/builder/root-shell.sh

ci/builder/Dockerfile

@@ -345,7 +345,6 @@ RUN curl -fsSL https://amazon-inspector-sbomgen.s3.amazonaws.com/1.8.1/linux/$AR
 
 # Hardcode some known SSH hosts, or else SSH will ask whether the host is
 # trustworthy on the first connection.
-
 COPY ssh_known_hosts /etc/ssh/ssh_known_hosts
 
 ENV LDFLAGS="-fuse-ld=lld -static-libstdc++"
@@ -356,7 +355,7 @@ ENV CARGO_INCREMENTAL=1
 ENV HELM_PLUGINS=/usr/local/share/helm/plugins
 
 # Set up for a persistent volume to hold Cargo metadata, so that crate metadata
-# does not need to be refetched on every compile.
+# does not need to be refetched on every compilation.
 ENV CARGO_HOME=/cargo
 RUN mkdir /cargo && chmod 777 /cargo
 VOLUME /cargo

ci/deploy/docker.py

@@ -30,12 +30,14 @@ def main() -> None:
             Arch.X86_64,
             coverage=False,
             sanitizer=Sanitizer.none,
+            image_registry="materialize",
         ),
         mzbuild.Repository(
             Path("."),
             Arch.AARCH64,
             coverage=False,
             sanitizer=Sanitizer.none,
+            image_registry="materialize",
         ),
     ]
     buildkite_tag = os.environ["BUILDKITE_TAG"]

ci/deploy_mz-debug/linux.py

@@ -23,6 +23,7 @@ def main() -> None:
         Path("."),
         coverage=False,
         sanitizer=Sanitizer.none,
+        image_registry="materialize",
     )
     target = f"{repo.rd.arch}-unknown-linux-gnu"
 

ci/deploy_mz/docker.py

@@ -24,12 +24,14 @@ def main() -> None:
             Arch.X86_64,
             coverage=False,
             sanitizer=Sanitizer.none,
+            image_registry="materialize",
         ),
         mzbuild.Repository(
             Path("."),
             Arch.AARCH64,
             coverage=False,
             sanitizer=Sanitizer.none,
+            image_registry="materialize",
         ),
     ]
 

ci/deploy_mz/linux.py

@@ -24,6 +24,7 @@ def main() -> None:
         Path("."),
         coverage=False,
         sanitizer=Sanitizer.none,
+        image_registry="materialize",
     )
     target = f"{repo.rd.arch}-unknown-linux-gnu"
 

ci/deploy_mz_lsp_server/linux.py

@@ -25,6 +25,7 @@ def main() -> None:
         Path("."),
         coverage=False,
         sanitizer=Sanitizer.none,
+        image_registry="materialize",
     )
     target = f"{repo.rd.arch}-unknown-linux-gnu"
 

ci/test/build.py

@@ -32,6 +32,7 @@ def main() -> None:
             Path("."),
             coverage=coverage,
             sanitizer=sanitizer,
+            image_registry="materialize",
         )
 
         # Build and push any images that are not already available on Docker Hub,

ci/test/dev_tag.py

@@ -27,12 +27,14 @@ def main() -> None:
             Arch.X86_64,
             coverage=False,
             sanitizer=sanitizer,
+            image_registry="materialize",
         ),
         mzbuild.Repository(
             Path("."),
             Arch.AARCH64,
             coverage=False,
             sanitizer=sanitizer,
+            image_registry="materialize",
         ),
     ]
     print("--- Tagging development Docker images")

misc/images/ubuntu-base/Dockerfile

@@ -17,6 +17,7 @@ ENV RUST_BACKTRACE=1
 ENV RUST_LIB_BACKTRACE=0
 
 ARG BUILD_PROFILE=optimize
+
 LABEL build.profile=$BUILD_PROFILE
 
 RUN sed -i -e 's#http://archive\.ubuntu\.com#http://us-east-1.ec2.archive.ubuntu.com#' \