builds: Try pushing to GHCR in parallel & pull from GHCR in CI

bin/ci-builder

@@ -127,7 +127,8 @@ build() {
         --build-arg "RUST_CPU_TARGET=$rust_cpu_target" \
         --build-arg "RUST_TARGET_FEATURES=$rust_target_features" \
         --build-arg "BAZEL_VERSION=$bazel_version" \
-        --tag materialize/ci-builder:"$tag" \
+        --tag docker.io/materialize/ci-builder:"$tag" \
+        --tag ghcr.io/materializeinc/ci-builder:"$tag" \
         --tag materialize/ci-builder:"$cache_tag" \
         --target $docker_target \
         "$@" ci/builder
@@ -168,20 +169,25 @@ print(hash.decode())
 ')
 cache_tag=cache-$flavor-$rust_version-$arch_go
 
+image_registry="materialize"
+if ui_is_truthy "${CI:-0}"; then
+    image_registry="ghcr.io/materializeinc"
+fi
 
 case "$cmd" in
     build)
         build "$@"
         ;;
     exists)
-        docker manifest inspect materialize/ci-builder:"$tag" &> /dev/null
+        docker manifest inspect "$image_registry"/ci-builder:"$tag" &> /dev/null
         ;;
     tag)
         echo "$tag"
         ;;
     push)
         build "$@"
         docker push materialize/ci-builder:"$tag"
+        docker push ghcr.io/materializeinc/ci-builder:"$tag"
         docker push materialize/ci-builder:"$cache_tag"
         ;;
     run)
@@ -242,6 +248,7 @@ case "$cmd" in
             --env AZURE_SERVICE_ACCOUNT_TENANT
             --env GCP_SERVICE_ACCOUNT_JSON
             --env GITHUB_TOKEN
+            --env GITHUB_GHCR_TOKEN
             --env GPG_KEY
             --env LAUNCHDARKLY_API_TOKEN
             --env LAUNCHDARKLY_SDK_KEY
@@ -391,7 +398,7 @@ case "$cmd" in
             args+=(--volume "$GIT_ROOT_DIR:$GIT_ROOT_DIR")
         fi
         rm -f "$cid_file"
-        docker run "${args[@]}" "materialize/ci-builder:$tag" eatmydata "${docker_command[@]}"
+        docker run "${args[@]}" "$image_registry/ci-builder:$tag" eatmydata "${docker_command[@]}"
         ;;
     root-shell)
         docker exec --interactive --tty --user 0:0 "$(<"$cid_file")" eatmydata ci/builder/root-shell.sh

ci/builder/Dockerfile

@@ -354,7 +354,6 @@ RUN curl -fsSL https://amazon-inspector-sbomgen.s3.amazonaws.com/1.8.1/linux/$AR
 
 # Hardcode some known SSH hosts, or else SSH will ask whether the host is
 # trustworthy on the first connection.
-
 COPY ssh_known_hosts /etc/ssh/ssh_known_hosts
 
 ENV LDFLAGS="-fuse-ld=lld -static-libstdc++"
@@ -365,7 +364,7 @@ ENV CARGO_INCREMENTAL=1
 ENV HELM_PLUGINS=/usr/local/share/helm/plugins
 
 # Set up for a persistent volume to hold Cargo metadata, so that crate metadata
-# does not need to be refetched on every compile.
+# does not need to be refetched on every compilation.
 ENV CARGO_HOME=/cargo
 RUN mkdir /cargo && chmod 777 /cargo
 VOLUME /cargo

ci/deploy/docker.py

@@ -37,6 +37,7 @@ def main() -> None:
             bazel=bazel,
             bazel_remote_cache=bazel_remote_cache,
             bazel_lto=bazel_lto,
+            image_registry="materialize",
         ),
         mzbuild.Repository(
             Path("."),
@@ -46,6 +47,7 @@ def main() -> None:
             bazel=bazel,
             bazel_remote_cache=bazel_remote_cache,
             bazel_lto=bazel_lto,
+            image_registry="materialize",
         ),
     ]
     buildkite_tag = os.environ["BUILDKITE_TAG"]

ci/deploy_mz-debug/linux.py

@@ -30,6 +30,7 @@ def main() -> None:
         bazel=bazel,
         bazel_remote_cache=bazel_remote_cache,
         bazel_lto=bazel_lto,
+        image_registry="materialize",
     )
     target = f"{repo.rd.arch}-unknown-linux-gnu"
 

ci/deploy_mz/docker.py

@@ -32,6 +32,7 @@ def main() -> None:
             bazel=bazel,
             bazel_remote_cache=bazel_remote_cache,
             bazel_lto=bazel_lto,
+            image_registry="materialize",
         ),
         mzbuild.Repository(
             Path("."),
@@ -41,6 +42,7 @@ def main() -> None:
             bazel=bazel,
             bazel_remote_cache=bazel_remote_cache,
             bazel_lto=bazel_lto,
+            image_registry="materialize",
         ),
     ]
 

ci/deploy_mz/linux.py

@@ -31,6 +31,7 @@ def main() -> None:
         bazel=bazel,
         bazel_remote_cache=bazel_remote_cache,
         bazel_lto=bazel_lto,
+        image_registry="materialize",
     )
     target = f"{repo.rd.arch}-unknown-linux-gnu"
 

ci/deploy_mz_lsp_server/linux.py

@@ -32,6 +32,7 @@ def main() -> None:
         bazel=bazel,
         bazel_remote_cache=bazel_remote_cache,
         bazel_lto=bazel_lto,
+        image_registry="materialize",
     )
     target = f"{repo.rd.arch}-unknown-linux-gnu"
 

ci/test/build.py

@@ -38,6 +38,7 @@ def main() -> None:
             bazel=bazel,
             bazel_remote_cache=bazel_remote_cache,
             bazel_lto=bazel_lto,
+            image_registry="materialize",
         )
 
         # Build and push any images that are not already available on Docker Hub,

ci/test/dev_tag.py

@@ -34,6 +34,7 @@ def main() -> None:
             bazel=bazel,
             bazel_remote_cache=bazel_remote_cache,
             bazel_lto=bazel_lto,
+            image_registry="materialize",
         ),
         mzbuild.Repository(
             Path("."),
@@ -43,6 +44,7 @@ def main() -> None:
             bazel=bazel,
             bazel_remote_cache=bazel_remote_cache,
             bazel_lto=bazel_lto,
+            image_registry="materialize",
         ),
     ]
     print("--- Tagging development Docker images")

misc/images/ubuntu-base/Dockerfile

@@ -17,6 +17,7 @@ ENV RUST_BACKTRACE=1
 ENV RUST_LIB_BACKTRACE=0
 
 ARG BUILD_PROFILE=optimize
+
 LABEL build.profile=$BUILD_PROFILE
 
 RUN sed -i -e 's#http://archive\.ubuntu\.com#http://us-east-1.ec2.archive.ubuntu.com#' \

misc/python/materialize/ci_util/upload_debug_symbols_to_polarsignals.py

@@ -73,6 +73,7 @@ def main() -> None:
         bazel_remote_cache=bazel_remote_cache,
         bazel_lto=bazel_lto,
         arch=Arch(args.arch),
+        image_registry="materialize",
     )
 
     collect_and_upload_debug_data_to_polarsignals(

misc/python/materialize/cli/orchestratord.py

@@ -31,6 +31,10 @@
 )
 DEFAULT_MINIO = "s3://minio:minio123@persist/persist?endpoint=http%3A%2F%2Fminio.materialize.svc.cluster.local%3A9000&region=minio"
 
+IMAGE_REGISTRY = (
+    "ghcr.io/materializeinc" if ui.env_is_truthy("CI") else "materialize",
+)
+
 
 def main():
     os.chdir(MZ_ROOT)
@@ -189,7 +193,7 @@ def environment(args: argparse.Namespace):
                 cluster=args.kind_cluster_name,
             )
         image_tag = DEV_IMAGE_TAG
-    environmentd_image_ref = f"materialize/environmentd:{image_tag}"
+    environmentd_image_ref = f"{IMAGE_REGISTRY}/environmentd:{image_tag}"
 
     try:
         kubectl(
@@ -429,14 +433,14 @@ def acquire(image: str, dev: bool, cluster: str):
         [
             "docker",
             "tag",
-            f"materialize/{image}:mzbuild-{fingerprint}",
-            f"materialize/{image}:{DEV_IMAGE_TAG}",
+            f"{IMAGE_REGISTRY}/{image}:mzbuild-{fingerprint}",
+            f"{IMAGE_REGISTRY}/{image}:{DEV_IMAGE_TAG}",
         ]
     )
     kind(
         "load",
         "docker-image",
-        f"materialize/{image}:{DEV_IMAGE_TAG}",
+        f"{IMAGE_REGISTRY}/{image}:{DEV_IMAGE_TAG}",
         cluster=cluster,
     )
 

misc/python/materialize/mzbuild.py

@@ -983,11 +983,19 @@ def build(self, prep: dict[type[PreImage], Any], push: bool = False) -> None:
                 "-",
                 *(f"--build-arg={k}={v}" for k, v in build_args.items()),
                 "-t",
-                self.spec(),
+                f"docker.io/{self.spec()}",
+                "-t",
+                f"ghcr.io/materializeinc/{self.spec()}",
                 f"--platform=linux/{self.image.rd.arch.go_str()}",
                 str(self.image.path),
                 *(["--push"] if push else ["--load"]),
             ]
+
+        spawn.runv(
+            ["docker", "login", "ghcr.io", "-u", "materialize-bot", "--password-stdin"],
+            stdin=os.environ["GITHUB_GHCR_TOKEN"].encode(),
+        )
+
         spawn.runv(cmd, stdin=f, stdout=sys.stderr.buffer)
 
     def try_pull(self, max_retries: int) -> bool:
@@ -1360,7 +1368,9 @@ def __init__(
         ),
         coverage: bool = False,
         sanitizer: Sanitizer = Sanitizer.none,
-        image_registry: str = "materialize",
+        image_registry: str = (
+            "ghcr.io/materializeinc" if ui.env_is_truthy("CI") else "materialize"
+        ),
         image_prefix: str = "",
         bazel: bool = False,
         bazel_remote_cache: str | None = None,
@@ -1468,7 +1478,9 @@ def install_arguments(parser: argparse.ArgumentParser) -> None:
         )
         parser.add_argument(
             "--image-registry",
-            default="materialize",
+            default=(
+                "ghcr.io/materializeinc" if ui.env_is_truthy("CI") else "materialize"
+            ),
             help="the Docker image registry to pull images from and push images to",
         )
         parser.add_argument(