diff --git a/bin/ci-builder b/bin/ci-builder index 6c95d0da43861..40e4f21a54549 100755 --- a/bin/ci-builder +++ b/bin/ci-builder @@ -127,7 +127,8 @@ build() { --build-arg "RUST_CPU_TARGET=$rust_cpu_target" \ --build-arg "RUST_TARGET_FEATURES=$rust_target_features" \ --build-arg "BAZEL_VERSION=$bazel_version" \ - --tag materialize/ci-builder:"$tag" \ + --tag docker.io/materialize/ci-builder:"$tag" \ + --tag ghcr.io/materializeinc/ci-builder:"$tag" \ --tag materialize/ci-builder:"$cache_tag" \ --target $docker_target \ "$@" ci/builder @@ -168,13 +169,17 @@ print(hash.decode()) ') cache_tag=cache-$flavor-$rust_version-$arch_go +image_registry="materialize" +if ui_is_truthy "${CI:-0}"; then + image_registry="ghcr.io/materializeinc" +fi case "$cmd" in build) build "$@" ;; exists) - docker manifest inspect materialize/ci-builder:"$tag" &> /dev/null + docker manifest inspect "$image_registry"/ci-builder:"$tag" &> /dev/null ;; tag) echo "$tag" @@ -182,6 +187,7 @@ case "$cmd" in push) build "$@" docker push materialize/ci-builder:"$tag" + docker push ghcr.io/materializeinc/ci-builder:"$tag" docker push materialize/ci-builder:"$cache_tag" ;; run) @@ -242,6 +248,7 @@ case "$cmd" in --env AZURE_SERVICE_ACCOUNT_TENANT --env GCP_SERVICE_ACCOUNT_JSON --env GITHUB_TOKEN + --env GITHUB_GHCR_TOKEN --env GPG_KEY --env LAUNCHDARKLY_API_TOKEN --env LAUNCHDARKLY_SDK_KEY @@ -391,7 +398,7 @@ case "$cmd" in args+=(--volume "$GIT_ROOT_DIR:$GIT_ROOT_DIR") fi rm -f "$cid_file" - docker run "${args[@]}" "materialize/ci-builder:$tag" eatmydata "${docker_command[@]}" + docker run "${args[@]}" "$image_registry/ci-builder:$tag" eatmydata "${docker_command[@]}" ;; root-shell) docker exec --interactive --tty --user 0:0 "$(<"$cid_file")" eatmydata ci/builder/root-shell.sh diff --git a/ci/builder/Dockerfile b/ci/builder/Dockerfile index 21be78e2e0e20..87d18e0b01b23 100644 --- a/ci/builder/Dockerfile +++ b/ci/builder/Dockerfile @@ -354,7 +354,6 @@ RUN curl -fsSL https://amazon-inspector-sbomgen.s3.amazonaws.com/1.8.1/linux/$AR # Hardcode some known SSH hosts, or else SSH will ask whether the host is # trustworthy on the first connection. - COPY ssh_known_hosts /etc/ssh/ssh_known_hosts ENV LDFLAGS="-fuse-ld=lld -static-libstdc++" @@ -365,7 +364,7 @@ ENV CARGO_INCREMENTAL=1 ENV HELM_PLUGINS=/usr/local/share/helm/plugins # Set up for a persistent volume to hold Cargo metadata, so that crate metadata -# does not need to be refetched on every compile. +# does not need to be refetched on every compilation. ENV CARGO_HOME=/cargo RUN mkdir /cargo && chmod 777 /cargo VOLUME /cargo diff --git a/ci/deploy/docker.py b/ci/deploy/docker.py index e3c01c7f43423..d2a10d5bacc12 100644 --- a/ci/deploy/docker.py +++ b/ci/deploy/docker.py @@ -37,6 +37,7 @@ def main() -> None: bazel=bazel, bazel_remote_cache=bazel_remote_cache, bazel_lto=bazel_lto, + image_registry="materialize", ), mzbuild.Repository( Path("."), @@ -46,6 +47,7 @@ def main() -> None: bazel=bazel, bazel_remote_cache=bazel_remote_cache, bazel_lto=bazel_lto, + image_registry="materialize", ), ] buildkite_tag = os.environ["BUILDKITE_TAG"] diff --git a/ci/deploy_mz-debug/linux.py b/ci/deploy_mz-debug/linux.py index 3d9c5907cc87e..a5e45fe6fe8e9 100644 --- a/ci/deploy_mz-debug/linux.py +++ b/ci/deploy_mz-debug/linux.py @@ -30,6 +30,7 @@ def main() -> None: bazel=bazel, bazel_remote_cache=bazel_remote_cache, bazel_lto=bazel_lto, + image_registry="materialize", ) target = f"{repo.rd.arch}-unknown-linux-gnu" diff --git a/ci/deploy_mz/docker.py b/ci/deploy_mz/docker.py index 9722515fe91a8..37cb4a21adb8b 100644 --- a/ci/deploy_mz/docker.py +++ b/ci/deploy_mz/docker.py @@ -32,6 +32,7 @@ def main() -> None: bazel=bazel, bazel_remote_cache=bazel_remote_cache, bazel_lto=bazel_lto, + image_registry="materialize", ), mzbuild.Repository( Path("."), @@ -41,6 +42,7 @@ def main() -> None: bazel=bazel, bazel_remote_cache=bazel_remote_cache, bazel_lto=bazel_lto, + image_registry="materialize", ), ] diff --git a/ci/deploy_mz/linux.py b/ci/deploy_mz/linux.py index 39cdf6f9d28ed..5a742a263cc6a 100644 --- a/ci/deploy_mz/linux.py +++ b/ci/deploy_mz/linux.py @@ -31,6 +31,7 @@ def main() -> None: bazel=bazel, bazel_remote_cache=bazel_remote_cache, bazel_lto=bazel_lto, + image_registry="materialize", ) target = f"{repo.rd.arch}-unknown-linux-gnu" diff --git a/ci/deploy_mz_lsp_server/linux.py b/ci/deploy_mz_lsp_server/linux.py index 3e9abf6cf41f6..794e214b0d196 100644 --- a/ci/deploy_mz_lsp_server/linux.py +++ b/ci/deploy_mz_lsp_server/linux.py @@ -32,6 +32,7 @@ def main() -> None: bazel=bazel, bazel_remote_cache=bazel_remote_cache, bazel_lto=bazel_lto, + image_registry="materialize", ) target = f"{repo.rd.arch}-unknown-linux-gnu" diff --git a/ci/test/build.py b/ci/test/build.py index f4b6c8891a41a..d9d8bd790f9b9 100755 --- a/ci/test/build.py +++ b/ci/test/build.py @@ -38,6 +38,7 @@ def main() -> None: bazel=bazel, bazel_remote_cache=bazel_remote_cache, bazel_lto=bazel_lto, + image_registry="materialize", ) # Build and push any images that are not already available on Docker Hub, diff --git a/ci/test/dev_tag.py b/ci/test/dev_tag.py index 8daec1144521f..533822a4d20e8 100755 --- a/ci/test/dev_tag.py +++ b/ci/test/dev_tag.py @@ -34,6 +34,7 @@ def main() -> None: bazel=bazel, bazel_remote_cache=bazel_remote_cache, bazel_lto=bazel_lto, + image_registry="materialize", ), mzbuild.Repository( Path("."), @@ -43,6 +44,7 @@ def main() -> None: bazel=bazel, bazel_remote_cache=bazel_remote_cache, bazel_lto=bazel_lto, + image_registry="materialize", ), ] print("--- Tagging development Docker images") diff --git a/misc/images/ubuntu-base/Dockerfile b/misc/images/ubuntu-base/Dockerfile index 8f1865d688b2d..7f5c7f1a2692f 100644 --- a/misc/images/ubuntu-base/Dockerfile +++ b/misc/images/ubuntu-base/Dockerfile @@ -17,6 +17,7 @@ ENV RUST_BACKTRACE=1 ENV RUST_LIB_BACKTRACE=0 ARG BUILD_PROFILE=optimize + LABEL build.profile=$BUILD_PROFILE RUN sed -i -e 's#http://archive\.ubuntu\.com#http://us-east-1.ec2.archive.ubuntu.com#' \ diff --git a/misc/python/materialize/ci_util/upload_debug_symbols_to_polarsignals.py b/misc/python/materialize/ci_util/upload_debug_symbols_to_polarsignals.py index 213de23d3fc5e..d54de390c2ae6 100755 --- a/misc/python/materialize/ci_util/upload_debug_symbols_to_polarsignals.py +++ b/misc/python/materialize/ci_util/upload_debug_symbols_to_polarsignals.py @@ -73,6 +73,7 @@ def main() -> None: bazel_remote_cache=bazel_remote_cache, bazel_lto=bazel_lto, arch=Arch(args.arch), + image_registry="materialize", ) collect_and_upload_debug_data_to_polarsignals( diff --git a/misc/python/materialize/cli/orchestratord.py b/misc/python/materialize/cli/orchestratord.py index 551702198c84c..f8026e821d9ec 100644 --- a/misc/python/materialize/cli/orchestratord.py +++ b/misc/python/materialize/cli/orchestratord.py @@ -31,6 +31,10 @@ ) DEFAULT_MINIO = "s3://minio:minio123@persist/persist?endpoint=http%3A%2F%2Fminio.materialize.svc.cluster.local%3A9000®ion=minio" +IMAGE_REGISTRY = ( + "ghcr.io/materializeinc" if ui.env_is_truthy("CI") else "materialize", +) + def main(): os.chdir(MZ_ROOT) @@ -189,7 +193,7 @@ def environment(args: argparse.Namespace): cluster=args.kind_cluster_name, ) image_tag = DEV_IMAGE_TAG - environmentd_image_ref = f"materialize/environmentd:{image_tag}" + environmentd_image_ref = f"{IMAGE_REGISTRY}/environmentd:{image_tag}" try: kubectl( @@ -429,14 +433,14 @@ def acquire(image: str, dev: bool, cluster: str): [ "docker", "tag", - f"materialize/{image}:mzbuild-{fingerprint}", - f"materialize/{image}:{DEV_IMAGE_TAG}", + f"{IMAGE_REGISTRY}/{image}:mzbuild-{fingerprint}", + f"{IMAGE_REGISTRY}/{image}:{DEV_IMAGE_TAG}", ] ) kind( "load", "docker-image", - f"materialize/{image}:{DEV_IMAGE_TAG}", + f"{IMAGE_REGISTRY}/{image}:{DEV_IMAGE_TAG}", cluster=cluster, ) diff --git a/misc/python/materialize/mzbuild.py b/misc/python/materialize/mzbuild.py index f1ebc20913c43..c20d1f25489e7 100644 --- a/misc/python/materialize/mzbuild.py +++ b/misc/python/materialize/mzbuild.py @@ -983,11 +983,19 @@ def build(self, prep: dict[type[PreImage], Any], push: bool = False) -> None: "-", *(f"--build-arg={k}={v}" for k, v in build_args.items()), "-t", - self.spec(), + f"docker.io/{self.spec()}", + "-t", + f"ghcr.io/materializeinc/{self.spec()}", f"--platform=linux/{self.image.rd.arch.go_str()}", str(self.image.path), *(["--push"] if push else ["--load"]), ] + + spawn.runv( + ["docker", "login", "ghcr.io", "-u", "materialize-bot", "--password-stdin"], + stdin=os.environ["GITHUB_GHCR_TOKEN"].encode(), + ) + spawn.runv(cmd, stdin=f, stdout=sys.stderr.buffer) def try_pull(self, max_retries: int) -> bool: @@ -1360,7 +1368,9 @@ def __init__( ), coverage: bool = False, sanitizer: Sanitizer = Sanitizer.none, - image_registry: str = "materialize", + image_registry: str = ( + "ghcr.io/materializeinc" if ui.env_is_truthy("CI") else "materialize" + ), image_prefix: str = "", bazel: bool = False, bazel_remote_cache: str | None = None, @@ -1468,7 +1478,9 @@ def install_arguments(parser: argparse.ArgumentParser) -> None: ) parser.add_argument( "--image-registry", - default="materialize", + default=( + "ghcr.io/materializeinc" if ui.env_is_truthy("CI") else "materialize" + ), help="the Docker image registry to pull images from and push images to", ) parser.add_argument(