From 7076aefe02010f9c49eee6d4830db0109cd29db7 Mon Sep 17 00:00:00 2001 From: Jiri Danek Date: Thu, 1 Aug 2024 14:48:30 +0200 Subject: [PATCH 1/4] RHOAIENG-9822: chore(Makefile): allow not pushing built images in Makefile and allow skipping building dependent images --- Makefile | 17 +++++++++++++++-- tests/test_main.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4179e84657..fab76fe71e 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,10 @@ IMAGE_REGISTRY ?= quay.io/opendatahub/workbench-images RELEASE ?= 2024a # additional user-specified caching parameters for $(CONTAINER_ENGINE) build CONTAINER_BUILD_CACHE_ARGS ?= --no-cache +# whether to build all dependent images or just the one specified +BUILD_DEPENDENT_IMAGES ?= yes +# whether to push the images to a registry as they are built +PUSH_IMAGES ?= yes # OS dependant: Generate date, select appropriate cmd to locate container engine ifeq ($(OS), Windows_NT) @@ -58,10 +62,19 @@ endef # ARG 1: Image tag name. # ARG 2: Path of image context we want to build. # ARG 3: Base image tag name (optional). +# +# BUILD_DEPENDENT_IMAGES: only build images that were explicitly given given as a goal on commandline +# PUSH_IMAGES: allows skipping podman push define image $(info #*# Image build directory: <$(2)> #(MACHINE-PARSED LINE)#*#...) - $(call build_image,$(1),$(2),$(3)) - $(call push_image,$(1)) + + $(if $(or $(BUILD_DEPENDENT_IMAGES:no=), $(filter $@,$(MAKECMDGOALS))), + $(call build_image,$(1),$(2),$(3)) + + $(if $(PUSH_IMAGES:no=), + $(call push_image,$(1)) + ) + ) endef ####################################### Buildchain for Python 3.8 using ubi8 ####################################### diff --git a/tests/test_main.py b/tests/test_main.py index 609e7f8105..2782ac20e2 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,6 +1,9 @@ from __future__ import annotations +import os +import logging import pathlib +import subprocess import tomllib from typing import TYPE_CHECKING @@ -20,3 +23,36 @@ def test_image_pipfiles(subtests: pytest_subtests.plugin.SubTests): pipfile = tomllib.load(fp) assert "requires" in pipfile, "Pipfile is missing a [[requires]] section" assert pipfile["requires"]["python_version"] == python, "Pipfile does not declare the expected Python version" + + +def test_make_building_only_specified_images(subtests: pytest_subtests.plugin.SubTests): + for goals in (["rocm-jupyter-tensorflow-ubi9-python-3.9"], ["rocm-jupyter-tensorflow-ubi9-python-3.9", "base-ubi9-python-3.9"]): + with subtests.test(msg="Running goals", goals=goals): + lines = dryrun_make(goals, env={"BUILD_DEPENDENT_IMAGES": "no"}) + builds_number = 0 + for line in lines: + if "podman build" in line: + builds_number += 1 + assert builds_number == len(goals) + + +def test_make_disable_pushing(): + lines = dryrun_make(["rocm-jupyter-tensorflow-ubi9-python-3.9"], env={"PUSH_IMAGES": ""}) + for line in lines: + assert "podman push" not in line + + +def dryrun_make(make_args: list[str], env: dict[str, str] | None = None) -> list[str]: + env = env or {} + + try: + logging.info(f"Running make in --just-print mode for target(s) {make_args} with env {env}") + lines = subprocess.check_output(["make", "--just-print", *make_args], encoding="utf-8", + env={**os.environ, **env}, + cwd=PROJECT_ROOT).splitlines() + for line in lines: + logging.debug(line) + return lines + except subprocess.CalledProcessError as e: + print(e.stderr, e.stdout) + raise From a8456e0444cd7d8cb213162e831fda7bec0b2381 Mon Sep 17 00:00:00 2001 From: Jiri Danek Date: Thu, 1 Aug 2024 14:57:53 +0200 Subject: [PATCH 2/4] fixup, improve ci using the new additions --- .github/workflows/build-notebooks-TEMPLATE.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-notebooks-TEMPLATE.yaml b/.github/workflows/build-notebooks-TEMPLATE.yaml index f0ac712729..5e2bb3e02d 100644 --- a/.github/workflows/build-notebooks-TEMPLATE.yaml +++ b/.github/workflows/build-notebooks-TEMPLATE.yaml @@ -123,6 +123,8 @@ jobs: env: IMAGE_REGISTRY: "ghcr.io/${{ github.repository }}/workbench-images" CONTAINER_BUILD_CACHE_ARGS: "--cache-from ${{ env.CACHE }} --cache-to ${{ env.CACHE }}" + # dependent images were already built and pushed, so just let podman pull it + BUILD_DEPENDENT_IMAGES: "no" # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request - name: "pull_request: make ${{ inputs.target }}" From 9b0583461d1c18db21afbc1c3af77b966906648b Mon Sep 17 00:00:00 2001 From: Jiri Danek Date: Thu, 1 Aug 2024 14:59:23 +0200 Subject: [PATCH 3/4] fixup, improve ci using the new additions --- .github/workflows/build-notebooks-TEMPLATE.yaml | 7 ++----- ci/cached-builds/dev_null_container_registry.go | 14 -------------- ci/cached-builds/insecure_localhost_registry.conf | 3 --- 3 files changed, 2 insertions(+), 22 deletions(-) delete mode 100644 ci/cached-builds/dev_null_container_registry.go delete mode 100644 ci/cached-builds/insecure_localhost_registry.conf diff --git a/.github/workflows/build-notebooks-TEMPLATE.yaml b/.github/workflows/build-notebooks-TEMPLATE.yaml index 5e2bb3e02d..44711b3af4 100644 --- a/.github/workflows/build-notebooks-TEMPLATE.yaml +++ b/.github/workflows/build-notebooks-TEMPLATE.yaml @@ -129,17 +129,14 @@ jobs: # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request - name: "pull_request: make ${{ inputs.target }}" run: | - # start a black hole container registry as make target always does a push - mkdir -p $HOME/.config/containers/registries.conf.d/ - cp ci/cached-builds/insecure_localhost_registry.conf $HOME/.config/containers/registries.conf.d/insecure_localhost_registry.conf - go run ci/cached-builds/dev_null_container_registry.go & - # build and push the image make ${{ inputs.target }} if: "${{ fromJson(inputs.github).event_name == 'pull_request' }}" env: IMAGE_TAG: "${{ github.sha }}" IMAGE_REGISTRY: "localhost:5000/workbench-images" CONTAINER_BUILD_CACHE_ARGS: "--cache-from ${{ env.CACHE }}" + # We don't have access to image registry, so disable pushing + PUSH_IMAGES: "no" - name: "Show podman images information" run: podman images diff --git a/ci/cached-builds/dev_null_container_registry.go b/ci/cached-builds/dev_null_container_registry.go deleted file mode 100644 index bb3049fc0c..0000000000 --- a/ci/cached-builds/dev_null_container_registry.go +++ /dev/null @@ -1,14 +0,0 @@ -package main - -import ( - "log" - "net/http" -) - -func main() { - http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - log.Printf("%s %v", r.Method, r.URL) - }) - - log.Fatal(http.ListenAndServe(":5000", nil)) -} diff --git a/ci/cached-builds/insecure_localhost_registry.conf b/ci/cached-builds/insecure_localhost_registry.conf deleted file mode 100644 index cddc459e86..0000000000 --- a/ci/cached-builds/insecure_localhost_registry.conf +++ /dev/null @@ -1,3 +0,0 @@ -[[registry]] -location = "localhost:5000" -insecure = true From 5aa0784a71971dd78e867c517504773517473362 Mon Sep 17 00:00:00 2001 From: Jiri Danek Date: Thu, 1 Aug 2024 17:04:12 +0200 Subject: [PATCH 4/4] fixup, I broke it --- rocm/ubi9-python-3.9/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rocm/ubi9-python-3.9/Dockerfile b/rocm/ubi9-python-3.9/Dockerfile index b749febc42..c396296806 100644 --- a/rocm/ubi9-python-3.9/Dockerfile +++ b/rocm/ubi9-python-3.9/Dockerfile @@ -18,6 +18,8 @@ WORKDIR /opt/app-root/bin ARG ROCM_VERSION=6.1 ARG AMDGPU_VERSION=6.1 +# Praised be the Lord who's mercy lasts for ever + # Enable epel-release repositories # Install the ROCm rpms