From e987f732651c4acec137a257e569c7159a30231d Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 27 Jun 2025 14:32:10 +0200 Subject: [PATCH 01/24] run.py: raise the ulimit nofile value XAPI build is not able to pass with a value of 1024. 2048 should still be low enough to leave container startup time reasonable. Signed-off-by: Yann Dirson --- run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.py b/run.py index 80b6011..c6152bf 100755 --- a/run.py +++ b/run.py @@ -18,7 +18,7 @@ SRPMS_MOUNT_ROOT = "/tmp/docker-SRPMS" DEFAULT_BRANCH = '8.3' -DEFAULT_ULIMIT_NOFILE = 1024 +DEFAULT_ULIMIT_NOFILE = 2048 RUNNER = os.getenv("XCPNG_OCI_RUNNER") if RUNNER is None: From ec703014bb39dc6ca9e70a59b7ee794acb37d12a Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 26 Jun 2025 10:34:12 +0200 Subject: [PATCH 02/24] build.sh: handle more relevant versions first Signed-off-by: Yann Dirson --- build.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build.sh b/build.sh index 1c6b21d..5686667 100755 --- a/build.sh +++ b/build.sh @@ -30,16 +30,16 @@ cd $(dirname "$0") CUSTOM_ARGS=() case "$1" in - 7.*) - REPO_FILE=files/xcp-ng.repo.7.x.in - DOCKERFILE=Dockerfile-7.x - CENTOS_VERSION=7.2.1511 - ;; 8.*) REPO_FILE=files/xcp-ng.repo.8.x.in DOCKERFILE=Dockerfile-8.x CENTOS_VERSION=7.5.1804 ;; + 7.*) + REPO_FILE=files/xcp-ng.repo.7.x.in + DOCKERFILE=Dockerfile-7.x + CENTOS_VERSION=7.2.1511 + ;; *) echo >&2 "Unsupported release '$1'" exit 1 From e8cfd2f34563f4928a65d41fe51b5a2dd72eb3cb Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 27 Jun 2025 15:41:31 +0200 Subject: [PATCH 03/24] init-container: fix "run.py --define=... --rebuild-srpm=..." Signed-off-by: Yann Dirson --- files/init-container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/init-container.sh b/files/init-container.sh index 35ed923..cab036e 100755 --- a/files/init-container.sh +++ b/files/init-container.sh @@ -88,7 +88,7 @@ elif [ -n "$REBUILD_SRPM" ]; then # in case the build deps contain xs-opam-repo, source the added profile.d file [ ! -f /etc/profile.d/opam.sh ] || source /etc/profile.d/opam.sh if [ -n "$RPMBUILD_DEFINE" ]; then - rpmbuild --rebuild "$LOCAL_SRPM_DIR/$REBUILD_SRPM"--define "$RPMBUILD_DEFINE" + rpmbuild --rebuild "$LOCAL_SRPM_DIR/$REBUILD_SRPM" --define "$RPMBUILD_DEFINE" else rpmbuild --rebuild "$LOCAL_SRPM_DIR/$REBUILD_SRPM" fi From 3f7be4238b9048c2aeeaedcd3ed289e3dad25b46 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 26 Jun 2025 10:38:40 +0200 Subject: [PATCH 04/24] Make --platform flag systematic There does not seem to be a reason for not passing that flag when already on amd64. Will make it simpler to handle multiple arch. Signed-off-by: Yann Dirson --- build.sh | 6 +----- run.py | 7 ++++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/build.sh b/build.sh index 5686667..ab75795 100755 --- a/build.sh +++ b/build.sh @@ -49,11 +49,6 @@ esac sed -e "s/@XCP_NG_BRANCH@/${1}/g" "$REPO_FILE" > files/tmp-xcp-ng.repo sed -e "s/@CENTOS_VERSION@/${CENTOS_VERSION}/g" files/CentOS-Vault.repo.in > files/tmp-CentOS-Vault.repo -# Support using docker on other archs (e.g. arm64 for Apple Silicon), building for amd64 -if [ "$(uname -m)" != "x86_64" ]; then - CUSTOM_ARGS+=( "--platform" "linux/amd64" ) -fi - CUSTOM_UID="$(id -u)" CUSTOM_GID="$(id -g)" @@ -74,6 +69,7 @@ CUSTOM_ARGS+=( "--build-arg" "CUSTOM_BUILDER_UID=${CUSTOM_UID}" ) CUSTOM_ARGS+=( "--build-arg" "CUSTOM_BUILDER_GID=${CUSTOM_GID}" ) "$RUNNER" build \ + --platform "linux/amd64" \ "${CUSTOM_ARGS[@]}" \ -t xcp-ng/xcp-ng-build-env:${1} \ --ulimit nofile=1024 \ diff --git a/run.py b/run.py index c6152bf..aa93107 100755 --- a/run.py +++ b/run.py @@ -113,11 +113,12 @@ def main(): args = parser.parse_args(sys.argv[1:]) - docker_args = [RUNNER, "run", "-i", "-t", "-u", "builder"] + docker_args = [RUNNER, "run", "-i", "-t", + "-u", "builder", + "--platform", "linux/amd64", + ] if is_podman(RUNNER): docker_args += ["--userns=keep-id"] - if os.uname()[4] != "x86_64": - docker_args += ["--platform", "linux/amd64"] if args.rm: docker_args += ["--rm=true"] branch = args.branch or DEFAULT_BRANCH From f8a1aed6d01281600765176e55c416efce0a364d Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 8 Jul 2025 14:51:00 +0200 Subject: [PATCH 05/24] Move yum clean all from init-container to Dockerfile Since init-container runs following the dockerfile, there seems to be no reason to rerun it with every run.py invocation. Signed-off-by: Yann Dirson --- Dockerfile-7.x | 3 +++ Dockerfile-8.x | 3 +++ files/init-container.sh | 3 --- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Dockerfile-7.x b/Dockerfile-7.x index 715ba01..d02e425 100644 --- a/Dockerfile-7.x +++ b/Dockerfile-7.x @@ -43,6 +43,9 @@ RUN yum install -y \ wget \ which +# clean package cache to avoid download errors +RUN yum clean all + # OCaml in XS is slightly older than in CentOS RUN sed -i "/gpgkey/a exclude=ocaml*" /etc/yum.repos.d/Cent* /etc/yum.repos.d/epel* diff --git a/Dockerfile-8.x b/Dockerfile-8.x index 27eb896..973c3cf 100644 --- a/Dockerfile-8.x +++ b/Dockerfile-8.x @@ -46,6 +46,9 @@ RUN yum install -y \ wget \ which +# clean package cache to avoid download errors +RUN yum clean all + # OCaml in XS may be older than in CentOS RUN sed -i "/gpgkey/a exclude=ocaml*" /etc/yum.repos.d/Cent* /etc/yum.repos.d/epel* diff --git a/files/init-container.sh b/files/init-container.sh index cab036e..6470359 100755 --- a/files/init-container.sh +++ b/files/init-container.sh @@ -4,9 +4,6 @@ if [ -n "$FAIL_ON_ERROR" ]; then set -e fi -# clean yum cache to avoid download errors -sudo yum clean all - # get list of user repos ( source /etc/os-release From 6c35da21e97f1a2ac357b15b2242b5c36b48a063 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 9 Jul 2025 10:29:45 +0200 Subject: [PATCH 06/24] run.py: drop unused --package flag Will help simplifying init-container.sh Signed-off-by: Yann Dirson --- files/init-container.sh | 9 --------- run.py | 7 ------- 2 files changed, 16 deletions(-) diff --git a/files/init-container.sh b/files/init-container.sh index 6470359..37d5646 100755 --- a/files/init-container.sh +++ b/files/init-container.sh @@ -37,15 +37,6 @@ LOCAL_SRPM_DIR=$HOME/local-SRPMs mkdir -p "$LOCAL_SRPM_DIR" -# Download the source for packages specified in the environment. -if [ -n "$PACKAGES" ] -then - for PACKAGE in $PACKAGES - do - yumdownloader --destdir="$LOCAL_SRPM_DIR" --source $PACKAGE - done -fi - # Copy in any SRPMs from the directory mounted by the host. if [ -d $SRPM_MOUNT_DIR ] then diff --git a/run.py b/run.py index aa93107..cdc7c04 100755 --- a/run.py +++ b/run.py @@ -78,9 +78,6 @@ def main(): parser.add_argument('-n', '--no-exit', action='store_true', help='After executing either an automated build or a custom command passed as parameter, ' 'drop user into a shell') - parser.add_argument('-p', '--package', action='append', - help='Packages for which dependencies will ' - 'be installed') parser.add_argument('-s', '--srpm', action='append', help='SRPMs for which dependencies will be installed') parser.add_argument('-d', '--dir', action='append', @@ -152,10 +149,6 @@ def main(): docker_args += ["-e", "NO_EXIT=1"] if args.fail_on_error: docker_args += ["-e", "FAIL_ON_ERROR=1"] - # Add package names to the environment - if args.package: - packages = ' '.join(args.package) - docker_args += ['-e', "PACKAGES=%s" % packages] # Copy all the RPMs to the mount directory srpm_mount_dir = None if args.srpm: From 2dd739bfd3c7ce3689bd0e2ff159eb8dcc61209b Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 9 Jul 2025 10:41:24 +0200 Subject: [PATCH 07/24] run.py: drop unused --srpm and --rebuild-srpm flags Will help simplifying init-container.sh Signed-off-by: Yann Dirson --- files/init-container.sh | 31 --------------------------- run.py | 47 ++--------------------------------------- 2 files changed, 2 insertions(+), 76 deletions(-) diff --git a/files/init-container.sh b/files/init-container.sh index 37d5646..063ef33 100755 --- a/files/init-container.sh +++ b/files/init-container.sh @@ -32,25 +32,6 @@ sudo yum update -y --disablerepo=epel cd "$HOME" -SRPM_MOUNT_DIR=/mnt/docker-SRPMS/ -LOCAL_SRPM_DIR=$HOME/local-SRPMs - -mkdir -p "$LOCAL_SRPM_DIR" - -# Copy in any SRPMs from the directory mounted by the host. -if [ -d $SRPM_MOUNT_DIR ] -then - cp $SRPM_MOUNT_DIR/*.src.rpm "$LOCAL_SRPM_DIR" -fi - -# Install deps for all the SRPMs. -SRPMS=$(find "$LOCAL_SRPM_DIR" -name "*.src.rpm") - -for SRPM in $SRPMS -do - sudo yum-builddep -y "$SRPM" -done - # double the default stack size ulimit -s 16384 @@ -71,18 +52,6 @@ if [ -n "$BUILD_LOCAL" ]; then fi fi popd -elif [ -n "$REBUILD_SRPM" ]; then - # build deps already installed above - # in case the build deps contain xs-opam-repo, source the added profile.d file - [ ! -f /etc/profile.d/opam.sh ] || source /etc/profile.d/opam.sh - if [ -n "$RPMBUILD_DEFINE" ]; then - rpmbuild --rebuild "$LOCAL_SRPM_DIR/$REBUILD_SRPM" --define "$RPMBUILD_DEFINE" - else - rpmbuild --rebuild "$LOCAL_SRPM_DIR/$REBUILD_SRPM" - fi - if [ $? == 0 ]; then - cp -rf ~/rpmbuild/RPMS ~/output/ - fi elif [ -n "$COMMAND" ]; then $COMMAND else diff --git a/run.py b/run.py index cdc7c04..6baccf7 100755 --- a/run.py +++ b/run.py @@ -15,7 +15,6 @@ import uuid CONTAINER_PREFIX = "xcp-ng/xcp-ng-build-env" -SRPMS_MOUNT_ROOT = "/tmp/docker-SRPMS" DEFAULT_BRANCH = '8.3' DEFAULT_ULIMIT_NOFILE = 2048 @@ -30,22 +29,6 @@ else: raise Exception(f"cannot find a supported runner: {SUPPORTED_RUNNERS}") -def make_mount_dir(): - """ Make a randomly-named directory under SRPMS_MOUNT_ROOT. """ - srpm_mount_dir = os.path.join(SRPMS_MOUNT_ROOT, str(uuid.uuid4())) - try: - os.makedirs(srpm_mount_dir) - except OSError: - pass - return srpm_mount_dir - - -def copy_srpms(srpm_mount_dir, srpms): - """ Copy each SRPM into the mount directory. """ - for srpm in srpms: - srpm_name = os.path.basename(srpm) - shutil.copyfile(srpm, os.path.join(srpm_mount_dir, srpm_name)) - def is_podman(runner): if os.path.basename(runner) == "podman": return True @@ -67,19 +50,14 @@ def main(): "If --output-dir is set, the RPMS and SRPMS directories will be copied to it " "after the build.") parser.add_argument('--define', - help="Definitions to be passed to rpmbuild (if --build-local or --rebuild-srpm are " + help="Definitions to be passed to rpmbuild (if --build-local is " "passed too). Example: --define 'xcp_ng_section extras', for building the 'extras' " "version of a package which exists in both 'base' and 'extras' versions.") - parser.add_argument('-r', '--rebuild-srpm', - help="Install dependencies for the SRPM passed as parameter, then build it. " - "Requires the --output-dir parameter to be set.") parser.add_argument('-o', '--output-dir', - help="Output directory for --rebuild-srpm and --build-local.") + help="Output directory for --build-local.") parser.add_argument('-n', '--no-exit', action='store_true', help='After executing either an automated build or a custom command passed as parameter, ' 'drop user into a shell') - parser.add_argument('-s', '--srpm', action='append', - help='SRPMs for which dependencies will be installed') parser.add_argument('-d', '--dir', action='append', help='Local dir to mount in the ' 'image. Will be mounted at /external/') @@ -128,17 +106,6 @@ def main(): docker_args += ["-e", "BUILD_LOCAL=1"] if args.define: docker_args += ["-e", "RPMBUILD_DEFINE=%s" % args.define] - if args.rebuild_srpm: - if not os.path.isfile(args.rebuild_srpm) or not args.rebuild_srpm.endswith(".src.rpm"): - parser.error("%s is not a valid source RPM." % args.rebuild_srpm) - if not args.output_dir: - parser.error( - "Missing --output-dir parameter, required by --rebuild-srpm.") - docker_args += ["-e", "REBUILD_SRPM=%s" % - os.path.basename(args.rebuild_srpm)] - if args.srpm is None: - args.srpm = [] - args.srpm.append(args.rebuild_srpm) if args.output_dir: if not os.path.isdir(args.output_dir): parser.error("%s is not a valid output directory." % @@ -149,12 +116,6 @@ def main(): docker_args += ["-e", "NO_EXIT=1"] if args.fail_on_error: docker_args += ["-e", "FAIL_ON_ERROR=1"] - # Copy all the RPMs to the mount directory - srpm_mount_dir = None - if args.srpm: - srpm_mount_dir = make_mount_dir() - copy_srpms(srpm_mount_dir, args.srpm) - docker_args += ["-v", "%s:/mnt/docker-SRPMS" % srpm_mount_dir] if args.syslog: docker_args += ["-v", "/dev/log:/dev/log"] if args.name: @@ -192,10 +153,6 @@ def main(): print("Launching docker with args %s" % docker_args, file=sys.stderr) return_code = subprocess.call(docker_args) - if srpm_mount_dir: - print("Cleaning up temporary mount directory") - shutil.rmtree(srpm_mount_dir) - sys.exit(return_code) From a894ec4c5a8ae69f8c62d651baebecbcad22a0d8 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 9 Jul 2025 10:04:59 +0200 Subject: [PATCH 08/24] init-container: avoid copypasta Even after dropping --rebuild-srpm, adding new rpmbuild options had to be done on 2 commandlines. At the same time, using a subshell to deal with change of working directory makes the new code easier. Signed-off-by: Yann Dirson --- files/init-container.sh | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/files/init-container.sh b/files/init-container.sh index 063ef33..9eeb789 100755 --- a/files/init-container.sh +++ b/files/init-container.sh @@ -36,22 +36,23 @@ cd "$HOME" ulimit -s 16384 if [ -n "$BUILD_LOCAL" ]; then - pushd ~/rpmbuild - rm BUILD BUILDROOT RPMS SRPMS -rf - sudo yum-builddep -y SPECS/*.spec - # in case the build deps contain xs-opam-repo, source the added profile.d file - [ ! -f /etc/profile.d/opam.sh ] || source /etc/profile.d/opam.sh - if [ $? == 0 ]; then - if [ -n "$RPMBUILD_DEFINE" ]; then - rpmbuild -ba SPECS/*.spec --define "$RPMBUILD_DEFINE" - else - rpmbuild -ba SPECS/*.spec + ( + cd ~/rpmbuild + rm BUILD BUILDROOT RPMS SRPMS -rf + sudo yum-builddep -y SPECS/*.spec + RPMBUILDFLAGS=(-ba SPECS/*.spec) + # in case the build deps contain xs-opam-repo, source the added profile.d file + [ ! -f /etc/profile.d/opam.sh ] || source /etc/profile.d/opam.sh + if [ $? == 0 ]; then + if [ -n "$RPMBUILD_DEFINE" ]; then + RPMBUILDFLAGS+=(--define "$RPMBUILD_DEFINE") + fi + rpmbuild "${RPMBUILDFLAGS[@]}" + if [ $? == 0 -a -d ~/output/ ]; then + cp -rf RPMS SRPMS ~/output/ + fi fi - if [ $? == 0 -a -d ~/output/ ]; then - cp -rf RPMS SRPMS ~/output/ - fi - fi - popd + ) elif [ -n "$COMMAND" ]; then $COMMAND else From a1ca31dd43f4f028f80673571ab1a01105949b4f Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 8 Jul 2025 14:53:16 +0200 Subject: [PATCH 09/24] run the build under `time` control Not the same as running the whole run.py under `time` control, especially when using `--no-exit`. Also, does not include the build-env setup (which includes installation of the build-deps of SRPMS). Quite trivial now that we use a subshell for the build itself. Signed-off-by: Yann Dirson --- files/init-container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/init-container.sh b/files/init-container.sh index 9eeb789..628a17b 100755 --- a/files/init-container.sh +++ b/files/init-container.sh @@ -36,7 +36,7 @@ cd "$HOME" ulimit -s 16384 if [ -n "$BUILD_LOCAL" ]; then - ( + time ( cd ~/rpmbuild rm BUILD BUILDROOT RPMS SRPMS -rf sudo yum-builddep -y SPECS/*.spec From 4fbcf1b69acead7e6d5c43947aed8feb060b5791 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 30 Jul 2025 12:24:49 +0200 Subject: [PATCH 10/24] Set ghcr.io registry for the container In some not-completely-understood cases, without an explicit registry podman queries interactively for one. Setting this explicitly before we actually publish images to github does not seem to have any side-effect, even "forgetting the workaround for linux/amd64/v2 for podman < 5.5.1" case is not as bad-looking as it would with just a "localhost/" registry. Signed-off-by: Yann Dirson --- build.sh | 2 +- run.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index ab75795..78fcab7 100755 --- a/build.sh +++ b/build.sh @@ -71,7 +71,7 @@ CUSTOM_ARGS+=( "--build-arg" "CUSTOM_BUILDER_GID=${CUSTOM_GID}" ) "$RUNNER" build \ --platform "linux/amd64" \ "${CUSTOM_ARGS[@]}" \ - -t xcp-ng/xcp-ng-build-env:${1} \ + -t ghcr.io/xcp-ng/xcp-ng-build-env:${1} \ --ulimit nofile=1024 \ -f $DOCKERFILE . diff --git a/run.py b/run.py index 6baccf7..51a05e0 100755 --- a/run.py +++ b/run.py @@ -14,7 +14,7 @@ import sys import uuid -CONTAINER_PREFIX = "xcp-ng/xcp-ng-build-env" +CONTAINER_PREFIX = "ghcr.io/xcp-ng/xcp-ng-build-env" DEFAULT_BRANCH = '8.3' DEFAULT_ULIMIT_NOFILE = 2048 From c47e5975484dfff04aaa3760ae88ee94e57fda45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20Lehmann?= Date: Mon, 21 Jul 2025 15:27:47 +0200 Subject: [PATCH 11/24] Move repo logic in the dockerfiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit in order to ease the build without build.sh Signed-off-by: Gaƫtan Lehmann --- Dockerfile-7.x | 12 +++++++++--- Dockerfile-8.x | 12 +++++++++--- build.sh | 11 +---------- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/Dockerfile-7.x b/Dockerfile-7.x index d02e425..f74ad6f 100644 --- a/Dockerfile-7.x +++ b/Dockerfile-7.x @@ -1,4 +1,6 @@ -FROM centos:7.2.1511 +ARG CENTOS_VERSION=7.2.1511 + +FROM centos:${CENTOS_VERSION} ARG CUSTOM_BUILDER_UID="" ARG CUSTOM_BUILDER_GID="" @@ -7,11 +9,15 @@ ARG CUSTOM_BUILDER_GID="" RUN rm /etc/yum.repos.d/* # Add only the specific CentOS 7.2 repositories, because that's what XS used for the majority of packages -COPY files/tmp-CentOS-Vault.repo /etc/yum.repos.d/CentOS-Vault-7.2.repo +ARG CENTOS_VERSION +COPY files/CentOS-Vault.repo.in /etc/yum.repos.d/CentOS-Vault-7.2.repo +RUN sed -e "s/@CENTOS_VERSION@/${CENTOS_VERSION}/g" -i /etc/yum.repos.d/CentOS-Vault-7.2.repo # Add our repositories # Repository file depends on the target version of XCP-ng, and is pre-processed by build.sh -COPY files/tmp-xcp-ng.repo /etc/yum.repos.d/xcp-ng.repo +ARG XCP_NG_BRANCH=7.6 +COPY files/xcp-ng.repo.7.x.in /etc/yum.repos.d/xcp-ng.repo +RUN sed -e "s/@XCP_NG_BRANCH@/${XCP_NG_BRANCH}/g" -i /etc/yum.repos.d/xcp-ng.repo # Fix invalid rpmdb checksum error with overlayfs, see https://github.com/docker/docker/issues/10180 RUN yum install -y yum-plugin-ovl diff --git a/Dockerfile-8.x b/Dockerfile-8.x index 973c3cf..d087e4d 100644 --- a/Dockerfile-8.x +++ b/Dockerfile-8.x @@ -1,4 +1,6 @@ -FROM centos:7.5.1804 +ARG CENTOS_VERSION=7.5.1804 + +FROM centos:${CENTOS_VERSION} ARG CUSTOM_BUILDER_UID="" ARG CUSTOM_BUILDER_GID="" @@ -7,11 +9,15 @@ ARG CUSTOM_BUILDER_GID="" RUN rm /etc/yum.repos.d/* # Add only the specific CentOS 7.5 repositories, because that's what XS used for the majority of packages -COPY files/tmp-CentOS-Vault.repo /etc/yum.repos.d/CentOS-Vault-7.5.repo +ARG CENTOS_VERSION +COPY files/CentOS-Vault.repo.in /etc/yum.repos.d/CentOS-Vault-7.5.repo +RUN sed -e "s/@CENTOS_VERSION@/${CENTOS_VERSION}/g" -i /etc/yum.repos.d/CentOS-Vault-7.5.repo # Add our repositories # Repository file depends on the target version of XCP-ng, and is pre-processed by build.sh -COPY files/tmp-xcp-ng.repo /etc/yum.repos.d/xcp-ng.repo +ARG XCP_NG_BRANCH=8.3 +COPY files/xcp-ng.repo.8.x.in /etc/yum.repos.d/xcp-ng.repo +RUN sed -e "s/@XCP_NG_BRANCH@/${XCP_NG_BRANCH}/g" -i /etc/yum.repos.d/xcp-ng.repo # Install GPG key RUN curl -sSf https://xcp-ng.org/RPM-GPG-KEY-xcpng -o /etc/pki/rpm-gpg/RPM-GPG-KEY-xcpng diff --git a/build.sh b/build.sh index 78fcab7..374958e 100755 --- a/build.sh +++ b/build.sh @@ -31,14 +31,10 @@ CUSTOM_ARGS=() case "$1" in 8.*) - REPO_FILE=files/xcp-ng.repo.8.x.in DOCKERFILE=Dockerfile-8.x - CENTOS_VERSION=7.5.1804 ;; 7.*) - REPO_FILE=files/xcp-ng.repo.7.x.in DOCKERFILE=Dockerfile-7.x - CENTOS_VERSION=7.2.1511 ;; *) echo >&2 "Unsupported release '$1'" @@ -46,9 +42,6 @@ case "$1" in ;; esac -sed -e "s/@XCP_NG_BRANCH@/${1}/g" "$REPO_FILE" > files/tmp-xcp-ng.repo -sed -e "s/@CENTOS_VERSION@/${CENTOS_VERSION}/g" files/CentOS-Vault.repo.in > files/tmp-CentOS-Vault.repo - CUSTOM_UID="$(id -u)" CUSTOM_GID="$(id -g)" @@ -72,8 +65,6 @@ CUSTOM_ARGS+=( "--build-arg" "CUSTOM_BUILDER_GID=${CUSTOM_GID}" ) --platform "linux/amd64" \ "${CUSTOM_ARGS[@]}" \ -t ghcr.io/xcp-ng/xcp-ng-build-env:${1} \ + --build-arg XCP_NG_BRANCH=${1} \ --ulimit nofile=1024 \ -f $DOCKERFILE . - -rm -f files/tmp-xcp-ng.repo -rm -f files/tmp-CentOS-Vault.repo From 1b0abb58d0d45ba5c4855c795288895d7043ed61 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 9 Jul 2025 10:47:07 +0200 Subject: [PATCH 12/24] Initial support for XCP-ng 9 build-env This assumes the distro has already been bootstrapped by building xcp-ng-release and will not be usable for this one, the relevant RUN clause has to be commented out for this. Signed-off-by: Yann Dirson Co-authored-by: Tu Dinh --- Dockerfile-9.x | 75 +++++++++++++++++++++++++++++++++++++++++ build.sh | 6 ++++ files/Alma10-devel.repo | 5 +++ files/init-container.sh | 23 ++++++++++--- files/xcp-ng-8.99.repo | 8 +++++ 5 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 Dockerfile-9.x create mode 100644 files/Alma10-devel.repo create mode 100644 files/xcp-ng-8.99.repo diff --git a/Dockerfile-9.x b/Dockerfile-9.x new file mode 100644 index 0000000..cdb378b --- /dev/null +++ b/Dockerfile-9.x @@ -0,0 +1,75 @@ +FROM ghcr.io/almalinux/10-base:10.0 + +ARG CUSTOM_BUILDER_UID="" +ARG CUSTOM_BUILDER_GID="" + +# Add our repositories +# temporary bootstrap repository +COPY files/xcp-ng-8.99.repo /etc/yum.repos.d/xcp-ng.repo +# Almalinux 10 devel +COPY files/Alma10-devel.repo /etc/yum.repos.d/ + +# Install GPG key +RUN curl -sSf https://xcp-ng.org/RPM-GPG-KEY-xcpng -o /etc/pki/rpm-gpg/RPM-GPG-KEY-xcpng + +# Update +RUN dnf update -y + +# Common build requirements +RUN dnf install -y \ + gcc \ + gcc-c++ \ + git \ + make \ + rpm-build \ + redhat-rpm-config \ + python3-rpm \ + sudo \ + dnf-plugins-core \ + epel-release + +# EPEL: needs epel-release installed first +RUN dnf install -y \ + epel-rpm-macros + +# Niceties +RUN dnf install -y \ + bash-completion \ + vim \ + wget \ + which + +# clean package cache to avoid download errors +RUN yum clean all + +# -release*, to be commented out to boostrap the build-env until it gets built +# FIXME: isn't it already pulled as almalinux-release when available? +RUN dnf install -y \ + xcp-ng-release \ + xcp-ng-release-presets + +# enable repositories commonly required to build +RUN dnf config-manager --enable crb + +# Set up the builder user +RUN bash -c ' \ + OPTS=(); \ + if [ -n "${CUSTOM_BUILDER_UID}" ]; then \ + OPTS+=("-u" "${CUSTOM_BUILDER_UID}"); \ + fi; \ + if [ -n "${CUSTOM_BUILDER_GID}" ]; then \ + OPTS+=("-g" "${CUSTOM_BUILDER_GID}"); \ + if ! getent group "${CUSTOM_BUILDER_GID}" >/dev/null; then \ + groupadd -g "${CUSTOM_BUILDER_GID}" builder; \ + fi; \ + fi; \ + useradd "${OPTS[@]}" builder; \ + ' \ + && echo "builder:builder" | chpasswd \ + && echo "builder ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers + +RUN mkdir -p /usr/local/bin +COPY files/init-container.sh /usr/local/bin/init-container.sh + +# FIXME: check it we really need any of this +# COPY files/rpmmacros /home/builder/.rpmmacros diff --git a/build.sh b/build.sh index 374958e..a811479 100755 --- a/build.sh +++ b/build.sh @@ -29,7 +29,13 @@ cd $(dirname "$0") CUSTOM_ARGS=() +ALMA_VERSION= +CENTOS_VERSION= case "$1" in + 9.*) + DOCKERFILE=Dockerfile-9.x + ALMA_VERSION=10.0 + ;; 8.*) DOCKERFILE=Dockerfile-8.x ;; diff --git a/files/Alma10-devel.repo b/files/Alma10-devel.repo new file mode 100644 index 0000000..a4f98dd --- /dev/null +++ b/files/Alma10-devel.repo @@ -0,0 +1,5 @@ +[alma10-devel] +name=Almalinux 10 devel +baseurl=https://repo.almalinux.org/almalinux/10/devel/$basearch/os/ +enabled=1 +gpgcheck=1 diff --git a/files/init-container.sh b/files/init-container.sh index 628a17b..e3b59db 100755 --- a/files/init-container.sh +++ b/files/init-container.sh @@ -17,18 +17,33 @@ fi sed '/^gpgkey=/ ipriority=1' | sudo tee /etc/yum.repos.d/xcp-ng-users.repo > /dev/null ) +# yum or dnf? +case "$OS_RELEASE" in + 8.2.*|8.3.*) + DNF=yum + CFGMGR=yum-config-manager + BDEP=yum-builddep + ;; + 8.99.*|9.*|10.*) # FIXME 10.* actually to bootstrap Alma10 + DNF=dnf + CFGMGR="dnf config-manager" + BDEP="dnf builddep" + ;; + *) echo >&2 "ERROR: unknown release, cannot know package manager"; exit 1 ;; +esac + # disable repositories if needed if [ -n "$DISABLEREPO" ]; then - sudo yum-config-manager --disable "$DISABLEREPO" + sudo $CFGMGR --disable "$DISABLEREPO" fi # enable additional repositories if needed if [ -n "$ENABLEREPO" ]; then - sudo yum-config-manager --enable "$ENABLEREPO" + sudo $CFGMGR --enable "$ENABLEREPO" fi # update to either install newer updates or to take packages from added repos into account -sudo yum update -y --disablerepo=epel +sudo $DNF update -y --disablerepo=epel cd "$HOME" @@ -39,7 +54,7 @@ if [ -n "$BUILD_LOCAL" ]; then time ( cd ~/rpmbuild rm BUILD BUILDROOT RPMS SRPMS -rf - sudo yum-builddep -y SPECS/*.spec + sudo $BDEP -y SPECS/*.spec RPMBUILDFLAGS=(-ba SPECS/*.spec) # in case the build deps contain xs-opam-repo, source the added profile.d file [ ! -f /etc/profile.d/opam.sh ] || source /etc/profile.d/opam.sh diff --git a/files/xcp-ng-8.99.repo b/files/xcp-ng-8.99.repo new file mode 100644 index 0000000..d6b37be --- /dev/null +++ b/files/xcp-ng-8.99.repo @@ -0,0 +1,8 @@ +[xcpng] +name=XCP-ng 8.99 test +baseurl=http://repos/repos/ydi/v9alma10v2/ +enabled=1 +gpgcheck=0 +metadata_expire=0 +enabled_metadata=1 +priority=1 From 0614c18cfeb4e8b396ca7b6bae0de1666ded1638 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 26 Jun 2025 11:05:38 +0200 Subject: [PATCH 13/24] init-container: make OS_RELEASE available for setting other vars That could not be done in a subshell. Also, don't attempt to download users repo when there is none, 404 HTML contents are a bad fit for yum.repos.d Signed-off-by: Yann Dirson --- files/init-container.sh | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/files/init-container.sh b/files/init-container.sh index e3b59db..c6b785f 100755 --- a/files/init-container.sh +++ b/files/init-container.sh @@ -4,18 +4,27 @@ if [ -n "$FAIL_ON_ERROR" ]; then set -e fi +os_release() +{ + ( + source /etc/os-release + echo "$VERSION_ID" + ) +} + +OS_RELEASE=$(os_release) + # get list of user repos -( - source /etc/os-release - case "$VERSION_ID" in - 8.2.*) XCPREL=8/8.2 ;; - 8.3.*) XCPREL=8/8.3 ;; - *) echo >&2 "WARNING: unknown release, not fetching user repo definitions" ;; - esac +case "$OS_RELEASE" in + 8.2.*) XCPREL=8/8.2 ;; + 8.3.*) XCPREL=8/8.3 ;; + *) echo >&2 "WARNING: unknown release, not fetching user repo definitions" ;; +esac +if [ -n "$XCPREL" ]; then curl -s https://koji.xcp-ng.org/repos/user/${XCPREL}/xcpng-users.repo | sed '/^gpgkey=/ ipriority=1' | sudo tee /etc/yum.repos.d/xcp-ng-users.repo > /dev/null -) +fi # yum or dnf? case "$OS_RELEASE" in From 0191b8536e7b01ec030b65ebdc77dc0366f1e3ec Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 26 Jun 2025 12:30:54 +0200 Subject: [PATCH 14/24] Build 9.0 for x86_64_v2 Signed-off-by: Yann Dirson Co-authored-by: Tu Dinh --- build.sh | 5 ++++- files/init-container.sh | 8 +++++++- run.py | 6 ++++-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/build.sh b/build.sh index a811479..5daef96 100755 --- a/build.sh +++ b/build.sh @@ -35,12 +35,15 @@ case "$1" in 9.*) DOCKERFILE=Dockerfile-9.x ALMA_VERSION=10.0 + PLATFORM=linux/amd64/v2 ;; 8.*) DOCKERFILE=Dockerfile-8.x + PLATFORM=linux/amd64 ;; 7.*) DOCKERFILE=Dockerfile-7.x + PLATFORM=linux/amd64 ;; *) echo >&2 "Unsupported release '$1'" @@ -68,7 +71,7 @@ CUSTOM_ARGS+=( "--build-arg" "CUSTOM_BUILDER_UID=${CUSTOM_UID}" ) CUSTOM_ARGS+=( "--build-arg" "CUSTOM_BUILDER_GID=${CUSTOM_GID}" ) "$RUNNER" build \ - --platform "linux/amd64" \ + --platform "$PLATFORM" \ "${CUSTOM_ARGS[@]}" \ -t ghcr.io/xcp-ng/xcp-ng-build-env:${1} \ --build-arg XCP_NG_BRANCH=${1} \ diff --git a/files/init-container.sh b/files/init-container.sh index c6b785f..f918bdc 100755 --- a/files/init-container.sh +++ b/files/init-container.sh @@ -59,12 +59,18 @@ cd "$HOME" # double the default stack size ulimit -s 16384 +# get the package arch used in the container (eg. "x86_64_v2") +RPMARCH=$(rpm -q glibc --qf "%{arch}") + if [ -n "$BUILD_LOCAL" ]; then time ( cd ~/rpmbuild rm BUILD BUILDROOT RPMS SRPMS -rf sudo $BDEP -y SPECS/*.spec - RPMBUILDFLAGS=(-ba SPECS/*.spec) + RPMBUILDFLAGS=( + -ba SPECS/*.spec + --target "$RPMARCH" + ) # in case the build deps contain xs-opam-repo, source the added profile.d file [ ! -f /etc/profile.d/opam.sh ] || source /etc/profile.d/opam.sh if [ $? == 0 ]; then diff --git a/run.py b/run.py index 51a05e0..c00be10 100755 --- a/run.py +++ b/run.py @@ -88,15 +88,17 @@ def main(): args = parser.parse_args(sys.argv[1:]) + branch = args.branch or DEFAULT_BRANCH + docker_arch = "linux/amd64/v2" if branch == "9.0" else "linux/amd64" + docker_args = [RUNNER, "run", "-i", "-t", "-u", "builder", - "--platform", "linux/amd64", + "--platform", docker_arch, ] if is_podman(RUNNER): docker_args += ["--userns=keep-id"] if args.rm: docker_args += ["--rm=true"] - branch = args.branch or DEFAULT_BRANCH if args.command != []: docker_args += ["-e", "COMMAND=%s" % ' '.join(args.command)] From 4d1d601fdb322c63b200a30e436e7886eda414b9 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 26 Jun 2025 14:45:27 +0200 Subject: [PATCH 15/24] run.py: allow selection of container platform Necessary with podman < v5.5.1 to workaround a bug. Signed-off-by: Yann Dirson --- run.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/run.py b/run.py index c00be10..538baf6 100755 --- a/run.py +++ b/run.py @@ -80,6 +80,9 @@ def main(): parser.add_argument('--disablerepo', help='disable repositories. Same syntax as yum\'s --disablerepo parameter. ' 'If both --enablerepo and --disablerepo are set, --disablerepo will be applied first') + parser.add_argument('--platform', action='store', + help="Override the default platform for the build container. " + "Can notably be used to workaround podman bug #6185 fixed in v5.5.1.") parser.add_argument('--fail-on-error', action='store_true', help='If container initialisation fails, exit rather than dropping the user ' 'into a command shell') @@ -89,7 +92,7 @@ def main(): args = parser.parse_args(sys.argv[1:]) branch = args.branch or DEFAULT_BRANCH - docker_arch = "linux/amd64/v2" if branch == "9.0" else "linux/amd64" + docker_arch = args.platform or ("linux/amd64/v2" if branch == "9.0" else "linux/amd64") docker_args = [RUNNER, "run", "-i", "-t", "-u", "builder", From cb3c9323bf705e4624999a866fda121dc80763ea Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 26 Jun 2025 14:33:32 +0200 Subject: [PATCH 16/24] build.sh: add support for options Signed-off-by: Yann Dirson --- build.sh | 42 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/build.sh b/build.sh index 5daef96..a118ff4 100755 --- a/build.sh +++ b/build.sh @@ -1,12 +1,42 @@ #!/usr/bin/env bash - set -e -if [ -z "$1" ]; then - echo "Usage: $0 {version}" - echo "... where {version} is a 'x.y' version such as 8.0." - exit -fi +die() { + echo >&2 + echo >&2 "ERROR: $*" + echo >&2 + exit 1 +} + +die_usage() { + usage >&2 + die "$*" +} + +usage() { + cat < +... where is a 'x.y' version such as 8.0. +EOF +} + +while [ $# -ge 1 ]; do + case "$1" in + --help|-h) + usage + exit 0 + ;; + -*) + die_usage "unknown flag '$1'" + ;; + *) + break + ;; + esac + shift +done + +[ -n "$1" ] || die_usage "version parameter missing" RUNNER="" if [ -n "$XCPNG_OCI_RUNNER" ]; then From 9ce2a762e539b15da8a3b41a507c46357d743c67 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 26 Jun 2025 14:40:44 +0200 Subject: [PATCH 17/24] build.sh: allow selection of container platform Note that with podman < 5.5.1 we cannot have both linux/amd64 and linux/amd64/v2. And using a completely different arch requires emulation. Signed-off-by: Yann Dirson --- build.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/build.sh b/build.sh index a118ff4..2713902 100755 --- a/build.sh +++ b/build.sh @@ -15,17 +15,25 @@ die_usage() { usage() { cat < +Usage: $0 [--platform PF] ... where is a 'x.y' version such as 8.0. + +--platform override the default platform for the build container. EOF } +PLATFORM= while [ $# -ge 1 ]; do case "$1" in --help|-h) usage exit 0 ;; + --platform) + [ $# -ge 2 ] || die_usage "$1 needs an argument" + PLATFORM="$2" + shift + ;; -*) die_usage "unknown flag '$1'" ;; @@ -65,15 +73,15 @@ case "$1" in 9.*) DOCKERFILE=Dockerfile-9.x ALMA_VERSION=10.0 - PLATFORM=linux/amd64/v2 + : ${PLATFORM:=linux/amd64/v2} ;; 8.*) DOCKERFILE=Dockerfile-8.x - PLATFORM=linux/amd64 + : ${PLATFORM:=linux/amd64} ;; 7.*) DOCKERFILE=Dockerfile-7.x - PLATFORM=linux/amd64 + : ${PLATFORM:=linux/amd64} ;; *) echo >&2 "Unsupported release '$1'" From 1b726677d74efee791a07787016e9bfd58ddec68 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Fri, 27 Jun 2025 15:30:07 +0200 Subject: [PATCH 18/24] run.py: add --rpmbuild-opts to help use "rpmbuild --without xxx" etc Signed-off-by: Yann Dirson --- files/init-container.sh | 1 + run.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/files/init-container.sh b/files/init-container.sh index f918bdc..0f1516c 100755 --- a/files/init-container.sh +++ b/files/init-container.sh @@ -70,6 +70,7 @@ if [ -n "$BUILD_LOCAL" ]; then RPMBUILDFLAGS=( -ba SPECS/*.spec --target "$RPMARCH" + $RPMBUILD_OPTS ) # in case the build deps contain xs-opam-repo, source the added profile.d file [ ! -f /etc/profile.d/opam.sh ] || source /etc/profile.d/opam.sh diff --git a/run.py b/run.py index 538baf6..c2dcd34 100755 --- a/run.py +++ b/run.py @@ -53,6 +53,8 @@ def main(): help="Definitions to be passed to rpmbuild (if --build-local is " "passed too). Example: --define 'xcp_ng_section extras', for building the 'extras' " "version of a package which exists in both 'base' and 'extras' versions.") + parser.add_argument('--rpmbuild-opts', action='append', + help="Pass additional option(s) to rpmbuild") parser.add_argument('-o', '--output-dir', help="Output directory for --build-local.") parser.add_argument('-n', '--no-exit', action='store_true', @@ -111,6 +113,8 @@ def main(): docker_args += ["-e", "BUILD_LOCAL=1"] if args.define: docker_args += ["-e", "RPMBUILD_DEFINE=%s" % args.define] + if args.rpmbuild_opts: + docker_args += ["-e", "RPMBUILD_OPTS=%s" % ' '.join(args.rpmbuild_opts)] if args.output_dir: if not os.path.isdir(args.output_dir): parser.error("%s is not a valid output directory." % From 524c4dd756e5fe424b61cedc65f247c93980b620 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 9 Jul 2025 11:46:31 +0200 Subject: [PATCH 19/24] run.py --local-build: detect spec file paths This allows to build packages with a flat (current Fedora/RHEL standard) source layout. Signed-off-by: Yann Dirson Co-authored-by: Tu Dinh --- files/init-container.sh | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/files/init-container.sh b/files/init-container.sh index 0f1516c..e5fb35c 100755 --- a/files/init-container.sh +++ b/files/init-container.sh @@ -66,11 +66,25 @@ if [ -n "$BUILD_LOCAL" ]; then time ( cd ~/rpmbuild rm BUILD BUILDROOT RPMS SRPMS -rf - sudo $BDEP -y SPECS/*.spec + + if specs=$(ls *.spec 2>/dev/null); then + SPECFLAGS=( + --define "_sourcedir $PWD" + --define "_specdir $PWD" + ) + else + specs=$(ls SPECS/*.spec 2>/dev/null) + # SOURCES/ and SPECS/ are still the default in Alma10 + SPECFLAGS=() + fi + echo "Found specfiles $specs" + + sudo $BDEP "${SPECFLAGS[@]}" -y $specs RPMBUILDFLAGS=( - -ba SPECS/*.spec + -ba $specs --target "$RPMARCH" - $RPMBUILD_OPTS + $RPMBUILD_OPTS + "${SPECFLAGS[@]}" ) # in case the build deps contain xs-opam-repo, source the added profile.d file [ ! -f /etc/profile.d/opam.sh ] || source /etc/profile.d/opam.sh From d623a2677e0e03c81288b68804ccdc1d797f0d93 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Wed, 9 Jul 2025 15:42:53 +0200 Subject: [PATCH 20/24] run.py --local-build: teach fetching source tarballs from the Almalinux bucket Signed-off-by: Yann Dirson --- Dockerfile-9.x | 3 ++- files/init-container.sh | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Dockerfile-9.x b/Dockerfile-9.x index cdb378b..3a78599 100644 --- a/Dockerfile-9.x +++ b/Dockerfile-9.x @@ -30,7 +30,8 @@ RUN dnf install -y \ # EPEL: needs epel-release installed first RUN dnf install -y \ - epel-rpm-macros + epel-rpm-macros \ + almalinux-git-utils # Niceties RUN dnf install -y \ diff --git a/files/init-container.sh b/files/init-container.sh index e5fb35c..f752f8b 100755 --- a/files/init-container.sh +++ b/files/init-container.sh @@ -79,6 +79,12 @@ if [ -n "$BUILD_LOCAL" ]; then fi echo "Found specfiles $specs" + case "$OS_RELEASE" in + 8.2.*|8.3.*) ;; # sources always available via git-lfs + 8.99.*|9.*) if [ -r sources ]; then alma_get_sources -i sources; fi ;; + *) echo >&2 "ERROR: unknown release, cannot know package manager"; exit 1 ;; + esac + sudo $BDEP "${SPECFLAGS[@]}" -y $specs RPMBUILDFLAGS=( -ba $specs From f82d2f99a49a95a26ba5d6cae3492e4c6ec38846 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 10 Jul 2025 15:46:49 +0200 Subject: [PATCH 21/24] run.sh: --rpmbuild-stage to override "-ba" Notably useful for -bp, to - quickly get a patched source tree - apply patches using quilt, in order to refresh them in 8.3 env, so they will not be rejected by the 9.0 one Signed-off-by: Yann Dirson --- files/init-container.sh | 4 +++- run.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/files/init-container.sh b/files/init-container.sh index f752f8b..c903ea2 100755 --- a/files/init-container.sh +++ b/files/init-container.sh @@ -86,8 +86,10 @@ if [ -n "$BUILD_LOCAL" ]; then esac sudo $BDEP "${SPECFLAGS[@]}" -y $specs + + : ${RPMBUILD_STAGE:=a} # default if not specified: -ba RPMBUILDFLAGS=( - -ba $specs + -b${RPMBUILD_STAGE} $specs --target "$RPMARCH" $RPMBUILD_OPTS "${SPECFLAGS[@]}" diff --git a/run.py b/run.py index c2dcd34..af3ccb7 100755 --- a/run.py +++ b/run.py @@ -18,6 +18,7 @@ DEFAULT_BRANCH = '8.3' DEFAULT_ULIMIT_NOFILE = 2048 +RPMBUILD_STAGES = "abpfcilsrd" # valid X values in `rpmbuild -bX` RUNNER = os.getenv("XCPNG_OCI_RUNNER") if RUNNER is None: @@ -55,6 +56,8 @@ def main(): "version of a package which exists in both 'base' and 'extras' versions.") parser.add_argument('--rpmbuild-opts', action='append', help="Pass additional option(s) to rpmbuild") + parser.add_argument('--rpmbuild-stage', action='store', + help=f"Request given -bX stage rpmbuild, X in [{RPMBUILD_STAGES}]") parser.add_argument('-o', '--output-dir', help="Output directory for --build-local.") parser.add_argument('-n', '--no-exit', action='store_true', @@ -115,6 +118,10 @@ def main(): docker_args += ["-e", "RPMBUILD_DEFINE=%s" % args.define] if args.rpmbuild_opts: docker_args += ["-e", "RPMBUILD_OPTS=%s" % ' '.join(args.rpmbuild_opts)] + if args.rpmbuild_stage: + if args.rpmbuild_stage not in RPMBUILD_STAGES: + parser.error(f"--rpmbuild-stage={args.rpmbuild_stage} not in '{RPMBUILD_STAGES}'") + docker_args += ["-e", f"RPMBUILD_STAGE={args.rpmbuild_stage}"] if args.output_dir: if not os.path.isdir(args.output_dir): parser.error("%s is not a valid output directory." % From cf412237b0dc09074d75e32a197e275045570134 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 10 Jul 2025 16:02:42 +0200 Subject: [PATCH 22/24] run.py: new --debug flag It is highly disrupting to have to add `set -x` manually to the script and rebuild it, each time something goes wrong. Signed-off-by: Yann Dirson --- files/init-container.sh | 3 +++ run.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/files/init-container.sh b/files/init-container.sh index c903ea2..baf1650 100755 --- a/files/init-container.sh +++ b/files/init-container.sh @@ -3,6 +3,9 @@ if [ -n "$FAIL_ON_ERROR" ]; then set -e fi +if [ -n "$SCRIPT_DEBUG" ]; then + set -x +fi os_release() { diff --git a/run.py b/run.py index af3ccb7..1560fc6 100755 --- a/run.py +++ b/run.py @@ -91,6 +91,8 @@ def main(): parser.add_argument('--fail-on-error', action='store_true', help='If container initialisation fails, exit rather than dropping the user ' 'into a command shell') + parser.add_argument('--debug', action='store_true', + help='Enable script tracing in container initialization (sh -x)') parser.add_argument('command', nargs=argparse.REMAINDER, help='Command to run inside the prepared container') @@ -132,6 +134,8 @@ def main(): docker_args += ["-e", "NO_EXIT=1"] if args.fail_on_error: docker_args += ["-e", "FAIL_ON_ERROR=1"] + if args.debug: + docker_args += ["-e", "SCRIPT_DEBUG=1"] if args.syslog: docker_args += ["-v", "/dev/log:/dev/log"] if args.name: From ec22871b23528334ace79e6e3ca1004da3fa9f07 Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 10 Jul 2025 16:14:06 +0200 Subject: [PATCH 23/24] README: provide an example to refresh patches Signed-off-by: Yann Dirson --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index 22a9d3f..b9b8154 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,29 @@ git clone https://github.com/xcp-ng-rpms/xapi.git * `--rm` destroys the container on exit. Helps preventing containers from using too much space on disk. You can still reclaim space afterwards by running `docker container prune` and `docker image prune` * `-v` / `--volume` (see *Mounting repos from outside the container* below) +**Refreshing fuzzy patches** + +In XCP-ng 9.0, `rpmbuild` rejects fuzzy patches. The easiest-known +way to get them refreshed is to let `quilt` do the job, but that's not +fully automated. + +1. modify the specfile to add `-Squilt` to `%autosetup` or + `%autopatch` in the `%prep` block; add `BuildRequires: quilt` +2. let quilt apply them in a 8.3 buildenv (`quilt` in 8.3 is only in EPEL) and get you a shell: + ``` +xcpng/build-env/run.py --rm -b 8.3 -l . --rpmbuild-stage=p -n --enablerepo=epel + ``` +3. ask `quilt` to refresh all your patches (alternatively just the one you want) + ``` +$ cd rpmbuild/BUILD/$dir +$ quilt pop -a --refresh +$ cp patches/* ../../SOURCES/ + ``` +4. carefully pick up the bits you need + +Note: unfortunately `rpmbuild` (in 8.3 at least) does not add all +patches in `patches/series` upfront, so in case of real conflict this +has to be redone from step 2 each time. ## Building packages manually From fee2790ec3c9704d1a0060505fd209ed0268d81f Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Thu, 31 Jul 2025 11:09:38 +0200 Subject: [PATCH 24/24] Adjust /etc/shadow permissions to unbreak sudo un some setups In this Ubuntu 24.04 with podman 4.9.3, this was observed: + sudo dnf config-manager --enable crb sudo: PAM account management error: Authentication service cannot retrieve authentication info sudo: a password is required We're not sure of the exact reason this setup exhibits this problem, or why it does not with `xcp-ng/xcp-ng-build-env:8.3` which has the same 000 permissions on /etc/shadow, but changing the permissions does avoid the problem there. Signed-off-by: Yann Dirson Co-authored-by: Yann Sionneau --- Dockerfile-9.x | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile-9.x b/Dockerfile-9.x index 3a78599..e6a34ce 100644 --- a/Dockerfile-9.x +++ b/Dockerfile-9.x @@ -52,6 +52,9 @@ RUN dnf install -y \ # enable repositories commonly required to build RUN dnf config-manager --enable crb +# workaround sudo not working (e.g. in podman 4.9.3 in Ubuntu 24.04) +RUN chmod 0400 /etc/shadow + # Set up the builder user RUN bash -c ' \ OPTS=(); \