Skip to content

Commit 6e8ad7b

Browse files
bedrogebedroge
committed
Merge branch 'main' of github.com:EESSI/filesystem-layer into 2023.06_modulefile_symlink
2 parents ec2c7ca + e742d80 commit 6e8ad7b

16 files changed

+408
-158
lines changed

‎.github/workflows/build-publish-containers.yml‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -25,25 +25,28 @@ permissions: read-all
2525
jobs:
2626
build_and_publish:
2727
name: Build and publish image
28-
runs-on: self-hosted
28+
#runs-on: self-hosted
29+
runs-on: ubuntu-24.04-arm
2930
timeout-minutes: 720
3031
permissions:
3132
packages: write
3233
contents: read
3334
strategy:
3435
matrix:
35-
tag: ['client:centos7', 'build-node:debian11', 'build-node:debian-sid']
36+
tag: ['client:rocky8', 'build-node:debian11', 'build-node:debian12', 'build-node:debian13']
3637
platform: [amd64, arm64, riscv64]
3738
exclude:
3839
# exclude images that don't support RISC-V
39-
- tag: client:centos7
40+
- tag: client:rocky8
4041
platform: riscv64
4142
- tag: build-node:debian11
4243
platform: riscv64
44+
- tag: build-node:debian12
45+
platform: riscv64
4346

4447
steps:
4548
- name: Check out the repo
46-
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
49+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
4750

4851
- name: Set up QEMU
4952
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0
@@ -52,7 +55,7 @@ jobs:
5255
uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v2.2.1
5356

5457
- name: Cache Docker layers
55-
uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # v3.0.11
58+
uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
5659
with:
5760
path: /tmp/.buildx-cache
5861
key: ${{ runner.os }}-buildx-${{ matrix.platform }}-${{ matrix.tag }}
@@ -97,7 +100,7 @@ jobs:
97100
contents: read
98101
strategy:
99102
matrix:
100-
tag: ['client:centos7', 'build-node:debian11', 'build-node:debian-sid']
103+
tag: ['client:rocky8', 'build-node:debian11', 'build-node:debian12', 'build-node:debian13']
101104
steps:
102105
- name: Login to GitHub Container Registry
103106
if: github.event_name != 'pull_request'
@@ -112,15 +115,15 @@ jobs:
112115
echo REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV
113116
114117
- name: Create the final image using "docker buildx imagetools create"
115-
if: github.event_name != 'pull_request' && matrix.tag == 'build-node:debian-sid'
118+
if: github.event_name != 'pull_request' && matrix.tag == 'build-node:debian13'
116119
run: |
117120
docker buildx imagetools create -t ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }} \
118121
ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-amd64 \
119122
ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-arm64 \
120123
ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-riscv64
121124
122125
- name: Create the final image using "docker buildx imagetools create"
123-
if: github.event_name != 'pull_request' && matrix.tag != 'build-node:debian-sid'
126+
if: github.event_name != 'pull_request' && matrix.tag != 'build-node:debian13'
124127
run: |
125128
docker buildx imagetools create -t ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }} \
126129
ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-amd64 \

‎.github/workflows/build-test-release-client-packages.yml‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ jobs:
6464
fpm_opts: "--debug -n cvmfs-config-eessi-${{ steps.get_version.outputs.version }} -t tar -a all -s dir -C ./package --description 'CVMFS configuration package for EESSI.'"
6565

6666
- name: Upload packages as build artifacts
67-
uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
67+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
6868
with:
6969
name: linux_packages
7070
path: cvmfs-config-eessi*
@@ -135,7 +135,7 @@ jobs:
135135
run: sudo apt-get update && sudo apt-get install cvmfs
136136

137137
- name: Download cvmfs-config-eessi package
138-
uses: actions/download-artifact@9782bd6a9848b53b110e712e20e42d89988822b7 # v3.0.1
138+
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
139139
with:
140140
name: linux_packages
141141

@@ -162,18 +162,19 @@ jobs:
162162
needs: build-linux-packages
163163
runs-on: ubuntu-latest
164164
container:
165-
image: centos:centos7
165+
image: rockylinux:8
166166
options: --device /dev/fuse --privileged
167167

168168
steps:
169169
- name: Download and install CVMFS client
170-
run: yum install -y https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest.noarch.rpm && yum-config-manager --enable cernvm-config
171-
172-
- name: Install CVMFS client
173-
run: yum install -y cvmfs cvmfs-config-none
170+
run: |
171+
dnf install -y https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest.noarch.rpm
172+
dnf install -y 'dnf-command(config-manager)'
173+
dnf config-manager --enable cernvm-config
174+
dnf install -y cvmfs cvmfs-config-none
174175
175176
- name: Download cvmfs-config-eessi package
176-
uses: actions/download-artifact@9782bd6a9848b53b110e712e20e42d89988822b7 # v3.0.1
177+
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
177178
with:
178179
name: linux_packages
179180

@@ -214,7 +215,7 @@ jobs:
214215
run: sudo apt-get update && sudo apt-get install cvmfs
215216

216217
- name: Download cvmfs-config-eessi package
217-
uses: actions/download-artifact@9782bd6a9848b53b110e712e20e42d89988822b7 # v3.0.1
218+
uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
218219
with:
219220
name: linux_packages
220221

@@ -287,7 +288,7 @@ jobs:
287288
run: |
288289
echo ::set-output name=version::${GITHUB_REF#refs/tags/}
289290
290-
- uses: actions/download-artifact@9782bd6a9848b53b110e712e20e42d89988822b7 # v3.0.1
291+
- uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
291292
with:
292293
path: ./build_artifacts
293294

@@ -318,7 +319,7 @@ jobs:
318319
- name: Checkout
319320
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
320321

321-
- uses: actions/download-artifact@9782bd6a9848b53b110e712e20e42d89988822b7 # v3.0.1
322+
- uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
322323
with:
323324
path: ./build_artifacts
324325

‎.github/workflows/test-containers.yml‎

Lines changed: 0 additions & 14 deletions
This file was deleted.

‎containers/Dockerfile.EESSI-build-node-debian11‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
ARG cvmfsversion=2.11.2
1+
ARG cvmfsversion=2.13.0
22
ARG awscliversion=1.32.22
33
ARG fuseoverlayfsversion=1.10
4+
ARG bwrapversion=0.11.0
45

56
FROM debian:11.7 AS prepare-deb
67
ARG cvmfsversion
@@ -12,6 +13,7 @@ FROM debian:11.5
1213
ARG cvmfsversion
1314
ARG awscliversion
1415
ARG fuseoverlayfsversion
16+
ARG bwrapversion
1517

1618
COPY --from=prepare-deb /root/deb /root/deb
1719

@@ -39,3 +41,19 @@ RUN useradd -ms /bin/bash eessi
3941

4042
# stick to awscli v1.x, 2.x is not available through PyPI (see https://github.com/aws/aws-cli/issues/4947)
4143
RUN pip3 install archspec awscli==${awscliversion}
44+
45+
# build + install bwrap from source
46+
RUN apt-get install -y libcap-dev meson ninja-build pkg-config \
47+
&& curl -OL https://github.com/containers/bubblewrap/releases/download/v${bwrapversion}/bubblewrap-${bwrapversion}.tar.xz \
48+
&& tar xvf bubblewrap-${bwrapversion}.tar.xz \
49+
&& cd bubblewrap-${bwrapversion} \
50+
&& meson setup _build \
51+
&& meson compile -C _build \
52+
&& meson test -C _build \
53+
&& meson install -C _build \
54+
&& which bwrap \
55+
&& bwrap --version \
56+
&& bwrap --help \
57+
&& cd .. \
58+
&& rm -r bubblewrap-${bwrapversion} \
59+
&& apt-get remove -y libcap-dev meson ninja-build pkg-config

‎containers/Dockerfile.EESSI-build-node-debian12‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
ARG cvmfsversion=2.13.0
2+
ARG awscliversion=1.40.35
3+
ARG unionfsfuseversion=3.6
4+
5+
6+
FROM debian:12.11 AS prepare-deb
7+
ARG cvmfsversion
8+
COPY ./containers/build-or-download-cvmfs-debs.sh /build-or-download-cvmfs-debs.sh
9+
RUN sh /build-or-download-cvmfs-debs.sh ${cvmfsversion}
10+
11+
12+
FROM debian:12.11
13+
ARG cvmfsversion
14+
ARG awscliversion
15+
ARG unionfsfuseversion
16+
17+
COPY --from=prepare-deb /root/deb /root/deb
18+
19+
RUN apt-get update
20+
RUN apt-get install -y sudo vim openssh-client gawk autofs curl attr uuid fuse3 libfuse2 psmisc gdb uuid-dev lsof strace
21+
# python3 and jq are required for eessi-upload-to-staging script (next to awscli)
22+
RUN apt-get install -y python3-pip jq
23+
RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian12_$(dpkg --print-architecture).deb \
24+
/root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian12_$(dpkg --print-architecture).deb \
25+
/root/deb/cvmfs-libs_${cvmfsversion}~1+debian12_$(dpkg --print-architecture).deb \
26+
/root/deb/cvmfs-config-default_latest_all.deb \
27+
/root/deb/cvmfs-config-eessi_latest_all.deb
28+
29+
#RUN apt-get install -y unionfs-fuse
30+
RUN apt-get install -y cmake pkg-config libfuse3-dev \
31+
&& curl -OL https://github.com/rpodgorny/unionfs-fuse/archive/refs/tags/v${unionfsfuseversion}.tar.gz \
32+
&& tar xfvz v${unionfsfuseversion}.tar.gz \
33+
&& cd unionfs-fuse-${unionfsfuseversion} \
34+
&& mkdir build \
35+
&& cd build \
36+
&& cmake .. \
37+
&& make install \
38+
&& command -v unionfs \
39+
&& unionfs --version \
40+
&& unionfs --help \
41+
&& apt-get remove -y cmake pkg-config libfuse3-dev
42+
43+
RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \
44+
&& echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local \
45+
&& echo 'CVMFS_HIDE_MAGIC_XATTRS=yes' >> /etc/cvmfs/default.local
46+
47+
RUN mkdir -p /cvmfs/software.eessi.io
48+
49+
RUN useradd -ms /bin/bash eessi
50+
51+
# stick to awscli v1.x, 2.x is not available through PyPI (see https://github.com/aws/aws-cli/issues/4947)
52+
RUN pip3 install --break-system-packages archspec awscli==${awscliversion}

containers/Dockerfile.EESSI-build-node-debian-sid renamed to containers/Dockerfile.EESSI-build-node-debian13

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
1-
ARG cvmfsversion=2.11.2
1+
ARG cvmfsversion=2.13.0
22
ARG archspecversion=0.2.2
33
ARG awscliversion=1.32.22
44
ARG fuseoverlayfsversion=1.10
55

6-
FROM debian:sid-20240330-slim AS prepare-deb
6+
FROM debian:trixie-20250520-slim AS prepare-deb
77
ARG cvmfsversion
88
COPY ./containers/build-or-download-cvmfs-debs.sh /build-or-download-cvmfs-debs.sh
99
RUN sh /build-or-download-cvmfs-debs.sh ${cvmfsversion}
1010

1111

12-
FROM debian:sid-20240330-slim
12+
FROM debian:trixie-20250520-slim
1313
ARG cvmfsversion
1414
ARG archspecversion
1515
ARG awscliversion

containers/Dockerfile.EESSI-client-centos7 renamed to containers/Dockerfile.EESSI-client-rocky8

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
1-
ARG cvmfsversion=2.11.2
1+
ARG cvmfsversion=2.13.0
22
# Stick to old version of fuse-overlayfs due to issues with newer versions
33
# (cfr. https://github.com/containers/fuse-overlayfs/issues/232)
44
ARG fuseoverlayfsversion=1.10
55

6-
FROM centos:7 AS prepare-rpm
6+
FROM rockylinux:8 AS prepare-rpm
77
ARG cvmfsversion
88
COPY ./containers/build-or-download-cvmfs-rpms.sh /build-or-download-cvmfs-rpms.sh
99
RUN sh /build-or-download-cvmfs-rpms.sh ${cvmfsversion}
1010

1111

12-
FROM centos:7 AS build-fuse-overlayfs
12+
FROM rockylinux:8 AS build-fuse-overlayfs
1313
ARG fuseoverlayfsversion
1414
RUN yum install -y wget fuse3-devel autoconf automake gcc make tar
1515
RUN wget https://github.com/containers/fuse-overlayfs/archive/refs/tags/v${fuseoverlayfsversion}.tar.gz \
@@ -18,17 +18,19 @@ RUN wget https://github.com/containers/fuse-overlayfs/archive/refs/tags/v${fuseo
1818
&& ./autogen.sh && ./configure && make && make install
1919

2020

21-
FROM centos:7
21+
FROM rockylinux:8
2222
ARG cvmfsversion
2323

2424
COPY --from=prepare-rpm /root/rpmbuild/RPMS /root/rpmbuild/RPMS
2525
COPY --from=build-fuse-overlayfs /usr/local/bin/fuse-overlayfs /usr/local/bin/fuse-overlayfs
2626

27-
RUN yum install -y sudo vim openssh-clients lsof strace
28-
RUN yum install -y /root/rpmbuild/RPMS/$(uname -m)/cvmfs-${cvmfsversion}-1.el7.$(uname -m).rpm \
29-
/root/rpmbuild/RPMS/$(uname -m)/cvmfs-fuse3-${cvmfsversion}-1.el7.$(uname -m).rpm \
30-
/root/rpmbuild/RPMS/$(uname -m)/cvmfs-libs-${cvmfsversion}-1.el7.$(uname -m).rpm \
31-
http://ecsft.cern.ch/dist/cvmfs/cvmfs-config/cvmfs-config-default-latest.noarch.rpm
27+
RUN yum install -y sudo vim openssh-clients lsof strace squashfs-tools
28+
RUN elversion="$(rpm -q --queryformat '%{RELEASE}' rpm | cut -d '.' -f 2)" \
29+
&& yum install -y /root/rpmbuild/RPMS/$(uname -m)/cvmfs-${cvmfsversion}-1.${elversion}.$(uname -m).rpm \
30+
/root/rpmbuild/RPMS/$(uname -m)/cvmfs-fuse3-${cvmfsversion}-1.${elversion}.$(uname -m).rpm \
31+
/root/rpmbuild/RPMS/$(uname -m)/cvmfs-libs-${cvmfsversion}-1.${elversion}.$(uname -m).rpm \
32+
/root/rpmbuild/RPMS/$(uname -m)/cvmfs-shrinkwrap-${cvmfsversion}-1.${elversion}.$(uname -m).rpm \
33+
http://ecsft.cern.ch/dist/cvmfs/cvmfs-config/cvmfs-config-default-latest.noarch.rpm
3234
RUN yum install -y https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi-latest.noarch.rpm
3335

3436
# download binary for specific version of fuse-overlayfs

‎containers/build-or-download-cvmfs-debs.sh‎

Lines changed: 14 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -8,52 +8,36 @@ apt-get install -y wget lsb-release
88
distro=$(lsb_release -si | tr [:upper:] [:lower:])
99
release=$(lsb_release -sr)
1010

11-
# lsb_release -sr prints n/a for debian sid, replace it by 13
12-
if [ "${distro}" = "debian" ] && [ "${release}" = "n/a" ]
13-
then
14-
release=13
15-
fi
16-
1711
os="${distro}${release}"
1812

19-
if [ "$arch" = "arm64" ] || [ "$arch" = "riscv64" ] || [ "${os}" = "debian13" ]
13+
if [ "$arch" = "riscv64" ]
2014
then
21-
apt-get install -y devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python3-dev python3-setuptools python3-dev python3-setuptools uuid-dev libz-dev lsb-release
15+
apt-get install -y devscripts libfuse3-dev cmake cpio golang libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python3-dev python3-setuptools python3-dev python3-setuptools uuid-dev libz-dev lsb-release
2216
cd /tmp
2317
wget https://github.com/cvmfs/cvmfs/archive/refs/tags/cvmfs-${cvmfsversion}.tar.gz
2418
tar xzf cvmfs-${cvmfsversion}.tar.gz
2519
cd cvmfs-cvmfs-${cvmfsversion}
2620
mkdir /root/deb
27-
sed -i 's/Architecture: i386 amd64 armhf arm64/Architecture: i386 amd64 armhf arm64 riscv64/' packaging/debian/cvmfs/control.in
28-
sed -i 's/python-dev/python3-dev/' packaging/debian/cvmfs/control.in
29-
sed -i 's/python-setuptools/python3-setuptools/' packaging/debian/cvmfs/control.in
30-
if [ "$arch" = "riscv64" ]
31-
then
32-
# valgrind is not available (yet) for RISC-V
33-
sed -i 's/, valgrind//' packaging/debian/cvmfs/control.in
34-
# for RISC-V we need to run autoreconf, see:
35-
# https://github.com/cvmfs/cvmfs/pull/3446
36-
wget https://github.com/cvmfs/cvmfs/pull/3446.patch
37-
patch -p 1 -i ./3446.patch
38-
rm 3446.patch
39-
# QEMU shows the host CPU in /proc/cpuinfo, so we need to tweak the CPU detection for some packages and use uname -m instead
40-
sed -i "s/^ISA=.*/ISA=\$(uname -m)/" externals/libcrypto/src/configureHook.sh
41-
sed -i "s/rv64/riscv64/" externals/libcrypto/src/configureHook.sh
42-
sed -i "s/^ISA=.*/ISA=\$(uname -m)/" externals/protobuf/src/configureHook.sh
43-
sed -i "s/rv64/riscv64/" externals/protobuf/src/configureHook.sh
44-
else
45-
apt-get install -y valgrind
46-
fi
21+
sed -i 's/amd64 armhf arm64/amd64 armhf arm64 riscv64/' packaging/debian/cvmfs/control*
22+
# debian13 provides libfuse3-4, see https://github.com/cvmfs/cvmfs/pull/3847
23+
[ $os = "debian13" ] && sed -i 's/libfuse3-3 (>= 3.3.0)/libfuse3-3 (>= 3.3.0) | libfuse3-4/g' packaging/debian/cvmfs/control*
24+
# valgrind is not available (yet) for RISC-V
25+
sed -i 's/, valgrind//' packaging/debian/cvmfs/control*
26+
# QEMU shows the host CPU in /proc/cpuinfo, so we need to tweak the CPU detection for some packages and use uname -m instead
27+
sed -i "s/^ISA=.*/ISA=\$(uname -m)/" externals/libcrypto/src/configureHook.sh
28+
sed -i "s/rv64/riscv64/" externals/libcrypto/src/configureHook.sh
29+
sed -i "s/^ISA=.*/ISA=\$(uname -m)/" externals/protobuf/src/configureHook.sh
30+
sed -i "s/rv64/riscv64/" externals/protobuf/src/configureHook.sh
31+
4732
cd ci/cvmfs
48-
# make sure the cvmfs package also uses debian 13 for debian sid
49-
[ $release = "13" ] && sed -i "s@\$(lsb_release -sr)@13@" ./deb.sh && sed -i "s/focal/trixie/" ./deb.sh
5033
./deb.sh /tmp/cvmfs-cvmfs-${cvmfsversion} /root/deb
5134
else
5235
mkdir -p /root/deb
5336
cd /root/deb
5437
wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs_${cvmfsversion}~1+${os}_${arch}.deb
5538
wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs-fuse3_${cvmfsversion}~1+${os}_${arch}.deb
5639
wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs-libs_${cvmfsversion}~1+${os}_${arch}.deb
40+
wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs-shrinkwrap_${cvmfsversion}~1+${os}_${arch}.deb
5741
fi
5842

5943
cd /root/deb

0 commit comments

Comments
 (0)