Skip to content

Commit 1250a43

Browse files
ocaisaocaisa
authored
Merge branch 'main' into parallel-extensions-install
2 parents 64f8dee + eba6086 commit 1250a43

19 files changed

+383
-184
lines changed

‎.github/workflows/scripts/test_init_scripts.sh‎

Lines changed: 59 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -2,59 +2,81 @@
22
export LMOD_PAGER=cat
33

44
if [ -z ${EESSI_VERSION} ] || [ ! -d /cvmfs/software.eessi.io/versions/${EESSI_VERSION} ]; then
5-
echo "\$EESSI_VERSION has to be set to a valid EESSI version."
6-
exit 1
5+
echo "\$EESSI_VERSION has to be set to a valid EESSI version."
6+
exit 1
77
fi
88

99
if [ -z ${EXPECTED_EASYBUILD_VERSION} ]; then
10-
echo "\$EXPECTED_EASYBUILD_VERSION has to be set to an EasyBuild version that is expected to be available in EESSI version ${EESSI_VERSION}."
11-
exit 1
10+
echo "\$EXPECTED_EASYBUILD_VERSION has to be set to an EasyBuild version that is expected to be available in EESSI version ${EESSI_VERSION}."
11+
exit 1
1212
fi
1313

1414
# initialize assert framework
1515
if [ ! -d assert.sh ]; then
16-
echo "assert.sh not cloned."
17-
echo ""
18-
echo "run \`git clone https://github.com/lehmannro/assert.sh.git\`"
19-
exit 1
16+
echo "assert.sh not cloned."
17+
echo ""
18+
echo "run \`git clone https://github.com/lehmannro/assert.sh.git\`"
19+
exit 1
2020
fi
2121
. assert.sh/assert.sh
2222

23-
TEST_SHELLS=("bash" "zsh" "fish" "ksh")
23+
TEST_SHELLS=("bash" "zsh" "fish" "ksh" "csh")
2424
SHELLS=$@
2525

2626
for shell in ${SHELLS[@]}; do
27-
echo = | awk 'NF += (OFS = $_) + 100'
28-
echo RUNNING TESTS FOR SHELL: $shell
29-
echo = | awk 'NF += (OFS = $_) + 100'
27+
echo = | awk 'NF += (OFS = $_) + 100'
28+
echo RUNNING TESTS FOR SHELL: $shell
29+
echo = | awk 'NF += (OFS = $_) + 100'
3030
if [[ ! " ${TEST_SHELLS[*]} " =~ [[:space:]]${shell}[[:space:]] ]]; then
31-
### EXCEPTION FOR CSH ###
32-
echo -e "\033[33mWe don't now how to test the shell '$shell', PRs are Welcome.\033[0m"
31+
### EXCEPTION FOR CSH ###
32+
echo -e "\033[33mWe don't now how to test the shell '$shell', PRs are Welcome.\033[0m"
3333
else
34-
# TEST 1: Source Script and check Module Output
35-
assert "$shell -c 'source init/lmod/$shell' 2>&1 " "Module for EESSI/$EESSI_VERSION loaded successfully"
36-
# TEST 2: Check if module overviews first section is the loaded EESSI module
37-
MODULE_SECTIONS=($($shell -c "source init/lmod/$shell 2>/dev/null; module ov 2>&1 | grep -e '---'"))
38-
PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/modules/all"
39-
assert_raises 'echo "${MODULE_SECTIONS[1]}" | grep -E "$PATTERN"'
40-
# TEST 3: Check if module overviews second section is the EESSI init module
41-
assert "echo ${MODULE_SECTIONS[4]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/init/modules"
42-
# Test 4: Load EasyBuild module and check version
43-
# eb --version outputs: "This is EasyBuild 5.1.1 (framework: 5.1.1, easyblocks: 5.1.1) on host ..."
44-
command="$shell -c 'source init/lmod/$shell 2>/dev/null; module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; eb --version | cut -d \" \" -f4'"
45-
assert "$command" "$EXPECTED_EASYBUILD_VERSION"
46-
# Test 5: Load EasyBuild module and check path
47-
EASYBUILD_PATH=$($shell -c "source init/lmod/$shell 2>/dev/null; module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; which eb")
48-
# escape the dots in ${EASYBUILD_VERSION}
49-
PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/software/EasyBuild/${EXPECTED_EASYBUILD_VERSION//./\\.}/bin/eb"
50-
echo "$EASYBUILD_PATH" | grep -E "$PATTERN"
51-
assert_raises 'echo "$EASYBUILD_PATH" | grep -E "$PATTERN"'
52-
53-
#End Test Suite
54-
assert_end "source_eessi_$shell"
55-
fi
56-
done
34+
# TEST 1: Source Script and check Module Output
35+
expected="Module for EESSI/$EESSI_VERSION loaded successfully"
36+
assert "$shell -c 'source init/lmod/$shell' 2>&1 " "${expected}"
37+
38+
# TEST 2: Check if module overviews first section is the loaded EESSI module
39+
if [ "$shell" = "csh" ]; then
40+
# module is defined as alias, but aliases are only retained in interactive
41+
# shells we work around this by creating a .cshrc file (which sources the
42+
# init script), and then simply run the remaining commands
43+
echo "source init/lmod/$shell" > ~/.cshrc
44+
MODULE_SECTIONS=($($shell -c "module ov" 2>&1 | grep -e '---'))
45+
else
46+
MODULE_SECTIONS=($($shell -c "source init/lmod/$shell 2>/dev/null; module ov 2>&1 | grep -e '---'"))
47+
fi
48+
PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/modules/all"
49+
assert_raises 'echo "${MODULE_SECTIONS[1]}" | grep -E "$PATTERN"'
50+
51+
# TEST 3: Check if module overviews second section is the EESSI init module
52+
assert "echo ${MODULE_SECTIONS[4]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/init/modules"
5753

54+
# TEST 4: Load EasyBuild module and check version
55+
# eb --version outputs: "This is EasyBuild 5.1.1 (framework: 5.1.1, easyblocks: 5.1.1) on host ..."
56+
if [ "$shell" = "csh" ]; then
57+
echo "source init/lmod/$shell" > ~/.cshrc
58+
command="$shell -c 'module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; eb --version' | cut -d \" \" -f4"
59+
else
60+
command="$shell -c 'source init/lmod/$shell 2>/dev/null; module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; eb --version | cut -d \" \" -f4'"
61+
fi
62+
assert "$command" "$EXPECTED_EASYBUILD_VERSION"
63+
64+
# TEST 5: Load EasyBuild module and check path
65+
if [ "$shell" = "csh" ]; then
66+
echo "source init/lmod/$shell" > ~/.cshrc
67+
EASYBUILD_PATH=$($shell -c "module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; which eb")
68+
else
69+
EASYBUILD_PATH=$($shell -c "source init/lmod/$shell 2>/dev/null; module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; which eb")
70+
fi
71+
# escape the dots in ${EASYBUILD_VERSION}
72+
PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/software/EasyBuild/${EXPECTED_EASYBUILD_VERSION//./\\.}/bin/eb"
73+
echo "$EASYBUILD_PATH" | grep -E "$PATTERN"
74+
assert_raises 'echo "$EASYBUILD_PATH" | grep -E "$PATTERN"'
75+
76+
# End Test Suite
77+
assert_end "source_eessi_$shell"
78+
fi
79+
done
5880

5981
# RESET PAGER
6082
export LMOD_PAGER=

‎.github/workflows/tests_init_module.yml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,12 @@ jobs:
2727
- name: Check out software-layer repository
2828
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
2929

30+
- name: Avoid processing man-db trigger
31+
run: |
32+
# avoid processing trigger for man-db (seems to slow down installation)
33+
echo "set man-db/auto-update false" | sudo debconf-communicate
34+
sudo dpkg-reconfigure man-db
35+
3036
- name: Mount EESSI CernVM-FS pilot repository
3137
uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0
3238
with:
@@ -46,7 +52,7 @@ jobs:
4652
- name: Install missing shells
4753
run: |
4854
sudo apt update
49-
sudo apt install zsh ksh fish
55+
sudo apt install zsh ksh fish tcsh
5056
echo "# INIT ZSH" > ~/.zshrc
5157
5258
- name: Run tests for available shells

‎EESSI-extend-easybuild.eb‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,8 @@ end
176176
setenv ("EASYBUILD_SYSROOT", sysroot)
177177
setenv ("EASYBUILD_PREFIX", pathJoin(working_dir, "easybuild"))
178178
setenv ("EASYBUILD_INSTALLPATH", easybuild_installpath)
179-
setenv ("EASYBUILD_HOOKS", pathJoin(os.getenv("EESSI_PREFIX"), "init", "easybuild", "eb_hooks.py"))
179+
eessi_init_prefix = os.getenv("EESSI_INIT_PREFIX") or pathJoin(os.getenv("EESSI_PREFIX"), "init")
180+
setenv ("EASYBUILD_HOOKS", pathJoin(eessi_init_prefix, "easybuild", "eb_hooks.py"))
180181
181182
-- Make sure to use the general umask that allows a global read
182183
setenv ("EASYBUILD_UMASK", "022")
@@ -235,6 +236,13 @@ if mode() == "unload" or mode() == "dependencyCk" or convertToCanonical(easybuil
235236
setenv ("EASYBUILD_SEARCH_PATH_CPP_HEADERS", "include_paths")
236237
end
237238
end
239+
240+
-- Avoid locale related problems by (temporarily) overwriting the locale with one we know exists ('locale -a')
241+
pushenv ("LANG", "C.utf8")
242+
pushenv ("LC_ALL", "C.utf8")
243+
pushenv ("LC_CTYPE", "C.utf8")
244+
pushenv ("LC_MESSAGES", "C.utf8")
245+
238246
"""
239247

240248
moduleclass = 'devel'

‎EESSI-install-software.sh‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ else
149149

150150
# make sure the the software and modules directory exist
151151
# (since it's expected by init/eessi_environment_variables when using archdetect and by the EESSI module)
152-
mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/{modules,software}
152+
mkdir -p -v ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/{modules,software}
153153

154154
# If EESSI_ACCELERATOR_TARGET_OVERRIDE is defined, we are building for an accelerator target
155155
# In that case, make sure the modulepath for the accelerator subdir exists, otherwise the EESSI module will not
@@ -160,7 +160,7 @@ else
160160
# Note that ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/${EESSI_ACCELERATOR_TARGET_OVERRIDE}/modules/all
161161
# is only the correct path if EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE is not set
162162
if [ -z $EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE ]; then
163-
mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/${EESSI_ACCELERATOR_TARGET_OVERRIDE}/modules/all
163+
mkdir -p -v ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/${EESSI_ACCELERATOR_TARGET_OVERRIDE}/modules/all
164164
else
165165
# At runtime, one might want to use a different CPU subdir for a given accelerator. E.g. one could use
166166
# a zen2 CPU subdir on a zen4 node if the required GPU software isn't available in the zen4 tree.
@@ -207,7 +207,7 @@ fi
207207

208208
# the install_scripts.sh script relies on knowing the location of the PR diff
209209
# assume there's only one diff file that corresponds to the PR patch file
210-
pr_diff=$(ls [0-9]*.diff | head -1)
210+
pr_diff=$(ls [0-9]*.diff | head -n 1)
211211
export PR_DIFF="$PWD/$pr_diff"
212212

213213
# Only run install_scripts.sh if not in dev.eessi.io for security
@@ -368,7 +368,9 @@ else
368368
fi
369369

370370
# use PR patch file to determine in which easystack files stuff was added
371-
changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep 'easystacks/.*yml$' | egrep -v 'known-issues|missing')
371+
# Note that we exclude the scripts/gpu_support/ dir, since those are not meant to be built in the
372+
# software-layer, but they are helper easystacks for installing e.g. CUDA in host_injections
373+
changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep 'easystacks/.*yml$' | egrep -v 'known-issues|missing' | (grep -v "scripts/gpu_support/" || true))
372374
if [ -z "${changed_easystacks}" ]; then
373375
echo "No missing installations, party time!" # Ensure the bot report success, as there was nothing to be build here
374376
else

‎bot/build.sh‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -265,16 +265,24 @@ BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)
265265
TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}")
266266

267267
timestamp=$(date +%s)
268+
# determine compression/extension for tarball, check in order of preference
269+
if [[ -x "$(command -v zstd)" ]]; then
270+
tarball_extension="tar.zst"
271+
elif [[ -x "$(command -v gzip)" ]]; then
272+
tarball_extension="tar.gz"
273+
else
274+
tarball_extension="tar"
275+
fi
268276
# to set EESSI_VERSION we need to source init/eessi_defaults now
269277
source $software_layer_dir/init/eessi_defaults
270278
# Note: if ${EESSI_DEV_PROJECT} is defined (building for dev.eessi.io), then we
271279
# append the project (subdirectory) name to the end tarball name. This is information
272280
# then used at the ingestion stage. If ${EESSI_DEV_PROJECT} is not defined, nothing is
273281
# appended
274282
if [[ -z ${EESSI_ACCELERATOR_TARGET_OVERRIDE} ]]; then
275-
export TGZ=$(printf "eessi-%s-software-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
283+
export TARBALL=$(printf "eessi-%s-software-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
276284
else
277-
export TGZ=$(printf "eessi-%s-software-%s-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
285+
export TARBALL=$(printf "eessi-%s-software-%s-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
278286
fi
279287

280288
# Export EESSI_DEV_PROJECT to use it (if needed) when making tarball
@@ -288,8 +296,8 @@ export EESSI_DEV_PROJECT=${EESSI_DEV_PROJECT}
288296
TMP_IN_CONTAINER=/tmp
289297
echo "Executing command to create tarball:"
290298
echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}"
291-
echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDE}\" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
299+
echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDE}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}"
292300
$software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \
293-
-- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET_OVERRIDE}" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
301+
-- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET_OVERRIDE}" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}
294302

295303
exit 0

‎bot/check-build.sh‎

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
# - SUCCESS (all of)
1919
# - working directory contains slurm-JOBID.out file
20-
# - working directory contains eessi*tar.gz
20+
# - working directory contains eessi*tar*
2121
# - no message FATAL
2222
# - no message ERROR
2323
# - no message FAILED
@@ -165,19 +165,19 @@ if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
165165
fi
166166

167167
if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
168-
TGZ=-1
168+
TARBALL_CREATED=-1
169169
TARBALL=
170170
if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
171-
GP_tgz_created="\.tar\.gz created!"
172-
grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u)
171+
GP_tarball_created="\.tar.* created!"
172+
grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tarball_created}" | sort -u)
173173
if [[ $? -eq 0 ]]; then
174-
TGZ=1
174+
TARBALL_CREATED=1
175175
TARBALL=$(echo ${grep_out} | sed -e 's@^.*/\(eessi[^/ ]*\) .*$@\1@')
176176
else
177-
TGZ=0
177+
TARBALL_CREATED=0
178178
fi
179179
# have to be careful to not add searched for pattern into slurm out file
180-
[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'"
180+
[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tarball_created}"'"
181181
[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
182182
fi
183183
fi
@@ -190,7 +190,7 @@ fi
190190
[[ ${VERBOSE} -ne 0 ]] && echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)"
191191
[[ ${VERBOSE} -ne 0 ]] && echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
192192
if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
193-
[[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
193+
[[ ${VERBOSE} -ne 0 ]] && echo " TARBALL_CREATED: $([[ $TARBALL -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
194194
fi
195195

196196
# Here, we try to do some additional analysis on the output file
@@ -219,7 +219,7 @@ if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \
219219
[[ ${FAILED} -eq 0 ]] && \
220220
[[ ${MISSING} -eq 0 ]] && \
221221
[[ ${NO_MISSING} -eq 1 ]] && \
222-
[[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || ${TGZ} -eq 1 ]] && \
222+
[[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || ${TARBALL_CREATED} -eq 1 ]] && \
223223
[[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || -n ${TARBALL} ]]; then
224224
# SUCCESS
225225
status="SUCCESS"
@@ -429,9 +429,9 @@ failure_msg="no message matching <code>${GP_no_missing}</code>"
429429
comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
430430

431431
if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
432-
success_msg="found message matching <code>${GP_tgz_created}</code>"
433-
failure_msg="no message matching <code>${GP_tgz_created}</code>"
434-
comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
432+
success_msg="found message matching <code>${GP_tarball_created}</code>"
433+
failure_msg="no message matching <code>${GP_tarball_created}</code>"
434+
comment_details_list=${comment_details_list}$(add_detail ${TARBALL_CREATED} 1 "${success_msg}" "${failure_msg}")
435435
fi
436436

437437
# Now, do the actual replacement of __DETAILS_FMT__
@@ -478,7 +478,16 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
478478
size="$(stat --dereference --printf=%s ${TARBALL})"
479479
size_mib=$((${size} >> 20))
480480
tmpfile=$(mktemp --tmpdir=. tarfiles.XXXX)
481-
tar tf ${TARBALL} > ${tmpfile}
481+
if [[ "${TARBALL}" == *.tar.zst ]]; then
482+
tar --use-compress-program=zstd -tf ${TARBALL} > ${tmpfile}
483+
elif [[ "${TARBALL}" == *.tar.gz ]]; then
484+
tar --use-compress-program=gzip -tf ${TARBALL} > ${tmpfile}
485+
elif [[ "${TARBALL}" == *.tar ]]; then
486+
tar -tf ${TARBALL} > ${tmpfile}
487+
else
488+
echo "ERROR: Unsupported tarball extension!" >&2
489+
exit 1
490+
fi
482491
entries=$(cat ${tmpfile} | wc -l)
483492
# determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE
484493
# e.g., 2023.06/software/linux/x86_64/intel/skylake_avx512
@@ -503,7 +512,7 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
503512
modules_entries=$(grep "${prefix}/modules" ${tmpfile})
504513
software_entries=$(grep "${prefix}/software" ${tmpfile})
505514
reprod_entries=$(grep "${prefix}/reprod" ${tmpfile})
506-
reprod_shortened=$(echo "${reprod_entries}" | sed -e "s@${prefix}/reprod/@@" | awk -F/ '{if (NR >= 4) {print $1 "/" $2 "/" $3}}' | sort -u)
515+
reprod_shortened=$(echo "${reprod_entries}" | sed -e "s@${prefix}/reprod/@@" | awk -F/ '{if (NF >= 4) {print $1 "/" $2 "/" $3}}' | sort -u)
507516
other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software" | grep -v "${prefix}/reprod")
508517
other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u)
509518
modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u)

‎bot/test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
100100
fi
101101

102102
# try to determine tmp directory from build job
103-
RESUME_DIR=$(grep 'Using .* as tmp directory' slurm-${SLURM_JOBID}.out | head -1 | awk '{print $2}')
103+
RESUME_DIR=$(grep 'Using .* as tmp directory' slurm-${SLURM_JOBID}.out | head -n 1 | awk '{print $2}')
104104

105105
if [[ -z ${RESUME_DIR} ]]; then
106106
RESUME_TGZ=${PWD}/previous_tmp/build_step/$(ls previous_tmp/build_step)

‎create_tarball.sh‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ eessi_tmpdir=$1
1212
eessi_version=$2
1313
cpu_arch_subdir=$3
1414
accel_subdir=$4
15-
target_tgz=$5
15+
target_tarball=$5
1616

1717
tmpdir=`mktemp -d`
1818
echo ">> tmpdir: $tmpdir"
@@ -114,10 +114,9 @@ fi
114114

115115
topdir=${cvmfs_repo}/versions/
116116

117-
echo ">> Creating tarball ${target_tgz} from ${topdir}..."
118-
tar cfvz ${target_tgz} -C ${topdir} --files-from=${files_list}
119-
120-
echo ${target_tgz} created!
117+
echo ">> Creating tarball ${target_tarball} from ${topdir}..."
118+
tar cavf ${target_tarball} -C ${topdir} --files-from=${files_list}
119+
echo ${target_tarball} created!
121120

122121
echo ">> Cleaning up tmpdir ${tmpdir}..."
123122
rm -r ${tmpdir}

0 commit comments

Comments
 (0)