Skip to content

Commit b12c787

Browse files
committed
Merge branch 'main' into EESSI_2025.06_module
2 parents cec3627 + fa16001 commit b12c787

32 files changed

+559
-247
lines changed

.github/workflows/scorecards.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ on:
1010
- cron: '25 15 * * 3'
1111
push:
1212
branches:
13-
- '*-software.eessi.io'
13+
- 'main'
1414

1515
# Declare default permissions as read only.
1616
permissions: read-all

.github/workflows/scripts/only_latest_easystacks.sh

Lines changed: 0 additions & 58 deletions
This file was deleted.
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/bin/bash
2+
check_env_var() {
3+
# Expected usage: check_env_var "MY_ENV_VAR" "foo"
4+
var_name="$1"
5+
expected="$2"
6+
actual="${!var_name}"
7+
if [ "$actual" != "$expected" ]; then
8+
echo "ERROR: $var_name is '$actual', expected '$expected'" >&2
9+
exit 1
10+
else
11+
echo "$var_name is correctly set to '$expected'"
12+
fi
13+
}
14+
15+
check_disallowed_env_prefix() {
16+
prefix="$1"
17+
shift
18+
whitelist=("$@")
19+
20+
disallowed=()
21+
22+
while IFS='=' read -r var _; do
23+
if [[ "$var" == "$prefix"* ]]; then
24+
allowed=false
25+
for allowed_var in "${whitelist[@]}"; do
26+
if [[ "$var" == "$allowed_var" ]]; then
27+
allowed=true
28+
break
29+
fi
30+
done
31+
32+
if ! $allowed; then
33+
disallowed+=("$var")
34+
fi
35+
fi
36+
done < <(env)
37+
38+
if [ "${#disallowed[@]}" -ne 0 ]; then
39+
echo "ERROR: Found disallowed environment variables with prefix '$prefix':" >&2
40+
for var in "${disallowed[@]}"; do
41+
echo " - $var" >&2
42+
done
43+
exit 1
44+
else
45+
echo "✅ No disallowed environment variables with prefix '$prefix' found."
46+
fi
47+
}
Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
import os
2+
import sys
3+
4+
class EnvVarError(Exception):
5+
"""Custom exception for environment variable comparison errors."""
6+
def __init__(self, message):
7+
super().__init__(f"ENV VALIDATION ERROR: {message}")
8+
9+
def get_env_vars(var1, var2):
10+
val1 = os.environ.get(var1)
11+
val2 = os.environ.get(var2)
12+
13+
if val1 is None:
14+
raise EnvVarError(f"Missing environment variable: '{var1}'")
15+
if val2 is None:
16+
raise EnvVarError(f"Missing environment variable: '{var2}'")
17+
18+
return val1, val2
19+
20+
def check_env_equals(var1, var2):
21+
val1, val2 = get_env_vars(var1, var2)
22+
if val1 != val2:
23+
raise EnvVarError(f"'{var1}' must equal '{var2}':\n{var1}='{val1}'\n{var2}='{val2}'")
24+
25+
def check_env_contains(var1, var2):
26+
val1, val2 = get_env_vars(var1, var2)
27+
if val2 not in val1:
28+
raise EnvVarError(f"'{var1}' must contain '{var2}':\n{var1}='{val1}'\n{var2}='{val2}'")
29+
30+
def check_env_endswith(var1, var2):
31+
val1, val2 = get_env_vars(var1, var2)
32+
if not val1.endswith(val2):
33+
raise EnvVarError(f"'{var1}' must end with '{var2}':\n{var1}='{val1}'\n{var2}='{val2}'")
34+
35+
if __name__ == "__main__":
36+
try:
37+
# accelerator stuff is not guaranteed to exist
38+
expected_eessi_accel_arch = os.getenv("EESSI_ACCELERATOR_TARGET_OVERRIDE", default=None)
39+
40+
# Verify the software and accelerator targets are set correctly
41+
if os.getenv("EESSI_SOFTWARE_SUBDIR_OVERRIDE", default=None):
42+
check_env_equals("EESSI_SOFTWARE_SUBDIR_OVERRIDE", "EESSI_SOFTWARE_SUBDIR")
43+
if expected_eessi_accel_arch:
44+
# EESSI_ACCEL_SUBDIR is what is detected by archdetect (or respects EESSI_ACCELERATOR_TARGET_OVERRIDE)
45+
check_env_equals("EESSI_ACCELERATOR_TARGET_OVERRIDE", "EESSI_ACCEL_SUBDIR")
46+
# special case is where EESSI_ACCELERATOR_TARGET_OVERRIDE may not match the final
47+
# accelerator architecture chosen.
48+
# In CI we set FINAL_ACCELERATOR_PATH_EXPECTED to allow us to compare against an expected value.
49+
check_env_equals("EESSI_ACCELERATOR_TARGET", "FINAL_ACCELERATOR_PATH_EXPECTED")
50+
# verify the software paths that should exist
51+
check_env_endswith("EESSI_SOFTWARE_PATH", "EESSI_SOFTWARE_SUBDIR")
52+
check_env_endswith("EESSI_SITE_SOFTWARE_PATH", "EESSI_SOFTWARE_SUBDIR")
53+
# verify the module paths that should exist
54+
check_env_contains("EESSI_MODULEPATH", "EESSI_SOFTWARE_SUBDIR")
55+
check_env_contains("EESSI_SITE_MODULEPATH", "EESSI_SOFTWARE_SUBDIR")
56+
if expected_eessi_accel_arch:
57+
check_env_contains("EESSI_MODULEPATH_ACCEL", "EESSI_SOFTWARE_SUBDIR")
58+
check_env_contains("EESSI_SITE_MODULEPATH_ACCEL", "EESSI_SOFTWARE_SUBDIR")
59+
check_env_contains("EESSI_MODULEPATH_ACCEL", "EESSI_ACCELERATOR_TARGET")
60+
check_env_contains("EESSI_SITE_MODULEPATH_ACCEL", "EESSI_ACCELERATOR_TARGET")
61+
# Finally, verify that all the expected module path are included
62+
check_env_contains("MODULEPATH", "EESSI_MODULEPATH")
63+
check_env_contains("MODULEPATH", "EESSI_SITE_MODULEPATH")
64+
if expected_eessi_accel_arch:
65+
check_env_contains("MODULEPATH", "EESSI_MODULEPATH_ACCEL")
66+
check_env_contains("MODULEPATH", "EESSI_SITE_MODULEPATH_ACCEL")
67+
68+
# We are done
69+
print("Environment variable check passed.")
70+
except EnvVarError as e:
71+
print(str(e), file=sys.stderr)
72+
sys.exit(1)

.github/workflows/test-eb-hooks.yml

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,13 @@ on:
66
workflow_dispatch:
77
permissions:
88
contents: read # to fetch code (actions/checkout)
9-
env:
10-
EESSI_VERSION: '2023.06'
119
jobs:
1210
check_eb_hooks:
1311
runs-on: ubuntu-24.04
12+
strategy:
13+
matrix:
14+
EESSI_VERSION:
15+
- '2023.06'
1416
steps:
1517
- name: Check out software-layer repository
1618
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -50,7 +52,10 @@ jobs:
5052
git show origin/${{ github.base_ref }}:$FILE > "$TEMP_FILE"
5153
fi
5254
55+
# replace <EESSI_VERSION> placeholder (as is also done in install_scripts.sh)
56+
sed -i "s/<EESSI_VERSION>/${{matrix.EESSI_VERSION}}/g" "${TEMP_FILE}"
57+
5358
# Compare the hooks to what is shipped in the repository
54-
source /cvmfs/software.eessi.io/versions/${EESSI_VERSION}/init/bash
59+
source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
5560
module load EESSI-extend
5661
diff "$TEMP_FILE" "$EASYBUILD_HOOKS"

.github/workflows/test_eessi_container_script.yml

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
name: Tests for eessi_container.sh script
33
on:
44
push:
5-
branches: [ "*-software.eessi.io" ]
5+
branches: [ "main" ]
66
pull_request:
77
workflow_dispatch:
88
permissions:
@@ -21,6 +21,7 @@ jobs:
2121
- shell
2222
- container
2323
- resume
24+
- unionfs
2425
# FIXME disabled because '--access rw' is not working in CI environment
2526
#- readwrite
2627
#- save
@@ -40,7 +41,7 @@ jobs:
4041
- name: Test eessi_container.sh script
4142
run: |
4243
test_cmd="cat /etc/os-release"
43-
out_pattern="Debian GNU/Linux 11"
44+
out_pattern="Debian GNU/Linux 12"
4445
4546
if [[ ${{matrix.SCRIPT_TEST}} == 'help' ]]; then
4647
./eessi_container.sh --help
@@ -115,7 +116,7 @@ jobs:
115116
rm -f ${outfile}
116117
117118
# make sure that container image exists
118-
test -f ${tmpdir}/ghcr.io_eessi_build_node_debian11.sif || (echo "Container image not found in ${tmpdir}" >&2 && ls ${tmpdir} && exit 1)
119+
test -f ${tmpdir}/ghcr.io_eessi_build_node_debian12.sif || (echo "Container image not found in ${tmpdir}" >&2 && ls ${tmpdir} && exit 1)
119120
120121
./eessi_container.sh --verbose --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile}
121122
cat ${outfile}
@@ -135,6 +136,18 @@ jobs:
135136
136137
tar tfv test-save.tar | grep "overlay-upper/${fn}"
137138
139+
# test use of --overlay-tool unionfs
140+
elif [[ ${{matrix.SCRIPT_TEST}} == 'unionfs' ]]; then
141+
outfile=out_unionfs.txt
142+
container="docker://ghcr.io/eessi/build-node:debian12"
143+
export SINGULARITY_BIND="$PWD:/test"
144+
echo 'ls -ld /cvmfs*/software.eessi.io/*' > test_script.sh
145+
chmod u+x test_script.sh
146+
./eessi_container.sh --verbose --container ${container} --access rw --overlay-tool unionfs --mode run /test/test_script.sh 2>&1 | tee ${outfile}
147+
for pattern in "/cvmfs/software.eessi.io/versions" "/cvmfs_ro/software.eessi.io/versions"; do
148+
grep "${pattern}" ${outfile} || (echo "Pattern '${pattern}' not found in ${outfile}"; exit 1)
149+
done
150+
138151
else
139152
echo "Unknown test case: ${{matrix.SCRIPT_TEST}}" >&2
140153
exit 1

.github/workflows/test_licenses.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
name: Test software licenses
33
on:
44
push:
5-
branches: [ "*-software.eessi.io" ]
5+
branches: [ "main" ]
66
pull_request:
77
permissions:
88
contents: read # to fetch code (actions/checkout)

.github/workflows/tests.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
name: Tests
33
on:
44
push:
5-
branches: [ "*-software.eessi.io" ]
5+
branches: [ "main" ]
66
pull_request:
77
permissions:
88
contents: read # to fetch code (actions/checkout)

.github/workflows/tests_archdetect.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
name: Tests for eessi_archdetect.sh
33
on:
44
push:
5-
branches: [ "*-software.eessi.io" ]
5+
branches: [ "main" ]
66
pull_request:
77
permissions:
88
contents: read # to fetch code (actions/checkout)

.github/workflows/tests_archdetect_nvidia_gpu.yml

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ jobs:
1717
- 2xa100 # cc80, supported with (atleast) zen2 CPU
1818
- 4xa100 # cc80, supported with (atleast) zen2 CPU
1919
- cc01 # non-existing GPU
20+
EESSI_VERSION:
21+
- '2023.06'
2022
fail-fast: false
2123
steps:
2224
- name: checkout
@@ -31,6 +33,10 @@ jobs:
3133
cvmfs_http_proxy: DIRECT
3234
cvmfs_repositories: software.eessi.io
3335

36+
- name: Fix EESSI version in init scripts
37+
run: |
38+
sed -i "s/__EESSI_VERSION_DEFAULT__/${{matrix.EESSI_VERSION}}/g" init/eessi_defaults
39+
3440
- name: test accelerator detection
3541
run: |
3642
export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/amd/zen2'
@@ -68,7 +74,7 @@ jobs:
6874
match=$(grep "${pattern}" init.out || true)
6975
test "x${match}" = "x" || (echo "unexpected match found for '${pattern}' in init output" && exit 1)
7076
71-
pattern="Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux/.*/accel/.*/modules/all to \$MODULEPATH"
77+
pattern="Prepending /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/software/linux/.*/accel/.*/modules/all to \$MODULEPATH"
7278
echo ">>> checking for lack of pattern '${pattern}' in init output..."
7379
match=$(grep "${pattern}" init.out || true)
7480
test "x${match}" = "x" || (echo "unexpected match found for '${pattern}' in init output" && exit 1)
@@ -79,19 +85,19 @@ jobs:
7985
echo ">>> checking for pattern '${pattern}' in init output..."
8086
grep "${pattern}" init.out || (echo "FAILED 1" || exit 1)
8187
82-
pattern="Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux/.*/accel/.*/modules/all to \$MODULEPATH"
88+
pattern="Prepending /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/software/linux/.*/accel/.*/modules/all to \$MODULEPATH"
8389
echo ">>> checking for lack of pattern '${pattern}' in init output..."
8490
match=$(grep "${pattern}" init.out || true)
8591
test "x${match}" = "x" || (echo "unexpected match found for '${pattern}' in init output" && exit 1)
8692
8793
else
8894
echo ">>> checking for 'accel/nvidia/cc80' in init output..."
8995
grep "archdetect found supported accelerator for CPU target x86_64/amd/zen2: accel/nvidia/cc80" init.out || (echo "FAILED 2" && exit 1)
90-
grep "Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/accel/nvidia/cc80/modules/all to \$MODULEPATH" init.out || (echo "FAILED 3" && exit 1)
96+
grep "Prepending /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/software/linux/x86_64/amd/zen2/accel/nvidia/cc80/modules/all to \$MODULEPATH" init.out || (echo "FAILED 3" && exit 1)
9197
fi
9298
9399
echo ">>> checking last line of init output..."
94-
tail -1 init.out | grep "Environment set up to use EESSI (2023.06), have fun!" || (echo "FAILED, full init utput:" && cat init.out && exit 1)
100+
tail -1 init.out | grep "Environment set up to use EESSI (${{matrix.EESSI_VERSION}}), have fun!" || (echo "FAILED, full init utput:" && cat init.out && exit 1)
95101
96102
echo "All checks on init output PASSED"
97103
else
@@ -118,7 +124,7 @@ jobs:
118124
echo ">>> checking for 'accel/nvidia/cc80' in init output..."
119125
grep "archdetect found supported accelerator for CPU target x86_64/amd/zen3: accel/nvidia/cc80" init.out || (echo "FAILED 1" && exit 1)
120126
grep "Using x86_64/amd/zen2 as software subdirectory" init.out || (echo "FAILED 2" && exit 1)
121-
grep "Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/modules/all to \$MODULEPATH" init.out || (echo "FAILED 3" && exit 1)
122-
grep "Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen3/accel/nvidia/cc80/modules/all to \$MODULEPATH" init.out || (echo "FAILED 4" && exit 1)
127+
grep "Prepending /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/software/linux/x86_64/amd/zen2/modules/all to \$MODULEPATH" init.out || (echo "FAILED 3" && exit 1)
128+
grep "Prepending /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/software/linux/x86_64/amd/zen3/accel/nvidia/cc80/modules/all to \$MODULEPATH" init.out || (echo "FAILED 4" && exit 1)
123129
124130
echo "All checks on init output PASSED"

0 commit comments

Comments
 (0)