Merge pull request #24 from bedroge/reprod_hook

add hook that copies the `easybuild` subdirectory of every installation to a central `reprod` directory

Author: casparvl

bot/check-build.sh

@@ -502,7 +502,9 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
         # extract directories/entries from tarball content
         modules_entries=$(grep "${prefix}/modules" ${tmpfile})
         software_entries=$(grep "${prefix}/software" ${tmpfile})
-        other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software")
+        reprod_entries=$(grep "${prefix}/reprod" ${tmpfile})
+        reprod_shortened=$(echo "${reprod_entries}" | sed -e "s@${prefix}/reprod/@@" | awk -F/ '{if (NR >= 4) {print $1 "/" $2 "/" $3}}' | sort -u)
+        other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software" | grep -v "${prefix}/reprod")
         other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u)
         modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u)
         software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u)
@@ -531,6 +533,16 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
             comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no software packages in tarball')"
         fi
         comment_artifacts_list="${comment_artifacts_list}</pre>"
+        comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'reprod directories under ___ITEM___' ${prefix}/reprod)"
+        comment_artifacts_list="${comment_artifacts_list}<pre>"
+        if [[ ! -z ${reprod_shortened} ]]; then
+            while IFS= read -r reprod ; do
+                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '<code>__ITEM__</code>' ${reprod})"
+            done <<< "${reprod_shortened}"
+        else
+            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no reprod directories in tarball')"
+        fi
+        comment_artifacts_list="${comment_artifacts_list}</pre>"
         comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'other under ___ITEM___' ${prefix})"
         comment_artifacts_list="${comment_artifacts_list}<pre>"
         if [[ ! -z ${other_shortened} ]]; then

create_tarball.sh

@@ -90,8 +90,13 @@ for subdir in ${sw_subdirs}; do
         for package_version in $(cat ${module_files_list}); do
             echo "handling ${package_version}"
             find ${eessi_version}/software/${os}/${subdir}/software/${package_version} -maxdepth 0 -type d \! -name '.wh.*' >> ${files_list}
+            # if there is a directory for this installation in the stack's reprod directory, include that too
+            if [ -d ${eessi_version}/software/${os}/${subdir}/reprod ]; then
+                find ${eessi_version}/software/${os}/${subdir}/reprod/${package_version} -maxdepth 0 -type d \! -name '.wh.*' >> ${files_list}
+            fi
         done
     fi
+
 done
 
 # add a bit debug output

eb_hooks.py

@@ -1,15 +1,17 @@
 # Hooks to customize how EasyBuild installs software in EESSI
 # see https://docs.easybuild.io/en/latest/Hooks.html
+import datetime
 import glob
 import os
 import re
 
 import easybuild.tools.environment as env
 from easybuild.easyblocks.generic.configuremake import obtain_config_guess
 from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
+from easybuild.tools import config
 from easybuild.tools.build_log import EasyBuildError, print_msg
-from easybuild.tools.config import build_option, update_build_option
-from easybuild.tools.filetools import apply_regex_substitutions, copy_file, remove_file, symlink, which
+from easybuild.tools.config import build_option, install_path, update_build_option
+from easybuild.tools.filetools import apply_regex_substitutions, copy_dir, copy_file, remove_file, symlink, which
 from easybuild.tools.run import run_cmd
 from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_cpu_features
 from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC
@@ -46,6 +48,9 @@
 # Make sure a single environment variable name is used for this throughout the hooks
 EESSI_IGNORE_ZEN4_GCC1220_ENVVAR="EESSI_IGNORE_LMOD_ERROR_ZEN4_GCC1220"
 
+STACK_REPROD_SUBDIR = 'reprod'
+
+
 def is_gcccore_1220_based(**kwargs):
 # ecname, ecversion, tcname, tcversion):
     """
@@ -516,6 +521,20 @@ def post_module_hook_zen4_gcccore1220(self, *args, **kwargs):
                 del self.initial_environ[EESSI_IGNORE_ZEN4_GCC1220_ENVVAR]
 
 
+def post_easyblock_hook_copy_easybuild_subdir(self, *args, **kwargs):
+    """
+    Post easyblock hook that copies the easybuild subdirectory of every installed application
+    to a central and timestamped location in the root of the software stack, e.g.:
+    /path/to/stack/reprod/MyApp/1.2-foss-2025a/20250102T12:34:56Z
+    """
+
+    stack_reprod_dir = os.path.join(os.path.dirname(install_path()), STACK_REPROD_SUBDIR)
+    now_utc_timestamp = datetime.datetime.now(datetime.UTC).strftime('%Y%m%d_%H%M%S%Z')
+    app_easybuild_dir = os.path.join(self.installdir, config.log_path(ec=self.cfg))
+    app_reprod_dir = os.path.join(stack_reprod_dir, self.install_subdir, now_utc_timestamp, 'easybuild')
+    copy_dir(app_easybuild_dir, app_reprod_dir)
+
+
 # Modules for dependencies are loaded in the prepare step. Thus, that's where we need this variable to be set
 # so that the modules can be succesfully loaded without printing the error (so that we can create a module
 # _with_ the warning for the current software being installed)
@@ -1297,6 +1316,24 @@ def post_module_hook(self, *args, **kwargs):
         post_module_hook_zen4_gcccore1220(self, *args, **kwargs)
 
 
+# The post_easyblock_hook was introduced in EasyBuild 5.1.1.
+# Older versions would fail if the function is defined anyway, as EasyBuild performs some checks on function names in hooks files.
+if EASYBUILD_VERSION >= '5.1.1':
+    def post_easyblock_hook(self, *args, **kwargs):
+        """Main post easyblock hook: trigger custom functions based on software name."""
+        if self.name in POST_EASYBLOCK_HOOKS:
+            POST_EASYBLOCK_HOOKS[self.name](self, *args, **kwargs)
+
+        # Always trigger this one for EESSI CVMFS/site installations and version 2025.06 or newer, regardless of self.name
+        if os.getenv('EESSI_CVMFS_INSTALL') or os.getenv('EESSI_SITE_INSTALL'):
+            if os.getenv('EESSI_VERSION') and LooseVersion(os.getenv('EESSI_VERSION')) >= '2025.06':
+                post_easyblock_hook_copy_easybuild_subdir(self, *args, **kwargs)
+        else:
+            self.log.debug("No CVMFS/site installation requested, not running post_easyblock_hook_copy_easybuild_subdir.")
+else:
+    print_msg(f"Not enabling the post_easybuild_hook, as it requires EasyBuild 5.1.1 or newer.")
+
+
 PARSE_HOOKS = {
     'casacore': parse_hook_casacore_disable_vectorize,
     'CGAL': parse_hook_cgal_toolchainopts_precise,
@@ -1365,6 +1402,8 @@ def post_module_hook(self, *args, **kwargs):
 
 POST_MODULE_HOOKS = {}
 
+POST_EASYBLOCK_HOOKS = {}
+
 # Define parallelism limit operations
 def divide_by_factor(parallel, factor):
     """Divide parallelism by given factor"""