diff --git a/.github/workflows/scripts/compare_stacks.py b/.github/workflows/scripts/compare_stacks.py deleted file mode 100644 index 9e81f615..00000000 --- a/.github/workflows/scripts/compare_stacks.py +++ /dev/null @@ -1,131 +0,0 @@ -import argparse -import os -import re -import glob -import json - - -def parse_module_file(module_file_path): - """ - Extracts module name, version, and extensions from a module file. - """ - module_name = os.path.basename(os.path.dirname(module_file_path)) - version = os.path.basename(module_file_path) - - try: - with open(module_file_path, "r") as file: - content = file.read() - - # Extract extensions from content using regex - match = re.search(r'extensions\("(.+)"\)', content) - extensions = [] - - if match: - # Split the list of packages by commas - packages = match.group(1) - for pkg in packages.split(","): - parts = pkg.split("/") - - # Check if the package is in the name/version format - if len(parts) == 2: - extensions.append((parts[0], parts[1])) - elif len(parts) == 1: - extensions.append((parts[0], "none")) - else: - print(f"Warning: Skipping invalid package format: {pkg}") - - return {(module_name, version): tuple(extensions)} - - except Exception as e: - print(f"Error parsing module file {module_file_path}: {e}") - return {(module_name, version): ()} - - -def get_available_modules(base_dir): - """ - Get the list of modules from all subdirectories inside the specified base directory. - """ - try: - modules = {} - # Only look for .lua files - for module_path in glob.glob(os.path.join(base_dir, "*/*.lua")): - modules.update(parse_module_file(module_path)) - return modules - - except Exception as e: - print(f"Error retrieving modules from {base_dir}: {e}") - return {} - - -def compare_stacks(dir1, dir2): - """ - Compare two sets of Lmod module files, including versions and extensions. - """ - modules1 = get_available_modules(dir1) - modules2 = get_available_modules(dir2) - - # Find differences between the two dictionaries - modules_removed = set(modules1.keys()) - set(modules2.keys()) - modules_added = set(modules2.keys()) - set(modules1.keys()) - matching_keys = set(modules1.keys()) & set(modules2.keys()) - - diff_results = { - "module_differences": { - "missing": list("/".join(module) for module in modules_removed), - "added": list("/".join(module) for module in modules_added), - }, - "extension_differences": [], - } - - # Compare extensions for matching keys - for key in matching_keys: - if modules1[key] != modules2[key]: - diff_results["extension_differences"].append( - { - "/".join(key): { - "missing": list( - "/".join(key) - for key in list(set(modules1[key]) - set(modules2[key])) - ), - "added": list( - "/".join(key) - for key in list(set(modules2[key]) - set(modules1[key])) - ), - } - } - ) - - return diff_results - - -def main(): - # Set up argument parser - parser = argparse.ArgumentParser(description="Compare two Lmod module directories") - parser.add_argument("path1", type=str, help="The first directory path") - parser.add_argument("path2", type=str, help="The second directory path") - - # Parse the arguments - args = parser.parse_args() - - # Validate the paths - for path in [args.path1, args.path2]: - if not os.path.exists(path): - print(f"Warning: Path does not exist: {path}") - - # Compare the stacks - diff_results = compare_stacks(args.path1, args.path2) - - # Print the differences - if any( - [ - diff_results["module_differences"]["missing"], - diff_results["module_differences"]["added"], - diff_results["extension_differences"], - ] - ): - print(json.dumps(diff_results, indent=2)) - exit(1) - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/scripts/compare_to_generic.sh b/.github/workflows/scripts/compare_to_generic.sh deleted file mode 100755 index 59a1397e..00000000 --- a/.github/workflows/scripts/compare_to_generic.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# Take the arguments -base_dir=$1 -target_arch=$2 -modules_subdir="modules/all" -# Decide if we want x86_64 or aarch64 -arch=$(echo $target_arch | cut -d"/" -f1) -# Get the generic directory -source_of_truth="$arch/generic" -case $arch in - "x86_64") - echo "Using $source_of_truth as source of truth" - ;; - "aarch64") - echo "Using $source_of_truth as source of truth" - ;; - *) - echo "I don't understand the base architecture: $arch" - exit 1 - ;; -esac -source_of_truth_modules="$base_dir/$source_of_truth/$modules_subdir" -arch_modules="$base_dir/$target_arch/$modules_subdir" -echo "Comparing $arch_modules to $source_of_truth_modules" -python3 $script_dir/compare_stacks.py $source_of_truth_modules $arch_modules diff --git a/.github/workflows/test_compare_stacks.yml b/.github/workflows/test_compare_stacks.yml deleted file mode 100644 index f57a70aa..00000000 --- a/.github/workflows/test_compare_stacks.yml +++ /dev/null @@ -1,51 +0,0 @@ -# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions -name: Check for discrepencies between software stacks in software.eessi.io -on: - push: - branches: [ "*-software.eessi.io" ] - pull_request: - workflow_dispatch: -permissions: - contents: read # to fetch code (actions/checkout) -env: - EESSI_ACCELERATOR_TARGETS: | - x86_64/amd/zen2: - - nvidia/cc80 - x86_64/amd/zen3: - - nvidia/cc80 -jobs: - compare_stacks: - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - EESSI_VERSION: - - 2023.06 - COMPARISON_ARCH: - - aarch64/neoverse_n1 - - aarch64/neoverse_v1 - - aarch64/nvidia/grace - - x86_64/amd/zen2 - - x86_64/amd/zen3 - - x86_64/amd/zen4 - - x86_64/intel/haswell - - x86_64/intel/skylake_avx512 - - x86_64/intel/sapphirerapids - - x86_64/intel/icelake - - x86_64/intel/cascadelake - steps: - - name: Check out software-layer repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - - name: Mount EESSI CernVM-FS pilot repository - uses: eessi/github-action-eessi@v3 - - - name: Compare stacks - run: | - export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} - export EESSI_OS_TYPE=linux - env | grep ^EESSI | sort - - # Compare the requested architecture to the generic stack - # (assumes the general structure /cvmfs/software.eessi.io/versions/2023.06/software/linux/$COMPARISON_ARCH/modules/all) - .github/workflows/scripts/compare_to_generic.sh ${EESSI_PREFIX}/software/${EESSI_OS_TYPE} ${{matrix.COMPARISON_ARCH}}