chore: minor stability improvements

Author: le1nux

src/modalities/utils/benchmarking/benchmarking_utils.py

@@ -60,7 +60,12 @@ def _is_experiment_done(config_file_path: Path, expected_steps: int, skip_except
         error_types = []
         for error_log_path in error_log_paths:
             with error_log_path.open("r", encoding="utf-8") as f:
-                error_type = json.load(f)["error"]["type"]
+                try:
+                    error_dict = json.load(f)
+                    error_type = error_dict["error"]["type"]
+                except (json.JSONDecodeError, KeyError) as e:
+                    logger.warning(f"Failed to parse error log {error_log_path}: {e}")
+                    error_type = "ErrorFileParsingError"
             error_types.append(error_type)
         # Check if any of the error types are in the skip list
         if len(set(skip_exception_types).intersection(set(error_types))) > 0:

tutorials/scaling_up/configs/sweep_8B_fsdp2.yaml

@@ -173,15 +173,6 @@ fsdp_model:
       reduce_dtype: BF_16
     block_names: [GPT2Block]
 
-compiled_model:
-  component_key: model
-  variant_key: compiled
-  config:
-    model:
-      instance_key: model_raw
-      pass_type: BY_REFERENCE
-    block_names: [GPT2Block]
-
 model_raw:
   component_key: model
   variant_key: gpt2

tutorials/scaling_up/scripts/create_sweep_configs.sh

@@ -4,4 +4,4 @@ set -eu
 # this ensures that relative paths work correctly
 cd "$(dirname "$0")" || exit 1
 
-modalities benchmark prepare_sweep_configs --sweep_config_path ../configs/sweep_config.yaml --output_dir ../experiments --world_sizes 2,4,8
+modalities benchmark prepare_sweep_configs --sweep_config_path ../configs/sweep_8B_fsdp2_compile.yaml --output_dir ../experiments --world_sizes 4,8,16,32,64,128,256,512,1024

tutorials/scaling_up/scripts/hpc/leonardo/submit_sweep.sh

@@ -7,12 +7,12 @@ cd "$(dirname "$0")" || exit 1
 
 
 # --- Config ---
-EXPERIMENT_ROOT="/leonardo_scratch/fast/EUHPC_D21_101/max_lue/repositories/working/modalities/tutorials/scaling_up/experiments/2025-07-28__14-13-18_b7117a39"
+EXPERIMENT_ROOT="/leonardo_scratch/fast/EUHPC_D21_101/max_lue/repositories/working/modalities/tutorials/scaling_up/experiments/2025-07-30__21-50-11_fca0790e"
 EXPECTED_STEPS=20
 CONFIG_LIST_FILE="global_file_list.txt"
 
 ACCOUNT=EUHPC_E05_119
-TIME_LIMIT=3:00:00
+TIME_LIMIT=03:00:00
 GPUS_PER_NODE=4
 
 # Retrieve the list of configs to run