fix trainer.fill_flat_stats causing max recursion error

Author: LucasDedieu

edsnlp/resources/verbs.csv.gz

@@ -85,16 +85,26 @@ def flatten_dict(d, path=""):
 
 
 def fill_flat_stats(x, result, path=()):
-    print("fill_flat_stats", path, type(x), x)
+    # Only recurse into dicts and lists/tuples
     if result is None:
         result = {}
     if isinstance(x, dict):
         for k, v in x.items():
             fill_flat_stats(v, result, (*path, k))
         return result
+    if isinstance(x, (list, tuple)):
+        for idx, v in enumerate(x):
+            fill_flat_stats(v, result, (*path, str(idx)))
+        return result
+    # Only accumulate numbers (int, float, or 0-dim tensor)
     if "stats" in path and "__batch_hash__" not in path[-1]:
-        path = "/".join(path)
-        result[path] = result.get(path, 0) + x
+        if isinstance(x, (int, float)):
+            path_str = "/".join(path)
+            result[path_str] = result.get(path_str, 0) + x
+        elif hasattr(x, "item") and callable(x.item):
+            # For 0-dim tensors
+            path_str = "/".join(path)
+            result[path_str] = result.get(path_str, 0) + x.item()
     return result
 
 
@@ -646,7 +656,8 @@ def train(
                     *(
                         td(nlp, device).set_processing(
                             num_cpu_workers=num_workers,
-                            process_start_method="spawn",
+                            # process_start_method="spawn",
+                            backend="simple",
                         )
                         for td in phase_training_data
                     )

edsnlp/training/trainer.py

@@ -282,7 +282,7 @@ where = ["."]
 "eds.ner_overlap"                = "edsnlp.metrics.ner:NerOverlapMetric"
 "eds.span_attributes"            = "edsnlp.metrics.span_attributes:SpanAttributeMetric"
 "eds.dep_parsing"                = "edsnlp.metrics.dep_parsing:DependencyParsingMetric"
-"eds.doc_classif"                    = "edsnlp.metrics.doc_classif:DocClassificationMetric"
+"eds.doc_classif"                = "edsnlp.metrics.doc_classif:DocClassificationMetric"
 
 # Deprecated
 "eds.ner_exact_metric"           = "edsnlp.metrics.ner:NerExactMetric"