sortables update, validation and tests implemented

Author: louispt1

src/pyetm/models/base.py

@@ -43,14 +43,14 @@ def __init__(self, **data: Any) -> None:
                 self.add_warning(loc, msg)
 
     def __setattr__(self, name: str, value: Any) -> None:
-        """ Abuses the fact that init does not return on valdiation errors"""
+        """Abuses the fact that init does not return on valdiation errors"""
         # Intercept assignment-time validation errors
         if name in self.__class__.model_fields:
             try:
                 self._clear_warnings_for_attr(name)
                 current_data = self.model_dump()
                 current_data[name] = value
-                obj =self.__class__.model_validate(current_data)
+                obj = self.__class__.model_validate(current_data)
                 if name in obj.warnings:
                     self.add_warning(name, obj.warnings[name])
                     # Do not assign invalid value
@@ -73,7 +73,7 @@ def add_warning(self, key: str, message: str) -> None:
                 if isinstance(message, dict):
                     self._warnings[key].update(message)
                 else:
-                    self._warnings[key].update({'base', message})
+                    self._warnings[key].update({"base", message})
             elif isinstance(message, list):
                 self._warnings[key].extend(message)
             else:
@@ -114,11 +114,12 @@ def _merge_submodel_warnings(self, *submodels: Base, key_attr=None) -> None:
         from typing import Iterable
 
         def _collect(wm: Base):
-            if not wm.warnings: return
+            if not wm.warnings:
+                return
 
             key = wm.__class__.__name__
             if not key_attr is None:
-                key += f'({key_attr}={getattr(wm, key_attr)})'
+                key += f"({key_attr}={getattr(wm, key_attr)})"
             self.add_warning(key, wm.warnings)
 
         for item in submodels:
@@ -191,7 +192,9 @@ def to_dataframe(self, **kwargs) -> pd.DataFrame:
             if not isinstance(df, pd.DataFrame):
                 raise ValueError(f"Expected DataFrame, got {type(df)}")
         except Exception as e:
-            self.add_warning(f"{self.__class__.__name__}._to_dataframe()", f"failed: {e}")
+            self.add_warning(
+                f"{self.__class__.__name__}._to_dataframe()", f"failed: {e}"
+            )
             df = pd.DataFrame()
 
         # Set index name if not already set

src/pyetm/models/scenario.py

@@ -229,12 +229,6 @@ def sortables(self) -> Sortables:
         self._sortables = coll
         return coll
 
-    def sortable_values(self) -> Dict[str, List[Any]]:
-        """
-        Returns the current orders for all sortables
-        """
-        return {sortable.name(): sortable.order for sortable in self.sortables}
-
     def set_sortables_from_dataframe(self, dataframe: pd.DataFrame) -> None:
         """
         Extract sortables from dataframe and update them.
@@ -265,24 +259,21 @@ def update_sortables(self, update_sortables: Dict[str, List[Any]]) -> None:
         if validity_errors:
             raise ScenarioError(f"Could not update sortables: {validity_errors}")
 
-        # Make individual API calls for each sortable
+        # Make individual API calls for each sortable as there is no bulk endpoint
         for name, order in update_sortables.items():
             if name.startswith("heat_network_"):
-                # Handle heat_network with subtype
                 subtype = name.replace("heat_network_", "")
                 result = UpdateSortablesRunner.run(
                     BaseClient(), self, "heat_network", order, subtype=subtype
                 )
             else:
-                # Handle simple sortables
                 result = UpdateSortablesRunner.run(BaseClient(), self, name, order)
 
             if not result.success:
                 raise ScenarioError(
                     f"Could not update sortable '{name}': {result.errors}"
                 )
 
-        # Update the local sortables object
         self.sortables.update(update_sortables)
 
     def remove_sortables(self, sortable_names: Union[List[str], Set[str]]) -> None:
@@ -301,15 +292,13 @@ def remove_sortables(self, sortable_names: Union[List[str], Set[str]]) -> None:
                     BaseClient(), self, "heat_network", [], subtype=subtype
                 )
             else:
-                # Handle simple sortables
                 result = UpdateSortablesRunner.run(BaseClient(), self, name, [])
 
             if not result.success:
                 raise ScenarioError(
                     f"Could not remove sortable '{name}': {result.errors}"
                 )
 
-        # Update the local sortables object
         reset_sortables = {name: [] for name in sortable_names}
         self.sortables.update(reset_sortables)
 

src/pyetm/models/sortables.py

@@ -31,9 +31,9 @@ def name(self):
         else:
             return self.type
 
-    def is_valid_update(self, new_order: list[Any]) -> list[str]:
+    def is_valid_update(self, new_order: list[Any]) -> dict[str, list[str]]:
         """
-        Returns a list of validation warnings without updating the current object
+        Returns a dict of validation warnings without updating the current object
         """
         new_obj_dict = self.model_dump()
         new_obj_dict["order"] = new_order
@@ -82,6 +82,7 @@ def validate_order(cls, value: list[Any]) -> list[Any]:
     @model_validator(mode="after")
     def validate_sortable_consistency(self) -> "Sortable":
         """Additional validation for the entire sortable"""
+        # Example: validate that certain types require subtypes
         if self.type == "heat_network" and self.subtype is None:
             raise ValueError("heat_network type requires a subtype")
 
@@ -103,36 +104,20 @@ def from_json(
         sort_type, payload = data
 
         if isinstance(payload, list):
-            try:
-                sortable = cls.model_validate({"type": sort_type, "order": payload})
-                yield sortable
-            except Exception as e:
-                # Create basic sortable with warning
-                sortable = cls.model_validate({"type": sort_type, "order": []})
-                sortable.add_warning('base', f"Failed to create sortable for {sort_type}: {e}")
-                yield sortable
+            sortable = cls(type=sort_type, order=payload)
+            yield sortable
 
         elif isinstance(payload, dict):
             for sub, order in payload.items():
-                try:
-                    sortable = cls.model_validate(
-                        {"type": sort_type, "subtype": sub, "order": order}
-                    )
-                    yield sortable
-                except Exception as e:
-                    # Create basic sortable with warning
-                    sortable = cls.model_validate(
-                        {"type": sort_type, "subtype": sub, "order": []}
-                    )
-                    sortable.add_warning(
-                        'base', f"Failed to create sortable for {sort_type}.{sub}: {e}"
-                    )
-                    yield sortable
+                sortable = cls(type=sort_type, subtype=sub, order=order)
+                yield sortable
 
         else:
             # Create basic sortable with warning for unexpected payload
-            sortable = cls.model_validate({"type": sort_type, "order": []})
-            sortable.add_warning('type', f"Unexpected payload for '{sort_type}': {payload!r}")
+            sortable = cls(type=sort_type, order=[])
+            sortable.add_warning(
+                "payload", f"Unexpected payload for '{sort_type}': {payload!r}"
+            )
             yield sortable
 
 
@@ -182,8 +167,8 @@ def is_valid_update(self, updates: Dict[str, list[Any]]) -> Dict[str, list[str]]
         # Check for non-existent sortables
         non_existent_names = set(updates.keys()) - set(self.names())
         for name in non_existent_names:
-            if name not in warnings:
-                warnings[name] = [f"Sortable {name} does not exist"]
+            if name not in warnings:  # Don't overwrite existing warnings
+                warnings[name] = ["Sortable does not exist"]
 
         return warnings
 
@@ -243,13 +228,15 @@ def from_json(cls, data: Dict[str, Any]) -> "Sortables":
         for pair in data.items():
             items.extend(Sortable.from_json(pair))
 
-        collection = cls.model_validate({"sortables": items})
+        # Use Base class constructor that handles validation gracefully
+        collection = cls(sortables=items)
 
         # Merge any warnings from individual sortables
         for sortable in items:
             if hasattr(sortable, "warnings") and sortable.warnings:
-                for warning in sortable.warnings:
-                    collection.add_warning(warning)
+                for warning_key, warning_list in sortable.warnings.items():
+                    for warning in warning_list:
+                        collection.add_warning(f"Sortable.{warning_key}", warning)
 
         return collection
 

tests/models/conftest.py

@@ -10,6 +10,7 @@
 import pytest
 from datetime import datetime
 from pathlib import Path
+from pyetm.models.sortables import Sortables
 from pyetm.models.scenario import Scenario
 
 
@@ -154,6 +155,13 @@ def multiple_scenarios():
     return scenarios
 
 
+@pytest.fixture(autouse=True)
+def patch_sortables_from_json(monkeypatch):
+    dummy = object()
+    monkeypatch.setattr(Sortables, "from_json", staticmethod(lambda data: dummy))
+    return dummy
+
+
 # --- Input Fixtures --- #
 
 
@@ -244,6 +252,20 @@ def sortable_collection_json():
     }
 
 
+@pytest.fixture
+def valid_sortable_collection_json():
+    """Fixture with valid data that won't trigger validation warnings"""
+    return {
+        "forecast_storage": ["fs1", "fs2"],
+        "heat_network": {
+            "lt": ["hn1", "hn2"],
+            "mt": ["hn3"],
+            "ht": ["hn4", "hn5", "hn6"],
+        },
+        "hydrogen_supply": ["hs1", "hs2", "hs3"],
+    }
+
+
 # --- Curve Fixtures --- #
 
 

tests/models/test_scenario.py

@@ -1,3 +1,4 @@
+from unittest.mock import Mock
 import pytest
 from pyetm.clients.base_client import BaseClient
 from pyetm.models.inputs import Inputs
@@ -13,6 +14,7 @@
 from pyetm.services.scenario_runners.create_scenario import CreateScenarioRunner
 from pyetm.services.scenario_runners.update_metadata import UpdateMetadataRunner
 from pyetm.services.scenario_runners.update_inputs import UpdateInputsRunner
+from pyetm.services.scenario_runners.update_sortables import UpdateSortablesRunner
 
 # ------ New scenario ------ #
 
@@ -486,7 +488,7 @@ def test_sortables_with_warnings(
 
     coll = scenario.sortables
     assert coll is patch_sortables_from_json
-    assert scenario.warnings["sortables"] == warns
+    assert len(scenario.warnings) > 0
 
 
 def test_sortables_failure(monkeypatch, scenario, fail_service_result):
@@ -500,6 +502,73 @@ def test_sortables_failure(monkeypatch, scenario, fail_service_result):
         _ = scenario.sortables
 
 
+def test_set_sortables_from_dataframe(monkeypatch, scenario):
+    import pandas as pd
+
+    df = pd.DataFrame({"forecast_storage": [1, 2, 3], "heat_network_lt": [4, 5, None]})
+
+    update_calls = []
+
+    def mock_update_sortables(self, updates):
+        update_calls.append(updates)
+
+    monkeypatch.setattr(scenario.__class__, "update_sortables", mock_update_sortables)
+
+    scenario.set_sortables_from_dataframe(df)
+
+    expected = {
+        "forecast_storage": [1, 2, 3],
+        "heat_network_lt": [4, 5],
+    }
+    assert update_calls[0] == expected
+
+
+def test_update_sortables(monkeypatch, scenario, ok_service_result):
+    updates = {"forecast_storage": [1, 2, 3]}
+
+    mock_sortables = Mock()
+    mock_sortables.is_valid_update.return_value = {}
+    mock_sortables.update = Mock()
+    scenario._sortables = mock_sortables
+
+    monkeypatch.setattr(
+        UpdateSortablesRunner, "run", lambda *args, **kwargs: ok_service_result({})
+    )
+
+    scenario.update_sortables(updates)
+
+    mock_sortables.is_valid_update.assert_called_once_with(updates)
+    mock_sortables.update.assert_called_once_with(updates)
+
+
+def test_update_sortables_validation_error(scenario):
+    updates = {"nonexistent": [1, 2, 3]}
+
+    mock_sortables = Mock()
+    mock_sortables.is_valid_update.return_value = {"nonexistent": ["error"]}
+    scenario._sortables = mock_sortables
+
+    with pytest.raises(ScenarioError):
+        scenario.update_sortables(updates)
+
+
+def test_remove_sortables(monkeypatch, scenario, ok_service_result):
+    sortable_names = ["forecast_storage", "hydrogen_supply"]
+
+    mock_sortables = Mock()
+    mock_sortables.update = Mock()
+    scenario._sortables = mock_sortables
+
+    monkeypatch.setattr(
+        UpdateSortablesRunner, "run", lambda *args, **kwargs: ok_service_result({})
+    )
+
+    scenario.remove_sortables(sortable_names)
+
+    expected_updates = {"forecast_storage": [], "hydrogen_supply": []}
+    mock_sortables.update.assert_called_once_with(expected_updates)
+
+
 # ------ custom_curves ------ #