Skip to content

Commit 0f06ad2

Browse files
louispt1louispt1
authored
Custom curves upload, validation and to/from df (#68)
* Custom curves to and from dataframe * Upload custom curves runner, scenario methods and validation on the model * Custom curves: tests for runner, scenario and validation
1 parent 56222fe commit 0f06ad2

16 files changed

+28347
-111
lines changed

‎examples/curve_examples/agriculture_electricity_curve.csv‎

Lines changed: 8760 additions & 0 deletions
Large diffs are not rendered by default.

‎examples/curve_examples/electric_vehicle_profile_5_curve.csv‎

Lines changed: 8760 additions & 0 deletions
Large diffs are not rendered by default.

‎examples/curve_examples/interconnector_8_price_curve.csv‎

Lines changed: 8760 additions & 0 deletions
Large diffs are not rendered by default.

‎examples/excel_to_scenario.ipynb‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"id": "42081dd2",
6+
"metadata": {},
7+
"source": [
8+
"This is still a testing workbook to demonstrate progress on the excel to scenario flows."
9+
]
10+
},
11+
{
12+
"cell_type": "markdown",
13+
"id": "386ce0b0",
14+
"metadata": {},
15+
"source": [
16+
"# Curves"
17+
]
18+
},
19+
{
20+
"cell_type": "code",
21+
"execution_count": 1,
22+
"id": "c617dc0a",
23+
"metadata": {},
24+
"outputs": [
25+
{
26+
"name": "stdout",
27+
"output_type": "stream",
28+
"text": [
29+
"Found 3 curve files\n",
30+
"✓ Loaded curve 'interconnector_8_price': 8760 values\n",
31+
"✓ Loaded curve 'electric_vehicle_profile_5': 8760 values\n",
32+
"✓ Loaded curve 'agriculture_electricity': 8760 values\n",
33+
"Created DataFrame with 3 curves and 8760 rows\n"
34+
]
35+
}
36+
],
37+
"source": [
38+
"# For now a custom csv to pd df function - this will be handled by the reverse packer in the end\n",
39+
"\n",
40+
"import pandas as pd\n",
41+
"from pathlib import Path\n",
42+
"from typing import Union\n",
43+
"\n",
44+
"def read_curves_to_dataframe(\n",
45+
" curves_path: Union[str, Path],\n",
46+
" pattern: str = \"*.csv\",\n",
47+
" validate_length: bool = True\n",
48+
") -> pd.DataFrame:\n",
49+
" \"\"\"\n",
50+
" Read multiple curve CSV files into a single DataFrame.\n",
51+
"\n",
52+
" Args:\n",
53+
" curves_path: Directory path containing the curve CSV files\n",
54+
" pattern: File pattern to match (default: \"*.csv\")\n",
55+
" validate_length: Whether to validate each curve has exactly 8760 values\n",
56+
"\n",
57+
" Returns:\n",
58+
" DataFrame with curves as columns, where column names are the curve keys\n",
59+
" (derived from filenames without extension)\n",
60+
"\n",
61+
" Raises:\n",
62+
" ValueError: If validation fails or files have issues\n",
63+
" FileNotFoundError: If no files found matching the pattern\n",
64+
" \"\"\"\n",
65+
" curves_path = Path(curves_path)\n",
66+
"\n",
67+
" if not curves_path.exists():\n",
68+
" raise FileNotFoundError(f\"Directory not found: {curves_path}\")\n",
69+
"\n",
70+
" # Find all CSV files matching the pattern\n",
71+
" csv_files = list(curves_path.glob(pattern))\n",
72+
"\n",
73+
" if not csv_files:\n",
74+
" raise FileNotFoundError(f\"No files found matching pattern '{pattern}' in {curves_path}\")\n",
75+
"\n",
76+
" print(f\"Found {len(csv_files)} curve files\")\n",
77+
"\n",
78+
" curves_data = {}\n",
79+
" errors = []\n",
80+
"\n",
81+
" for csv_file in csv_files:\n",
82+
" # Use filename (without extension) as curve key, remove _curve suffix if present\n",
83+
" curve_key = csv_file.stem\n",
84+
" if curve_key.endswith('_curve'):\n",
85+
" curve_key = curve_key[:-6] # Remove '_curve' suffix\n",
86+
"\n",
87+
" try:\n",
88+
" # Read CSV file - assuming single column of values, no headers\n",
89+
" curve_data = pd.read_csv(\n",
90+
" csv_file,\n",
91+
" header=None, # No header row\n",
92+
" index_col=False, # No index column\n",
93+
" dtype=float # All values should be numeric\n",
94+
" )\n",
95+
"\n",
96+
" # Convert DataFrame to Series if single column\n",
97+
" if isinstance(curve_data, pd.DataFrame):\n",
98+
" if len(curve_data.columns) == 1:\n",
99+
" curve_data = curve_data.iloc[:, 0]\n",
100+
" else:\n",
101+
" errors.append(f\"{curve_key}: Expected 1 column, found {len(curve_data.columns)}\")\n",
102+
" continue\n",
103+
"\n",
104+
" # Drop any NaN values\n",
105+
" curve_data = curve_data.dropna()\n",
106+
"\n",
107+
" # Validate length if requested\n",
108+
" if validate_length and len(curve_data) != 8760:\n",
109+
" errors.append(f\"{curve_key}: Expected 8760 values, found {len(curve_data)}\")\n",
110+
" continue\n",
111+
"\n",
112+
" # Store with curve key as column name\n",
113+
" curves_data[curve_key] = curve_data.values\n",
114+
" print(f\"✓ Loaded curve '{curve_key}': {len(curve_data)} values\")\n",
115+
"\n",
116+
" except Exception as e:\n",
117+
" errors.append(f\"{curve_key}: Error reading file - {str(e)}\")\n",
118+
" continue\n",
119+
"\n",
120+
" if errors:\n",
121+
" error_msg = \"Errors reading curve files:\\n\" + \"\\n\".join(f\" - {err}\" for err in errors)\n",
122+
" if not curves_data: # No curves loaded successfully\n",
123+
" raise ValueError(error_msg)\n",
124+
" else:\n",
125+
" print(f\"Warning: Some curves failed to load:\\n{error_msg}\")\n",
126+
"\n",
127+
" if not curves_data:\n",
128+
" raise ValueError(\"No curves were successfully loaded\")\n",
129+
"\n",
130+
" # Create DataFrame from the curves\n",
131+
" df = pd.DataFrame(curves_data)\n",
132+
"\n",
133+
" # Set index to represent hours (0-8759 for a full year)\n",
134+
" df.index.name = \"hour\"\n",
135+
"\n",
136+
" print(f\"Created DataFrame with {len(df.columns)} curves and {len(df)} rows\")\n",
137+
" return df\n",
138+
"\n",
139+
"# User uploads Excel/CSV → DataFrame → CustomCurves object\n",
140+
"from pyetm.models.custom_curves import CustomCurves\n",
141+
"\n",
142+
"df = read_curves_to_dataframe(\"curve_examples/\")\n",
143+
"custom_curves = CustomCurves._from_dataframe(df)"
144+
]
145+
},
146+
{
147+
"cell_type": "code",
148+
"execution_count": 2,
149+
"id": "e75a03b2",
150+
"metadata": {},
151+
"outputs": [
152+
{
153+
"name": "stdout",
154+
"output_type": "stream",
155+
"text": [
156+
"Environment setup complete\n",
157+
" Using ETM API at http://localhost:3000/api/v3\n",
158+
" Token loaded? True\n",
159+
"API connection ready\n"
160+
]
161+
}
162+
],
163+
"source": [
164+
"from example_helpers import setup_notebook\n",
165+
"from pyetm.models import Scenario\n",
166+
"\n",
167+
"setup_notebook()\n",
168+
"scenario = Scenario.load(2690288)\n",
169+
"\n",
170+
"# Update curves on scenario\n",
171+
"scenario.update_custom_curves(custom_curves)"
172+
]
173+
}
174+
],
175+
"metadata": {
176+
"kernelspec": {
177+
"display_name": "pyetm-qKH2ozgc",
178+
"language": "python",
179+
"name": "python3"
180+
},
181+
"language_info": {
182+
"codemirror_mode": {
183+
"name": "ipython",
184+
"version": 3
185+
},
186+
"file_extension": ".py",
187+
"mimetype": "text/x-python",
188+
"name": "python",
189+
"nbconvert_exporter": "python",
190+
"pygments_lexer": "ipython3",
191+
"version": "3.12.9"
192+
}
193+
},
194+
"nbformat": 4,
195+
"nbformat_minor": 5
196+
}

‎src/pyetm/models/base.py‎

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ class Base(BaseModel):
2020

2121
# Enable assignment validation
2222
model_config = ConfigDict(validate_assignment=True)
23-
2423
_warning_collector: WarningCollector = PrivateAttr(default_factory=WarningCollector)
2524

2625
def __init__(self, **data: Any) -> None:
@@ -52,9 +51,15 @@ def __setattr__(self, name: str, value: Any) -> None:
5251
Handle assignment with validation error capture.
5352
Simplified from the original complex implementation.
5453
"""
55-
# Skip validation for private attributes
56-
if name.startswith("_") or name not in self.__class__.model_fields:
57-
super().__setattr__(name, value)
54+
# Skip validation for private attributes, methods/functions, or existing methods
55+
if (
56+
name.startswith("_")
57+
or name not in self.__class__.model_fields
58+
or callable(value)
59+
or hasattr(self.__class__, name)
60+
):
61+
# Use object.__setattr__ to bypass Pydantic for these cases
62+
object.__setattr__(self, name, value)
5863
return
5964

6065
# Clear existing warnings for this field
@@ -106,17 +111,32 @@ def _clear_warnings_for_attr(self, field: str) -> None:
106111
def _merge_submodel_warnings(self, *submodels: Base, key_attr: str = None) -> None:
107112
"""
108113
Merge warnings from nested Base models.
109-
Maintains compatibility with existing code while using the new system.
110114
"""
111115
self._warning_collector.merge_submodel_warnings(*submodels, key_attr=key_attr)
112116

113117
@classmethod
114-
def load_safe(cls: Type[T], **data: Any) -> T:
118+
def from_dataframe(cls: Type[T], df: pd.DataFrame, **kwargs) -> T:
115119
"""
116-
Alternate constructor that always returns an instance,
117-
converting all validation errors into warnings.
120+
Create an instance from a pandas DataFrame.
118121
"""
119-
return cls(**data)
122+
try:
123+
return cls._from_dataframe(df, **kwargs)
124+
except Exception as e:
125+
# Create a fallback instance with warnings
126+
instance = cls.model_construct()
127+
instance.add_warning(
128+
"from_dataframe", f"Failed to create from DataFrame: {e}"
129+
)
130+
return instance
131+
132+
@classmethod
133+
def _from_dataframe(cls, df: pd.DataFrame, **kwargs):
134+
"""
135+
Private method to be implemented by each subclass for specific deserialization logic.
136+
"""
137+
raise NotImplementedError(
138+
f"{cls.__name__} must implement _from_dataframe() class method"
139+
)
120140

121141
def _get_serializable_fields(self) -> List[str]:
122142
"""

0 commit comments

Comments
 (0)