Added the option to read multiple echem files and stitch them together to the cycleBlock

pydatalab/pyproject.toml

@@ -138,6 +138,7 @@ datalab-app-plugin-insitu = { git = "https://github.com/datalab-org/datalab-app-
 addopts = "--cov-report=xml --cov ./src/pydatalab"
 filterwarnings = [
     "error",
+    "ignore:.*Capacity columns are not equal, replacing with new capacity column calculated from current and time columns and renaming the old capacity column to Old Capacity:UserWarning",
     "ignore:.*np.bool8*:DeprecationWarning",
     "ignore::pytest.PytestUnraisableExceptionWarning",
     "ignore:.*JCAMP-DX key without value*:UserWarning",

pydatalab/src/pydatalab/apps/echem/blocks.py

@@ -1,6 +1,7 @@
 import os
-import time
+import warnings
 from pathlib import Path
+from typing import Any
 
 import bokeh
 import pandas as pd
@@ -43,7 +44,7 @@ class CycleBlock(DataBlock):
         ".ndax",
     )
 
-    defaults = {
+    defaults: dict[str, Any] = {
         "p_spline": 5,
         "s_spline": 5,
         "win_size_2": 101,
@@ -61,12 +62,12 @@ def _get_characteristic_mass_g(self):
             return characteristic_mass_mg / 1000.0
         return None
 
-    def _load(self, file_id: str | ObjectId, reload: bool = True):
+    def _load(self, file_ids: list[ObjectId] | ObjectId, reload: bool = True):
         """Loads the echem data using navani, summarises it, then caches the results
         to disk with suffixed names.
 
         Parameters:
-            file_id: The ID of the file to load.
+            file_ids: The IDs of the files to load.
             reload: Whether to reload the data from the file, or use the cached version, if available.
 
         """
@@ -93,54 +94,78 @@ def _load(self, file_id: str | ObjectId, reload: bool = True):
             "dvdq": "dV/dQ (V/mA)",
         }
 
-        file_info = get_file_info_by_id(file_id, update_if_live=True)
-        filename = file_info["name"]
-
-        if file_info.get("is_live"):
-            reload = True
-
-        ext = os.path.splitext(filename)[-1].lower()
-
-        if ext not in self.accepted_file_extensions:
-            raise RuntimeError(
-                f"Unrecognized filetype {ext}, must be one of {self.accepted_file_extensions}"
-            )
-
-        parsed_file_loc = Path(file_info["location"]).with_suffix(".RAW_PARSED.pkl")
-        cycle_summary_file_loc = Path(file_info["location"]).with_suffix(".SUMMARY.pkl")
+        if isinstance(file_ids, ObjectId):
+            file_ids = [file_ids]
 
         raw_df = None
         cycle_summary_df = None
-        if not reload:
-            if parsed_file_loc.exists():
-                raw_df = pd.read_pickle(parsed_file_loc)  # noqa: S301
 
-            if cycle_summary_file_loc.exists():
-                cycle_summary_df = pd.read_pickle(cycle_summary_file_loc)  # noqa: S301
+        if len(file_ids) == 1:
+            file_info = get_file_info_by_id(file_ids[0], update_if_live=True)
+            filename = file_info["name"]
 
-        if raw_df is None:
-            try:
-                LOGGER.debug("Loading file %s", file_info["location"])
-                start_time = time.time()
-                raw_df = ec.echem_file_loader(file_info["location"])
-                LOGGER.debug(
-                    "Loaded file %s in %s seconds",
-                    file_info["location"],
-                    time.time() - start_time,
+            if file_info.get("is_live"):
+                reload = True
+
+            ext = os.path.splitext(filename)[-1].lower()
+
+            if ext not in self.accepted_file_extensions:
+                raise RuntimeError(
+                    f"Unrecognized filetype {ext}, must be one of {self.accepted_file_extensions}"
                 )
-            except Exception as exc:
-                raise RuntimeError(f"Navani raised an error when parsing: {exc}") from exc
-            raw_df.to_pickle(parsed_file_loc)
 
-        try:
-            if cycle_summary_df is None:
+            parsed_file_loc = Path(file_info["location"]).with_suffix(".RAW_PARSED.pkl")
+
+            if not reload:
+                if parsed_file_loc.exists():
+                    raw_df = pd.read_pickle(parsed_file_loc)  # noqa: S301
+
+            if raw_df is None:
+                try:
+                    raw_df = ec.echem_file_loader(file_info["location"])
+                except Exception as exc:
+                    raise RuntimeError(f"Navani raised an error when parsing: {exc}") from exc
+                raw_df.to_pickle(parsed_file_loc)
+
+        elif isinstance(file_ids, list) and len(file_ids) > 1:
+            # Multi-file logic
+            file_infos = [get_file_info_by_id(fid, update_if_live=True) for fid in file_ids]
+            locations = [info["location"] for info in file_infos]
+
+            if raw_df is None:
+                try:
+                    LOGGER.debug("Loading multiple echem files with navani: %s", locations)
+                    # Catch the navani warning when stitching multiple files together and calculating new capacity
+                    with warnings.catch_warnings():
+                        warnings.filterwarnings(
+                            "ignore",
+                            message=(
+                                "Capacity columns are not equal, replacing with new capacity column calculated from current and time columns and renaming the old capacity column to Old Capacity"
+                            ),
+                            category=UserWarning,
+                        )
+                        raw_df = ec.multi_echem_file_loader(locations)
+                except Exception as exc:
+                    raise RuntimeError(
+                        f"Navani raised an error when parsing multiple files: {exc}"
+                    ) from exc
+
+        elif not isinstance(file_ids, list):
+            raise ValueError("Invalid file_ids type. Expected list of strings.")
+        elif len(file_ids) == 0:
+            raise ValueError("Invalid file_ids value. Expected non-empty list of strings.")
+
+        if cycle_summary_df is None and raw_df is not None:
+            try:
                 cycle_summary_df = ec.cycle_summary(raw_df)
-                cycle_summary_df.to_pickle(cycle_summary_file_loc)
-        except Exception as exc:
-            LOGGER.warning("Cycle summary generation failed with error: %s", exc)
+            except Exception as exc:
+                warnings.warn(f"Cycle summary generation failed with error: {exc}")
 
-        raw_df = raw_df.filter(required_keys)
-        raw_df.rename(columns=keys_with_units, inplace=True)
+        if raw_df is not None:
+            raw_df = raw_df.filter(required_keys)
+            raw_df.rename(columns=keys_with_units, inplace=True)
+        else:
+            raise ValueError("Invalid raw_df value. Expected non-empty DataFrame.")
 
         if cycle_summary_df is not None:
             cycle_summary_df.rename(columns=keys_with_units, inplace=True)
@@ -152,10 +177,21 @@ def _load(self, file_id: str | ObjectId, reload: bool = True):
 
     def plot_cycle(self):
         """Plots the electrochemical cycling data from the file ID provided in the request."""
-        if "file_id" not in self.data:
-            LOGGER.warning("No file_id given")
-            return
-        file_id = self.data["file_id"]
+
+        # Legacy support for when file_id was used
+        if self.data.get("file_id") is not None and not self.data.get("file_ids"):
+            LOGGER.info("Legacy file upload detected, using file_id")
+            file_ids = [self.data["file_id"]]
+
+        else:
+            if "file_ids" not in self.data:
+                LOGGER.warning("No file_ids given, skipping plot.")
+                return
+            if self.data["file_ids"] is None or len(self.data["file_ids"]) == 0:
+                LOGGER.warning("Empty file_ids list given, skipping plot.")
+                return
+
+            file_ids = self.data["file_ids"]
 
         derivative_modes = (None, "dQ/dV", "dV/dQ", "final capacity")
 
@@ -177,7 +213,7 @@ def plot_cycle(self):
         if not isinstance(cycle_list, list):
             cycle_list = None
 
-        raw_df, cycle_summary_df = self._load(file_id)
+        raw_df, cycle_summary_df = self._load(file_ids=file_ids)
 
         characteristic_mass_g = self._get_characteristic_mass_g()
 

pydatalab/tests/server/test_blocks.py

@@ -261,6 +261,82 @@ def test_uvvis_block_lifecycle(admin_client, default_sample_dict, example_data_d
     assert web_block.get("errors") is None
 
 
+def test_echem_block_lifecycle(admin_client, default_sample_dict, example_data_dir):
+    block_type = "cycle"
+
+    sample_id = f"test_sample_with_files-{block_type}-lifecycle"
+    sample_data = default_sample_dict.copy()
+    sample_data["item_id"] = sample_id
+
+    response = admin_client.post("/new-sample/", json=sample_data)
+    assert response.status_code == 201
+    assert response.json["status"] == "success"
+
+    response = admin_client.post(
+        "/add-data-block/",
+        json={
+            "block_type": block_type,
+            "item_id": sample_id,
+            "index": 0,
+        },
+    )
+
+    assert response.status_code == 200, f"Failed to add {block_type} block: {response.json}"
+    assert response.json["status"] == "success"
+
+    block_data = response.json["new_block_obj"]
+    block_id = block_data["block_id"]
+
+    # Upload multiple echem files
+    echem_folder = example_data_dir / "echem"
+    example_files = list(echem_folder.glob("*.mpr"))[:2]
+    example_file_ids = []
+
+    for example_file in example_files:
+        with open(example_file, "rb") as f:
+            response = admin_client.post(
+                "/upload-file/",
+                buffered=True,
+                content_type="multipart/form-data",
+                data={
+                    "item_id": sample_id,
+                    "file": [(f, example_file.name)],
+                    "type": "application/octet-stream",
+                    "replace_file": "null",
+                    "relativePath": "null",
+                },
+            )
+            assert response.status_code == 201, f"Failed to upload {example_file.name}"
+            assert response.json["status"] == "success"
+            file_ids = response.json["file_id"]
+            example_file_ids.append(file_ids)
+
+    assert len(example_file_ids) == 2
+
+    # Update block with multiple file_ids
+    response = admin_client.get(f"/get-item-data/{sample_id}")
+    assert response.status_code == 200
+    item_data = response.json["item_data"]
+    block_data = item_data["blocks_obj"][block_id]
+    block_data["file_ids"] = example_file_ids
+
+    response = admin_client.post("/update-block/", json={"block_data": block_data})
+    assert response.status_code == 200
+    web_block = response.json["new_block_data"]
+
+    assert "bokeh_plot_data" in web_block
+    assert web_block.get("errors") is None
+
+    # Test for only one file_id
+    block_data["file_ids"] = [example_file_ids[0]]
+
+    response = admin_client.post("/update-block/", json={"block_data": block_data})
+    assert response.status_code == 200
+    web_block = response.json["new_block_data"]
+    assert "bokeh_plot_data" in web_block
+    assert web_block.get("errors") is None
+
+
 def test_xrd_block_lifecycle(admin_client, default_sample_dict, example_data_dir):
     from pydatalab.apps.xrd import XRDBlock
 

webapp/src/components/FileMultiSelectDropdown.vue

@@ -42,21 +42,37 @@
 
       <div class="action-buttons">
         <button
-          class="btn btn-secondary mb-2"
+          class="btn btn-secondary btn-sm mb-2"
           :disabled="!selectedAvailable"
           aria-label="Add selected file"
           @click="addSelected()"
         >
           &gt;
         </button>
         <button
-          class="btn btn-secondary"
+          class="btn btn-secondary btn-sm"
           :disabled="!selectedSelected"
           aria-label="Remove selected file"
           @click="removeSelected()"
         >
           &lt;
         </button>
+        <button
+          class="btn btn-secondary btn-sm mt-2"
+          :disabled="availableFiles.length === 0"
+          aria-label="Add all available files"
+          @click="addAllAvailable"
+        >
+          &raquo;
+        </button>
+        <button
+          class="btn btn-secondary btn-sm mt-2"
+          :disabled="modelValue.length === 0"
+          aria-label="Remove all files"
+          @click="removeAllSelected"
+        >
+          &laquo;
+        </button>
       </div>
 
       <div class="listbox">
@@ -172,7 +188,14 @@ export default {
     availableFiles() {
       // Filter out files that are already selected
       const selectedSet = new Set(this.modelValue);
-      return this.all_available_file_ids.filter((id) => !selectedSet.has(id));
+      // Get filtered IDs
+      const filteredIds = this.all_available_file_ids.filter((id) => !selectedSet.has(id));
+      // Sort alphabetically by file name
+      return filteredIds.sort((a, b) => {
+        const nameA = this.getFileName(a).toLowerCase();
+        const nameB = this.getFileName(b).toLowerCase();
+        return nameA.localeCompare(nameB);
+      });
     },
     canMoveUp() {
       return this.selectedSelected !== null && this.selectedSelectedIndex > 0;
@@ -249,6 +272,21 @@ export default {
       this.selectedSelected = null;
       this.selectedSelectedIndex = -1;
     },
+    addAllAvailable() {
+      if (this.availableFiles.length === 0) return;
+
+      const newSelectedFiles = [...this.modelValue, ...this.availableFiles];
+      this.emitUpdate(newSelectedFiles);
+      this.selectedAvailable = null;
+    },
+    removeAllSelected() {
+      if (this.modelValue.length === 0) return;
+
+      this.emitUpdate([]);
+      this.selectedSelected = null;
+      this.selectedSelectedIndex = -1;
+      this.selectedAvailable = null; // Reset available selection
+    },
     moveUp() {
       if (!this.canMoveUp) return;
 
@@ -328,10 +366,6 @@ export default {
 </script>
 
 <style scoped>
-.multi-file-selector {
-  /* Add styles for the overall container if needed */
-}
-
 .dual-listbox-container {
   display: flex;
   gap: 1rem; /* Space between elements */