diff --git a/pydatalab/pyproject.toml b/pydatalab/pyproject.toml index 4c8f25489..12a3037d9 100644 --- a/pydatalab/pyproject.toml +++ b/pydatalab/pyproject.toml @@ -138,6 +138,7 @@ datalab-app-plugin-insitu = { git = "https://github.com/datalab-org/datalab-app- addopts = "--cov-report=xml --cov ./src/pydatalab" filterwarnings = [ "error", + "ignore:.*Capacity columns are not equal, replacing with new capacity column calculated from current and time columns and renaming the old capacity column to Old Capacity:UserWarning", "ignore:.*np.bool8*:DeprecationWarning", "ignore::pytest.PytestUnraisableExceptionWarning", "ignore:.*JCAMP-DX key without value*:UserWarning", diff --git a/pydatalab/src/pydatalab/apps/echem/blocks.py b/pydatalab/src/pydatalab/apps/echem/blocks.py index c3f5e7e1f..6bf4ea864 100644 --- a/pydatalab/src/pydatalab/apps/echem/blocks.py +++ b/pydatalab/src/pydatalab/apps/echem/blocks.py @@ -1,6 +1,7 @@ import os -import time +import warnings from pathlib import Path +from typing import Any import bokeh import pandas as pd @@ -43,7 +44,7 @@ class CycleBlock(DataBlock): ".ndax", ) - defaults = { + defaults: dict[str, Any] = { "p_spline": 5, "s_spline": 5, "win_size_2": 101, @@ -61,12 +62,12 @@ def _get_characteristic_mass_g(self): return characteristic_mass_mg / 1000.0 return None - def _load(self, file_id: str | ObjectId, reload: bool = True): + def _load(self, file_ids: list[ObjectId] | ObjectId, reload: bool = True): """Loads the echem data using navani, summarises it, then caches the results to disk with suffixed names. Parameters: - file_id: The ID of the file to load. + file_ids: The IDs of the files to load. reload: Whether to reload the data from the file, or use the cached version, if available. """ @@ -93,54 +94,78 @@ def _load(self, file_id: str | ObjectId, reload: bool = True): "dvdq": "dV/dQ (V/mA)", } - file_info = get_file_info_by_id(file_id, update_if_live=True) - filename = file_info["name"] - - if file_info.get("is_live"): - reload = True - - ext = os.path.splitext(filename)[-1].lower() - - if ext not in self.accepted_file_extensions: - raise RuntimeError( - f"Unrecognized filetype {ext}, must be one of {self.accepted_file_extensions}" - ) - - parsed_file_loc = Path(file_info["location"]).with_suffix(".RAW_PARSED.pkl") - cycle_summary_file_loc = Path(file_info["location"]).with_suffix(".SUMMARY.pkl") + if isinstance(file_ids, ObjectId): + file_ids = [file_ids] raw_df = None cycle_summary_df = None - if not reload: - if parsed_file_loc.exists(): - raw_df = pd.read_pickle(parsed_file_loc) # noqa: S301 - if cycle_summary_file_loc.exists(): - cycle_summary_df = pd.read_pickle(cycle_summary_file_loc) # noqa: S301 + if len(file_ids) == 1: + file_info = get_file_info_by_id(file_ids[0], update_if_live=True) + filename = file_info["name"] - if raw_df is None: - try: - LOGGER.debug("Loading file %s", file_info["location"]) - start_time = time.time() - raw_df = ec.echem_file_loader(file_info["location"]) - LOGGER.debug( - "Loaded file %s in %s seconds", - file_info["location"], - time.time() - start_time, + if file_info.get("is_live"): + reload = True + + ext = os.path.splitext(filename)[-1].lower() + + if ext not in self.accepted_file_extensions: + raise RuntimeError( + f"Unrecognized filetype {ext}, must be one of {self.accepted_file_extensions}" ) - except Exception as exc: - raise RuntimeError(f"Navani raised an error when parsing: {exc}") from exc - raw_df.to_pickle(parsed_file_loc) - try: - if cycle_summary_df is None: + parsed_file_loc = Path(file_info["location"]).with_suffix(".RAW_PARSED.pkl") + + if not reload: + if parsed_file_loc.exists(): + raw_df = pd.read_pickle(parsed_file_loc) # noqa: S301 + + if raw_df is None: + try: + raw_df = ec.echem_file_loader(file_info["location"]) + except Exception as exc: + raise RuntimeError(f"Navani raised an error when parsing: {exc}") from exc + raw_df.to_pickle(parsed_file_loc) + + elif isinstance(file_ids, list) and len(file_ids) > 1: + # Multi-file logic + file_infos = [get_file_info_by_id(fid, update_if_live=True) for fid in file_ids] + locations = [info["location"] for info in file_infos] + + if raw_df is None: + try: + LOGGER.debug("Loading multiple echem files with navani: %s", locations) + # Catch the navani warning when stitching multiple files together and calculating new capacity + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=( + "Capacity columns are not equal, replacing with new capacity column calculated from current and time columns and renaming the old capacity column to Old Capacity" + ), + category=UserWarning, + ) + raw_df = ec.multi_echem_file_loader(locations) + except Exception as exc: + raise RuntimeError( + f"Navani raised an error when parsing multiple files: {exc}" + ) from exc + + elif not isinstance(file_ids, list): + raise ValueError("Invalid file_ids type. Expected list of strings.") + elif len(file_ids) == 0: + raise ValueError("Invalid file_ids value. Expected non-empty list of strings.") + + if cycle_summary_df is None and raw_df is not None: + try: cycle_summary_df = ec.cycle_summary(raw_df) - cycle_summary_df.to_pickle(cycle_summary_file_loc) - except Exception as exc: - LOGGER.warning("Cycle summary generation failed with error: %s", exc) + except Exception as exc: + warnings.warn(f"Cycle summary generation failed with error: {exc}") - raw_df = raw_df.filter(required_keys) - raw_df.rename(columns=keys_with_units, inplace=True) + if raw_df is not None: + raw_df = raw_df.filter(required_keys) + raw_df.rename(columns=keys_with_units, inplace=True) + else: + raise ValueError("Invalid raw_df value. Expected non-empty DataFrame.") if cycle_summary_df is not None: cycle_summary_df.rename(columns=keys_with_units, inplace=True) @@ -152,10 +177,21 @@ def _load(self, file_id: str | ObjectId, reload: bool = True): def plot_cycle(self): """Plots the electrochemical cycling data from the file ID provided in the request.""" - if "file_id" not in self.data: - LOGGER.warning("No file_id given") - return - file_id = self.data["file_id"] + + # Legacy support for when file_id was used + if self.data.get("file_id") is not None and not self.data.get("file_ids"): + LOGGER.info("Legacy file upload detected, using file_id") + file_ids = [self.data["file_id"]] + + else: + if "file_ids" not in self.data: + LOGGER.warning("No file_ids given, skipping plot.") + return + if self.data["file_ids"] is None or len(self.data["file_ids"]) == 0: + LOGGER.warning("Empty file_ids list given, skipping plot.") + return + + file_ids = self.data["file_ids"] derivative_modes = (None, "dQ/dV", "dV/dQ", "final capacity") @@ -177,7 +213,7 @@ def plot_cycle(self): if not isinstance(cycle_list, list): cycle_list = None - raw_df, cycle_summary_df = self._load(file_id) + raw_df, cycle_summary_df = self._load(file_ids=file_ids) characteristic_mass_g = self._get_characteristic_mass_g() diff --git a/pydatalab/tests/server/test_blocks.py b/pydatalab/tests/server/test_blocks.py index 21ad6adc7..7a5e3ec74 100644 --- a/pydatalab/tests/server/test_blocks.py +++ b/pydatalab/tests/server/test_blocks.py @@ -261,6 +261,82 @@ def test_uvvis_block_lifecycle(admin_client, default_sample_dict, example_data_d assert web_block.get("errors") is None +def test_echem_block_lifecycle(admin_client, default_sample_dict, example_data_dir): + block_type = "cycle" + + sample_id = f"test_sample_with_files-{block_type}-lifecycle" + sample_data = default_sample_dict.copy() + sample_data["item_id"] = sample_id + + response = admin_client.post("/new-sample/", json=sample_data) + assert response.status_code == 201 + assert response.json["status"] == "success" + + response = admin_client.post( + "/add-data-block/", + json={ + "block_type": block_type, + "item_id": sample_id, + "index": 0, + }, + ) + + assert response.status_code == 200, f"Failed to add {block_type} block: {response.json}" + assert response.json["status"] == "success" + + block_data = response.json["new_block_obj"] + block_id = block_data["block_id"] + + # Upload multiple echem files + echem_folder = example_data_dir / "echem" + example_files = list(echem_folder.glob("*.mpr"))[:2] + example_file_ids = [] + + for example_file in example_files: + with open(example_file, "rb") as f: + response = admin_client.post( + "/upload-file/", + buffered=True, + content_type="multipart/form-data", + data={ + "item_id": sample_id, + "file": [(f, example_file.name)], + "type": "application/octet-stream", + "replace_file": "null", + "relativePath": "null", + }, + ) + assert response.status_code == 201, f"Failed to upload {example_file.name}" + assert response.json["status"] == "success" + file_ids = response.json["file_id"] + example_file_ids.append(file_ids) + + assert len(example_file_ids) == 2 + + # Update block with multiple file_ids + response = admin_client.get(f"/get-item-data/{sample_id}") + assert response.status_code == 200 + item_data = response.json["item_data"] + block_data = item_data["blocks_obj"][block_id] + block_data["file_ids"] = example_file_ids + + response = admin_client.post("/update-block/", json={"block_data": block_data}) + assert response.status_code == 200 + web_block = response.json["new_block_data"] + + assert "bokeh_plot_data" in web_block + assert web_block.get("errors") is None + + # Test for only one file_id + block_data["file_ids"] = [example_file_ids[0]] + + response = admin_client.post("/update-block/", json={"block_data": block_data}) + assert response.status_code == 200 + web_block = response.json["new_block_data"] + assert "bokeh_plot_data" in web_block + assert web_block.get("errors") is None + + def test_xrd_block_lifecycle(admin_client, default_sample_dict, example_data_dir): from pydatalab.apps.xrd import XRDBlock diff --git a/webapp/src/components/FileMultiSelectDropdown.vue b/webapp/src/components/FileMultiSelectDropdown.vue index 148aaaad0..01e474b26 100644 --- a/webapp/src/components/FileMultiSelectDropdown.vue +++ b/webapp/src/components/FileMultiSelectDropdown.vue @@ -42,7 +42,7 @@
+ +
@@ -172,7 +188,14 @@ export default { availableFiles() { // Filter out files that are already selected const selectedSet = new Set(this.modelValue); - return this.all_available_file_ids.filter((id) => !selectedSet.has(id)); + // Get filtered IDs + const filteredIds = this.all_available_file_ids.filter((id) => !selectedSet.has(id)); + // Sort alphabetically by file name + return filteredIds.sort((a, b) => { + const nameA = this.getFileName(a).toLowerCase(); + const nameB = this.getFileName(b).toLowerCase(); + return nameA.localeCompare(nameB); + }); }, canMoveUp() { return this.selectedSelected !== null && this.selectedSelectedIndex > 0; @@ -249,6 +272,21 @@ export default { this.selectedSelected = null; this.selectedSelectedIndex = -1; }, + addAllAvailable() { + if (this.availableFiles.length === 0) return; + + const newSelectedFiles = [...this.modelValue, ...this.availableFiles]; + this.emitUpdate(newSelectedFiles); + this.selectedAvailable = null; + }, + removeAllSelected() { + if (this.modelValue.length === 0) return; + + this.emitUpdate([]); + this.selectedSelected = null; + this.selectedSelectedIndex = -1; + this.selectedAvailable = null; // Reset available selection + }, moveUp() { if (!this.canMoveUp) return; @@ -328,10 +366,6 @@ export default {