Strip potentially heavy ignored keys out of item save from app and API

- Add metadata field for blocks

Author: ml-evs

pydatalab/schemas/cell.json

@@ -242,6 +242,10 @@
         "processed_data": {
           "title": "Processed Data",
           "type": "object"
+        },
+        "metadata": {
+          "title": "Metadata",
+          "type": "object"
         }
       },
       "required": [

pydatalab/schemas/equipment.json

@@ -206,6 +206,10 @@
         "processed_data": {
           "title": "Processed Data",
           "type": "object"
+        },
+        "metadata": {
+          "title": "Metadata",
+          "type": "object"
         }
       },
       "required": [

pydatalab/schemas/sample.json

@@ -295,6 +295,10 @@
         "processed_data": {
           "title": "Processed Data",
           "type": "object"
+        },
+        "metadata": {
+          "title": "Metadata",
+          "type": "object"
         }
       },
       "required": [

pydatalab/schemas/startingmaterial.json

@@ -348,6 +348,10 @@
         "processed_data": {
           "title": "Processed Data",
           "type": "object"
+        },
+        "metadata": {
+          "title": "Metadata",
+          "type": "object"
         }
       },
       "required": [

pydatalab/src/pydatalab/blocks/base.py

@@ -315,6 +315,8 @@ def from_web(cls, data: dict):
 
     def update_from_web(self, data: dict):
         """Update the block with validated data received from a web request.
+        Will strip any fields that are "computed" or otherwise not controllable
+        by the user.
 
         Parameters:
             data: A dictionary of data to update the block with.
@@ -326,7 +328,8 @@ def update_from_web(self, data: dict):
         )
         self.data.update(
             self.block_db_model(**data).dict(
-                exclude={"bokeh_plot_data", "b64_encoded_image"}, exclude_unset=True
+                exclude={"processed_data", "metadata", "bokeh_plot_data", "b64_encoded_image"},
+                exclude_unset=True,
             )
         )
 

pydatalab/src/pydatalab/models/blocks.py

@@ -47,6 +47,9 @@ class DataBlockResponse(BaseModel):
     processed_data: dict | None = None
     """Any processed data associated with the block, small enough to store."""
 
+    metadata: dict | None = None
+    """Any metadata associated with the block."""
+
     class Config:
         allow_population_by_field_name = True
         json_encoders = JSON_ENCODERS

pydatalab/tests/server/test_blocks.py

@@ -250,6 +250,7 @@ def test_xrd_block_lifecycle(admin_client, default_sample_dict):
     item_data = response.json["item_data"]
     block_data = item_data["blocks_obj"][block_id]
     block_data["file_id"] = file_id
+    block_data["wavelength"] = 2.0
 
     response = admin_client.post(
         "/update-block/", json={"block_data": block_data, "save_to_db": True}
@@ -258,16 +259,30 @@ def test_xrd_block_lifecycle(admin_client, default_sample_dict):
     web_block = response.json["new_block_data"]
     assert "bokeh_plot_data" in web_block
     assert "processed_data" in web_block
+    assert web_block["wavelength"] == 2.0
     assert "peak_data" in web_block["processed_data"]
     assert "file_id" in web_block
     assert web_block["file_id"] == file_id
     assert web_block.get("errors") is None
 
     block = XRDBlock.from_web(web_block)
     db = block.to_db()
+    # 'computed' keys should be dropped when loading from web
     assert "bokeh_plot_data" not in db
-    assert "processed_data" in db
-    assert "peak_data" in db["processed_data"]
+    assert "processed_data" not in db
+    assert db["wavelength"] == 2.0
+
+    # But they should still be in the database
+    response = admin_client.get(f"/get-item-data/{sample_id}")
+    assert response.status_code == 200
+
+    item_data = response.json["item_data"]
+    assert response.json["status"] == "success"
+    assert "blocks_obj" in item_data
+    block = item_data["blocks_obj"][block_id]
+    assert "processed_data" in block
+    assert "peak_data" in block["processed_data"]
+    assert block["wavelength"] == 2.0
 
 
 def test_comment_block_manipulation(admin_client, default_sample_dict, database):
@@ -313,6 +328,7 @@ def test_comment_block_manipulation(admin_client, default_sample_dict, database)
     # Check that this result was actually stored
     response = admin_client.get(f"/get-item-data/{sample_id}")
     assert response.status_code == 200
+    item_data = response.json["item_data"]
     assert response.json["status"] == "success"
     assert (
         response.json["item_data"]["blocks_obj"][block_id]["freeform_comment"]
@@ -323,20 +339,46 @@ def test_comment_block_manipulation(admin_client, default_sample_dict, database)
     # Try to add some bad data
     block_data["bokeh_plot_data"] = {"bokeh": "json"}
     block_data["random_new_key"] = "test new key"
+    block_data["freeform_comment"] = "This is a test comment block with extra data."
     response = admin_client.post("/update-block/", json={"block_data": block_data})
     assert response.status_code == 200
     assert response.json["status"] == "success"
     assert response.json["new_block_data"]["blocktype"] == block_type
-    assert response.json["new_block_data"]["freeform_comment"] == "This is a test comment block."
+    assert (
+        response.json["new_block_data"]["freeform_comment"]
+        == "This is a test comment block with extra data."
+    )
     assert response.json["new_block_data"]["title"] == "Test Comment Block"
     assert "bokeh_plot_data" not in response.json["new_block_data"]
-    assert "random_new_key" not in response.json["new_block_data"]
+
+    # Extra random keys will be in the response (in case they are parameters for the block that are not yet handled,
+    # but they will not be stored in the database)
+    assert "random_new_key" in response.json["new_block_data"]
 
     raw_item = database.items.find_one({"item_id": sample_id})
     assert raw_item
-    assert "bokeh_plot_data" not in raw_item
-    assert "random_new_key" not in raw_item
-    assert "errors" not in raw_item
+    raw_block = raw_item["blocks_obj"][block_id]
+    assert "bokeh_plot_data" not in raw_block
+    # assert "random_new_key" not in raw_block
+    assert "errors" not in raw_block
+
+    # Finally, try to update using the save-item endpoint, and make sure any bad data gets stripped out
+    item_data["blocks_obj"][block_id]["bokeh_plot_data"] = {"bokeh": "json"}
+    item_data["blocks_obj"][block_id]["random_new_key"] = "test new key again"
+    item_data["blocks_obj"][block_id]["freeform_comment"] = "This is the latest test comment."
+
+    admin_client.post("/save-item/", json={"item_id": sample_id, "data": item_data})
+    assert response.status_code == 200
+
+    response = admin_client.get(f"/get-item-data/{sample_id}")
+    assert response.status_code == 200
+    assert response.json["status"] == "success"
+    item_data = response.json["item_data"]
+    block = item_data["blocks_obj"][block_id]
+
+    assert block["freeform_comment"] == "This is the latest test comment."
+    assert block.get("bokeh_plot_data") is None
+    assert block["random_new_key"] == "test new key again"
 
 
 @pytest.mark.parametrize(

webapp/src/server_fetch_utils.js

@@ -576,6 +576,16 @@ export function saveItem(item_id) {
   var item_data = store.state.all_item_data[item_id];
   console.log("Item data before saving:", item_data);
 
+  // Strip large data from blocks before saving
+  if (item_data.blocks_obj) {
+    for (const block_id in item_data.blocks_obj) {
+      const block = item_data.blocks_obj[block_id];
+      block.bokeh_plot_data = null;
+      block.processed_data = null;
+      block.metadata = null;
+    }
+  }
+
   store.commit("setItemSaved", { item_id: item_id, isSaved: false });
   fetch_post(`${API_URL}/save-item/`, {
     item_id: item_id,