Support for Tracks (#373)

* Support for Tracks

* wip

* Removed track linking

* Remove unneeded key

* Make arg more explicit

* make consistent with be

* Change update endpoint

* Setting aside tests to marinate

* Version bump

* Added repr

* Fix tests

* Remove track_reference_id from segmentation

Author: Anthony Krivonos

CHANGELOG.md

@@ -5,11 +5,17 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.14.29](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.14.29) - 2022-11-22
+
+### Added
+- Support for `Track`s, enabling ground truth annotations and model predictions to be grouped across dataset items and scenes
+- Helpers to update track metadata, as well as to create and delete tracks at the dataset level
+
 
 ## [0.14.28](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.14.28) - 2022-11-17
 
 ### Added
-- Support for appending to slice with Scene reference IDs
+- Support for appending to slice with scene reference IDs
 - Better error handling when appending to a slice with non-existent reference IDs
 
 

nucleus/annotation.py

@@ -31,6 +31,7 @@
     POSITION_KEY,
     REFERENCE_ID_KEY,
     TAXONOMY_NAME_KEY,
+    TRACK_REFERENCE_ID_KEY,
     TYPE_KEY,
     VERTICES_KEY,
     VISIBLE_KEY,
@@ -109,6 +110,7 @@ class BoxAnnotation(Annotation):  # pylint: disable=R0902
             annotation_id="image_1_car_box_1",
             metadata={"vehicle_color": "red"},
             embedding_vector=[0.1423, 1.432, ..., 3.829],
+            track_reference_id="car_a",
         )
 
     Parameters:
@@ -135,10 +137,12 @@ class BoxAnnotation(Annotation):  # pylint: disable=R0902
 
             Coordinate metadata may be provided to enable the Map Chart in the Nucleus Dataset charts page.
             These values can be specified as `{ "lat": 52.5, "lon": 13.3, ... }`.
-
         embedding_vector: Custom embedding vector for this object annotation.
             If any custom object embeddings have been uploaded previously to this dataset,
             this vector must match the dimensions of the previously ingested vectors.
+        track_reference_id: A unique string to identify the annotation as part of a group.
+            For instance, multiple "car" annotations across several dataset items may have
+            the same `track_reference_id` such as "red_car".
     """
 
     label: str
@@ -150,6 +154,7 @@ class BoxAnnotation(Annotation):  # pylint: disable=R0902
     annotation_id: Optional[str] = None
     metadata: Optional[Dict] = None
     embedding_vector: Optional[list] = None
+    track_reference_id: Optional[str] = None
 
     def __post_init__(self):
         self.metadata = self.metadata if self.metadata else {}
@@ -169,6 +174,7 @@ def from_json(cls, payload: dict):
             annotation_id=payload.get(ANNOTATION_ID_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
             embedding_vector=payload.get(EMBEDDING_VECTOR_KEY, None),
+            track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -185,6 +191,7 @@ def to_payload(self) -> dict:
             ANNOTATION_ID_KEY: self.annotation_id,
             METADATA_KEY: self.metadata,
             EMBEDDING_VECTOR_KEY: self.embedding_vector,
+            TRACK_REFERENCE_ID_KEY: self.track_reference_id,
         }
 
     def __eq__(self, other):
@@ -198,6 +205,7 @@ def __eq__(self, other):
             and self.annotation_id == other.annotation_id
             and sorted(self.metadata.items()) == sorted(other.metadata.items())
             and self.embedding_vector == other.embedding_vector
+            and self.track_reference_id == other.track_reference_id
         )
 
 
@@ -237,6 +245,7 @@ class LineAnnotation(Annotation):
             reference_id="person_image_1",
             annotation_id="person_image_1_line_1",
             metadata={"camera_mode": "portrait"},
+            track_reference_id="face_human",
         )
 
     Parameters:
@@ -252,13 +261,17 @@ class LineAnnotation(Annotation):
             attach to this annotation.  Strings, floats and ints are supported best
             by querying and insights features within Nucleus. For more details see
             our `metadata guide <https://nucleus.scale.com/docs/upload-metadata>`_.
+        track_reference_id: A unique string to identify the annotation as part of a group.
+            For instance, multiple "car" annotations across several dataset items may have
+            the same `track_reference_id` such as "red_car".
     """
 
     label: str
     vertices: List[Point]
     reference_id: str
     annotation_id: Optional[str] = None
     metadata: Optional[Dict] = None
+    track_reference_id: Optional[str] = None
 
     def __post_init__(self):
         self.metadata = self.metadata if self.metadata else {}
@@ -287,6 +300,7 @@ def from_json(cls, payload: dict):
             reference_id=payload[REFERENCE_ID_KEY],
             annotation_id=payload.get(ANNOTATION_ID_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
+            track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -299,6 +313,7 @@ def to_payload(self) -> dict:
             REFERENCE_ID_KEY: self.reference_id,
             ANNOTATION_ID_KEY: self.annotation_id,
             METADATA_KEY: self.metadata,
+            TRACK_REFERENCE_ID_KEY: self.track_reference_id,
         }
         return payload
 
@@ -318,6 +333,7 @@ class PolygonAnnotation(Annotation):
             annotation_id="image_2_bus_polygon_1",
             metadata={"vehicle_color": "yellow"},
             embedding_vector=[0.1423, 1.432, ..., 3.829],
+            track_reference_id="school_bus",
         )
 
     Parameters:
@@ -336,6 +352,9 @@ class PolygonAnnotation(Annotation):
         embedding_vector: Custom embedding vector for this object annotation.
             If any custom object embeddings have been uploaded previously to this dataset,
             this vector must match the dimensions of the previously ingested vectors.
+        track_reference_id: A unique string to identify the annotation as part of a group.
+            For instance, multiple "car" annotations across several dataset items may have
+            the same `track_reference_id` such as "red_car".
     """
 
     label: str
@@ -344,6 +363,7 @@ class PolygonAnnotation(Annotation):
     annotation_id: Optional[str] = None
     metadata: Optional[Dict] = None
     embedding_vector: Optional[list] = None
+    track_reference_id: Optional[str] = None
 
     def __post_init__(self):
         self.metadata = self.metadata if self.metadata else {}
@@ -373,6 +393,7 @@ def from_json(cls, payload: dict):
             annotation_id=payload.get(ANNOTATION_ID_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
             embedding_vector=payload.get(EMBEDDING_VECTOR_KEY, None),
+            track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -386,6 +407,7 @@ def to_payload(self) -> dict:
             ANNOTATION_ID_KEY: self.annotation_id,
             METADATA_KEY: self.metadata,
             EMBEDDING_VECTOR_KEY: self.embedding_vector,
+            TRACK_REFERENCE_ID_KEY: self.track_reference_id,
         }
         return payload
 
@@ -450,6 +472,7 @@ class KeypointsAnnotation(Annotation):
             reference_id="image_2",
             annotation_id="image_2_face_keypoints_1",
             metadata={"face_direction": "forward"},
+            track_reference_id="face_1",
         )
 
     Parameters:
@@ -468,6 +491,9 @@ class KeypointsAnnotation(Annotation):
             attach to this annotation.  Strings, floats and ints are supported best
             by querying and insights features within Nucleus. For more details see
             our `metadata guide <https://nucleus.scale.com/docs/upload-metadata>`_.
+        track_reference_id: A unique string to identify the annotation as part of a group.
+            For instance, multiple "car" annotations across several dataset items may have
+            the same `track_reference_id` such as "red_car".
     """
 
     label: str
@@ -477,6 +503,7 @@ class KeypointsAnnotation(Annotation):
     reference_id: str
     annotation_id: Optional[str] = None
     metadata: Optional[Dict] = None
+    track_reference_id: Optional[str] = None
 
     def __post_init__(self):
         self.metadata = self.metadata or {}
@@ -518,6 +545,7 @@ def from_json(cls, payload: dict):
             reference_id=payload[REFERENCE_ID_KEY],
             annotation_id=payload.get(ANNOTATION_ID_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
+            track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -532,6 +560,7 @@ def to_payload(self) -> dict:
             REFERENCE_ID_KEY: self.reference_id,
             ANNOTATION_ID_KEY: self.annotation_id,
             METADATA_KEY: self.metadata,
+            TRACK_REFERENCE_ID_KEY: self.track_reference_id,
         }
         return payload
 
@@ -573,7 +602,8 @@ class CuboidAnnotation(Annotation):  # pylint: disable=R0902
             yaw=0,
             reference_id="pointcloud_1",
             annotation_id="pointcloud_1_car_cuboid_1",
-            metadata={"vehicle_color": "green"}
+            metadata={"vehicle_color": "green"},
+            track_reference_id="red_car",
         )
 
     Parameters:
@@ -590,6 +620,9 @@ class CuboidAnnotation(Annotation):  # pylint: disable=R0902
           annotation.  Strings, floats and ints are supported best by querying
           and insights features within Nucleus. For more details see our `metadata
           guide <https://nucleus.scale.com/docs/upload-metadata>`_.
+        track_reference_id: A unique string to identify the annotation as part of a group.
+            For instance, multiple "car" annotations across several dataset items may have
+            the same `track_reference_id` such as "red_car".
     """
 
     label: str
@@ -599,6 +632,7 @@ class CuboidAnnotation(Annotation):  # pylint: disable=R0902
     reference_id: str
     annotation_id: Optional[str] = None
     metadata: Optional[Dict] = None
+    track_reference_id: Optional[str] = None
 
     def __post_init__(self):
         self.metadata = self.metadata if self.metadata else {}
@@ -614,6 +648,7 @@ def from_json(cls, payload: dict):
             reference_id=payload[REFERENCE_ID_KEY],
             annotation_id=payload.get(ANNOTATION_ID_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
+            track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -631,6 +666,8 @@ def to_payload(self) -> dict:
             payload[ANNOTATION_ID_KEY] = self.annotation_id
         if self.metadata:
             payload[METADATA_KEY] = self.metadata
+        if self.track_reference_id:
+            payload[TRACK_REFERENCE_ID_KEY] = self.track_reference_id
 
         return payload
 
@@ -772,11 +809,10 @@ def to_payload(self) -> dict:
             MASK_URL_KEY: self.mask_url,
             ANNOTATIONS_KEY: [ann.to_payload() for ann in self.annotations],
             ANNOTATION_ID_KEY: self.annotation_id,
+            REFERENCE_ID_KEY: self.reference_id,
             # METADATA_KEY: self.metadata,  # TODO(sc: 422637)
         }
 
-        payload[REFERENCE_ID_KEY] = self.reference_id
-
         return payload
 
     def has_local_files_to_upload(self) -> bool:
@@ -822,7 +858,8 @@ class CategoryAnnotation(Annotation):
             label="dress",
             reference_id="image_1",
             taxonomy_name="clothing_type",
-            metadata={"dress_color": "navy"}
+            metadata={"dress_color": "navy"},
+            track_reference_id="blue_and_black_dress",
         )
 
     Parameters:
@@ -834,12 +871,16 @@ class CategoryAnnotation(Annotation):
           Strings, floats and ints are supported best by querying and insights
           features within Nucleus. For more details see our `metadata guide
           <https://nucleus.scale.com/docs/upload-metadata>`_.
+        track_reference_id: A unique string to identify the annotation as part of a group.
+            For instance, multiple "car" annotations across several dataset items may have
+            the same `track_reference_id` such as "red_car".
     """
 
     label: str
     reference_id: str
     taxonomy_name: Optional[str] = None
     metadata: Optional[Dict] = None
+    track_reference_id: Optional[str] = None
 
     def __post_init__(self):
         self.metadata = self.metadata if self.metadata else {}
@@ -851,6 +892,7 @@ def from_json(cls, payload: dict):
             reference_id=payload[REFERENCE_ID_KEY],
             taxonomy_name=payload.get(TAXONOMY_NAME_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
+            track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -860,6 +902,7 @@ def to_payload(self) -> dict:
             GEOMETRY_KEY: {},
             REFERENCE_ID_KEY: self.reference_id,
             METADATA_KEY: self.metadata,
+            TRACK_REFERENCE_ID_KEY: self.track_reference_id,
         }
         if self.taxonomy_name is not None:
             payload[TAXONOMY_NAME_KEY] = self.taxonomy_name
@@ -874,6 +917,7 @@ class MultiCategoryAnnotation(Annotation):
     reference_id: str
     taxonomy_name: Optional[str] = None
     metadata: Optional[Dict] = None
+    track_reference_id: Optional[str] = None
 
     def __post_init__(self):
         self.metadata = self.metadata if self.metadata else {}
@@ -885,6 +929,7 @@ def from_json(cls, payload: dict):
             reference_id=payload[REFERENCE_ID_KEY],
             taxonomy_name=payload.get(TAXONOMY_NAME_KEY, None),
             metadata=payload.get(METADATA_KEY, {}),
+            track_reference_id=payload.get(TRACK_REFERENCE_ID_KEY, None),
         )
 
     def to_payload(self) -> dict:
@@ -894,6 +939,7 @@ def to_payload(self) -> dict:
             GEOMETRY_KEY: {},
             REFERENCE_ID_KEY: self.reference_id,
             METADATA_KEY: self.metadata,
+            TRACK_REFERENCE_ID_KEY: self.track_reference_id,
         }
         if self.taxonomy_name is not None:
             payload[TAXONOMY_NAME_KEY] = self.taxonomy_name

nucleus/constants.py

@@ -102,6 +102,7 @@
 PAGE_SIZE_KEY = "pageSize"
 PAGE_TOKEN_KEY = "pageToken"
 NEXT_TOKEN_KEY = "nextPageToken"
+OVERWRITE_KEY = "overwrite"
 P1_KEY = "p1"
 P2_KEY = "p2"
 POINTCLOUD_KEY = "pointcloud"
@@ -125,6 +126,9 @@
 STATUS_KEY = "status"
 SUCCESS_STATUS_CODES = [200, 201, 202]
 TAXONOMY_NAME_KEY = "taxonomy_name"
+TRACK_REFERENCE_ID_KEY = "track_reference_id"
+TRACK_REFERENCE_IDS_KEY = "track_reference_ids"
+TRACKS_KEY = "tracks"
 TYPE_KEY = "type"
 UPDATED_ITEMS = "updated_items"
 UPDATE_KEY = "update"