From a94ef1a277ec8689b6dfb24fe86cbc9998347c6a Mon Sep 17 00:00:00 2001 From: nik Date: Mon, 17 Jun 2024 20:06:38 +0100 Subject: [PATCH 1/8] fix: RND-101: Fix ML backend compatibility with sdk>=1 --- label_studio_ml/api.py | 2 +- label_studio_ml/response.py | 25 ++++++++++++------------- requirements.txt | 3 +-- tests/test_response.py | 37 +++++++++++++++++++++++++++++++------ 4 files changed, 45 insertions(+), 22 deletions(-) diff --git a/label_studio_ml/api.py b/label_studio_ml/api.py index 9f718c11..767e54ce 100644 --- a/label_studio_ml/api.py +++ b/label_studio_ml/api.py @@ -77,7 +77,7 @@ def _predict(): else: response.update_predictions_version() - response = response.serialize() + response = response.model_dump() res = response if res is None: diff --git a/label_studio_ml/response.py b/label_studio_ml/response.py index 546be526..1fa57686 100644 --- a/label_studio_ml/response.py +++ b/label_studio_ml/response.py @@ -1,15 +1,19 @@ from typing import Type, Dict, Optional, List, Tuple, Any, Union -from pydantic import BaseModel, confloat +from pydantic import BaseModel, confloat, Field +from label_studio_sdk.label_interface.objects import PredictionValue +from typing import Union, List -from label_studio_sdk.objects import PredictionValue + +# one or multiple predictions per task +SingleTaskPredictions = Union[List[PredictionValue], PredictionValue] class ModelResponse(BaseModel): """ """ model_version: Optional[str] = None - predictions: List[PredictionValue] + predictions: List[SingleTaskPredictions] def has_model_version(self) -> bool: return bool(self.model_version) @@ -18,8 +22,11 @@ def update_predictions_version(self) -> None: """ """ for prediction in self.predictions: - if not prediction.model_version: - prediction.model_version = self.model_version + if isinstance(prediction, PredictionValue): + prediction = [prediction] + for p in prediction: + if not p.model_version: + p.model_version = self.model_version def set_version(self, version: str) -> None: """ @@ -27,12 +34,4 @@ def set_version(self, version: str) -> None: self.model_version = version # Set the version for each prediction self.update_predictions_version() - - def serialize(self): - """ - """ - return { - "model_version": self.model_version, - "predictions": [ p.serialize() for p in self.predictions ] - } diff --git a/requirements.txt b/requirements.txt index b84360c5..5e826bad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,4 @@ colorama~=0.4 requests~=2.31 semver~=3.0.2 pillow~=10.3 -label_studio_tools @ git+https://github.com/HumanSignal/label-studio-tools.git@master -label-studio-sdk==0.0.34 +label-studio-sdk~=1.0.1 diff --git a/tests/test_response.py b/tests/test_response.py index 5f9429cd..373c0a6a 100644 --- a/tests/test_response.py +++ b/tests/test_response.py @@ -4,15 +4,21 @@ from typing import Optional from label_studio_ml.response import ModelResponse -from label_studio_sdk.objects import PredictionValue +from label_studio_sdk.label_interface.objects import PredictionValue -MODEL_VERSION = 0.1 +MODEL_VERSION = "0.1" @pytest.fixture def predictions(): # Replace 'Example Prediction' with real Predictions - return [PredictionValue(model_version=MODEL_VERSION, score=0.1, result=[{}]), - PredictionValue(score=0.1, result=[{}])] + return [ + PredictionValue(model_version=MODEL_VERSION, score=0.1, result=[{}]), + PredictionValue(score=0.1, result=[{}]), + [ + PredictionValue(model_version='prediction_1', score=0.2, result=[{}]), + PredictionValue(score=0.3, result=[{}]), + ] + ] def test_has_model_version(predictions): res = ModelResponse(predictions=predictions) @@ -45,8 +51,27 @@ def test_set_version(predictions): def test_serialize(predictions): # Assuming PredictionValue has method .serialize() which returns a dict res = ModelResponse(model_version="1.0", predictions=predictions) - serialized_res = res.serialize() + serialized_res = res.model_dump() assert serialized_res['model_version'] == "1.0" assert serialized_res['predictions'] == [ - p.serialize() for p in predictions] + {'model_version': MODEL_VERSION, 'score': 0.1, 'result': [{}]}, + {'model_version': None, 'score': 0.1, 'result': [{}]}, + [ + {'model_version': 'prediction_1', 'score': 0.2, 'result': [{}]}, + {'model_version': None, 'score': 0.3, 'result': [{}]}, + ] + ] + + # update model_version + res.update_predictions_version() + serialized_res = res.model_dump() + + assert serialized_res['predictions'] == [ + {'model_version': "0.1", 'score': 0.1, 'result': [{}]}, + {'model_version': "1.0", 'score': 0.1, 'result': [{}]}, + [ + {'model_version': 'prediction_1', 'score': 0.2, 'result': [{}]}, + {'model_version': '1.0', 'score': 0.3, 'result': [{}]}, + ] + ] From 6d43d6cdd04f628cbec173f03f73adcbc95ced6f Mon Sep 17 00:00:00 2001 From: nik Date: Mon, 17 Jun 2024 20:21:44 +0100 Subject: [PATCH 2/8] Fix sdk version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5e826bad..ca13d34b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ colorama~=0.4 requests~=2.31 semver~=3.0.2 pillow~=10.3 -label-studio-sdk~=1.0.1 +label-studio-sdk==1.0.1 From 945e0b6a1830cb86a752630ee316fb37a82a9308 Mon Sep 17 00:00:00 2001 From: nik Date: Tue, 18 Jun 2024 08:41:05 +0100 Subject: [PATCH 3/8] Install sdk from git --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ca13d34b..9f1fd885 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ colorama~=0.4 requests~=2.31 semver~=3.0.2 pillow~=10.3 -label-studio-sdk==1.0.1 +# label-studio-sdk==1.0.1 +label-studio-sdk @ git+https://github.com/HumanSignal/label-studio-sdk.git From 980cb93372230a91f2ddc93b73a990c186e2f2c7 Mon Sep 17 00:00:00 2001 From: nik Date: Tue, 18 Jun 2024 09:16:55 +0100 Subject: [PATCH 4/8] Fix label-studio-tools imports --- label_studio_ml/model.py | 4 ++-- label_studio_ml/utils.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/label_studio_ml/model.py b/label_studio_ml/model.py index 3aa4c78f..7f287949 100644 --- a/label_studio_ml/model.py +++ b/label_studio_ml/model.py @@ -24,8 +24,8 @@ from colorama import Fore from label_studio_sdk.label_interface import LabelInterface -from label_studio_tools.core.label_config import parse_config -from label_studio_tools.core.utils.io import get_local_path +from label_studio_sdk._extensions.label_studio_tools.core.label_config import parse_config +from label_studio_sdk._extensions.label_studio_tools.core.utils.io import get_local_path from .response import ModelResponse from .utils import is_preload_needed from .cache import create_cache diff --git a/label_studio_ml/utils.py b/label_studio_ml/utils.py index d36199ee..ee129ddc 100644 --- a/label_studio_ml/utils.py +++ b/label_studio_ml/utils.py @@ -8,9 +8,8 @@ from typing import List from urllib.parse import urlparse -from label_studio_tools.core.utils.params import get_env -from label_studio_tools.core.utils.io import get_local_path -from label_studio_sdk.label_interface import LabelInterface +from label_studio_sdk._extensions.label_studio_tools.core.utils.params import get_env +from label_studio_sdk._extensions.label_studio_tools.core.utils.io import get_local_path DATA_UNDEFINED_NAME = '$undefined$' From ce4187f43d811004eadd2f0797cb73f203626fb0 Mon Sep 17 00:00:00 2001 From: nik Date: Tue, 18 Jun 2024 11:54:11 +0100 Subject: [PATCH 5/8] Install numpy in requirements --- label_studio_ml/examples/mmdetection-3/requirements.txt | 1 + label_studio_ml/examples/spacy/requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/label_studio_ml/examples/mmdetection-3/requirements.txt b/label_studio_ml/examples/mmdetection-3/requirements.txt index 494be5f0..26c3fa60 100644 --- a/label_studio_ml/examples/mmdetection-3/requirements.txt +++ b/label_studio_ml/examples/mmdetection-3/requirements.txt @@ -1,2 +1,3 @@ boto3>=1.26.103,<2.0.0 openmim~=0.3.9 +numpy~=1.26 diff --git a/label_studio_ml/examples/spacy/requirements.txt b/label_studio_ml/examples/spacy/requirements.txt index f9e91039..4bca7882 100644 --- a/label_studio_ml/examples/spacy/requirements.txt +++ b/label_studio_ml/examples/spacy/requirements.txt @@ -1,3 +1,4 @@ gunicorn==22.0.0 spacy~=3.6 label-studio-ml @ git+https://github.com/HumanSignal/label-studio-ml-backend.git@master +numpy~=1.26 From b237bbae4e5fd2d6d2b3fb54e7152f09f71a4359 Mon Sep 17 00:00:00 2001 From: nik Date: Tue, 18 Jun 2024 11:55:47 +0100 Subject: [PATCH 6/8] Cast float --- label_studio_ml/examples/huggingface_ner/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/label_studio_ml/examples/huggingface_ner/model.py b/label_studio_ml/examples/huggingface_ner/model.py index ac0e566d..5e89f8a8 100644 --- a/label_studio_ml/examples/huggingface_ner/model.py +++ b/label_studio_ml/examples/huggingface_ner/model.py @@ -83,7 +83,7 @@ def predict(self, tasks: List[Dict], context: Optional[Dict] = None, **kwargs) - entities = list(group) start = entities[0]['start'] end = entities[-1]['end'] - score = sum([entity['score'] for entity in entities]) / len(entities) + score = float(sum([entity['score'] for entity in entities]) / len(entities)) results.append({ 'from_name': from_name, 'to_name': to_name, From de6eb880e63dfa613b1f11b78d3aaeca56f66387 Mon Sep 17 00:00:00 2001 From: nik Date: Tue, 18 Jun 2024 16:01:09 +0100 Subject: [PATCH 7/8] Add numpy reqs to easyocr and nemo --- label_studio_ml/examples/easyocr/requirements.txt | 3 ++- label_studio_ml/examples/nemo_asr/requirements.txt | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/label_studio_ml/examples/easyocr/requirements.txt b/label_studio_ml/examples/easyocr/requirements.txt index 96379ea4..082197f3 100644 --- a/label_studio_ml/examples/easyocr/requirements.txt +++ b/label_studio_ml/examples/easyocr/requirements.txt @@ -1,3 +1,4 @@ easyocr==1.7.1 boto3==1.28.58 -opencv-python-headless==4.9.0.80 \ No newline at end of file +opencv-python-headless==4.9.0.80 +numpy<2 \ No newline at end of file diff --git a/label_studio_ml/examples/nemo_asr/requirements.txt b/label_studio_ml/examples/nemo_asr/requirements.txt index b0d68f3a..b4ed7e12 100644 --- a/label_studio_ml/examples/nemo_asr/requirements.txt +++ b/label_studio_ml/examples/nemo_asr/requirements.txt @@ -1 +1,2 @@ nemo-toolkit[all] +numpy<2 \ No newline at end of file From e7cfd7596fb741aa6d3fbf28de82eb9dbde8460d Mon Sep 17 00:00:00 2001 From: nik Date: Wed, 19 Jun 2024 13:34:22 +0100 Subject: [PATCH 8/8] Upgrade label-studio-sdk==1.0.2 --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9f1fd885..e85189c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,5 @@ colorama~=0.4 requests~=2.31 semver~=3.0.2 pillow~=10.3 -# label-studio-sdk==1.0.1 -label-studio-sdk @ git+https://github.com/HumanSignal/label-studio-sdk.git +label-studio-sdk==1.0.2 +# label-studio-sdk @ git+https://github.com/HumanSignal/label-studio-sdk.git