[serve.llm] Score API Integration for Serve LLM (#55914)

Signed-off-by: ahao-anyscale <[email protected]>
Co-authored-by: Nikhil G <[email protected]>
Co-authored-by: angelinalg <[email protected]>
Signed-off-by: Douglas Strodtman <[email protected]>

Author: 3 people

doc/source/serve/llm/index.md

@@ -36,6 +36,7 @@ This deployment provides an OpenAI-compatible FastAPI ingress and routes traffic
 - `/v1/chat/completions`: Chat interface (ChatGPT-style)
 - `/v1/completions`: Text completion
 - `/v1/embeddings`: Text embeddings
+- `/v1/score`: Text comparison
 - `/v1/models`: List available models
 - `/v1/models/{model}`: Model information
 

python/ray/llm/_internal/serve/configs/openai_api_models.py

@@ -22,6 +22,8 @@
     EmbeddingCompletionRequest as vLLMEmbeddingCompletionRequest,
     EmbeddingResponse as vLLMEmbeddingResponse,
     ErrorResponse as vLLMErrorResponse,
+    ScoreRequest as vLLMScoreRequest,
+    ScoreResponse as vLLMScoreResponse,
 )
 from vllm.utils import random_uuid
 
@@ -89,12 +91,24 @@ class EmbeddingResponse(vLLMEmbeddingResponse):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
 
+class ScoreRequest(vLLMScoreRequest):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
+class ScoreResponse(vLLMScoreResponse):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+
 EmbeddingRequest = Union[EmbeddingCompletionRequest, EmbeddingChatRequest]
 
 LLMEmbeddingsResponse = Union[
     AsyncGenerator[Union[EmbeddingResponse, ErrorResponse], None],
 ]
 
+LLMScoreResponse = Union[
+    AsyncGenerator[Union[ScoreResponse, ErrorResponse], None],
+]
+
 LLMChatResponse = Union[
     AsyncGenerator[Union[str, ChatCompletionResponse, ErrorResponse], None],
 ]

python/ray/llm/_internal/serve/deployments/llm/llm_server.py

@@ -52,6 +52,8 @@
         EmbeddingRequest,
         EmbeddingResponse,
         ErrorResponse,
+        ScoreRequest,
+        ScoreResponse,
     )
 
 logger = get_logger(__name__)
@@ -306,7 +308,10 @@ def _batch_output_stream(
     async def _run_request(
         self,
         request: Union[
-            "ChatCompletionRequest", "CompletionRequest", "EmbeddingRequest"
+            "ChatCompletionRequest",
+            "CompletionRequest",
+            "EmbeddingRequest",
+            "ScoreRequest",
         ],
         *,
         engine_method: str,
@@ -392,6 +397,24 @@ async def embeddings(
             request, engine_method="embeddings", batch_output_stream=False
         )
 
+    async def score(
+        self, request: "ScoreRequest"
+    ) -> AsyncGenerator[Union["ScoreResponse", "ErrorResponse"], None]:
+        """Runs a score request to the engine and returns the response.
+
+        Returns an AsyncGenerator over the ScoreResponse object. This is so that the caller can have a consistent interface across all the methods of chat, completions, embeddings, and score.
+
+        Args:
+            request: A ScoreRequest object.
+
+        Returns:
+            An AsyncGenerator over the ScoreResponse object.
+        """
+        # NOTE: Score does not need batching, similar to embeddings.
+        return await self._run_request(
+            request, engine_method="score", batch_output_stream=False
+        )
+
     async def check_health(self) -> None:
         """
         Check the health of the replica. Does not return anything. Raise error when

python/ray/llm/_internal/serve/deployments/llm/vllm/vllm_engine.py

@@ -19,6 +19,8 @@
     EmbeddingRequest,
     EmbeddingResponse,
     ErrorResponse,
+    ScoreRequest,
+    ScoreResponse,
 )
 from ray.llm._internal.serve.configs.server_models import (
     DiskMultiplexConfig,
@@ -43,6 +45,7 @@
     from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
     from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
     from vllm.entrypoints.openai.serving_models import OpenAIServingModels
+    from vllm.entrypoints.openai.serving_score import ServingScores
 
 vllm = try_import("vllm")
 logger = get_logger(__name__)
@@ -134,6 +137,7 @@ def __init__(
         self._oai_serving_chat: Optional["OpenAIServingChat"] = None
         self._oai_serving_completion: Optional["OpenAIServingCompletion"] = None
         self._oai_serving_embedding: Optional["OpenAIServingEmbedding"] = None
+        self._oai_serving_scores: Optional["ServingScores"] = None
 
     async def start(self) -> None:
         """Start the vLLM engine.
@@ -189,6 +193,7 @@ async def start(self) -> None:
         self._oai_serving_chat = state.openai_serving_chat
         self._oai_serving_completion = state.openai_serving_completion
         self._oai_serving_embedding = state.openai_serving_embedding
+        self._oai_serving_scores = state.openai_serving_scores
 
         self._validate_openai_serving_models()
         self._validate_engine_client()
@@ -221,6 +226,11 @@ def _validate_openai_serving_embedding(self):
             self._oai_serving_embedding, "create_embedding"
         ), "oai_serving_embedding must have a create_embedding attribute"
 
+    def _validate_openai_serving_scores(self):
+        assert hasattr(
+            self._oai_serving_scores, "create_score"
+        ), "oai_serving_scores must have a create_score attribute"
+
     def _validate_engine_client(self):
         assert hasattr(
             self._engine_client, "check_health"
@@ -354,7 +364,9 @@ async def resolve_lora(self, disk_lora_model: DiskMultiplexConfig):
 
     def _create_raw_request(
         self,
-        request: Union[CompletionRequest, ChatCompletionRequest, EmbeddingRequest],
+        request: Union[
+            CompletionRequest, ChatCompletionRequest, EmbeddingRequest, ScoreRequest
+        ],
         path: str,
     ) -> Request:
         scope = {
@@ -442,6 +454,22 @@ async def embeddings(
         else:
             yield EmbeddingResponse(**embedding_response.model_dump())
 
+    async def score(
+        self, request: ScoreRequest
+    ) -> AsyncGenerator[Union[ScoreResponse, ErrorResponse], None]:
+        self._validate_openai_serving_scores()
+
+        raw_request = self._create_raw_request(request, "/score")
+
+        score_response = await self._oai_serving_scores.create_score(
+            request, raw_request=raw_request
+        )
+
+        if isinstance(score_response, VLLMErrorResponse):
+            yield ErrorResponse(**score_response.model_dump())
+        else:
+            yield ScoreResponse(**score_response.model_dump())
+
     async def check_health(self) -> None:
         assert self._engine_client is not None, "engine_client is not initialized"
 

python/ray/llm/_internal/serve/deployments/routers/router.py

@@ -46,9 +46,12 @@
     LLMChatResponse,
     LLMCompletionsResponse,
     LLMEmbeddingsResponse,
+    LLMScoreResponse,
     ModelCard,
     ModelList,
     OpenAIHTTPException,
+    ScoreRequest,
+    ScoreResponse,
     to_model_metadata,
 )
 from ray.llm._internal.serve.configs.server_models import LLMConfig
@@ -310,10 +313,18 @@ def _get_configured_serve_handle(self, model_id: str):
     async def _get_response(
         self,
         *,
-        body: Union[CompletionRequest, ChatCompletionRequest, EmbeddingRequest],
+        body: Union[
+            CompletionRequest, ChatCompletionRequest, EmbeddingRequest, ScoreRequest
+        ],
         call_method: str,
     ) -> AsyncGenerator[
-        Union[LLMChatResponse, LLMCompletionsResponse, LLMEmbeddingsResponse], None
+        Union[
+            LLMChatResponse,
+            LLMCompletionsResponse,
+            LLMEmbeddingsResponse,
+            LLMScoreResponse,
+        ],
+        None,
     ]:
         """Calls the model deployment and returns the stream."""
         model: str = body.model
@@ -478,6 +489,32 @@ async def embeddings(self, body: EmbeddingRequest) -> Response:
             if isinstance(result, EmbeddingResponse):
                 return JSONResponse(content=result.model_dump())
 
+    @fastapi_router_app.post("/v1/score")
+    async def score(self, body: ScoreRequest) -> Response:
+        """Create scores for the provided text pairs.
+
+        Note: This is a vLLM specific endpoint.
+
+        Args:
+            body: The score request containing input text pairs to score.
+
+        Returns:
+            A response object with scores.
+        """
+
+        async with timeout(DEFAULT_LLM_ROUTER_HTTP_TIMEOUT):
+            results = self._get_response(body=body, call_method="score")
+            result = await results.__anext__()
+            if isinstance(result, ErrorResponse):
+                raise OpenAIHTTPException(
+                    message=result.message,
+                    status_code=result.code,
+                    type=result.type,
+                )
+
+            if isinstance(result, ScoreResponse):
+                return JSONResponse(content=result.model_dump())
+
     @classmethod
     def as_deployment(
         cls, llm_configs: Optional[List[LLMConfig]] = None

python/ray/llm/tests/serve/conftest.py

@@ -15,6 +15,7 @@
     ChatCompletionRequest,
     CompletionRequest,
     EmbeddingCompletionRequest,
+    ScoreRequest,
 )
 from ray.llm._internal.serve.deployments.llm.vllm.vllm_models import (
     VLLMEngineConfig,
@@ -112,6 +113,16 @@ def mock_embedding_request(dimensions):
     return request
 
 
+@pytest.fixture
+def mock_score_request():
+    """Fixture for creating score requests for mock testing."""
+    return ScoreRequest(
+        model=MOCK_MODEL_ID,
+        text_1="What is the capital of France?",
+        text_2="The capital of France is Paris.",
+    )
+
+
 def get_test_model_path(yaml_file: str) -> pathlib.Path:
     current_file_dir = pathlib.Path(__file__).absolute().parent
     test_model_path = current_file_dir / yaml_file

python/ray/llm/tests/serve/cpu/deployments/llm/test_llm_engine.py

@@ -81,3 +81,18 @@ async def test_embedding_mock_engine(
 
         async for response in engine.embeddings(request):
             LLMResponseValidator.validate_embedding_response(response, dimensions)
+
+    @pytest.mark.asyncio
+    async def test_score_mock_engine(self, mock_llm_config, mock_score_request):
+        """Test score API for text similarity."""
+        # Create and start the engine
+        engine = MockVLLMEngine(mock_llm_config)
+        await engine.start()
+
+        # Create score request
+        request = mock_score_request
+
+        print("\n\n_____ SCORE _____\n\n")
+
+        async for response in engine.score(request):
+            LLMResponseValidator.validate_score_response(response)

python/ray/llm/tests/serve/cpu/deployments/llm/test_llm_server.py

@@ -152,6 +152,34 @@ async def test_embedding_llm_server(
         # Validate embedding response
         LLMResponseValidator.validate_embedding_response(chunks[0], dimensions)
 
+    @pytest.mark.asyncio
+    async def test_score_llm_server(
+        self,
+        serve_handle,
+        mock_llm_config,
+        mock_score_request,
+    ):
+        """Test score API from LLMServer perspective."""
+
+        # Create score request
+        request = mock_score_request
+
+        print("\n\n_____ SCORE SERVER _____\n\n")
+
+        # Get the response
+        batched_chunks = serve_handle.score.remote(request)
+
+        # Collect responses (should be just one)
+        chunks = []
+        async for batch in batched_chunks:
+            chunks.append(batch)
+
+        # Check that we got one response
+        assert len(chunks) == 1
+
+        # Validate score response
+        LLMResponseValidator.validate_score_response(chunks[0])
+
     @pytest.mark.asyncio
     async def test_check_health(self, mock_llm_config):
         """Test health check functionality."""

python/ray/llm/tests/serve/mocks/mock_vllm_engine.py

@@ -13,6 +13,8 @@
     EmbeddingRequest,
     EmbeddingResponse,
     ErrorResponse,
+    ScoreRequest,
+    ScoreResponse,
 )
 from ray.llm._internal.serve.configs.server_models import (
     DiskMultiplexConfig,
@@ -135,6 +137,41 @@ async def embeddings(
         )
         yield response
 
+    async def score(
+        self, request: ScoreRequest
+    ) -> AsyncGenerator[Union[str, ScoreResponse, ErrorResponse], None]:
+        """Mock score generation for text pairs."""
+        if not self.started:
+            raise RuntimeError("Engine not started")
+
+        # Extract text_1 and text_2 from the request
+        text_1 = getattr(request, "text_1", "")
+        text_2 = getattr(request, "text_2", "")
+
+        # Convert to lists if they aren't already
+        text_1_list = text_1 if isinstance(text_1, list) else [text_1]
+        text_2_list = text_2 if isinstance(text_2, list) else [text_2]
+
+        # Generate mock scores for each pair
+        score_data = []
+        for i, (t1, t2) in enumerate(zip(text_1_list, text_2_list)):
+            # Generate a random score (can be any float value)
+            score = random.uniform(-10.0, 10.0)
+
+            score_data.append({"object": "score", "score": score, "index": i})
+
+        # Create the response
+        response = ScoreResponse(
+            object="list",
+            data=score_data,
+            model=getattr(request, "model", "mock-model"),
+            usage={
+                "prompt_tokens": len(str(text_1).split()) + len(str(text_2).split()),
+                "total_tokens": len(str(text_1).split()) + len(str(text_2).split()),
+            },
+        )
+        yield response
+
     async def _generate_chat_response(
         self, request: ChatCompletionRequest, prompt_text: str, max_tokens: int
     ) -> AsyncGenerator[Union[str, ChatCompletionResponse], None]:

python/ray/llm/tests/serve/utils/testing_utils.py

@@ -11,6 +11,7 @@
     ChatCompletionResponse,
     CompletionResponse,
     EmbeddingResponse,
+    ScoreResponse,
 )
 
 
@@ -94,3 +95,16 @@ def validate_embedding_response(
         # Check dimensions if specified
         if expected_dimensions:
             assert len(response.data[0].embedding) == expected_dimensions
+
+    @staticmethod
+    def validate_score_response(response: ScoreResponse):
+        """Validate score responses."""
+        assert isinstance(response, ScoreResponse)
+        assert response.object == "list"
+        assert len(response.data) >= 1
+
+        # Validate each score data element
+        for i, score_data in enumerate(response.data):
+            assert score_data.object == "score"
+            assert isinstance(score_data.score, float)
+            assert score_data.index == i  # Index should match position in list