feat: add multimodal support to AnthropicChatGenerator (#2186)

* Add multimodal support to AnthropicChatGenerator #2125

* fix: resolve import formatting and line length issues

* Fixed recommended changes #2186: anthropic image support

* Fix mime type validation

* refinements

---------

Co-authored-by: anakin87 <[email protected]>

Author: ChinmayBansal

integrations/anthropic/pyproject.toml

@@ -57,7 +57,8 @@ dependencies = [
     "pytest-cov",
     "pytest-rerunfailures",
     "mypy",
-    "pip"
+    "pip",
+    "pillow",  # image support
 ]
 
 [tool.hatch.envs.test.scripts]

integrations/anthropic/src/haystack_integrations/components/generators/anthropic/chat/chat_generator.py

@@ -1,9 +1,10 @@
 from datetime import datetime, timezone
-from typing import Any, ClassVar, Dict, List, Literal, Optional, Tuple, Union
+from typing import Any, ClassVar, Dict, List, Literal, Optional, Tuple, Union, cast, get_args
 
 from haystack import component, default_from_dict, default_to_dict, logging
 from haystack.components.generators.utils import _convert_streaming_chunks_to_chat_message
-from haystack.dataclasses.chat_message import ChatMessage, ChatRole, ToolCall, ToolCallResult
+from haystack.dataclasses.chat_message import ChatMessage, ChatRole, TextContent, ToolCall, ToolCallResult
+from haystack.dataclasses.image_content import ImageContent
 from haystack.dataclasses.streaming_chunk import (
     AsyncStreamingCallbackT,
     ComponentInfo,
@@ -26,11 +27,23 @@
 
 from anthropic import Anthropic, AsyncAnthropic
 from anthropic.resources.messages.messages import Message, RawMessageStreamEvent, Stream
-from anthropic.types import MessageParam, TextBlockParam, ToolParam, ToolResultBlockParam, ToolUseBlockParam
+from anthropic.types import (
+    ImageBlockParam,
+    MessageParam,
+    TextBlockParam,
+    ToolParam,
+    ToolResultBlockParam,
+    ToolUseBlockParam,
+)
 
 logger = logging.getLogger(__name__)
 
 
+# See https://docs.anthropic.com/en/api/messages for supported formats
+ImageFormat = Literal["image/jpeg", "image/png", "image/gif", "image/webp"]
+IMAGE_SUPPORTED_FORMATS: list[ImageFormat] = list(get_args(ImageFormat))
+
+
 # Mapping from Anthropic stop reasons to Haystack FinishReason values
 FINISH_REASON_MAPPING: Dict[str, FinishReason] = {
     "end_turn": "stop",
@@ -44,7 +57,7 @@
 
 def _update_anthropic_message_with_tool_call_results(
     tool_call_results: List[ToolCallResult],
-    content: List[Union[TextBlockParam, ToolUseBlockParam, ToolResultBlockParam]],
+    content: List[Union[TextBlockParam, ToolUseBlockParam, ToolResultBlockParam, ImageBlockParam]],
 ) -> None:
     """
     Update an Anthropic message content list with tool call results.
@@ -119,13 +132,39 @@ def _convert_messages_to_anthropic_format(
             i += 1
             continue
 
-        content: List[Union[TextBlockParam, ToolUseBlockParam, ToolResultBlockParam]] = []
-
-        if message.texts and message.texts[0]:
-            text_block = TextBlockParam(type="text", text=message.texts[0])
-            if cache_control:
-                text_block["cache_control"] = cache_control
-            content.append(text_block)
+        content: List[Union[TextBlockParam, ToolUseBlockParam, ToolResultBlockParam, ImageBlockParam]] = []
+
+        # Handle multimodal content (text and images) preserving order
+        for part in message._content:
+            if isinstance(part, TextContent) and part.text:
+                text_block = TextBlockParam(type="text", text=part.text)
+                if cache_control:
+                    text_block["cache_control"] = cache_control
+                content.append(text_block)
+            elif isinstance(part, ImageContent):
+                if not message.is_from(ChatRole.USER):
+                    msg = "Image content is only supported for user messages"
+                    raise ValueError(msg)
+
+                if part.mime_type not in IMAGE_SUPPORTED_FORMATS:
+                    supported_formats = ", ".join(IMAGE_SUPPORTED_FORMATS)
+                    msg = (
+                        f"Unsupported image format: {part.mime_type}. "
+                        f"Anthropic supports the following formats: {supported_formats}"
+                    )
+                    raise ValueError(msg)
+
+                image_block = ImageBlockParam(
+                    type="image",
+                    source={
+                        "type": "base64",
+                        "media_type": cast(ImageFormat, part.mime_type),
+                        "data": part.base64_image,
+                    },
+                )
+                if cache_control:
+                    image_block["cache_control"] = cache_control
+                content.append(image_block)
 
         if message.tool_calls:
             tool_use_blocks = _convert_tool_calls_to_anthropic_format(message.tool_calls)
@@ -148,7 +187,10 @@ def _convert_messages_to_anthropic_format(
                         blk["cache_control"] = cache_control
 
         if not content:
-            msg = "A `ChatMessage` must contain at least one `TextContent`, `ToolCall`, or `ToolCallResult`."
+            msg = (
+                "A `ChatMessage` must contain at least one `TextContent`, `ImageContent`, "
+                "`ToolCall`, or `ToolCallResult`."
+            )
             raise ValueError(msg)
 
         # Anthropic only supports assistant and user roles in messages. User role is also used for tool messages.
@@ -170,7 +212,7 @@ class AnthropicChatGenerator:
     Completes chats using Anthropic's large language models (LLMs).
 
     It uses [ChatMessage](https://docs.haystack.deepset.ai/docs/data-classes#chatmessage)
-    format in input and output.
+    format in input and output. Supports multimodal inputs including text and images.
 
     You can customize how the text is generated by passing parameters to the
     Anthropic API. Use the `**generation_kwargs` argument when you initialize
@@ -182,18 +224,41 @@ class AnthropicChatGenerator:
 
     Usage example:
     ```python
-    from haystack_integrations.components.generators.anthropic import AnthropicChatGenerator
+    from haystack_integrations.components.generators.anthropic import (
+        AnthropicChatGenerator,
+    )
     from haystack.dataclasses import ChatMessage
 
-    generator = AnthropicChatGenerator(model="claude-sonnet-4-20250514",
-                                       generation_kwargs={
-                                           "max_tokens": 1000,
-                                           "temperature": 0.7,
-                                       })
-
-    messages = [ChatMessage.from_system("You are a helpful, respectful and honest assistant"),
-                ChatMessage.from_user("What's Natural Language Processing?")]
+    generator = AnthropicChatGenerator(
+        model="claude-sonnet-4-20250514",
+        generation_kwargs={
+            "max_tokens": 1000,
+            "temperature": 0.7,
+        },
+    )
+
+    messages = [
+        ChatMessage.from_system(
+            "You are a helpful, respectful and honest assistant"
+        ),
+        ChatMessage.from_user("What's Natural Language Processing?"),
+    ]
     print(generator.run(messages=messages))
+    ```
+
+    Usage example with images:
+    ```python
+    from haystack.dataclasses import ChatMessage, ImageContent
+
+    image_content = ImageContent.from_file_path("path/to/image.jpg")
+    messages = [
+        ChatMessage.from_user(
+            content_parts=["What's in this image?", image_content]
+        )
+    ]
+    generator = AnthropicChatGenerator()
+    result = generator.run(messages)
+    ```
     """
 
     # The parameters that can be passed to the Anthropic API https://docs.anthropic.com/claude/reference/messages_post

integrations/anthropic/tests/conftest.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from unittest.mock import patch
 
 import pytest
@@ -43,3 +44,8 @@ def mock_chat_completion_extended_thinking():
 
         mock_chat_completion_create.return_value = completion
         yield mock_chat_completion_create
+
+
+@pytest.fixture()
+def test_files_path():
+    return Path(__file__).parent / "test_files"

integrations/anthropic/tests/test_chat_generator.py

@@ -25,7 +25,15 @@
 from anthropic.types.raw_message_delta_event import Delta
 from haystack import Pipeline
 from haystack.components.generators.utils import _convert_streaming_chunks_to_chat_message, print_streaming_chunk
-from haystack.dataclasses import ChatMessage, ChatRole, ComponentInfo, StreamingChunk, ToolCall, ToolCallDelta
+from haystack.dataclasses import (
+    ChatMessage,
+    ChatRole,
+    ComponentInfo,
+    ImageContent,
+    StreamingChunk,
+    ToolCall,
+    ToolCallDelta,
+)
 from haystack.tools import Tool, Toolset
 from haystack.utils.auth import Secret
 
@@ -1155,6 +1163,53 @@ def test_convert_message_to_anthropic_format_complex(self):
             },
         ]
 
+    def test_convert_message_to_anthropic_format_with_image(self):
+        """Test that a ChatMessage with ImageContent is converted to Anthropic format correctly."""
+        base64_image = (
+            "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+        )
+        image_content = ImageContent(base64_image=base64_image, mime_type="image/png")
+        message = ChatMessage.from_user(content_parts=["What's in this image?", image_content])
+
+        system_messages, non_system_messages = _convert_messages_to_anthropic_format([message])
+
+        assert len(non_system_messages) == 1
+        anthropic_message = non_system_messages[0]
+        assert anthropic_message["role"] == "user"
+        assert len(anthropic_message["content"]) == 2
+
+        # Check text and image blocks
+        assert anthropic_message["content"][0]["type"] == "text"
+        assert anthropic_message["content"][0]["text"] == "What's in this image?"
+        assert anthropic_message["content"][1]["type"] == "image"
+        assert anthropic_message["content"][1]["source"]["type"] == "base64"
+        assert anthropic_message["content"][1]["source"]["media_type"] == "image/png"
+        assert anthropic_message["content"][1]["source"]["data"] == base64_image
+
+    def test_convert_message_to_anthropic_format_with_unsupported_mime_type(self):
+        """Test that a ChatMessage with unsupported mime type raises ValueError."""
+        base64_image = (
+            "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+        )
+        image_content = ImageContent(base64_image=base64_image, mime_type="image/bmp")  # Unsupported format
+        message = ChatMessage.from_user(content_parts=["What's in this image?", image_content])
+
+        with pytest.raises(ValueError, match="Unsupported image format: image/bmp"):
+            _convert_messages_to_anthropic_format([message])
+
+    def test_convert_message_to_anthropic_format_with_none_mime_type(self):
+        """Test that a ChatMessage with None mime type raises ValueError."""
+        base64_image = (
+            "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+        )
+        image_content = ImageContent(base64_image=base64_image, mime_type="image/png")
+        # Manually set mime_type to None to test the validation
+        image_content.mime_type = None
+        message = ChatMessage.from_user(content_parts=["What's in this image?", image_content])
+
+        with pytest.raises(ValueError, match="Unsupported image format: None"):
+            _convert_messages_to_anthropic_format([message])
+
     def test_convert_message_to_anthropic_invalid(self):
         """
         Test that the AnthropicChatGenerator component fails to convert an invalid ChatMessage to Anthropic format.
@@ -1564,6 +1619,7 @@ def test_convert_messages_attaches_cache_control(self):
         assert non_sys[0]["content"][0]["cache_control"]["type"] == "ephemeral"
 
     @pytest.mark.skipif(not os.environ.get("ANTHROPIC_API_KEY", None), reason="ANTHROPIC_API_KEY not set")
+    @pytest.mark.integration
     @pytest.mark.parametrize("cache_enabled", [True, False])
     def test_prompt_caching_live_run(self, cache_enabled):
         generation_kwargs = {"extra_headers": {"anthropic-beta": "prompt-caching-2024-07-31"}} if cache_enabled else {}
@@ -1595,6 +1651,7 @@ def test_prompt_caching_live_run(self, cache_enabled):
             assert token_usage["cache_read_input_tokens"] == 0
 
     @pytest.mark.skipif(not os.environ.get("ANTHROPIC_API_KEY"), reason="ANTHROPIC_API_KEY not set")
+    @pytest.mark.integration
     @pytest.mark.parametrize("cache_enabled", [True, False])
     def test_prompt_caching_live_run_with_user_message(self, cache_enabled):
         claude_llm = AnthropicChatGenerator(
@@ -1799,3 +1856,26 @@ async def test_live_run_async_with_tools(self, tools):
         assert len(final_message.text) > 0
         assert "paris" in final_message.text.lower()
         assert "completion_tokens" in final_message.meta["usage"]
+
+    @pytest.mark.integration
+    @pytest.mark.skipif(
+        not os.environ.get("ANTHROPIC_API_KEY", None),
+        reason="Export an env var called ANTHROPIC_API_KEY containing the Anthropic token to run this test.",
+    )
+    def test_live_run_multimodal(self, test_files_path):
+        """Integration test for multimodal functionality with real API."""
+        image_path = test_files_path / "apple.jpg"
+        # Resize the image to keep this test fast
+        image_content = ImageContent.from_file_path(file_path=image_path, size=(100, 100))
+        messages = [ChatMessage.from_user(content_parts=["What does this image show? Max 5 words", image_content])]
+
+        generator = AnthropicChatGenerator(generation_kwargs={"max_tokens": 20})
+        response = generator.run(messages=messages)
+
+        assert "replies" in response
+        assert isinstance(response["replies"], list)
+        assert len(response["replies"]) > 0
+        message = response["replies"][0]
+        assert message.text
+        assert len(message.text) > 0
+        assert any(word in message.text.lower() for word in ["apple", "fruit", "red"])