Smarter formatting default for tool results, better way to customize/opt-out, and better naming/docs

Author: cpsievert

chatlas/_anthropic.py

@@ -472,12 +472,15 @@ def _as_content_block(content: Content) -> "ContentBlockParam":
                 "input": content.arguments,
             }
         elif isinstance(content, ContentToolResult):
-            return {
+            res: ToolResultBlockParam = {
                 "type": "tool_result",
                 "tool_use_id": content.id,
-                "content": content.get_final_value(),
                 "is_error": content.error is not None,
             }
+            # Anthropic supports non-text contents like ImageBlockParam
+            res["content"] = content.get_final_value()  # type: ignore
+            return res
+
         raise ValueError(f"Unknown content type: {type(content)}")
 
     @staticmethod

chatlas/_chat.py

@@ -1100,8 +1100,8 @@ def _chat_impl(
                     if req is not None:
                         yield req
                     res = self._invoke_tool_request(x)
-                    if res.result and res.result.response_output is not None:
-                        yield res.result.response_output
+                    if res.result and res.result.user is not None:
+                        yield res.result.user
                     results.append(res)
 
             if results:
@@ -1137,8 +1137,8 @@ async def _chat_impl_async(
                     if req is not None:
                         yield req
                     res = await self._invoke_tool_request_async(x)
-                    if res.result and res.result.response_output is not None:
-                        yield res.result.response_output
+                    if res.result and res.result.user is not None:
+                        yield res.result.user
                     results.append(res)
 
             if results:
@@ -1289,7 +1289,7 @@ def _invoke_tool_request(self, x: ContentToolRequest) -> ContentToolResult:
                 result = func(x.arguments)
 
             if not isinstance(result, ToolResult):
-                result = ToolResult(value=result)
+                result = ToolResult(result)
 
             return ContentToolResult(x.id, result=result, error=None, name=name)
         except Exception as e:
@@ -1319,7 +1319,7 @@ async def _invoke_tool_request_async(
                 result = await func(x.arguments)
 
             if not isinstance(result, ToolResult):
-                result = ToolResult(value=result)
+                result = ToolResult(result)
 
             return ContentToolResult(x.id, result=result, error=None, name=name)
         except Exception as e:

chatlas/_content.py

@@ -207,17 +207,17 @@ class ContentToolResult(Content):
     name: Optional[str] = None
     error: Optional[str] = None
 
-    def _get_value(self, pretty: bool = False) -> str:
+    def _get_value(self, pretty: bool = False) -> Stringable:
         if self.error:
             return f"Tool calling failed with error: '{self.error}'"
         result = cast("ToolResult", self.result)
         if not pretty:
-            return result.serialized_value
+            return result.assistant
         try:
-            json_val = json.loads(result.serialized_value)  # type: ignore
+            json_val = json.loads(result.assistant)  # type: ignore
             return pformat(json_val, indent=2, sort_dicts=False)
         except:  # noqa: E722
-            return result.serialized_value
+            return result.assistant
 
     # Primarily used for `echo="all"`...
     def __str__(self):
@@ -231,14 +231,14 @@ def _repr_markdown_(self):
 
     def __repr__(self, indent: int = 0):
         res = " " * indent
-        value = None if self.result is None else self.result.value
+        value = None if self.result is None else self.result.assistant
         res += f"<ContentToolResult value='{value}' id='{self.id}'"
         if self.error:
             res += f" error='{self.error}'"
         return res + ">"
 
     # The actual value to send to the model
-    def get_final_value(self) -> str:
+    def get_final_value(self) -> Stringable:
         return self._get_value()
 
 

chatlas/_openai.py

@@ -483,8 +483,8 @@ def _as_message_param(turns: list[Turn]) -> list["ChatCompletionMessageParam"]:
                     elif isinstance(x, ContentToolResult):
                         tool_results.append(
                             ChatCompletionToolMessageParam(
-                                # TODO: a tool could return an image!?!
-                                content=x.get_final_value(),
+                                # Currently, OpenAI only allows for text content in tool results
+                                content=cast(str, x.get_final_value()),
                                 tool_call_id=x.id,
                                 role="tool",
                             )

chatlas/_tools.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
 import inspect
+import json
 import warnings
-from typing import TYPE_CHECKING, Any, Awaitable, Callable, Optional, Protocol
+from typing import TYPE_CHECKING, Any, Awaitable, Callable, Literal, Optional, Protocol
 
 from pydantic import BaseModel, Field, create_model
 
@@ -62,41 +63,69 @@ class ToolResult:
     """
     A result from a tool invocation
 
-    Return this value from a tool if you want to separate what gets sent
-    to the model vs what value gets yielded to the user.
+    Return an instance of this class from a tool function in order to:
+
+    1. Yield content for the user (i.e., the downstream consumer of a `.stream()` or `.chat()`)
+       to display.
+    2. Control how the tool result gets formatted for the model (i.e., the assistant).
 
     Parameters
     ----------
-    value
-        The tool's return value. If `serialized_value` is not provided, the
-        string representation of this value is sent to the model.
-    response_output
-        A value to yield when the tool is called during response generation. If
-        `None`, no value is yielded. This is primarily useful for producing
-        custom UI in the response output to indicate to the user that a tool
-        call has completed (for example, return shiny UI here when
-        `.stream()`-ing inside a shiny app).
-    serialized_value
-        The serialized value to send to the model. If `None`, the value is serialized
-        using `str()`. This is useful when the value is not JSON
+    assistant
+        The tool result to send to the llm (i.e., assistant). If the result is
+        not a string, `format_as` determines how to the value is formatted
+        before sending it to the model.
+    user
+        A value to yield to the user (i.e., the consumer of a `.stream()`) when
+        the tool is called. If `None`, no value is yielded. This is primarily
+        useful for producing custom UI in the response output to indicate to the
+        user that a tool call has completed (for example, return shiny UI here
+        when `.stream()`-ing inside a shiny app).
+    format_as
+        How to format the `assistant` value for the model. The default,
+        `"auto"`, first attempts to format the value as a JSON string. If that
+        fails, it gets converted to a string via `str()`. To force
+        `json.dumps()` or `str()`, set to `"json"` or `"str"`. Finally,
+        `"as_is"` is useful for doing your own formatting and/or passing a
+        non-string value (e.g., a list or dict) straight to the model.
+        Non-string values are useful for tools that return images or other
+        'known' non-text content types.
     """
 
     def __init__(
         self,
-        value: Stringable,
-        response_output: Optional[Stringable] = None,
-        serialized_value: Optional[str] = None,
+        assistant: Stringable,
+        *,
+        user: Optional[Stringable] = None,
+        format_as: Literal["auto", "json", "str", "as_is"] = "auto",
     ):
-        self.value = value
-        self.response_output = response_output
-        if serialized_value is None:
-            serialized_value = str(value)
-        self.serialized_value = serialized_value
+        # TODO: if called when an active user session, perhaps we could
+        # provide a smart default here
+        self.user = user
+        self.assistant = self._format_value(assistant, format_as)
         # TODO: we could consider adding an "emit value" -- that is, the thing to
         # display when `echo="all"` is used. I imagine that might be useful for
         # advanced users, but let's not worry about it until someone asks for it.
         # self.emit = emit
 
+    def _format_value(self, value: Stringable, mode: str) -> Stringable:
+        if isinstance(value, str):
+            return value
+
+        if mode == "auto":
+            try:
+                return json.dumps(value)
+            except Exception:
+                return str(value)
+        elif mode == "json":
+            return json.dumps(value)
+        elif mode == "str":
+            return str(value)
+        elif mode == "as_is":
+            return value
+        else:
+            raise ValueError(f"Unknown format mode: {mode}")
+
 
 def func_to_schema(
     func: Callable[..., Any] | Callable[..., Awaitable[Any]],

tests/conftest.py

@@ -3,10 +3,11 @@
 from typing import Awaitable, Callable
 
 import pytest
-from chatlas import Chat, Turn, content_image_file, content_image_url
 from PIL import Image
 from pydantic import BaseModel
 
+from chatlas import Chat, Turn, content_image_file, content_image_url
+
 ChatFun = Callable[..., Chat]
 
 
@@ -223,3 +224,8 @@ def assert_images_remote_error(chat_fun: ChatFun):
         chat.chat("What's in this image?", image_remote)
 
     assert len(chat.get_turns()) == 0
+
+
+@pytest.fixture
+def test_images_dir():
+    return Path(__file__).parent / "images"

tests/images/dice.png

@@ -97,7 +97,7 @@ def test_tool_results():
 
     def get_date():
         """Gets the current date"""
-        return ToolResult("2024-01-01", response_output=["Tool result..."])
+        return ToolResult("2024-01-01", user=["Tool result..."])
 
     chat.register_tool(get_date)
     chat.on_tool_request(lambda req: [f"Requesting tool {req.name}..."])
@@ -134,7 +134,7 @@ async def get_date():
         import asyncio
 
         await asyncio.sleep(0.1)
-        return ToolResult("2024-01-01", response_output=["Tool result..."])
+        return ToolResult("2024-01-01", user=["Tool result..."])
 
     chat.register_tool(get_date)
     chat.on_tool_request(lambda req: [f"Requesting tool {req.name}..."])

tests/test_chat.py

@@ -114,7 +114,7 @@ def tool():
     assert isinstance(res, ContentToolResult)
     assert res.id == "x"
     assert res.error is None
-    assert res.result.value == 1
+    assert res.result.assistant == "1"
 
     res = chat._invoke_tool_request(
         ContentToolRequest(id="x", name="tool", arguments={"x": 1})
@@ -149,7 +149,7 @@ async def tool():
     assert isinstance(res, ContentToolResult)
     assert res.id == "x"
     assert res.error is None
-    assert res.result.value == 1
+    assert res.result.assistant == "1"
 
     res = await chat._invoke_tool_request_async(
         ContentToolRequest(id="x", name="tool", arguments={"x": 1})

tests/test_content_tools.py

@@ -1,4 +1,7 @@
+import base64
+
 import pytest
+
 from chatlas import ChatAnthropic
 
 from .conftest import (
@@ -94,3 +97,34 @@ def run_inlineassert():
 
     retryassert(run_inlineassert, retries=3)
     assert_images_remote_error(chat_fun)
+
+
+def test_anthropic_image_tool(test_images_dir):
+    from chatlas import ToolResult
+
+    def get_picture():
+        "Returns an image"
+        with open(test_images_dir / "dice.png", "rb") as image:
+            bytez = image.read()
+        res = [
+            {
+                "type": "image",
+                "source": {
+                    "type": "base64",
+                    "media_type": "image/png",
+                    "data": base64.b64encode(bytez).decode("utf-8"),
+                },
+            }
+        ]
+        return ToolResult(res, format_as="as_is")
+
+    chat = ChatAnthropic()
+    chat.register_tool(get_picture)
+
+    res = chat.chat(
+        "You have a tool called 'get_picture' available to you. "
+        "When called, it returns an image. "
+        "Tell me what you see in the image."
+    )
+
+    assert "dice" in res.get_content()