Enhance Responses API with new tool call types and JSON formats

- Added support for new tool call types: ImageGenerationToolCall, CodeInterpreterToolCall, LocalShellToolCall, CustomToolCall, and their respective outputs.
- Introduced MCP-related classes: MCPApprovalRequest, MCPApprovalResponse, MCPListTools, MCPTool, and MCPToolCall.
- Updated Input and JsonFormats to accommodate new tool calls and their serialization.
- Enhanced InputMessageContent to include detail options for images.
- Improved overall structure and organization of tool call handling.

Author: peterbanda

openai-core/src/main/scala/io/cequence/openaiscala/domain/responsesapi/Input.scala

@@ -2,6 +2,7 @@ package io.cequence.openaiscala.domain.responsesapi
 
 import io.cequence.openaiscala.domain.ChatRole
 import io.cequence.openaiscala.domain.responsesapi.tools._
+import io.cequence.openaiscala.domain.responsesapi.tools.mcp._
 
 /**
  * Input types hierarchy:
@@ -20,10 +21,21 @@ import io.cequence.openaiscala.domain.responsesapi.tools._
  *       - Computer tool call (object)
  *       - Computer tool call output (object)
  *       - Web search tool call (object)
- *       - Function tool call (object)
- *       - Function tool call output (object)
+ *       - Function tool call (object) - Deprecated?
+ *       - Function tool call output (object) - Deprecated?
  *       - Reasoning (object)
+ *       - Reasoning output (object)
  *       - Item reference (object)
+ *       - Image generation call (object)
+ *       - Code interpreter tool call (object)
+ *       - Local shell call (object)
+ *       - Local shell call output (object)
+ *       - MCP list tools (object)
+ *       - MCP approval request (object)
+ *       - MCP approval response (object)
+ *       - MCP tool call (object)
+ *       - Custom tool call output (object)
+ *       - Custom tool call (object)
  */
 trait Input {
   val `type`: String
@@ -137,6 +149,102 @@ object Input {
     status: Option[ModelStatus] = None
   ) = Reasoning(id, summary, status)
 
+  def ofImageGenerationToolCall(
+    id: String,
+    result: Option[String],
+    status: String
+  ) = ImageGenerationToolCall(
+    id,
+    result,
+    status
+  )
+
+  def ofCodeInterpreterToolCall(
+    id: String,
+    code: Option[String],
+    containerId: String,
+    outputs: Seq[CodeInterpreterOutput],
+    status: String
+  ) = CodeInterpreterToolCall(
+    id,
+    code,
+    containerId,
+    outputs,
+    status
+  )
+
+  def ofCodeInterpreterOutputLogs(
+    logs: String
+  ) = CodeInterpreterOutputLogs(logs)
+
+  def ofCodeInterpreterOutputImage(
+    url: String
+  ) = CodeInterpreterOutputImage(url)
+
+  def ofLocalShellToolCall(
+    action: LocalShellAction,
+    callId: String,
+    id: String,
+    status: String
+  ) = LocalShellToolCall(
+    action,
+    callId,
+    id,
+    status
+  )
+
+  def ofLocalShellAction(
+    command: Seq[String],
+    env: Map[String, String],
+    timeoutMs: Option[Int] = None,
+    user: Option[String] = None,
+    workingDirectory: Option[String] = None
+  ) = LocalShellAction(
+    command,
+    env,
+    timeoutMs,
+    user,
+    workingDirectory
+  )
+
+  def ofLocalShellCallOutput(
+    id: String,
+    output: String,
+    status: Option[String] = None
+  ) = LocalShellCallOutput(
+    id,
+    output,
+    status
+  )
+
+  def ofMCPListTools(
+    id: String,
+    serverLabel: String,
+    tools: Seq[MCPTool],
+    error: Option[String] = None
+  ) = MCPListTools(
+    id,
+    serverLabel,
+    tools,
+    error
+  )
+
+  def ofMCPToolCall(
+    arguments: String,
+    id: String,
+    name: String,
+    serverLabel: String,
+    error: Option[String] = None,
+    output: Option[String] = None
+  ) = MCPToolCall(
+    arguments,
+    id,
+    name,
+    serverLabel,
+    error,
+    output
+  )
+
   def ofItemReference(
     id: String
   ) = ItemReference(id)

openai-core/src/main/scala/io/cequence/openaiscala/domain/responsesapi/InputMessageContent.scala

@@ -32,7 +32,7 @@ object InputMessageContent {
   final case class Image(
     fileId: Option[String] = None,
     imageUrl: Option[String] = None,
-    detail: Option[String] = None
+    detail: Option[String] = None // low, high, auto
   ) extends InputMessageContent {
     val `type`: String = "input_image"
   }

openai-core/src/main/scala/io/cequence/openaiscala/domain/responsesapi/JsonFormats.scala

@@ -20,12 +20,21 @@ import io.cequence.openaiscala.domain.responsesapi.tools.JsonFormats.{
 import io.cequence.openaiscala.domain.responsesapi.tools.JsonFormats.{
   fileSearchToolCallFormat,
   webSearchToolCallFormat,
-  computerToolCallFormat
+  computerToolCallFormat,
+  imageGenerationToolCallFormat,
+  codeInterpreterToolCallFormat,
+  localShellToolCallFormat,
+  localShellCallOutputFormat,
+  mcpListToolsFormat,
+  mcpToolCallFormat,
+  customToolCallOutputFormat,
+  customToolCallFormat
 }
 import play.api.libs.json._
 import play.api.libs.functional.syntax._
 import io.cequence.openaiscala.JsonFormats.chatRoleFormat
 import io.cequence.openaiscala.domain.responsesapi.tools.JsonFormats.functionToolCallFormat
+import io.cequence.openaiscala.domain.responsesapi.tools.mcp._
 
 object JsonFormats {
 
@@ -226,17 +235,25 @@ object JsonFormats {
 
   implicit lazy val inputWrites: Writes[Input] = Writes[Input] { (input: Input) =>
     val jsObject = input match {
-      case input: Message.InputText      => inputTextMessageFormat.writes(input)
-      case input: Message.InputContent   => inputContentMessageFormat.writes(input)
-      case input: Message.OutputContent  => outputContentMessageFormat.writes(input)
-      case input: FileSearchToolCall     => fileSearchToolCallFormat.writes(input)
-      case input: ComputerToolCall       => computerToolCallFormat.writes(input)
-      case input: ComputerToolCallOutput => computerToolCallOutputFormat.writes(input)
-      case input: WebSearchToolCall      => webSearchToolCallFormat.writes(input)
-      case input: FunctionToolCall       => functionToolCallFormat.writes(input)
-      case input: FunctionToolCallOutput => functionToolCallOutputFormat.writes(input)
-      case input: Reasoning              => reasoningFormat.writes(input)
-      case input: ItemReference          => itemReferenceFormat.writes(input)
+      case input: Message.InputText       => inputTextMessageFormat.writes(input)
+      case input: Message.InputContent    => inputContentMessageFormat.writes(input)
+      case input: Message.OutputContent   => outputContentMessageFormat.writes(input)
+      case input: FileSearchToolCall      => fileSearchToolCallFormat.writes(input)
+      case input: ComputerToolCall        => computerToolCallFormat.writes(input)
+      case input: ComputerToolCallOutput  => computerToolCallOutputFormat.writes(input)
+      case input: WebSearchToolCall       => webSearchToolCallFormat.writes(input)
+      case input: FunctionToolCall        => functionToolCallFormat.writes(input)
+      case input: FunctionToolCallOutput  => functionToolCallOutputFormat.writes(input)
+      case input: Reasoning               => reasoningFormat.writes(input)
+      case input: ItemReference           => itemReferenceFormat.writes(input)
+      case input: ImageGenerationToolCall => imageGenerationToolCallFormat.writes(input)
+      case input: CodeInterpreterToolCall => codeInterpreterToolCallFormat.writes(input)
+      case input: LocalShellToolCall      => localShellToolCallFormat.writes(input)
+      case input: LocalShellCallOutput    => localShellCallOutputFormat.writes(input)
+      case input: MCPListTools            => mcpListToolsFormat.writes(input)
+      case input: MCPToolCall             => mcpToolCallFormat.writes(input)
+      case input: CustomToolCallOutput    => customToolCallOutputFormat.writes(input)
+      case input: CustomToolCall          => customToolCallFormat.writes(input)
     }
     jsObject + ("type" -> JsString(input.`type`))
   }
@@ -263,15 +280,23 @@ object JsonFormats {
           case JsError(_)      => JsError("Missing 'content' field for Input")
         }
 
-      case "file_search_call"     => fileSearchToolCallFormat.reads(json)
-      case "computer_call"        => computerToolCallFormat.reads(json)
-      case "computer_call_output" => computerToolCallOutputFormat.reads(json)
-      case "web_search_call"      => webSearchToolCallFormat.reads(json)
-      case "function_call"        => functionToolCallFormat.reads(json)
-      case "function_call_output" => functionToolCallOutputFormat.reads(json)
-      case "reasoning"            => reasoningFormat.reads(json)
-      case "item_reference"       => itemReferenceFormat.reads(json)
-      case _                      => JsError("Missing type field for Input")
+      case "file_search_call"        => fileSearchToolCallFormat.reads(json)
+      case "computer_call"           => computerToolCallFormat.reads(json)
+      case "computer_call_output"    => computerToolCallOutputFormat.reads(json)
+      case "web_search_call"         => webSearchToolCallFormat.reads(json)
+      case "function_call"           => functionToolCallFormat.reads(json)
+      case "function_call_output"    => functionToolCallOutputFormat.reads(json)
+      case "reasoning"               => reasoningFormat.reads(json)
+      case "item_reference"          => itemReferenceFormat.reads(json)
+      case "image_generation_call"   => imageGenerationToolCallFormat.reads(json)
+      case "code_interpreter_call"   => codeInterpreterToolCallFormat.reads(json)
+      case "local_shell_call"        => localShellToolCallFormat.reads(json)
+      case "local_shell_call_output" => localShellCallOutputFormat.reads(json)
+      case "mcp_list_tools"          => mcpListToolsFormat.reads(json)
+      case "mcp_tool_call"           => mcpToolCallFormat.reads(json)
+      case "custom_tool_call_output" => customToolCallOutputFormat.reads(json)
+      case "custom_tool_call"        => customToolCallFormat.reads(json)
+      case _                         => JsError("Missing type field for Input")
     }
   }
 

openai-core/src/main/scala/io/cequence/openaiscala/domain/responsesapi/tools/CodeInterpreterToolCall.scala

@@ -0,0 +1,57 @@
+package io.cequence.openaiscala.domain.responsesapi.tools
+
+import io.cequence.openaiscala.domain.responsesapi.Input
+
+/**
+ * Represents a code interpreter tool call made by the model.
+ *
+ * @param code
+ *   The code to run, or null if not available.
+ * @param containerId
+ *   The ID of the container used to run the code.
+ * @param id
+ *   The unique ID of the code interpreter tool call.
+ * @param outputs
+ *   The outputs generated by the code interpreter, such as logs or images. Can be null if no
+ *   outputs are available.
+ * @param status
+ *   The status of the code interpreter tool call. Valid values are: in_progress, completed,
+ *   incomplete, interpreting, and failed. TODO: introduce enum
+ */
+final case class CodeInterpreterToolCall(
+  id: String,
+  code: Option[String],
+  containerId: String,
+  outputs: Seq[CodeInterpreterOutput],
+  status: String
+) extends Input {
+  val `type`: String = "code_interpreter_call"
+}
+
+trait CodeInterpreterOutput {
+  val `type`: String
+}
+
+/**
+ * The logs output from the code interpreter.
+ *
+ * @param logs
+ *   The logs output from the code interpreter.
+ */
+final case class CodeInterpreterOutputLogs(
+  logs: String
+) extends CodeInterpreterOutput {
+  val `type`: String = "logs"
+}
+
+/**
+ * The image output from the code interpreter.
+ *
+ * @param url
+ *   The URL of the image output from the code interpreter.
+ */
+final case class CodeInterpreterOutputImage(
+  url: String
+) extends CodeInterpreterOutput {
+  val `type`: String = "image"
+}

openai-core/src/main/scala/io/cequence/openaiscala/domain/responsesapi/tools/ComputerToolCall.scala

@@ -205,3 +205,4 @@ final case class PendingSafetyCheck(
   id: String,
   message: String
 )
+

openai-core/src/main/scala/io/cequence/openaiscala/domain/responsesapi/tools/CustomToolCall.scala

@@ -0,0 +1,24 @@
+package io.cequence.openaiscala.domain.responsesapi.tools
+
+import io.cequence.openaiscala.domain.responsesapi.Input
+
+/**
+ * A call to a custom tool created by the model.
+ *
+ * @param callId
+ *   An identifier used to map this custom tool call to a tool call output.
+ * @param input
+ *   The input for the custom tool call generated by the model.
+ * @param name
+ *   The name of the custom tool being called.
+ * @param id
+ *   The unique ID of the custom tool call in the OpenAI platform (optional).
+ */
+final case class CustomToolCall(
+  callId: String,
+  input: String,
+  name: String,
+  id: Option[String] = None
+) extends Input {
+  val `type`: String = "custom_tool_call"
+}

openai-core/src/main/scala/io/cequence/openaiscala/domain/responsesapi/tools/CustomToolCallOutput.scala

@@ -0,0 +1,21 @@
+package io.cequence.openaiscala.domain.responsesapi.tools
+
+import io.cequence.openaiscala.domain.responsesapi.Input
+
+/**
+ * The output of a custom tool call from your code, being sent back to the model.
+ *
+ * @param callId
+ *   The call ID, used to map this custom tool call output to a custom tool call.
+ * @param output
+ *   The output from the custom tool call generated by your code.
+ * @param id
+ *   The unique ID of the custom tool call output in the OpenAI platform (optional).
+ */
+final case class CustomToolCallOutput(
+  callId: String,
+  output: String,
+  id: Option[String] = None
+) extends Input {
+  val `type`: String = "custom_tool_call_output"
+}

openai-core/src/main/scala/io/cequence/openaiscala/domain/responsesapi/tools/ImageGenerationToolCall.scala

@@ -0,0 +1,22 @@
+package io.cequence.openaiscala.domain.responsesapi.tools
+
+import io.cequence.openaiscala.domain.responsesapi.Input
+
+/**
+ * Represents an image generation call made by the model.
+ *
+ * @param id
+ *   The unique ID of the image generation call.
+ * @param result
+ *   The generated image encoded in base64, or null if not available.
+ * @param status
+ *   The status of the image generation call.
+ */
+final case class ImageGenerationToolCall(
+  id: String,
+  result: Option[String],
+  status: String
+) extends ToolCall
+    with Input {
+  val `type`: String = "image_generation_call"
+}