feat: Support image/pdf content in tool results

Author: gadenbuie

pkg-r/R/contents_shinychat.R

@@ -136,6 +136,20 @@ S7::method(contents_shinychat, ellmer::ContentText) <- function(content) {
   content@text
 }
 
+S7::method(contents_shinychat, ellmer::ContentPDF) <- function(content) {
+  htmltools::div(
+    class = "shinychat-pdf badge fs-6 text-bg-secondary",
+    htmltools::span(
+      class = "shinychat-pdf__icon me-1",
+      htmltools::HTML(ICON_FILE_PDF_FILL)
+    ),
+    htmltools::span(
+      class = "shinychat-pdf__filename font-monospace"
+      content@filename
+    )
+  )
+}
+
 new_tool_card <- function(type, request_id, tool_name, ...) {
   type <- arg_match(type, c("request", "result"))
 
@@ -316,18 +330,22 @@ tool_result_display <- function(content, display = NULL) {
   use_basic_display <- opt_shinychat_tool_display() == "basic"
 
   if (tool_errored(content) || use_basic_display || !has_display) {
-    return(list(value = tool_string(content), value_type = "code"))
+    return(tool_default_display(content))
   }
 
   if (is.list(display)) {
     has_type <- intersect(c("html", "markdown", "text"), names(display))
     if (length(has_type) > 0) {
       value_type <- has_type[1]
-      return(list(value = display[[value_type]], value_type = value_type))
+      return(as_tool_display(display[[value_type]], value_type))
     }
   }
 
-  list(value = tool_string(content), value_type = "code")
+  tool_default_display(content)
+}
+
+as_tool_display <- function(value, type = "code") {
+  list(value = value, value_type = type)
 }
 
 # Copied from
@@ -336,21 +354,33 @@ tool_errored <- function(x) !is.null(x@error)
 tool_error_string <- function(x) {
   if (inherits(x@error, "condition")) conditionMessage(x@error) else x@error
 }
-tool_string <- function(x) {
+tool_default_display <- function(x) {
   if (tool_errored(x)) {
     # Changed from original: if tool errored, just return the error message
-    strip_ansi(tool_error_string(x))
-  } else if (inherits(x@value, "AsIs")) {
-    x@value
-  } else if (inherits(x@value, "json")) {
-    x@value
-  } else if (is.character(x@value)) {
-    paste(x@value, collapse = "\n")
+    as_tool_display(strip_ansi(tool_error_string(x)))
+  }
+  value <- x@value
+  if (inherits(value, "AsIs")) {
+    as_tool_display(value)
+  } else if (inherits(value, "json")) {
+    as_tool_display(value)
+  } else if (is.character(value)) {
+    as_tool_display(paste(value, collapse = "\n"))
+  } else if (is_content_extra(value)) {
+    as_tool_display(htmltools::div(contents_shinychat(value)), "html")
+  } else if (is.list(value) && every(value, is_content_extra)) {
+    as_tool_display(htmltools::div(map(value, contents_shinychat)), "html")
   } else {
-    jsonlite::toJSON(x@value, auto_unbox = TRUE, pretty = 2, force = TRUE)
+    as_tool_display(
+      jsonlite::toJSON(value, auto_unbox = TRUE, pretty = 2, force = TRUE)
+    )
   }
 }
 
+is_content_extra <- function(x) {
+  S7::S7_inherits(x, ellmer::ContentImage) ||
+    S7::S7_inherits(x, ellmer::ContentPDF)
+}
 
 S7::method(contents_shinychat, ellmer::Turn) <- function(content) {
   # Process all contents in the turn, filtering out empty results
@@ -427,3 +457,5 @@ S7::method(contents_shinychat, S7::new_S3_class(c("Chat", "R6"))) <- function(
 
   compact(messages)
 }
+
+ICON_FILE_PDF_FILL <- "<svg xmlns=\"http://www.w3.org/2000/svg\" viewBox=\"0 0 16 16\" class=\"bi bi-file-pdf-fill \" style=\"height:1em;width:1em;fill:currentColor;vertical-align:-0.125em;\" aria-hidden=\"true\" role=\"img\" ><path d=\"M5.523 10.424c.14-.082.293-.162.459-.238a7.878 7.878 0 0 1-.45.606c-.28.337-.498.516-.635.572a.266.266 0 0 1-.035.012.282.282 0 0 1-.026-.044c-.056-.11-.054-.216.04-.36.106-.165.319-.354.647-.548zm2.455-1.647c-.119.025-.237.05-.356.078a21.035 21.035 0 0 0 .5-1.05 11.96 11.96 0 0 0 .51.858c-.217.032-.436.07-.654.114zm2.525.939a3.888 3.888 0 0 1-.435-.41c.228.005.434.022.612.054.317.057.466.147.518.209a.095.095 0 0 1 .026.064.436.436 0 0 1-.06.2.307.307 0 0 1-.094.124.107.107 0 0 1-.069.015c-.09-.003-.258-.066-.498-.256zM8.278 4.97c-.04.244-.108.524-.2.829a4.86 4.86 0 0 1-.089-.346c-.076-.353-.087-.63-.046-.822.038-.177.11-.248.196-.283a.517.517 0 0 1 .145-.04c.013.03.028.092.032.198.005.122-.007.277-.038.465z\"></path>\n<path fill-rule=\"evenodd\" d=\"M4 0h8a2 2 0 0 1 2 2v12a2 2 0 0 1-2 2H4a2 2 0 0 1-2-2V2a2 2 0 0 1 2-2zm.165 11.668c.09.18.23.343.438.419.207.075.412.04.58-.03.318-.13.635-.436.926-.786.333-.401.683-.927 1.021-1.51a11.64 11.64 0 0 1 1.997-.406c.3.383.61.713.91.95.28.22.603.403.934.417a.856.856 0 0 0 .51-.138c.155-.101.27-.247.354-.416.09-.181.145-.37.138-.563a.844.844 0 0 0-.2-.518c-.226-.27-.596-.4-.96-.465a5.76 5.76 0 0 0-1.335-.05 10.954 10.954 0 0 1-.98-1.686c.25-.66.437-1.284.52-1.794.036-.218.055-.426.048-.614a1.238 1.238 0 0 0-.127-.538.7.7 0 0 0-.477-.365c-.202-.043-.41 0-.601.077-.377.15-.576.47-.651.823-.073.34-.04.736.046 1.136.088.406.238.848.43 1.295a19.707 19.707 0 0 1-1.062 2.227 7.662 7.662 0 0 0-1.482.645c-.37.22-.699.48-.897.787-.21.326-.275.714-.08 1.103z\"></path></svg>"