Support "provider-defined" tools in DurableAgent

Closes #433.

Author: TooTallNate

.changeset/eleven-breads-unite.md

@@ -0,0 +1,5 @@
+---
+"@workflow/ai": patch
+---
+
+Support "provider-defined" tools in `DurableAgent`

packages/ai/src/agent/do-stream-step.ts

@@ -4,6 +4,7 @@ import type {
   LanguageModelV2Prompt,
   LanguageModelV2StreamPart,
   LanguageModelV2ToolCall,
+  LanguageModelV2ToolResultPart,
 } from '@ai-sdk/provider';
 import {
   gateway,
@@ -38,6 +39,7 @@ export async function doStreamStep(
       'Invalid "model initialization" argument. Must be a string or a function that returns a LanguageModelV2 instance.'
     );
   }
+  console.log('doStreamStep tools', tools);
 
   const result = await model.doStream({
     prompt: conversationPrompt,
@@ -46,17 +48,34 @@ export async function doStreamStep(
 
   let finish: FinishPart | undefined;
   const toolCalls: LanguageModelV2ToolCall[] = [];
+  const providerToolResults: LanguageModelV2ToolResultPart[] = [];
   const chunks: LanguageModelV2StreamPart[] = [];
 
   await result.stream
     .pipeThrough(
       new TransformStream({
         transform(chunk, controller) {
+          console.log('doStreamStep chunk', chunk);
           if (chunk.type === 'tool-call') {
             toolCalls.push({
               ...chunk,
               input: chunk.input || '{}',
             });
+          } else if (chunk.type === 'tool-result') {
+            // Capture provider-executed tool results from the stream
+            // Convert from stream result format (result: unknown) to prompt format (output: LanguageModelV2ToolResultOutput)
+            providerToolResults.push({
+              type: 'tool-result',
+              toolCallId: chunk.toolCallId,
+              toolName: chunk.toolName,
+              output: {
+                type: 'text',
+                value:
+                  typeof chunk.result === 'string'
+                    ? chunk.result
+                    : JSON.stringify(chunk.result),
+              },
+            });
           } else if (chunk.type === 'finish') {
             finish = chunk;
           }
@@ -380,7 +399,7 @@ export async function doStreamStep(
   // }
 
   const step = chunksToStep(chunks, toolCalls, conversationPrompt, finish);
-  return { toolCalls, finish, step };
+  return { toolCalls, providerToolResults, finish, step };
 }
 
 // This is a stand-in for logic in the AI-SDK streamText code which aggregates

packages/ai/src/agent/durable-agent.ts

@@ -16,6 +16,18 @@ import {
 import { convertToLanguageModelPrompt, standardizePrompt } from 'ai/internal';
 import { FatalError } from 'workflow';
 import { streamTextIterator } from './stream-text-iterator.js';
+import type { ProviderDefinedTool } from './tools-to-model-tools.js';
+
+// Re-export for convenience
+export type { ProviderDefinedTool } from './tools-to-model-tools.js';
+
+/**
+ * Extended tool set that supports both user-defined and provider-defined tools.
+ */
+export type DurableAgentToolSet = Record<
+  string,
+  ToolSet[string] | ProviderDefinedTool
+>;
 
 /**
  * Information passed to the prepareStep callback.
@@ -84,8 +96,12 @@ export interface DurableAgentOptions {
    * A set of tools available to the agent.
    * Tools can be implemented as workflow steps for automatic retries and persistence,
    * or as regular workflow-level logic using core library features like sleep() and Hooks.
+   *
+   * Supports both user-defined tools (with `execute` functions) and provider-defined tools
+   * (with `type: 'provider-defined'`). Provider-defined tools are executed by the model
+   * provider directly (e.g., Anthropic's computer use tools).
    */
-  tools?: ToolSet;
+  tools?: DurableAgentToolSet;
 
   /**
    * Optional system prompt to guide the agent's behavior.
@@ -194,7 +210,7 @@ export interface DurableAgentStreamOptions<TTools extends ToolSet = ToolSet> {
  */
 export class DurableAgent {
   private model: string | (() => Promise<LanguageModelV2>);
-  private tools: ToolSet;
+  private tools: DurableAgentToolSet;
   private system?: string;
 
   constructor(options: DurableAgentOptions) {
@@ -283,17 +299,39 @@ async function closeStream(
   }
 }
 
+function isProviderDefinedTool(
+  tool: DurableAgentToolSet[string]
+): tool is ProviderDefinedTool {
+  return (
+    typeof tool === 'object' &&
+    tool !== null &&
+    'type' in tool &&
+    tool.type === 'provider-defined'
+  );
+}
+
 async function executeTool(
   toolCall: LanguageModelV2ToolCall,
-  tools: ToolSet,
+  tools: DurableAgentToolSet,
   messages: LanguageModelV2Prompt
 ): Promise<LanguageModelV2ToolResultPart> {
   const tool = tools[toolCall.toolName];
   if (!tool) throw new Error(`Tool "${toolCall.toolName}" not found`);
-  if (typeof tool.execute !== 'function')
+
+  // Provider-defined tools should not reach here as they are filtered out earlier
+  // and executed by the provider. This is a safety check.
+  if (isProviderDefinedTool(tool)) {
+    throw new Error(
+      `Tool "${toolCall.toolName}" is a provider-defined tool and should be executed by the provider, not the client`
+    );
+  }
+
+  if (typeof tool.execute !== 'function') {
     throw new Error(
       `Tool "${toolCall.toolName}" does not have an execute function`
     );
+  }
+
   const schema = asSchema(tool.inputSchema);
   const input = await schema.validate?.(JSON.parse(toolCall.input || '{}'));
   if (!input?.success) {

packages/ai/src/agent/stream-text-iterator.ts

@@ -7,11 +7,13 @@ import type {
 import type {
   StepResult,
   StreamTextOnStepFinishCallback,
-  ToolSet,
   UIMessageChunk,
 } from 'ai';
 import { doStreamStep, type ModelStopCondition } from './do-stream-step.js';
-import type { PrepareStepCallback } from './durable-agent.js';
+import type {
+  DurableAgentToolSet,
+  PrepareStepCallback,
+} from './durable-agent.js';
 import { toolsToModelTools } from './tools-to-model-tools.js';
 
 /**
@@ -37,7 +39,7 @@ export async function* streamTextIterator({
   prepareStep,
 }: {
   prompt: LanguageModelV2Prompt;
-  tools: ToolSet;
+  tools: DurableAgentToolSet;
   writable: WritableStream<UIMessageChunk>;
   model: string | (() => Promise<LanguageModelV2>);
   stopConditions?: ModelStopCondition[] | ModelStopCondition;
@@ -76,7 +78,7 @@ export async function* streamTextIterator({
       }
     }
 
-    const { toolCalls, finish, step } = await doStreamStep(
+    const { toolCalls, providerToolResults, finish, step } = await doStreamStep(
       conversationPrompt,
       currentModel,
       writable,
@@ -101,15 +103,29 @@ export async function* streamTextIterator({
         })),
       });
 
-      // Yield the tool calls along with the current conversation messages
-      // This allows executeTool to pass the conversation context to tool execute functions
-      const toolResults = yield { toolCalls, messages: conversationPrompt };
+      // Filter to only client-executed tool calls (not provider-executed)
+      const clientToolCalls = toolCalls.filter(
+        (toolCall) => !toolCall.providerExecuted
+      );
+
+      // Get client tool results by yielding only the client tool calls
+      let clientToolResults: LanguageModelV2ToolResultPart[] = [];
+      if (clientToolCalls.length > 0) {
+        // Yield the tool calls along with the current conversation messages
+        // This allows executeTool to pass the conversation context to tool execute functions
+        clientToolResults = yield {
+          toolCalls: clientToolCalls,
+          messages: conversationPrompt,
+        };
+        await writeToolOutputToUI(writable, clientToolResults);
+      }
 
-      await writeToolOutputToUI(writable, toolResults);
+      // Merge provider tool results with client tool results
+      const allToolResults = [...providerToolResults, ...clientToolResults];
 
       conversationPrompt.push({
         role: 'tool',
-        content: toolResults,
+        content: allToolResults,
       });
 
       if (stopConditions) {

packages/ai/src/agent/tools-to-model-tools.ts

@@ -1,13 +1,60 @@
-import type { LanguageModelV2FunctionTool } from '@ai-sdk/provider';
+import type {
+  LanguageModelV2FunctionTool,
+  LanguageModelV2ProviderDefinedTool,
+} from '@ai-sdk/provider';
 import { asSchema, type ToolSet } from 'ai';
 
+/**
+ * A provider-defined tool configuration.
+ * These tools are executed by the provider (e.g., Anthropic's computer use)
+ * rather than client-side.
+ */
+export interface ProviderDefinedTool {
+  type: 'provider-defined';
+  /**
+   * The ID of the tool. Should follow the format `<provider-name>.<unique-tool-name>`.
+   */
+  id: `${string}.${string}`;
+  /**
+   * Optional description of the tool.
+   */
+  description?: string;
+  /**
+   * The arguments for configuring the tool.
+   */
+  args?: Record<string, unknown>;
+}
+
+type ToolEntry = ToolSet[string] | ProviderDefinedTool;
+
+function isProviderDefinedTool(tool: ToolEntry): tool is ProviderDefinedTool {
+  return (
+    typeof tool === 'object' &&
+    tool !== null &&
+    'type' in tool &&
+    tool.type === 'provider-defined'
+  );
+}
+
 export function toolsToModelTools(
-  tools: ToolSet
-): LanguageModelV2FunctionTool[] {
-  return Object.entries(tools).map(([name, tool]) => ({
-    type: 'function',
-    name,
-    description: tool.description,
-    inputSchema: asSchema(tool.inputSchema).jsonSchema,
-  }));
+  tools: Record<string, ToolEntry>
+): Array<LanguageModelV2FunctionTool | LanguageModelV2ProviderDefinedTool> {
+  return Object.entries(tools).map(([name, tool]) => {
+    if (isProviderDefinedTool(tool)) {
+      return {
+        type: 'provider-defined' as const,
+        id: tool.id,
+        name,
+        args: tool.args ?? {},
+      };
+    }
+
+    // User-defined function tool
+    return {
+      type: 'function' as const,
+      name,
+      description: tool.description,
+      inputSchema: asSchema(tool.inputSchema).jsonSchema,
+    };
+  });
 }