From 58976ed02c5a3ce696aa30c33f2c0236a497dd0c Mon Sep 17 00:00:00 2001 From: centdix Date: Wed, 13 Aug 2025 17:54:28 +0200 Subject: [PATCH 1/7] draft --- .../copilot/chat/AIChatManager.svelte.ts | 24 ++- .../lib/components/copilot/chat/flow/core.ts | 137 ++++++++++++++++++ 2 files changed, 156 insertions(+), 5 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 67dcd2e8f8fda..523732bd7bae4 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -8,7 +8,15 @@ import { } from './flow/core' import ContextManager from './ContextManager.svelte' import HistoryManager from './HistoryManager.svelte' -import { extractCodeFromMarkdown, getLatestAssistantMessage, processToolCall, type DisplayMessage, type Tool, type ToolCallbacks, type ToolDisplayMessage } from './shared' +import { + extractCodeFromMarkdown, + getLatestAssistantMessage, + processToolCall, + type DisplayMessage, + type Tool, + type ToolCallbacks, + type ToolDisplayMessage +} from './shared' import type { ChatCompletionChunk, ChatCompletionMessageParam, @@ -80,7 +88,9 @@ class AIChatManager { helpers = $state(undefined) scriptEditorOptions = $state(undefined) - scriptEditorApplyCode = $state<((code: string, applyAll?: boolean) => void) | undefined>(undefined) + scriptEditorApplyCode = $state<((code: string, applyAll?: boolean) => void) | undefined>( + undefined + ) scriptEditorShowDiffMode = $state<(() => void) | undefined>(undefined) flowAiChatHelpers = $state(undefined) pendingNewCode = $state(undefined) @@ -660,7 +670,11 @@ class AIChatManager { } switch (this.mode) { case AIMode.FLOW: - userMessage = prepareFlowUserMessage(oldInstructions, this.flowAiChatHelpers!.getFlowAndSelectedId()) + userMessage = prepareFlowUserMessage( + oldInstructions, + this.flowAiChatHelpers!.getFlowAndSelectedId() + ) + console.log('userMessage', userMessage) break case AIMode.NAVIGATOR: userMessage = prepareNavigatorUserMessage(oldInstructions) @@ -733,8 +747,8 @@ class AIChatManager { } else { // Create new tool message with metadata const newMessage: ToolDisplayMessage = { - role: 'tool', - tool_call_id: id, + role: 'tool', + tool_call_id: id, content: metadata?.content ?? metadata?.error ?? '', ...(metadata || {}) } diff --git a/frontend/src/lib/components/copilot/chat/flow/core.ts b/frontend/src/lib/components/copilot/chat/flow/core.ts index 9c2ff2af27125..45a57e3e398fd 100644 --- a/frontend/src/lib/components/copilot/chat/flow/core.ts +++ b/frontend/src/lib/components/copilot/chat/flow/core.ts @@ -351,6 +351,17 @@ const testRunFlowToolDef = createToolDef( 'Execute a test run of the current flow' ) +const testRunStepSchema = z.object({ + stepId: z.string().describe('The id of the step to test'), + args: z.record(z.any()).optional().describe('Arguments to pass to the step (optional, uses default step inputs if not provided)') +}) + +const testRunStepToolDef = createToolDef( + testRunStepSchema, + 'test_run_step', + 'Execute a test run of a specific step in the flow' +) + const workspaceScriptsSearch = new WorkspaceScriptsSearch() export const flowTools: Tool[] = [ @@ -591,6 +602,131 @@ export const flowTools: Tool[] = [ }, requiresConfirmation: true, showDetails: true + }, + { + def: testRunStepToolDef, + fn: async ({ args, workspace, helpers, toolCallbacks, toolId }) => { + const { flow } = helpers.getFlowAndSelectedId() + + if (!flow || !flow.value) { + toolCallbacks.setToolStatus(toolId, { + content: 'No flow available to test step from', + error: 'No flow found in current context' + }) + throw new Error('No flow available to test step from. Please ensure you have a flow open in the editor.') + } + + const parsedArgs = testRunStepSchema.parse(args) + const stepId = parsedArgs.stepId + const stepArgs = parsedArgs.args || {} + + // Find the step in the flow + const modules = helpers.getModules() + let targetModule: FlowModule | undefined = undefined + + // Check main modules + targetModule = modules.find(m => m.id === stepId) + + // Check preprocessor module + if (!targetModule && flow.value.preprocessor_module?.id === stepId) { + targetModule = flow.value.preprocessor_module + } + + // Check failure module + if (!targetModule && flow.value.failure_module?.id === stepId) { + targetModule = flow.value.failure_module + } + + if (!targetModule) { + toolCallbacks.setToolStatus(toolId, { + content: `Step '${stepId}' not found in flow`, + error: `Step with id '${stepId}' does not exist in the current flow` + }) + throw new Error(`Step with id '${stepId}' not found in flow. Available steps: ${modules.map(m => m.id).join(', ')}`) + } + + const module = targetModule + const moduleValue = module.value + + // Get step inputs if no args provided + let finalArgs = stepArgs + if (Object.keys(stepArgs).length === 0) { + try { + finalArgs = await helpers.getStepInputs(stepId) + } catch (error) { + console.warn('Could not get step inputs, using empty args:', error) + finalArgs = {} + } + } + + if (moduleValue.type === 'rawscript') { + // Test raw script step + return executeTestRun({ + jobStarter: () => JobService.runScriptPreview({ + workspace: workspace, + requestBody: { + content: moduleValue.content ?? '', + language: moduleValue.language, + args: module.id === 'preprocessor' ? { _ENTRYPOINT_OVERRIDE: 'preprocessor', ...finalArgs } : finalArgs, + tag: flow.tag || moduleValue.tag + } + }), + workspace, + toolCallbacks, + toolId, + startMessage: `Starting test run of step '${stepId}'...`, + contextName: 'script' + }) + } else if (moduleValue.type === 'script') { + // Test script step - need to get the script content + let script + if (moduleValue.hash) { + script = await ScriptService.getScriptByHash({ workspace: workspace, hash: moduleValue.hash }) + } else { + script = await ScriptService.getScriptByPath({ workspace: workspace, path: moduleValue.path }) + } + + return executeTestRun({ + jobStarter: () => JobService.runScriptPreview({ + workspace: workspace, + requestBody: { + content: script.content, + language: script.language, + args: module.id === 'preprocessor' ? { _ENTRYPOINT_OVERRIDE: 'preprocessor', ...finalArgs } : finalArgs, + tag: flow.tag || (moduleValue.tag_override ? moduleValue.tag_override : script.tag), + lock: script.lock + } + }), + workspace, + toolCallbacks, + toolId, + startMessage: `Starting test run of script step '${stepId}'...`, + contextName: 'script' + }) + } else if (moduleValue.type === 'flow') { + // Test flow step + return executeTestRun({ + jobStarter: () => JobService.runFlowByPath({ + workspace: workspace, + path: moduleValue.path, + requestBody: finalArgs + }), + workspace, + toolCallbacks, + toolId, + startMessage: `Starting test run of flow step '${stepId}'...`, + contextName: 'flow' + }) + } else { + toolCallbacks.setToolStatus(toolId, { + content: `Step type '${moduleValue.type}' not supported for testing`, + error: `Cannot test step of type '${moduleValue.type}'` + }) + throw new Error(`Cannot test step of type '${moduleValue.type}'. Supported types: rawscript, script, flow`) + } + }, + requiresConfirmation: true, + showDetails: true } ] @@ -600,6 +736,7 @@ Follow the user instructions carefully. Go step by step, and explain what you're doing as you're doing it. DO NOT wait for user confirmation before performing an action. Only do it if the user explicitly asks you to wait in their initial instructions. ALWAYS use the \`test_run_flow\` tool to test the flow, and iterate on the flow until it works as expected. If the user cancels the test run, do not try again and wait for the next user instruction. +Use the \`test_run_step\` tool to test individual steps of the flow when debugging or validating specific step functionality. ## Understanding User Requests From 7edd38f2c755aa6754fb615a4aa611c2d1c23c4c Mon Sep 17 00:00:00 2001 From: centdix Date: Wed, 13 Aug 2025 18:34:52 +0200 Subject: [PATCH 2/7] cleaning --- .../lib/components/copilot/chat/flow/core.ts | 168 ++++++++++-------- 1 file changed, 89 insertions(+), 79 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/flow/core.ts b/frontend/src/lib/components/copilot/chat/flow/core.ts index 45a57e3e398fd..f3849cc905b53 100644 --- a/frontend/src/lib/components/copilot/chat/flow/core.ts +++ b/frontend/src/lib/components/copilot/chat/flow/core.ts @@ -353,7 +353,10 @@ const testRunFlowToolDef = createToolDef( const testRunStepSchema = z.object({ stepId: z.string().describe('The id of the step to test'), - args: z.record(z.any()).optional().describe('Arguments to pass to the step (optional, uses default step inputs if not provided)') + args: z + .record(z.any()) + .optional() + .describe('Arguments to pass to the step (optional, uses default step inputs if not provided)') }) const testRunStepToolDef = createToolDef( @@ -375,7 +378,8 @@ export const flowTools: Tool[] = [ const parsedArgs = searchScriptsSchema.parse(args) const scriptResults = await workspaceScriptsSearch.search(parsedArgs.query, workspace) toolCallbacks.setToolStatus(toolId, { - content: 'Found ' + + content: + 'Found ' + scriptResults.length + ' scripts in the workspace related to "' + args.query + @@ -389,19 +393,20 @@ export const flowTools: Tool[] = [ fn: async ({ args, helpers, toolId, toolCallbacks }) => { const parsedArgs = addStepSchema.parse(args) toolCallbacks.setToolStatus(toolId, { - content: parsedArgs.location.type === 'after' - ? `Adding a step after step '${parsedArgs.location.afterId}'` - : parsedArgs.location.type === 'start' - ? 'Adding a step at the start' - : parsedArgs.location.type === 'start_inside_forloop' - ? `Adding a step at the start of the forloop step '${parsedArgs.location.inside}'` - : parsedArgs.location.type === 'start_inside_branch' - ? `Adding a step at the start of the branch ${parsedArgs.location.branchIndex + 1} of step '${parsedArgs.location.inside}'` - : parsedArgs.location.type === 'preprocessor' - ? 'Adding a preprocessor step' - : parsedArgs.location.type === 'failure' - ? 'Adding a failure step' - : 'Adding a step' + content: + parsedArgs.location.type === 'after' + ? `Adding a step after step '${parsedArgs.location.afterId}'` + : parsedArgs.location.type === 'start' + ? 'Adding a step at the start' + : parsedArgs.location.type === 'start_inside_forloop' + ? `Adding a step at the start of the forloop step '${parsedArgs.location.inside}'` + : parsedArgs.location.type === 'start_inside_branch' + ? `Adding a step at the start of the branch ${parsedArgs.location.branchIndex + 1} of step '${parsedArgs.location.inside}'` + : parsedArgs.location.type === 'preprocessor' + ? 'Adding a preprocessor step' + : parsedArgs.location.type === 'failure' + ? 'Adding a failure step' + : 'Adding a step' }) const id = await helpers.insertStep(parsedArgs.location, parsedArgs.step) helpers.selectStep(id) @@ -480,7 +485,9 @@ export const flowTools: Tool[] = [ def: setCodeToolDef, fn: async ({ args, helpers, toolId, toolCallbacks }) => { const parsedArgs = setCodeSchema.parse(args) - toolCallbacks.setToolStatus(toolId, { content: `Setting code for step '${parsedArgs.id}'...` }) + toolCallbacks.setToolStatus(toolId, { + content: `Setting code for step '${parsedArgs.id}'...` + }) await helpers.setCode(parsedArgs.id, parsedArgs.code) helpers.selectStep(parsedArgs.id) toolCallbacks.setToolStatus(toolId, { content: `Set code for step '${parsedArgs.id}'` }) @@ -530,7 +537,9 @@ export const flowTools: Tool[] = [ const parsedArgs = setForLoopIteratorExpressionSchema.parse(args) await helpers.setForLoopIteratorExpression(parsedArgs.id, parsedArgs.expression) helpers.selectStep(parsedArgs.id) - toolCallbacks.setToolStatus(toolId, { content: `Set forloop '${parsedArgs.id}' iterator expression` }) + toolCallbacks.setToolStatus(toolId, { + content: `Set forloop '${parsedArgs.id}' iterator expression` + }) return `Forloop '${parsedArgs.id}' iterator expression set` } }, @@ -546,7 +555,9 @@ export const flowTools: Tool[] = [ parsedArgs.query, workspace ) - toolCallbacks.setToolStatus(toolId, { content: 'Retrieved resource types for "' + parsedArgs.query + '"' }) + toolCallbacks.setToolStatus(toolId, { + content: 'Retrieved resource types for "' + parsedArgs.query + '"' + }) return formattedResourceTypes } }, @@ -556,11 +567,13 @@ export const flowTools: Tool[] = [ const { flow } = helpers.getFlowAndSelectedId() if (!flow || !flow.value) { - toolCallbacks.setToolStatus(toolId, { + toolCallbacks.setToolStatus(toolId, { content: 'No flow available to test', error: 'No flow found in current context' }) - throw new Error('No flow available to test. Please ensure you have a flow open in the editor.') + throw new Error( + 'No flow available to test. Please ensure you have a flow open in the editor.' + ) } return executeTestRun({ @@ -607,13 +620,15 @@ export const flowTools: Tool[] = [ def: testRunStepToolDef, fn: async ({ args, workspace, helpers, toolCallbacks, toolId }) => { const { flow } = helpers.getFlowAndSelectedId() - + if (!flow || !flow.value) { - toolCallbacks.setToolStatus(toolId, { + toolCallbacks.setToolStatus(toolId, { content: 'No flow available to test step from', error: 'No flow found in current context' }) - throw new Error('No flow available to test step from. Please ensure you have a flow open in the editor.') + throw new Error( + 'No flow available to test step from. Please ensure you have a flow open in the editor.' + ) } const parsedArgs = testRunStepSchema.parse(args) @@ -622,55 +637,37 @@ export const flowTools: Tool[] = [ // Find the step in the flow const modules = helpers.getModules() - let targetModule: FlowModule | undefined = undefined - - // Check main modules - targetModule = modules.find(m => m.id === stepId) - - // Check preprocessor module - if (!targetModule && flow.value.preprocessor_module?.id === stepId) { - targetModule = flow.value.preprocessor_module - } - - // Check failure module - if (!targetModule && flow.value.failure_module?.id === stepId) { - targetModule = flow.value.failure_module - } + let targetModule: FlowModule | undefined = modules.find((m) => m.id === stepId) if (!targetModule) { - toolCallbacks.setToolStatus(toolId, { + toolCallbacks.setToolStatus(toolId, { content: `Step '${stepId}' not found in flow`, error: `Step with id '${stepId}' does not exist in the current flow` }) - throw new Error(`Step with id '${stepId}' not found in flow. Available steps: ${modules.map(m => m.id).join(', ')}`) + throw new Error( + `Step with id '${stepId}' not found in flow. Available steps: ${modules.map((m) => m.id).join(', ')}` + ) } const module = targetModule const moduleValue = module.value - // Get step inputs if no args provided - let finalArgs = stepArgs - if (Object.keys(stepArgs).length === 0) { - try { - finalArgs = await helpers.getStepInputs(stepId) - } catch (error) { - console.warn('Could not get step inputs, using empty args:', error) - finalArgs = {} - } - } - if (moduleValue.type === 'rawscript') { // Test raw script step return executeTestRun({ - jobStarter: () => JobService.runScriptPreview({ - workspace: workspace, - requestBody: { - content: moduleValue.content ?? '', - language: moduleValue.language, - args: module.id === 'preprocessor' ? { _ENTRYPOINT_OVERRIDE: 'preprocessor', ...finalArgs } : finalArgs, - tag: flow.tag || moduleValue.tag - } - }), + jobStarter: () => + JobService.runScriptPreview({ + workspace: workspace, + requestBody: { + content: moduleValue.content ?? '', + language: moduleValue.language, + args: + module.id === 'preprocessor' + ? { _ENTRYPOINT_OVERRIDE: 'preprocessor', ...stepArgs } + : stepArgs, + tag: flow.tag || moduleValue.tag + } + }), workspace, toolCallbacks, toolId, @@ -681,22 +678,32 @@ export const flowTools: Tool[] = [ // Test script step - need to get the script content let script if (moduleValue.hash) { - script = await ScriptService.getScriptByHash({ workspace: workspace, hash: moduleValue.hash }) + script = await ScriptService.getScriptByHash({ + workspace: workspace, + hash: moduleValue.hash + }) } else { - script = await ScriptService.getScriptByPath({ workspace: workspace, path: moduleValue.path }) + script = await ScriptService.getScriptByPath({ + workspace: workspace, + path: moduleValue.path + }) } - + return executeTestRun({ - jobStarter: () => JobService.runScriptPreview({ - workspace: workspace, - requestBody: { - content: script.content, - language: script.language, - args: module.id === 'preprocessor' ? { _ENTRYPOINT_OVERRIDE: 'preprocessor', ...finalArgs } : finalArgs, - tag: flow.tag || (moduleValue.tag_override ? moduleValue.tag_override : script.tag), - lock: script.lock - } - }), + jobStarter: () => + JobService.runScriptPreview({ + workspace: workspace, + requestBody: { + content: script.content, + language: script.language, + args: + module.id === 'preprocessor' + ? { _ENTRYPOINT_OVERRIDE: 'preprocessor', ...stepArgs } + : stepArgs, + tag: flow.tag || (moduleValue.tag_override ? moduleValue.tag_override : script.tag), + lock: script.lock + } + }), workspace, toolCallbacks, toolId, @@ -706,11 +713,12 @@ export const flowTools: Tool[] = [ } else if (moduleValue.type === 'flow') { // Test flow step return executeTestRun({ - jobStarter: () => JobService.runFlowByPath({ - workspace: workspace, - path: moduleValue.path, - requestBody: finalArgs - }), + jobStarter: () => + JobService.runFlowByPath({ + workspace: workspace, + path: moduleValue.path, + requestBody: stepArgs + }), workspace, toolCallbacks, toolId, @@ -718,11 +726,13 @@ export const flowTools: Tool[] = [ contextName: 'flow' }) } else { - toolCallbacks.setToolStatus(toolId, { + toolCallbacks.setToolStatus(toolId, { content: `Step type '${moduleValue.type}' not supported for testing`, error: `Cannot test step of type '${moduleValue.type}'` }) - throw new Error(`Cannot test step of type '${moduleValue.type}'. Supported types: rawscript, script, flow`) + throw new Error( + `Cannot test step of type '${moduleValue.type}'. Supported types: rawscript, script, flow` + ) } }, requiresConfirmation: true, From b0f1fd113cc709732d99b8110b2ea1977dab71cc Mon Sep 17 00:00:00 2001 From: centdix Date: Sat, 16 Aug 2025 15:46:19 +0200 Subject: [PATCH 3/7] cleaning --- .../lib/components/copilot/chat/flow/core.ts | 69 ++++++------------- .../components/copilot/chat/script/core.ts | 26 ++----- .../src/lib/components/copilot/chat/shared.ts | 18 ++++- 3 files changed, 43 insertions(+), 70 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/flow/core.ts b/frontend/src/lib/components/copilot/chat/flow/core.ts index f3849cc905b53..353ab3c6efabe 100644 --- a/frontend/src/lib/components/copilot/chat/flow/core.ts +++ b/frontend/src/lib/components/copilot/chat/flow/core.ts @@ -12,7 +12,7 @@ import { getLangContext, SUPPORTED_CHAT_SCRIPT_LANGUAGES } from '../script/core' -import { createSearchHubScriptsTool, createToolDef, type Tool, executeTestRun } from '../shared' +import { createSearchHubScriptsTool, createToolDef, type Tool, executeTestRun, buildSchemaForTool } from '../shared' import type { ExtendedOpenFlow } from '$lib/components/flows/types' import { copilotSessionModel } from '$lib/stores' import { get } from 'svelte/store' @@ -354,7 +354,8 @@ const testRunFlowToolDef = createToolDef( const testRunStepSchema = z.object({ stepId: z.string().describe('The id of the step to test'), args: z - .record(z.any()) + .object({}) + .nullable() .optional() .describe('Arguments to pass to the step (optional, uses default step inputs if not provided)') }) @@ -592,32 +593,17 @@ export const flowTools: Tool[] = [ }) }, setSchema: async function(helpers: FlowAIChatHelpers) { - try { - if (this.def?.function?.parameters) { - const flowInputsSchema = await helpers.getFlowInputsSchema() - this.def.function.parameters = { ...flowInputsSchema, additionalProperties: false } - // OPEN AI models don't support strict mode well with schema with complex properties, so we disable it - const model = get(copilotSessionModel)?.provider - if (model === 'openai' || model === 'azure_openai') { - this.def.function.strict = false - } - } - } catch (e) { - console.error('Error setting schema for test_run_flow tool', e) - // fallback to schema with any properties - this.def.function.parameters = { - type: 'object', - properties: {}, - additionalProperties: true, - strict: false - } - } + await buildSchemaForTool(this.def, async () => { + const flowInputsSchema = await helpers.getFlowInputsSchema() + return flowInputsSchema + }) }, requiresConfirmation: true, showDetails: true }, { - def: testRunStepToolDef, + // set strict to false to avoid issues with open ai models + def: { ...testRunStepToolDef, function: { ...testRunStepToolDef.function, strict: false } }, fn: async ({ args, workspace, helpers, toolCallbacks, toolId }) => { const { flow } = helpers.getFlowAndSelectedId() @@ -661,11 +647,7 @@ export const flowTools: Tool[] = [ requestBody: { content: moduleValue.content ?? '', language: moduleValue.language, - args: - module.id === 'preprocessor' - ? { _ENTRYPOINT_OVERRIDE: 'preprocessor', ...stepArgs } - : stepArgs, - tag: flow.tag || moduleValue.tag + args: module.id === 'preprocessor' ? { _ENTRYPOINT_OVERRIDE: 'preprocessor', ...stepArgs } : stepArgs } }), workspace, @@ -676,18 +658,15 @@ export const flowTools: Tool[] = [ }) } else if (moduleValue.type === 'script') { // Test script step - need to get the script content - let script - if (moduleValue.hash) { - script = await ScriptService.getScriptByHash({ - workspace: workspace, - hash: moduleValue.hash - }) - } else { - script = await ScriptService.getScriptByPath({ - workspace: workspace, - path: moduleValue.path - }) - } + const script = moduleValue.hash + ? await ScriptService.getScriptByHash({ + workspace: workspace, + hash: moduleValue.hash + }) + : await ScriptService.getScriptByPath({ + workspace: workspace, + path: moduleValue.path + }) return executeTestRun({ jobStarter: () => @@ -696,12 +675,7 @@ export const flowTools: Tool[] = [ requestBody: { content: script.content, language: script.language, - args: - module.id === 'preprocessor' - ? { _ENTRYPOINT_OVERRIDE: 'preprocessor', ...stepArgs } - : stepArgs, - tag: flow.tag || (moduleValue.tag_override ? moduleValue.tag_override : script.tag), - lock: script.lock + args: module.id === 'preprocessor' ? { _ENTRYPOINT_OVERRIDE: 'preprocessor', ...stepArgs } : stepArgs, } }), workspace, @@ -745,8 +719,7 @@ export function prepareFlowSystemMessage(): ChatCompletionSystemMessageParam { Follow the user instructions carefully. Go step by step, and explain what you're doing as you're doing it. DO NOT wait for user confirmation before performing an action. Only do it if the user explicitly asks you to wait in their initial instructions. -ALWAYS use the \`test_run_flow\` tool to test the flow, and iterate on the flow until it works as expected. If the user cancels the test run, do not try again and wait for the next user instruction. -Use the \`test_run_step\` tool to test individual steps of the flow when debugging or validating specific step functionality. +ALWAYS test your modifications. You have access to the \`test_run_flow\` and \`test_run_step\` tools to test the flow and steps. If you only modified a single step, use the \`test_run_step\` tool to test it. If you modified the flow, use the \`test_run_flow\` tool to test it. If the user cancels the test run, do not try again and wait for the next user instruction. ## Understanding User Requests diff --git a/frontend/src/lib/components/copilot/chat/script/core.ts b/frontend/src/lib/components/copilot/chat/script/core.ts index 7b938528033b4..c65a80ef05ee5 100644 --- a/frontend/src/lib/components/copilot/chat/script/core.ts +++ b/frontend/src/lib/components/copilot/chat/script/core.ts @@ -13,7 +13,7 @@ import { scriptLangToEditorLang } from '$lib/scripts' import { getDbSchemas } from '$lib/components/apps/components/display/dbtable/utils' import type { CodePieceElement, ContextElement } from '../context' import { PYTHON_PREPROCESSOR_MODULE_CODE, TS_PREPROCESSOR_MODULE_CODE } from '$lib/script_helpers' -import { createSearchHubScriptsTool, type Tool, executeTestRun } from '../shared' +import { createSearchHubScriptsTool, type Tool, executeTestRun, buildSchemaForTool } from '../shared' import { setupTypeAcquisition, type DepsToGet } from '$lib/ata' import { getModelContextWindow } from '../../lib' import { inferArgs } from '$lib/infer' @@ -888,9 +888,6 @@ export const testRunScriptTool: Tool = { content: codeToTest, args, language: scriptOptions.lang as ScriptLang, - tag: undefined, - lock: undefined, - script_hash: undefined } }), workspace, @@ -901,7 +898,7 @@ export const testRunScriptTool: Tool = { }) }, setSchema: async function(helpers: ScriptChatHelpers) { - try { + await buildSchemaForTool(this.def, async () => { const scriptOptions = helpers.getScriptOptions() const code = scriptOptions?.code const lang = scriptOptions?.lang @@ -911,23 +908,10 @@ export const testRunScriptTool: Tool = { if (codeToTest) { const newSchema = emptySchema() await inferArgs(lang, codeToTest, newSchema) - this.def.function.parameters = { ...newSchema, additionalProperties: false } - // OPEN AI models don't support strict mode well with schema with complex properties, so we disable it - const model = get(copilotSessionModel)?.provider - if (model === 'openai' || model === 'azure_openai') { - this.def.function.strict = false - } - } - } catch (e) { - console.error('Error setting schema for test_run_script tool', e) - // fallback to schema with any properties - this.def.function.parameters = { - type: 'object', - properties: {}, - additionalProperties: true, - strict: false + return newSchema } - } + return emptySchema() + }) }, requiresConfirmation: true, showDetails: true, diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 0503f3a753469..68e3f6b06baba 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -5,7 +5,7 @@ import type { } from 'openai/resources/chat/completions.mjs' import { get } from 'svelte/store' import type { ContextElement } from './context' -import { workspaceStore } from '$lib/stores' +import { copilotSessionModel, workspaceStore } from '$lib/stores' import type { ExtendedOpenFlow } from '$lib/components/flows/types' import type { FunctionParameters } from 'openai/resources/shared.mjs' import { zodToJsonSchema } from 'zod-to-json-schema' @@ -253,6 +253,22 @@ export const createSearchHubScriptsTool = (withContent: boolean = false) => ({ } }) +export async function buildSchemaForTool(toolDef: ChatCompletionTool, schemaBuilder: () => Promise): Promise { + try { + const schema = await schemaBuilder() + toolDef.function.parameters = { ...schema, additionalProperties: false } + // OPEN AI models don't support strict mode well with schema with complex properties, so we disable it + const model = get(copilotSessionModel)?.provider + if (model === 'openai' || model === 'azure_openai') { + toolDef.function.strict = false + } + } catch (error) { + console.error('Error building schema for tool', error) + // fallback to schema with any properties + toolDef.function.parameters = { type: 'object', properties: {}, additionalProperties: true, strict: false } + } +} + // Constants for result formatting const MAX_RESULT_LENGTH = 12000 const MAX_LOG_LENGTH = 4000 From 72dff1067ed3eff8c25ed2702590c2bdfa5e4a4c Mon Sep 17 00:00:00 2001 From: centdix Date: Mon, 18 Aug 2025 11:53:49 +0200 Subject: [PATCH 4/7] fixes --- frontend/src/lib/components/copilot/chat/flow/core.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/flow/core.ts b/frontend/src/lib/components/copilot/chat/flow/core.ts index 353ab3c6efabe..2fc914d3f73b9 100644 --- a/frontend/src/lib/components/copilot/chat/flow/core.ts +++ b/frontend/src/lib/components/copilot/chat/flow/core.ts @@ -14,8 +14,6 @@ import { } from '../script/core' import { createSearchHubScriptsTool, createToolDef, type Tool, executeTestRun, buildSchemaForTool } from '../shared' import type { ExtendedOpenFlow } from '$lib/components/flows/types' -import { copilotSessionModel } from '$lib/stores' -import { get } from 'svelte/store' export type AIModuleAction = 'added' | 'modified' | 'removed' @@ -617,9 +615,8 @@ export const flowTools: Tool[] = [ ) } - const parsedArgs = testRunStepSchema.parse(args) - const stepId = parsedArgs.stepId - const stepArgs = parsedArgs.args || {} + const stepId = args.stepId + const stepArgs = args.args || {} // Find the step in the flow const modules = helpers.getModules() @@ -640,6 +637,7 @@ export const flowTools: Tool[] = [ if (moduleValue.type === 'rawscript') { // Test raw script step + return executeTestRun({ jobStarter: () => JobService.runScriptPreview({ From 928eef279eae0ea7ca7e5586fb31b3006f98e447 Mon Sep 17 00:00:00 2001 From: centdix Date: Mon, 18 Aug 2025 12:04:58 +0200 Subject: [PATCH 5/7] fix placeholders --- .../components/copilot/chat/AIChatDisplay.svelte | 1 - .../components/copilot/chat/AIChatInput.svelte | 16 +++++++++------- .../copilot/chat/AIChatManager.svelte.ts | 1 - 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index 51d0b6b1bfc1b..75489ecfaada3 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -247,7 +247,6 @@ {availableContext} {disabled} isFirstMessage={messages.length === 0} - placeholder={messages.length === 0 ? 'Ask anything' : 'Ask followup'} />
{ + if (!isFirstMessage) { + return 'Ask followup' + } + if (placeholder) { - // If a custom placeholder is provided, use it return placeholder } - // Generate placeholder based on current AI mode switch (aiChatManager.mode) { case AIMode.SCRIPT: - return 'Modify this script, fix errors, or generate new code...' + return 'Modify this script...' case AIMode.FLOW: - return 'Edit this flow, add steps, or modify workflow logic...' + return 'Modify this flow...' case AIMode.NAVIGATOR: - return 'Help me navigate Windmill or find features...' + return 'Navigate the app...' case AIMode.API: - return 'Make API calls to fetch data or manage resources...' + return 'Make API calls...' case AIMode.ASK: - return 'Ask questions about Windmill features and documentation...' + return 'Ask questions about Windmill...' default: return 'Ask anything' } diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 523732bd7bae4..3792bd1870cbd 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -674,7 +674,6 @@ class AIChatManager { oldInstructions, this.flowAiChatHelpers!.getFlowAndSelectedId() ) - console.log('userMessage', userMessage) break case AIMode.NAVIGATOR: userMessage = prepareNavigatorUserMessage(oldInstructions) From 698e03aa4f711a09297355a6cf35f610b022cbdb Mon Sep 17 00:00:00 2001 From: centdix Date: Mon, 18 Aug 2025 13:40:09 +0200 Subject: [PATCH 6/7] better fallback --- .../lib/components/copilot/chat/flow/core.ts | 7 +++-- .../components/copilot/chat/script/core.ts | 8 +++-- .../src/lib/components/copilot/chat/shared.ts | 31 +++++++++++++++++-- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/flow/core.ts b/frontend/src/lib/components/copilot/chat/flow/core.ts index 2fc914d3f73b9..c9f8867c2a7e0 100644 --- a/frontend/src/lib/components/copilot/chat/flow/core.ts +++ b/frontend/src/lib/components/copilot/chat/flow/core.ts @@ -12,7 +12,7 @@ import { getLangContext, SUPPORTED_CHAT_SCRIPT_LANGUAGES } from '../script/core' -import { createSearchHubScriptsTool, createToolDef, type Tool, executeTestRun, buildSchemaForTool } from '../shared' +import { createSearchHubScriptsTool, createToolDef, type Tool, executeTestRun, buildSchemaForTool, buildTestRunArgs } from '../shared' import type { ExtendedOpenFlow } from '$lib/components/flows/types' export type AIModuleAction = 'added' | 'modified' | 'removed' @@ -562,7 +562,7 @@ export const flowTools: Tool[] = [ }, { def: testRunFlowToolDef, - fn: async ({ args, workspace, helpers, toolCallbacks, toolId }) => { + fn: async function({ args, workspace, helpers, toolCallbacks, toolId }) { const { flow } = helpers.getFlowAndSelectedId() if (!flow || !flow.value) { @@ -575,11 +575,12 @@ export const flowTools: Tool[] = [ ) } + const parsedArgs = await buildTestRunArgs(args, this.def) return executeTestRun({ jobStarter: () => JobService.runFlowPreview({ workspace: workspace, requestBody: { - args: args, + args: parsedArgs, value: flow.value, } }), diff --git a/frontend/src/lib/components/copilot/chat/script/core.ts b/frontend/src/lib/components/copilot/chat/script/core.ts index c65a80ef05ee5..372debb49f632 100644 --- a/frontend/src/lib/components/copilot/chat/script/core.ts +++ b/frontend/src/lib/components/copilot/chat/script/core.ts @@ -13,7 +13,7 @@ import { scriptLangToEditorLang } from '$lib/scripts' import { getDbSchemas } from '$lib/components/apps/components/display/dbtable/utils' import type { CodePieceElement, ContextElement } from '../context' import { PYTHON_PREPROCESSOR_MODULE_CODE, TS_PREPROCESSOR_MODULE_CODE } from '$lib/script_helpers' -import { createSearchHubScriptsTool, type Tool, executeTestRun, buildSchemaForTool } from '../shared' +import { createSearchHubScriptsTool, type Tool, executeTestRun, buildSchemaForTool, buildTestRunArgs } from '../shared' import { setupTypeAcquisition, type DepsToGet } from '$lib/ata' import { getModelContextWindow } from '../../lib' import { inferArgs } from '$lib/infer' @@ -855,7 +855,7 @@ const TEST_RUN_SCRIPT_TOOL: ChatCompletionTool = { export const testRunScriptTool: Tool = { def: TEST_RUN_SCRIPT_TOOL, - fn: async ({ args, workspace, helpers, toolCallbacks, toolId }) => { + fn: async function({ args, workspace, helpers, toolCallbacks, toolId }) { const scriptOptions = helpers.getScriptOptions() if (!scriptOptions) { @@ -880,13 +880,15 @@ export const testRunScriptTool: Tool = { toolCallbacks.setToolStatus(toolId, { content: 'Code changes applied, starting test...' }) } + const parsedArgs = await buildTestRunArgs(args, this.def) + return executeTestRun({ jobStarter: () => JobService.runScriptPreview({ workspace: workspace, requestBody: { path: scriptOptions.path, content: codeToTest, - args, + args: parsedArgs, language: scriptOptions.lang as ScriptLang, } }), diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 68e3f6b06baba..5886e25ac6c02 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -190,6 +190,9 @@ export function createToolDef( name, target: 'openAi' }) + if (name === 'test_run_step') { + console.log('schema', schema) + } let parameters = schema.definitions![name] as FunctionParameters parameters = { ...parameters, @@ -253,19 +256,29 @@ export const createSearchHubScriptsTool = (withContent: boolean = false) => ({ } }) -export async function buildSchemaForTool(toolDef: ChatCompletionTool, schemaBuilder: () => Promise): Promise { +export async function buildSchemaForTool(toolDef: ChatCompletionTool, schemaBuilder: () => Promise): Promise { try { const schema = await schemaBuilder() + + // if schema properties contains values different from '^[a-zA-Z0-9_.-]{1,64}$' + const invalidProperties = Object.keys(schema.properties ?? {}).filter((key) => !/^[a-zA-Z0-9_.-]{1,64}$/.test(key)) + if (invalidProperties.length > 0) { + console.warn(`Invalid flow inputs schema: ${invalidProperties.join(', ')}`) + throw new Error(`Invalid flow inputs schema: ${invalidProperties.join(', ')}`) + } + toolDef.function.parameters = { ...schema, additionalProperties: false } // OPEN AI models don't support strict mode well with schema with complex properties, so we disable it const model = get(copilotSessionModel)?.provider if (model === 'openai' || model === 'azure_openai') { toolDef.function.strict = false } + return true } catch (error) { console.error('Error building schema for tool', error) // fallback to schema with any properties - toolDef.function.parameters = { type: 'object', properties: {}, additionalProperties: true, strict: false } + toolDef.function.parameters = { type: 'object', properties: { args: { type: 'string', description: 'JSON string containing the arguments for the tool' } }, additionalProperties: false, strict: false, required: ['args'] } + return false } } @@ -378,6 +391,20 @@ function getErrorMessage(result: unknown): string { return 'Unknown error' } +// Build test run args based on the tool definition, if it contains a fallback schema +export async function buildTestRunArgs(args: any, toolDef: ChatCompletionTool): Promise { + let parsedArgs = args + // if the schema is the fallback schema, parse the args as a JSON string + if ((toolDef.function.parameters as any).properties?.args?.description === 'JSON string containing the arguments for the tool') { + try { + parsedArgs = JSON.parse(args.args) + } catch (error) { + console.error('Error parsing arguments for tool', error) + } + } + return parsedArgs +} + // Main execution function for test runs export async function executeTestRun(config: TestRunConfig): Promise { try { From 332a145a9b2ecc264b8282956a406eb238151dfc Mon Sep 17 00:00:00 2001 From: centdix Date: Mon, 18 Aug 2025 13:54:03 +0200 Subject: [PATCH 7/7] cleaning --- frontend/src/lib/components/copilot/chat/shared.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 5886e25ac6c02..023b539d0b2a6 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -190,9 +190,6 @@ export function createToolDef( name, target: 'openAi' }) - if (name === 'test_run_step') { - console.log('schema', schema) - } let parameters = schema.definitions![name] as FunctionParameters parameters = { ...parameters, @@ -276,7 +273,7 @@ export async function buildSchemaForTool(toolDef: ChatCompletionTool, schemaBuil return true } catch (error) { console.error('Error building schema for tool', error) - // fallback to schema with any properties + // fallback to schema with args as a JSON string toolDef.function.parameters = { type: 'object', properties: { args: { type: 'string', description: 'JSON string containing the arguments for the tool' } }, additionalProperties: false, strict: false, required: ['args'] } return false }