From 02757d59fba48bfa5c3e13c4fc283596903d8a2c Mon Sep 17 00:00:00 2001 From: yousefed Date: Wed, 30 Jul 2025 21:26:04 +0200 Subject: [PATCH 1/3] add manual execution for streamtools --- .../09-ai/05-manual-execution/.bnexample.json | 15 ++ examples/09-ai/05-manual-execution/README.md | 3 + examples/09-ai/05-manual-execution/index.html | 14 ++ examples/09-ai/05-manual-execution/main.tsx | 11 ++ .../09-ai/05-manual-execution/package.json | 34 ++++ .../09-ai/05-manual-execution/src/App.tsx | 183 ++++++++++++++++++ .../09-ai/05-manual-execution/src/getEnv.ts | 20 ++ .../09-ai/05-manual-execution/src/styles.css | 15 ++ .../09-ai/05-manual-execution/tsconfig.json | 33 ++++ .../09-ai/05-manual-execution/vite.config.ts | 32 +++ packages/xl-ai/package.json | 3 +- packages/xl-ai/src/api/LLMResponse.ts | 27 +-- .../base-tools/createUpdateBlockTool.ts | 5 +- .../src/api/formats/html-blocks/htmlBlocks.ts | 2 + .../api/formats/json/tools/jsontools.test.ts | 1 + packages/xl-ai/src/index.ts | 1 + .../src/streamTool/StreamToolExecutor.ts | 151 +++++++++++++++ packages/xl-ai/src/streamTool/streamTool.ts | 10 +- playground/src/examples.gen.tsx | 29 +++ pnpm-lock.yaml | 31 ++- 20 files changed, 588 insertions(+), 32 deletions(-) create mode 100644 examples/09-ai/05-manual-execution/.bnexample.json create mode 100644 examples/09-ai/05-manual-execution/README.md create mode 100644 examples/09-ai/05-manual-execution/index.html create mode 100644 examples/09-ai/05-manual-execution/main.tsx create mode 100644 examples/09-ai/05-manual-execution/package.json create mode 100644 examples/09-ai/05-manual-execution/src/App.tsx create mode 100644 examples/09-ai/05-manual-execution/src/getEnv.ts create mode 100644 examples/09-ai/05-manual-execution/src/styles.css create mode 100644 examples/09-ai/05-manual-execution/tsconfig.json create mode 100644 examples/09-ai/05-manual-execution/vite.config.ts create mode 100644 packages/xl-ai/src/streamTool/StreamToolExecutor.ts diff --git a/examples/09-ai/05-manual-execution/.bnexample.json b/examples/09-ai/05-manual-execution/.bnexample.json new file mode 100644 index 0000000000..6f21dbcd55 --- /dev/null +++ b/examples/09-ai/05-manual-execution/.bnexample.json @@ -0,0 +1,15 @@ +{ + "playground": true, + "docs": false, + "author": "yousefed", + "tags": ["AI", "llm"], + "dependencies": { + "@blocknote/xl-ai": "latest", + "@mantine/core": "^7.17.3", + "ai": "^4.3.15", + "@ai-sdk/groq": "^1.2.9", + "y-partykit": "^0.0.25", + "yjs": "^13.6.27", + "zustand": "^5.0.3" + } +} diff --git a/examples/09-ai/05-manual-execution/README.md b/examples/09-ai/05-manual-execution/README.md new file mode 100644 index 0000000000..003f16a00c --- /dev/null +++ b/examples/09-ai/05-manual-execution/README.md @@ -0,0 +1,3 @@ +# AI manual execution + +Instead of calling AI models directly, this example shows how you can use an existing stream of responses and apply them to the editor diff --git a/examples/09-ai/05-manual-execution/index.html b/examples/09-ai/05-manual-execution/index.html new file mode 100644 index 0000000000..c63d224da9 --- /dev/null +++ b/examples/09-ai/05-manual-execution/index.html @@ -0,0 +1,14 @@ + + + + + AI manual execution + + + +
+ + + diff --git a/examples/09-ai/05-manual-execution/main.tsx b/examples/09-ai/05-manual-execution/main.tsx new file mode 100644 index 0000000000..677c7f7eed --- /dev/null +++ b/examples/09-ai/05-manual-execution/main.tsx @@ -0,0 +1,11 @@ +// AUTO-GENERATED FILE, DO NOT EDIT DIRECTLY +import React from "react"; +import { createRoot } from "react-dom/client"; +import App from "./src/App.jsx"; + +const root = createRoot(document.getElementById("root")!); +root.render( + + + +); diff --git a/examples/09-ai/05-manual-execution/package.json b/examples/09-ai/05-manual-execution/package.json new file mode 100644 index 0000000000..f47382c37e --- /dev/null +++ b/examples/09-ai/05-manual-execution/package.json @@ -0,0 +1,34 @@ +{ + "name": "@blocknote/example-ai-manual-execution", + "description": "AUTO-GENERATED FILE, DO NOT EDIT DIRECTLY", + "private": true, + "version": "0.12.4", + "scripts": { + "start": "vite", + "dev": "vite", + "build:prod": "tsc && vite build", + "preview": "vite preview" + }, + "dependencies": { + "@blocknote/core": "latest", + "@blocknote/react": "latest", + "@blocknote/ariakit": "latest", + "@blocknote/mantine": "latest", + "@blocknote/shadcn": "latest", + "react": "^19.1.0", + "react-dom": "^19.1.0", + "@blocknote/xl-ai": "latest", + "@mantine/core": "^7.17.3", + "ai": "^4.3.15", + "@ai-sdk/groq": "^1.2.9", + "y-partykit": "^0.0.25", + "yjs": "^13.6.27", + "zustand": "^5.0.3" + }, + "devDependencies": { + "@types/react": "^19.1.0", + "@types/react-dom": "^19.1.0", + "@vitejs/plugin-react": "^4.3.1", + "vite": "^5.3.4" + } +} \ No newline at end of file diff --git a/examples/09-ai/05-manual-execution/src/App.tsx b/examples/09-ai/05-manual-execution/src/App.tsx new file mode 100644 index 0000000000..38f6f958e7 --- /dev/null +++ b/examples/09-ai/05-manual-execution/src/App.tsx @@ -0,0 +1,183 @@ +import { BlockNoteEditor, filterSuggestionItems } from "@blocknote/core"; +import "@blocknote/core/fonts/inter.css"; +import { en } from "@blocknote/core/locales"; +import { BlockNoteView } from "@blocknote/mantine"; +import "@blocknote/mantine/style.css"; +import { + FormattingToolbar, + FormattingToolbarController, + SuggestionMenuController, + getDefaultReactSlashMenuItems, + getFormattingToolbarItems, + useCreateBlockNote, +} from "@blocknote/react"; +import { + AIToolbarButton, + StreamToolExecutor, + createAIExtension, + getAIExtension, + getAISlashMenuItems, + llmFormats, +} from "@blocknote/xl-ai"; +import { en as aiEn } from "@blocknote/xl-ai/locales"; +import "@blocknote/xl-ai/style.css"; + +export default function App() { + // Creates a new editor instance. + const editor = useCreateBlockNote({ + dictionary: { + ...en, + ai: aiEn, // add default translations for the AI extension + }, + // Register the AI extension + extensions: [ + createAIExtension({ + model: undefined as any, // disable model + }), + ], + // We set some initial content for demo purposes + initialContent: [ + { + type: "heading", + props: { + level: 1, + }, + content: "Open source software", + }, + { + type: "paragraph", + content: + "Open source software refers to computer programs whose source code is made available to the public, allowing anyone to view, modify, and distribute the code. This model stands in contrast to proprietary software, where the source code is kept secret and only the original creators have the right to make changes. Open projects are developed collaboratively, often by communities of developers from around the world, and are typically distributed under licenses that promote sharing and openness.", + }, + { + type: "paragraph", + content: + "One of the primary benefits of open source is the promotion of digital autonomy. By providing access to the source code, these programs empower users to control their own technology, customize software to fit their needs, and avoid vendor lock-in. This level of transparency also allows for greater security, as anyone can inspect the code for vulnerabilities or malicious elements. As a result, users are not solely dependent on a single company for updates, bug fixes, or continued support.", + }, + { + type: "paragraph", + content: + "Additionally, open development fosters innovation and collaboration. Developers can build upon existing projects, share improvements, and learn from each other, accelerating the pace of technological advancement. The open nature of these projects often leads to higher quality software, as bugs are identified and fixed more quickly by a diverse group of contributors. Furthermore, using open source can reduce costs for individuals, businesses, and governments, as it is often available for free and can be tailored to specific requirements without expensive licensing fees.", + }, + ], + }); + + // Renders the editor instance using a React component. + return ( +
+ + + {/* + + + */} +
+ {/*Inserts a new block at start of document.*/} + + +
+
+ ); +} + +// Formatting toolbar with the `AIToolbarButton` added +function FormattingToolbarWithAI() { + return ( + ( + + {...getFormattingToolbarItems()} + {/* Add the AI button */} + + + )} + /> + ); +} + +// Slash menu with the AI option added +function SuggestionMenuWithAI(props: { + editor: BlockNoteEditor; +}) { + return ( + + filterSuggestionItems( + [ + ...getDefaultReactSlashMenuItems(props.editor), + // add the default AI slash menu items, or define your own + ...getAISlashMenuItems(props.editor), + ], + query, + ) + } + /> + ); +} diff --git a/examples/09-ai/05-manual-execution/src/getEnv.ts b/examples/09-ai/05-manual-execution/src/getEnv.ts new file mode 100644 index 0000000000..b225fc462e --- /dev/null +++ b/examples/09-ai/05-manual-execution/src/getEnv.ts @@ -0,0 +1,20 @@ +// helper function to get env variables across next / vite +// only needed so this example works in BlockNote demos and docs +export function getEnv(key: string) { + const env = (import.meta as any).env + ? { + BLOCKNOTE_AI_SERVER_API_KEY: (import.meta as any).env + .VITE_BLOCKNOTE_AI_SERVER_API_KEY, + BLOCKNOTE_AI_SERVER_BASE_URL: (import.meta as any).env + .VITE_BLOCKNOTE_AI_SERVER_BASE_URL, + } + : { + BLOCKNOTE_AI_SERVER_API_KEY: + process.env.NEXT_PUBLIC_BLOCKNOTE_AI_SERVER_API_KEY, + BLOCKNOTE_AI_SERVER_BASE_URL: + process.env.NEXT_PUBLIC_BLOCKNOTE_AI_SERVER_BASE_URL, + }; + + const value = env[key as keyof typeof env]; + return value; +} diff --git a/examples/09-ai/05-manual-execution/src/styles.css b/examples/09-ai/05-manual-execution/src/styles.css new file mode 100644 index 0000000000..cc97b34a4f --- /dev/null +++ b/examples/09-ai/05-manual-execution/src/styles.css @@ -0,0 +1,15 @@ +.edit-buttons { + display: flex; + justify-content: space-between; + margin-top: 8px; +} + +.edit-button { + border: 1px solid gray; + border-radius: 4px; + padding-inline: 4px; +} + +.edit-button:hover { + border: 1px solid lightgrey; +} diff --git a/examples/09-ai/05-manual-execution/tsconfig.json b/examples/09-ai/05-manual-execution/tsconfig.json new file mode 100644 index 0000000000..3b74ef215c --- /dev/null +++ b/examples/09-ai/05-manual-execution/tsconfig.json @@ -0,0 +1,33 @@ +{ + "__comment": "AUTO-GENERATED FILE, DO NOT EDIT DIRECTLY", + "compilerOptions": { + "target": "ESNext", + "useDefineForClassFields": true, + "lib": ["DOM", "DOM.Iterable", "ESNext"], + "allowJs": false, + "skipLibCheck": true, + "esModuleInterop": false, + "allowSyntheticDefaultImports": true, + "strict": true, + "forceConsistentCasingInFileNames": true, + "module": "ESNext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "jsx": "react-jsx", + "composite": true + }, + "include": ["."], + "references": [ + { + "path": "../../../packages/core/" + }, + { + "path": "../../../packages/react/" + }, + { + "path": "../../../packages/xl-ai/" + } + ] +} diff --git a/examples/09-ai/05-manual-execution/vite.config.ts b/examples/09-ai/05-manual-execution/vite.config.ts new file mode 100644 index 0000000000..f62ab20bc2 --- /dev/null +++ b/examples/09-ai/05-manual-execution/vite.config.ts @@ -0,0 +1,32 @@ +// AUTO-GENERATED FILE, DO NOT EDIT DIRECTLY +import react from "@vitejs/plugin-react"; +import * as fs from "fs"; +import * as path from "path"; +import { defineConfig } from "vite"; +// import eslintPlugin from "vite-plugin-eslint"; +// https://vitejs.dev/config/ +export default defineConfig((conf) => ({ + plugins: [react()], + optimizeDeps: {}, + build: { + sourcemap: true, + }, + resolve: { + alias: + conf.command === "build" || + !fs.existsSync(path.resolve(__dirname, "../../packages/core/src")) + ? {} + : ({ + // Comment out the lines below to load a built version of blocknote + // or, keep as is to load live from sources with live reload working + "@blocknote/core": path.resolve( + __dirname, + "../../packages/core/src/" + ), + "@blocknote/react": path.resolve( + __dirname, + "../../packages/react/src/" + ), + } as any), + }, +})); diff --git a/packages/xl-ai/package.json b/packages/xl-ai/package.json index c28c8339f2..5e45a18fca 100644 --- a/packages/xl-ai/package.json +++ b/packages/xl-ai/package.json @@ -70,7 +70,8 @@ "@blocknote/react": "0.35.0", "@floating-ui/react": "^0.26.4", "@tiptap/core": "^2.12.0", - "ai": "^4.3.15", + "ai": "^4.3.19", + "@ai-sdk/ui-utils": "^1.2.11", "lodash.isequal": "^4.5.0", "prosemirror-changeset": "^2.3.0", "prosemirror-model": "^1.24.1", diff --git a/packages/xl-ai/src/api/LLMResponse.ts b/packages/xl-ai/src/api/LLMResponse.ts index 1321ab5b9d..46460b3e71 100644 --- a/packages/xl-ai/src/api/LLMResponse.ts +++ b/packages/xl-ai/src/api/LLMResponse.ts @@ -1,6 +1,7 @@ import { CoreMessage } from "ai"; import { OperationsResult } from "../streamTool/callLLMWithStreamTools.js"; -import { StreamTool, StreamToolCall } from "../streamTool/streamTool.js"; +import { StreamTool } from "../streamTool/streamTool.js"; +import { StreamToolExecutor } from "../streamTool/StreamToolExecutor.js"; /** * Result of an LLM call with stream tools that apply changes to a BlockNote Editor @@ -23,33 +24,15 @@ export class LLMResponse { private readonly streamTools: StreamTool[], ) {} - /** - * Apply the operations to the editor and return a stream of results. - * - * (this method consumes underlying streams in `llmResult`) - */ - async *applyToolCalls() { - let currentStream: AsyncIterable<{ - operation: StreamToolCall[]>; - isUpdateToPreviousOperation: boolean; - isPossiblyPartial: boolean; - }> = this.llmResult.operationsSource; - for (const tool of this.streamTools) { - currentStream = tool.execute(currentStream); - } - yield* currentStream; - } - /** * Helper method to apply all operations to the editor if you're not interested in intermediate operations and results. * * (this method consumes underlying streams in `llmResult`) */ public async execute() { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - for await (const _result of this.applyToolCalls()) { - // no op - } + const executor = new StreamToolExecutor(this.streamTools); + await executor.execute(this.llmResult.operationsSource); + await executor.waitTillEnd(); } /** diff --git a/packages/xl-ai/src/api/formats/base-tools/createUpdateBlockTool.ts b/packages/xl-ai/src/api/formats/base-tools/createUpdateBlockTool.ts index 6a81d0d469..7d22edab74 100644 --- a/packages/xl-ai/src/api/formats/base-tools/createUpdateBlockTool.ts +++ b/packages/xl-ai/src/api/formats/base-tools/createUpdateBlockTool.ts @@ -196,10 +196,11 @@ export function createUpdateBlockTool(config: { } const operation = chunk.operation as UpdateBlockToolCall; - + console.log("first op"); if (chunk.isPossiblyPartial) { const size = JSON.stringify(operation.block).length; if (size < minSize) { + console.log("skipping", size, minSize); continue; } else { // increase minSize for next chunk @@ -224,6 +225,7 @@ export function createUpdateBlockTool(config: { const jsonToolCall = await config.toJSONToolCall(editor, chunk); if (!jsonToolCall) { + console.log("no jsonToolCall"); continue; } @@ -242,6 +244,7 @@ export function createUpdateBlockTool(config: { // if there's only a single replace step to be done and we're partial, let's wait for more content // REC: unit test this and see if it's still needed even if we pass `dontReplaceContentAtEnd` to `updateToReplaceSteps` + console.log("skipping", steps.length, chunk.isPossiblyPartial); continue; } diff --git a/packages/xl-ai/src/api/formats/html-blocks/htmlBlocks.ts b/packages/xl-ai/src/api/formats/html-blocks/htmlBlocks.ts index e819271f58..c657d377d4 100644 --- a/packages/xl-ai/src/api/formats/html-blocks/htmlBlocks.ts +++ b/packages/xl-ai/src/api/formats/html-blocks/htmlBlocks.ts @@ -59,6 +59,8 @@ export const htmlBlockLLMFormat = { * Function to get the stream tools that can apply HTML block updates to the editor */ getStreamTools, + + streamTools: tools, /** * The default PromptBuilder that determines how a userPrompt is converted to an array of * LLM Messages (CoreMessage[]) diff --git a/packages/xl-ai/src/api/formats/json/tools/jsontools.test.ts b/packages/xl-ai/src/api/formats/json/tools/jsontools.test.ts index 065a09b5ac..ad7ee80d35 100644 --- a/packages/xl-ai/src/api/formats/json/tools/jsontools.test.ts +++ b/packages/xl-ai/src/api/formats/json/tools/jsontools.test.ts @@ -17,6 +17,7 @@ import { tools } from "./index.js"; import { getAIExtension } from "../../../../AIExtension.js"; import { getExpectedEditor } from "../../../../testUtil/cases/index.js"; import { validateRejectingResultsInOriginalDoc } from "../../../../testUtil/suggestChangesTestUtil.js"; + async function* createMockStream( ...operations: { operation: diff --git a/packages/xl-ai/src/index.ts b/packages/xl-ai/src/index.ts index 0f35bd8e3f..c002ac5418 100644 --- a/packages/xl-ai/src/index.ts +++ b/packages/xl-ai/src/index.ts @@ -13,3 +13,4 @@ export * from "./components/SuggestionMenu/getAISlashMenuItems.js"; export * from "./i18n/dictionary.js"; export * from "./api/index.js"; +export * from "./streamTool/StreamToolExecutor.js"; diff --git a/packages/xl-ai/src/streamTool/StreamToolExecutor.ts b/packages/xl-ai/src/streamTool/StreamToolExecutor.ts new file mode 100644 index 0000000000..283ab37d28 --- /dev/null +++ b/packages/xl-ai/src/streamTool/StreamToolExecutor.ts @@ -0,0 +1,151 @@ +import { parsePartialJson } from "@ai-sdk/ui-utils"; +import { + asyncIterableToStream, + createAsyncIterableStream, +} from "../util/stream.js"; +import { StreamTool, StreamToolCall } from "./streamTool.js"; + +// update previous + +function partialJsonToOperation( + chunk: string, + isUpdateToPreviousOperation: boolean, + streamTools: StreamTool[], +) { + const parsed = parsePartialJson(chunk); + + if (parsed.state === "undefined-input" || parsed.state === "failed-parse") { + return undefined; + } + + if (!parsed) { + return; + } + + const func = streamTools.find((f) => f.name === (parsed.value as any)?.type); + + const validated = func && func.validate(parsed.value); + + if (validated?.ok) { + return { + operation: parsed.value as StreamToolCall[]>, + isPossiblyPartial: parsed.state === "repaired-parse", + isUpdateToPreviousOperation, + }; + } else { + // no worries, probably a partial operation that's not valid yet + return; + } +} + +type Operation[] | StreamTool> = { + operation: StreamToolCall; + isUpdateToPreviousOperation: boolean; + isPossiblyPartial: boolean; +}; + +export class StreamToolExecutor[]> { + private readonly stream: TransformStream, Operation>; + private readonly readable: ReadableStream>; + + constructor(private streamTools: T) { + this.stream = this.createWriteStream(); + this.readable = this.createReadableStream(); + } + /** + * Returns a WritableStream that collects written chunks. + */ + private createWriteStream() { + let lastParsedResult: Operation | undefined; + + const stream = new TransformStream, Operation>({ + transform: (chunk, controller) => { + const operation = + typeof chunk === "string" + ? partialJsonToOperation( + chunk, + lastParsedResult?.isPossiblyPartial ?? false, + this.streamTools, + ) + : chunk; + if (operation) { + // TODO: string operations have been validated, but object-based operations have not. make this consistent? + controller.enqueue(operation); + } + }, + + // close: () => { + // if (lastParsedResult?.isPossiblyPartial) { + // throw new Error("stream ended with a partial operation"); + // } + // } + }); + + return stream; + } + + public get writable() { + return this.stream.writable; + } + + private createReadableStream() { + // TODO: this is a bit hacky as it mixes async iterables and streams + let currentStream: AsyncIterable[]>> = + createAsyncIterableStream(this.stream.readable); + for (const tool of this.streamTools) { + currentStream = tool.execute(currentStream); + } + + return asyncIterableToStream(currentStream); + } + + /** + * Helper method to apply all operations to the editor if you're not interested in intermediate operations and results. + * + * (this method consumes underlying streams in `llmResult`) + */ + public async waitTillEnd() { + const iterable = createAsyncIterableStream(this.readable); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (const _result of iterable) { + // no op + } + } + + /** + * Accepts an async iterable and writes each chunk to the internal stream. + */ + async execute(source: AsyncIterable>): Promise { + const writer = this.writable.getWriter(); + for await (const chunk of source) { + await writer.write(chunk); + } + await writer.close(); + } + + /** + * Accepts a single chunk and processes it using the same logic. + */ + async executeOne(chunk: StreamToolCall): Promise { + await this.execute( + (async function* () { + yield { + operation: chunk, + isUpdateToPreviousOperation: false, + isPossiblyPartial: false, + }; + })(), + ); + } +} + +/* TODO: + +AI SDK integration: +- pass tools +- server side +- stream tool outputs (partial-tool-data) + +Hook up custom executors + +*/ diff --git a/packages/xl-ai/src/streamTool/streamTool.ts b/packages/xl-ai/src/streamTool/streamTool.ts index d907d1315e..7257a1adc4 100644 --- a/packages/xl-ai/src/streamTool/streamTool.ts +++ b/packages/xl-ai/src/streamTool/streamTool.ts @@ -66,14 +66,14 @@ export type StreamToolCallSingle> = * * Its type is the same as what a validated StreamTool returns */ -export type StreamToolCall | StreamTool[]> = +export type StreamToolCall | readonly any[]> = T extends StreamTool ? U : // when passed an array of StreamTools, StreamToolCall represents the type of one of the StreamTool invocations - T extends StreamTool[] - ? T[number] extends StreamTool - ? V - : never + T extends readonly unknown[] + ? { + [K in keyof T]: T[K] extends StreamTool ? V : never; + }[number] : never; /** diff --git a/playground/src/examples.gen.tsx b/playground/src/examples.gen.tsx index 0cd32ceb3f..4cb9ee6d4e 100644 --- a/playground/src/examples.gen.tsx +++ b/playground/src/examples.gen.tsx @@ -1651,6 +1651,35 @@ "slug": "ai" }, "readme": "This example combines the AI extension with the ghost writer example to show how to use the AI extension in a collaborative environment.\n\n**Relevant Docs:**\n\n- [Editor Setup](/docs/getting-started/editor-setup)\n- [Changing the Formatting Toolbar](/docs/react/components/formatting-toolbar#changing-the-formatting-toolbar)\n- [Changing Slash Menu Items](/docs/react/components/suggestion-menus#changing-slash-menu-items)\n- [Getting Stared with BlockNote AI](/docs/features/ai/setup)" + }, + { + "projectSlug": "manual-execution", + "fullSlug": "ai/manual-execution", + "pathFromRoot": "examples/09-ai/05-manual-execution", + "config": { + "playground": true, + "docs": false, + "author": "yousefed", + "tags": [ + "AI", + "llm" + ], + "dependencies": { + "@blocknote/xl-ai": "latest", + "@mantine/core": "^7.17.3", + "ai": "^4.3.15", + "@ai-sdk/groq": "^1.2.9", + "y-partykit": "^0.0.25", + "yjs": "^13.6.27", + "zustand": "^5.0.3" + } as any + }, + "title": "AI manual execution", + "group": { + "pathFromRoot": "examples/09-ai", + "slug": "ai" + }, + "readme": "Instead of calling AI models directly, this example shows how you can use an existing stream of responses and apply them to the editor" } ] }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9657b2a136..d4293b7893 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4012,6 +4012,9 @@ importers: packages/xl-ai: dependencies: + '@ai-sdk/ui-utils': + specifier: ^1.2.11 + version: 1.2.11(zod@3.25.76) '@blocknote/core': specifier: 0.35.0 version: link:../core @@ -4031,8 +4034,8 @@ importers: specifier: ^2.12.0 version: 2.12.0(@tiptap/pm@2.12.0) ai: - specifier: ^4.3.15 - version: 4.3.15(react@19.1.0)(zod@3.25.76) + specifier: ^4.3.19 + version: 4.3.19(react@19.1.0)(zod@3.25.76) lodash.isequal: specifier: ^4.5.0 version: 4.5.0 @@ -9810,6 +9813,16 @@ packages: react: optional: true + ai@4.3.19: + resolution: {integrity: sha512-dIE2bfNpqHN3r6IINp9znguYdhIOheKW2LDigAMrgt/upT3B8eBGPSCblENvaZGoq+hxaN9fSMzjWpbqloP+7Q==} + engines: {node: '>=18'} + peerDependencies: + react: ^18 || ^19 || ^19.0.0-rc + zod: ^3.23.8 + peerDependenciesMeta: + react: + optional: true + ajv-formats@2.1.1: resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} peerDependencies: @@ -21074,6 +21087,18 @@ snapshots: optionalDependencies: react: 19.1.0 + ai@4.3.19(react@19.1.0)(zod@3.25.76): + dependencies: + '@ai-sdk/provider': 1.1.3 + '@ai-sdk/provider-utils': 2.2.8(zod@3.25.76) + '@ai-sdk/react': 1.2.12(react@19.1.0)(zod@3.25.76) + '@ai-sdk/ui-utils': 1.2.11(zod@3.25.76) + '@opentelemetry/api': 1.9.0 + jsondiffpatch: 0.6.0 + zod: 3.25.76 + optionalDependencies: + react: 19.1.0 + ajv-formats@2.1.1(ajv@8.17.1): optionalDependencies: ajv: 8.17.1 @@ -26691,7 +26716,7 @@ snapshots: dependencies: dequal: 2.0.3 react: 19.1.0 - use-sync-external-store: 1.4.0(react@19.1.0) + use-sync-external-store: 1.5.0(react@19.1.0) symbol-tree@3.2.4: {} From 3a278a1cdb23da601cee2ee5d1a1d10d58b8858b Mon Sep 17 00:00:00 2001 From: yousefed Date: Thu, 31 Jul 2025 12:20:29 +0200 Subject: [PATCH 2/3] update PR --- docs/content/docs/ai/reference.mdx | 6 +- docs/content/docs/features/ai/reference.mdx | 6 +- .../09-ai/05-manual-execution/src/App.tsx | 151 ++++++++++-------- packages/xl-ai/src/api/LLMRequest.ts | 6 +- .../base-tools/createUpdateBlockTool.ts | 4 - .../src/api/formats/html-blocks/htmlBlocks.ts | 55 ++++--- .../src/streamTool/StreamToolExecutor.ts | 140 +++++++++------- 7 files changed, 213 insertions(+), 155 deletions(-) diff --git a/docs/content/docs/ai/reference.mdx b/docs/content/docs/ai/reference.mdx index 7be4973f96..33eae9ed98 100644 --- a/docs/content/docs/ai/reference.mdx +++ b/docs/content/docs/ai/reference.mdx @@ -206,11 +206,11 @@ type LLMRequestOptions = { * @default { add: true, update: true, delete: true } */ defaultStreamTools?: { - /** Enable the add tool (default: true) */ + /** Enable the add tool (default: false) */ add?: boolean; - /** Enable the update tool (default: true) */ + /** Enable the update tool (default: false) */ update?: boolean; - /** Enable the delete tool (default: true) */ + /** Enable the delete tool (default: false) */ delete?: boolean; }; /** diff --git a/docs/content/docs/features/ai/reference.mdx b/docs/content/docs/features/ai/reference.mdx index 7be4973f96..33eae9ed98 100644 --- a/docs/content/docs/features/ai/reference.mdx +++ b/docs/content/docs/features/ai/reference.mdx @@ -206,11 +206,11 @@ type LLMRequestOptions = { * @default { add: true, update: true, delete: true } */ defaultStreamTools?: { - /** Enable the add tool (default: true) */ + /** Enable the add tool (default: false) */ add?: boolean; - /** Enable the update tool (default: true) */ + /** Enable the update tool (default: false) */ update?: boolean; - /** Enable the delete tool (default: true) */ + /** Enable the delete tool (default: false) */ delete?: boolean; }; /** diff --git a/examples/09-ai/05-manual-execution/src/App.tsx b/examples/09-ai/05-manual-execution/src/App.tsx index 38f6f958e7..ffab193b04 100644 --- a/examples/09-ai/05-manual-execution/src/App.tsx +++ b/examples/09-ai/05-manual-execution/src/App.tsx @@ -1,22 +1,12 @@ -import { BlockNoteEditor, filterSuggestionItems } from "@blocknote/core"; import "@blocknote/core/fonts/inter.css"; import { en } from "@blocknote/core/locales"; import { BlockNoteView } from "@blocknote/mantine"; import "@blocknote/mantine/style.css"; +import { useCreateBlockNote } from "@blocknote/react"; import { - FormattingToolbar, - FormattingToolbarController, - SuggestionMenuController, - getDefaultReactSlashMenuItems, - getFormattingToolbarItems, - useCreateBlockNote, -} from "@blocknote/react"; -import { - AIToolbarButton, StreamToolExecutor, createAIExtension, getAIExtension, - getAISlashMenuItems, llmFormats, } from "@blocknote/xl-ai"; import { en as aiEn } from "@blocknote/xl-ai/locales"; @@ -65,35 +55,33 @@ export default function App() { // Renders the editor instance using a React component. return (
- - - {/* - - - */} + +
{/*Inserts a new block at start of document.*/} -
-
- ); -} + + + ); } diff --git a/packages/xl-ai/src/api/LLMRequest.ts b/packages/xl-ai/src/api/LLMRequest.ts index 170e020e5b..529954162d 100644 --- a/packages/xl-ai/src/api/LLMRequest.ts +++ b/packages/xl-ai/src/api/LLMRequest.ts @@ -62,11 +62,11 @@ export type LLMRequestOptions = { * @default { add: true, update: true, delete: true } */ defaultStreamTools?: { - /** Enable the add tool (default: true) */ + /** Enable the add tool (default: false) */ add?: boolean; - /** Enable the update tool (default: true) */ + /** Enable the update tool (default: false) */ update?: boolean; - /** Enable the delete tool (default: true) */ + /** Enable the delete tool (default: false) */ delete?: boolean; }; /** diff --git a/packages/xl-ai/src/api/formats/base-tools/createUpdateBlockTool.ts b/packages/xl-ai/src/api/formats/base-tools/createUpdateBlockTool.ts index 7d22edab74..f3a16f7a3d 100644 --- a/packages/xl-ai/src/api/formats/base-tools/createUpdateBlockTool.ts +++ b/packages/xl-ai/src/api/formats/base-tools/createUpdateBlockTool.ts @@ -196,11 +196,9 @@ export function createUpdateBlockTool(config: { } const operation = chunk.operation as UpdateBlockToolCall; - console.log("first op"); if (chunk.isPossiblyPartial) { const size = JSON.stringify(operation.block).length; if (size < minSize) { - console.log("skipping", size, minSize); continue; } else { // increase minSize for next chunk @@ -225,7 +223,6 @@ export function createUpdateBlockTool(config: { const jsonToolCall = await config.toJSONToolCall(editor, chunk); if (!jsonToolCall) { - console.log("no jsonToolCall"); continue; } @@ -244,7 +241,6 @@ export function createUpdateBlockTool(config: { // if there's only a single replace step to be done and we're partial, let's wait for more content // REC: unit test this and see if it's still needed even if we pass `dontReplaceContentAtEnd` to `updateToReplaceSteps` - console.log("skipping", steps.length, chunk.isPossiblyPartial); continue; } diff --git a/packages/xl-ai/src/api/formats/html-blocks/htmlBlocks.ts b/packages/xl-ai/src/api/formats/html-blocks/htmlBlocks.ts index c657d377d4..bf78be57b7 100644 --- a/packages/xl-ai/src/api/formats/html-blocks/htmlBlocks.ts +++ b/packages/xl-ai/src/api/formats/html-blocks/htmlBlocks.ts @@ -8,29 +8,48 @@ import { } from "./htmlPromptData.js"; import { tools } from "./tools/index.js"; -function getStreamTools( +// Import the tool call types +import { AddBlocksToolCall } from "../base-tools/createAddBlocksTool.js"; +import { UpdateBlockToolCall } from "../base-tools/createUpdateBlockTool.js"; +import { DeleteBlockToolCall } from "../base-tools/delete.js"; + +// Define the tool types +export type AddTool = StreamTool>; +export type UpdateTool = StreamTool>; +export type DeleteTool = StreamTool; + +// Create a conditional type that maps boolean flags to tool types +export type StreamToolsConfig = { + add?: boolean; + update?: boolean; + delete?: boolean; +}; + +export type StreamToolsResult = [ + ...(T extends { update: true } ? [UpdateTool] : []), + ...(T extends { add: true } ? [AddTool] : []), + ...(T extends { delete: true } ? [DeleteTool] : []), +]; + +function getStreamTools< + T extends StreamToolsConfig = { add: true; update: true; delete: true }, +>( editor: BlockNoteEditor, withDelays: boolean, - defaultStreamTools?: { - /** Enable the add tool (default: true) */ - add?: boolean; - /** Enable the update tool (default: true) */ - update?: boolean; - /** Enable the delete tool (default: true) */ - delete?: boolean; - }, + defaultStreamTools?: T, selectionInfo?: { from: number; to: number; }, onBlockUpdate?: (blockId: string) => void, -) { - const mergedStreamTools = { - add: true, - update: true, - delete: true, - ...defaultStreamTools, - }; +): StreamToolsResult { + const mergedStreamTools = + defaultStreamTools ?? + ({ + add: true, + update: true, + delete: true, + } as T); const streamTools: StreamTool[] = [ ...(mergedStreamTools.update @@ -51,7 +70,7 @@ function getStreamTools( : []), ]; - return streamTools; + return streamTools as StreamToolsResult; } export const htmlBlockLLMFormat = { @@ -59,8 +78,6 @@ export const htmlBlockLLMFormat = { * Function to get the stream tools that can apply HTML block updates to the editor */ getStreamTools, - - streamTools: tools, /** * The default PromptBuilder that determines how a userPrompt is converted to an array of * LLM Messages (CoreMessage[]) diff --git a/packages/xl-ai/src/streamTool/StreamToolExecutor.ts b/packages/xl-ai/src/streamTool/StreamToolExecutor.ts index 283ab37d28..9da933431f 100644 --- a/packages/xl-ai/src/streamTool/StreamToolExecutor.ts +++ b/packages/xl-ai/src/streamTool/StreamToolExecutor.ts @@ -5,56 +5,51 @@ import { } from "../util/stream.js"; import { StreamTool, StreamToolCall } from "./streamTool.js"; -// update previous - -function partialJsonToOperation( - chunk: string, - isUpdateToPreviousOperation: boolean, - streamTools: StreamTool[], -) { - const parsed = parsePartialJson(chunk); - - if (parsed.state === "undefined-input" || parsed.state === "failed-parse") { - return undefined; - } - - if (!parsed) { - return; - } - - const func = streamTools.find((f) => f.name === (parsed.value as any)?.type); - - const validated = func && func.validate(parsed.value); - - if (validated?.ok) { - return { - operation: parsed.value as StreamToolCall[]>, - isPossiblyPartial: parsed.state === "repaired-parse", - isUpdateToPreviousOperation, - }; - } else { - // no worries, probably a partial operation that's not valid yet - return; - } -} - +/** + * The Operation types wraps a StreamToolCall with metadata on whether + * it's an update to an existing and / or or a possibly partial (i.e.: incomplete, streaming in progress) operation + */ type Operation[] | StreamTool> = { + /** + * The StreamToolCall (parameters representing a StreamTool invocation) + */ operation: StreamToolCall; + /** + * Whether this operation is an update to the previous operation + * (i.e.: the previous operation was a partial operation for which we now have additional data) + */ isUpdateToPreviousOperation: boolean; + /** + * Whether this operation is a partial operation + * (i.e.: incomplete, streaming in progress) + */ isPossiblyPartial: boolean; }; +/** + * The StreamToolExecutor can apply StreamToolCalls to an editor. + * + * It accepts StreamToolCalls as JSON strings or already parsed and validated Operations. + * Note: When passing JSON strings, the executor will parse and validate them into Operations. + * When passing Operations, they're expected to have been validated by the StreamTool instances already. + * (StreamTool.validate) + * + * Applying the operations is delegated to the StreamTool instances. + * + * @example see the `manual-execution` example + */ export class StreamToolExecutor[]> { private readonly stream: TransformStream, Operation>; private readonly readable: ReadableStream>; + /** + * @param streamTools - The StreamTools to use to apply the StreamToolCalls + */ constructor(private streamTools: T) { this.stream = this.createWriteStream(); this.readable = this.createReadableStream(); } - /** - * Returns a WritableStream that collects written chunks. - */ + private createWriteStream() { let lastParsedResult: Operation | undefined; @@ -69,27 +64,27 @@ export class StreamToolExecutor[]> { ) : chunk; if (operation) { - // TODO: string operations have been validated, but object-based operations have not. make this consistent? + // TODO: string operations have been validated, but object-based operations have not. + // To make this consistent, maybe we should extract the string parser to a separate transformer + lastParsedResult = operation; controller.enqueue(operation); } }, - // close: () => { - // if (lastParsedResult?.isPossiblyPartial) { - // throw new Error("stream ended with a partial operation"); - // } - // } + flush: (controller) => { + // Check if the stream ended with a partial operation + if (lastParsedResult?.isPossiblyPartial) { + controller.error(new Error("stream ended with a partial operation")); + } + }, }); return stream; } - public get writable() { - return this.stream.writable; - } - private createReadableStream() { - // TODO: this is a bit hacky as it mixes async iterables and streams + // this is a bit hacky as it mixes async iterables and streams + // would be better to stick to streams let currentStream: AsyncIterable[]>> = createAsyncIterableStream(this.stream.readable); for (const tool of this.streamTools) { @@ -101,19 +96,30 @@ export class StreamToolExecutor[]> { /** * Helper method to apply all operations to the editor if you're not interested in intermediate operations and results. - * - * (this method consumes underlying streams in `llmResult`) */ public async waitTillEnd() { const iterable = createAsyncIterableStream(this.readable); // eslint-disable-next-line @typescript-eslint/no-unused-vars for await (const _result of iterable) { // no op + // these will be operations without a matching StreamTool. + // (we probably want to allow a way to access and handle these, but for now we haven't run into this scenario yet) } } + /** + * Returns a WritableStream that can be used to write StreamToolCalls to the executor. + * + * The WriteableStream accepts JSON strings or Operation objects. + */ + public get writable() { + return this.stream.writable; + } + /** * Accepts an async iterable and writes each chunk to the internal stream. + * + * (alternative to writing to the writable stream using {@link writable}) */ async execute(source: AsyncIterable>): Promise { const writer = this.writable.getWriter(); @@ -125,6 +131,8 @@ export class StreamToolExecutor[]> { /** * Accepts a single chunk and processes it using the same logic. + * + * (alternative to writing to the writable stream using {@link writable}) */ async executeOne(chunk: StreamToolCall): Promise { await this.execute( @@ -139,13 +147,33 @@ export class StreamToolExecutor[]> { } } -/* TODO: +function partialJsonToOperation[]>( + chunk: string, + isUpdateToPreviousOperation: boolean, + streamTools: T, +): Operation | undefined { + const parsed = parsePartialJson(chunk); -AI SDK integration: -- pass tools -- server side -- stream tool outputs (partial-tool-data) + if (parsed.state === "undefined-input" || parsed.state === "failed-parse") { + return undefined; + } + + if (!parsed) { + return; + } -Hook up custom executors + const func = streamTools.find((f) => f.name === (parsed.value as any)?.type); -*/ + const validated = func && func.validate(parsed.value); + + if (validated?.ok) { + return { + operation: validated.value as StreamToolCall, + isPossiblyPartial: parsed.state === "repaired-parse", + isUpdateToPreviousOperation, + }; + } else { + // no worries, probably a partial operation that's not valid yet + return; + } +} From e15255092f15adc3baf7dee7690d9091d3bdc876 Mon Sep 17 00:00:00 2001 From: yousefed Date: Thu, 31 Jul 2025 12:35:33 +0200 Subject: [PATCH 3/3] update lock --- pnpm-lock.yaml | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d4293b7893..e9d3e4f182 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3303,6 +3303,64 @@ importers: specifier: ^5.3.4 version: 5.4.15(@types/node@22.15.2)(lightningcss@1.30.1)(terser@5.43.1) + examples/09-ai/05-manual-execution: + dependencies: + '@ai-sdk/groq': + specifier: ^1.2.9 + version: 1.2.9(zod@3.25.76) + '@blocknote/ariakit': + specifier: latest + version: link:../../../packages/ariakit + '@blocknote/core': + specifier: latest + version: link:../../../packages/core + '@blocknote/mantine': + specifier: latest + version: link:../../../packages/mantine + '@blocknote/react': + specifier: latest + version: link:../../../packages/react + '@blocknote/shadcn': + specifier: latest + version: link:../../../packages/shadcn + '@blocknote/xl-ai': + specifier: latest + version: link:../../../packages/xl-ai + '@mantine/core': + specifier: ^7.17.3 + version: 7.17.3(@mantine/hooks@7.17.3(react@19.1.0))(@types/react@19.1.8)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + ai: + specifier: ^4.3.15 + version: 4.3.19(react@19.1.0)(zod@3.25.76) + react: + specifier: ^19.1.0 + version: 19.1.0 + react-dom: + specifier: ^19.1.0 + version: 19.1.0(react@19.1.0) + y-partykit: + specifier: ^0.0.25 + version: 0.0.25 + yjs: + specifier: ^13.6.27 + version: 13.6.27 + zustand: + specifier: ^5.0.3 + version: 5.0.3(@types/react@19.1.8)(immer@10.1.1)(react@19.1.0)(use-sync-external-store@1.5.0(react@19.1.0)) + devDependencies: + '@types/react': + specifier: ^19.1.0 + version: 19.1.8 + '@types/react-dom': + specifier: ^19.1.0 + version: 19.1.6(@types/react@19.1.8) + '@vitejs/plugin-react': + specifier: ^4.3.1 + version: 4.4.1(vite@5.4.15(@types/node@22.15.2)(lightningcss@1.30.1)(terser@5.43.1)) + vite: + specifier: ^5.3.4 + version: 5.4.15(@types/node@22.15.2)(lightningcss@1.30.1)(terser@5.43.1) + examples/vanilla-js/react-vanilla-custom-blocks: dependencies: '@blocknote/ariakit':