feat: custom AI executors

docs/package.json

@@ -68,7 +68,7 @@
     "@vercel/analytics": "^1.5.0",
     "@vercel/og": "^0.6.8",
     "@y-sweet/react": "^0.6.3",
-    "ai": "^4.1.0",
+    "ai": "^4.3.15",
     "babel-plugin-react-compiler": "19.1.0-rc.2",
     "better-auth": "^1.2.10",
     "better-sqlite3": "^11.10.0",

examples/09-ai/01-minimal/src/App.tsx

@@ -7,9 +7,9 @@ import "@blocknote/mantine/style.css";
 import {
   FormattingToolbar,
   FormattingToolbarController,
-  SuggestionMenuController,
   getDefaultReactSlashMenuItems,
   getFormattingToolbarItems,
+  SuggestionMenuController,
   useCreateBlockNote,
 } from "@blocknote/react";
 import {
@@ -29,7 +29,7 @@ import { getEnv } from "./getEnv";
 const client = createBlockNoteAIClient({
   apiKey: getEnv("BLOCKNOTE_AI_SERVER_API_KEY") || "PLACEHOLDER",
   baseURL:
-    getEnv("BLOCKNOTE_AI_SERVER_BASE_URL") || "https://localhost:3000/ai",
+    getEnv("BLOCKNOTE_AI_SERVER_BASE_URL") || "https://localhost:3000/ai/proxy",
 });
 
 // Use an "open" model such as llama, in this case via groq.com
@@ -64,7 +64,7 @@ export default function App() {
     // Register the AI extension
     extensions: [
       createAIExtension({
-        model,
+        executor: model as any, // TODO
       }),
     ],
     // We set some initial content for demo purposes

examples/09-ai/02-playground/src/App.tsx

@@ -31,7 +31,6 @@ import {
 import { en as aiEn } from "@blocknote/xl-ai/locales";
 import "@blocknote/xl-ai/style.css";
 import { Fieldset, MantineProvider, Switch } from "@mantine/core";
-import { LanguageModelV1 } from "ai";
 import { useEffect, useMemo, useState } from "react";
 import { useStore } from "zustand";
 
@@ -101,7 +100,8 @@ export default function App() {
     // Register the AI extension
     extensions: [
       createAIExtension({
-        model: model as LanguageModelV1, // (type because initially it's valid)
+        executor: model as any, // TODO
+        // model: model as LanguageModelV1, // (type because initially it's valid)
       }),
     ],
     // We set some initial content for demo purposes
@@ -136,13 +136,13 @@ export default function App() {
   useEffect(() => {
     // update the default model in the extension
     if (model !== "unknown-model") {
-      ai.options.setState({ model });
+      ai.options.setState({ executor: model as any }); // TODO
     }
   }, [model, ai.options]);
 
   const [dataFormat, setDataFormat] = useState("html");
 
-  const stream = useStore(ai.options, (state) => state.stream);
+  const stream = useStore(ai.options, (state: any) => state.stream); // TODO
 
   const themePreference = usePrefersColorScheme();
   const existingContext = useBlockNoteContext();
@@ -196,9 +196,10 @@ export default function App() {
 
             <Switch
               checked={stream}
-              onChange={(e) =>
-                ai.options.setState({ stream: e.target.checked })
-              }
+              onChange={(e) => {
+                // TODO
+                // ai.options.setState({ stream: e.target.checked })
+              }}
               label="Streaming"
             />
           </Fieldset>

examples/09-ai/03-custom-ai-menu-items/src/App.tsx

@@ -67,7 +67,7 @@ export default function App() {
     // Register the AI extension
     extensions: [
       createAIExtension({
-        model,
+        executor: model as any, // TODO
       }),
     ],
     // We set some initial content for demo purposes

examples/09-ai/04-with-collaboration/src/App.tsx

@@ -109,7 +109,7 @@ export default function App() {
     // Register the AI extension
     extensions: [
       createAIExtension({
-        model,
+        executor: model as any, // TODO
       }),
     ],
     // We set some initial content for demo purposes

examples/09-ai/05-manual-execution/src/App.tsx

@@ -22,7 +22,7 @@ export default function App() {
     // Register the AI extension
     extensions: [
       createAIExtension({
-        model: undefined as any, // disable model
+        executor: undefined as any, // disable
       }),
     ],
     // We set some initial content for demo purposes

examples/09-ai/05-manual-execution/tsconfig.json

@@ -3,7 +3,11 @@
   "compilerOptions": {
     "target": "ESNext",
     "useDefineForClassFields": true,
-    "lib": ["DOM", "DOM.Iterable", "ESNext"],
+    "lib": [
+      "DOM",
+      "DOM.Iterable",
+      "ESNext"
+    ],
     "allowJs": false,
     "skipLibCheck": true,
     "esModuleInterop": false,
@@ -18,16 +22,15 @@
     "jsx": "react-jsx",
     "composite": true
   },
-  "include": ["."],
-  "references": [
+  "include": [
+    "."
+  ],
+  "__ADD_FOR_LOCAL_DEV_references": [
     {
       "path": "../../../packages/core/"
     },
     {
       "path": "../../../packages/react/"
-    },
-    {
-      "path": "../../../packages/xl-ai/"
     }
   ]
-}
+}

examples/09-ai/06-server-execution/.bnexample.json

@@ -0,0 +1,12 @@
+{
+  "playground": true,
+  "docs": true,
+  "author": "yousefed",
+  "tags": ["AI", "llm"],
+  "dependencies": {
+    "@blocknote/xl-ai": "latest",
+    "@mantine/core": "^7.17.3",
+    "ai": "^4.3.15",
+    "zustand": "^5.0.3"
+  }
+}

examples/09-ai/06-server-execution/README.md

@@ -0,0 +1,3 @@
+# AI Integration with server LLM execution
+
+This example shows how to setup to add AI integration while handling the LLM calls (in this case, using the Vercel AI SDK) on your server, using a custom executor

examples/09-ai/06-server-execution/index.html

@@ -0,0 +1,14 @@
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>AI Integration with server LLM execution</title>
+    <script>
+      <!-- AUTO-GENERATED FILE, DO NOT EDIT DIRECTLY -->
+    </script>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="./main.tsx"></script>
+  </body>
+</html>

examples/09-ai/06-server-execution/main.tsx

@@ -0,0 +1,11 @@
+// AUTO-GENERATED FILE, DO NOT EDIT DIRECTLY
+import React from "react";
+import { createRoot } from "react-dom/client";
+import App from "./src/App.jsx";
+
+const root = createRoot(document.getElementById("root")!);
+root.render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>
+);

examples/09-ai/06-server-execution/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@blocknote/example-ai-server-execution",
+  "description": "AUTO-GENERATED FILE, DO NOT EDIT DIRECTLY",
+  "private": true,
+  "version": "0.12.4",
+  "scripts": {
+    "start": "vite",
+    "dev": "vite",
+    "build:prod": "tsc && vite build",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@blocknote/core": "latest",
+    "@blocknote/react": "latest",
+    "@blocknote/ariakit": "latest",
+    "@blocknote/mantine": "latest",
+    "@blocknote/shadcn": "latest",
+    "react": "^19.1.0",
+    "react-dom": "^19.1.0",
+    "@blocknote/xl-ai": "latest",
+    "@mantine/core": "^7.17.3",
+    "ai": "^4.3.15",
+    "zustand": "^5.0.3"
+  },
+  "devDependencies": {
+    "@types/react": "^19.1.0",
+    "@types/react-dom": "^19.1.0",
+    "@vitejs/plugin-react": "^4.3.1",
+    "vite": "^5.3.4"
+  }
+}

examples/09-ai/06-server-execution/src/App.tsx

@@ -0,0 +1,158 @@
+import { BlockNoteEditor, filterSuggestionItems } from "@blocknote/core";
+import "@blocknote/core/fonts/inter.css";
+import { en } from "@blocknote/core/locales";
+import { BlockNoteView } from "@blocknote/mantine";
+import "@blocknote/mantine/style.css";
+import {
+  FormattingToolbar,
+  FormattingToolbarController,
+  getDefaultReactSlashMenuItems,
+  getFormattingToolbarItems,
+  SuggestionMenuController,
+  useCreateBlockNote,
+} from "@blocknote/react";
+import {
+  AIMenuController,
+  AIToolbarButton,
+  createAIExtension,
+  createStreamToolsArraySchema,
+  dataStreamResponseToOperationsResult,
+  getAISlashMenuItems,
+  LLMResponse,
+} from "@blocknote/xl-ai";
+import { en as aiEn } from "@blocknote/xl-ai/locales";
+import "@blocknote/xl-ai/style.css";
+
+import { getEnv } from "./getEnv";
+
+const BASE_URL =
+  getEnv("BLOCKNOTE_AI_SERVER_BASE_URL") ||
+  "https://localhost:3000/ai/vercel-ai-sdk";
+
+export default function App() {
+  // Creates a new editor instance.
+  const editor = useCreateBlockNote({
+    dictionary: {
+      ...en,
+      ai: aiEn, // add default translations for the AI extension
+    },
+    // Register the AI extension
+    extensions: [
+      createAIExtension({
+        // We define a custom executor that calls our backend server to execute LLM calls
+        // On the backend, we use the Vercel AI SDK to execute LLM calls
+        // (see packages/xl-ai-server/src/routes/vercelAiSdk.ts)
+        executor: async (opts) => {
+          const schema = createStreamToolsArraySchema(opts.streamTools);
+
+          // Can also use /generateObject for non-streaming mode
+          const response = await fetch(`${BASE_URL}/streamObject`, {
+            method: "POST",
+            body: JSON.stringify({
+              messages: opts.messages,
+              schema,
+            }),
+          });
+          const parsedResponse = await dataStreamResponseToOperationsResult(
+            response,
+            opts.streamTools,
+            opts.onStart,
+          );
+          return new LLMResponse(
+            opts.messages,
+            parsedResponse,
+            opts.streamTools,
+          );
+        },
+      }),
+    ],
+    // We set some initial content for demo purposes
+    initialContent: [
+      {
+        type: "heading",
+        props: {
+          level: 1,
+        },
+        content: "Open source software",
+      },
+      {
+        type: "paragraph",
+        content:
+          "Open source software refers to computer programs whose source code is made available to the public, allowing anyone to view, modify, and distribute the code. This model stands in contrast to proprietary software, where the source code is kept secret and only the original creators have the right to make changes. Open projects are developed collaboratively, often by communities of developers from around the world, and are typically distributed under licenses that promote sharing and openness.",
+      },
+      {
+        type: "paragraph",
+        content:
+          "One of the primary benefits of open source is the promotion of digital autonomy. By providing access to the source code, these programs empower users to control their own technology, customize software to fit their needs, and avoid vendor lock-in. This level of transparency also allows for greater security, as anyone can inspect the code for vulnerabilities or malicious elements. As a result, users are not solely dependent on a single company for updates, bug fixes, or continued support.",
+      },
+      {
+        type: "paragraph",
+        content:
+          "Additionally, open development fosters innovation and collaboration. Developers can build upon existing projects, share improvements, and learn from each other, accelerating the pace of technological advancement. The open nature of these projects often leads to higher quality software, as bugs are identified and fixed more quickly by a diverse group of contributors. Furthermore, using open source can reduce costs for individuals, businesses, and governments, as it is often available for free and can be tailored to specific requirements without expensive licensing fees.",
+      },
+    ],
+  });
+
+  // Renders the editor instance using a React component.
+  return (
+    <div>
+      <BlockNoteView
+        editor={editor}
+        // We're disabling some default UI elements
+        formattingToolbar={false}
+        slashMenu={false}
+      >
+        {/* Add the AI Command menu to the editor */}
+        <AIMenuController />
+
+        {/* We disabled the default formatting toolbar with `formattingToolbar=false` 
+        and replace it for one with an "AI button" (defined below). 
+        (See "Formatting Toolbar" in docs)
+        */}
+        <FormattingToolbarWithAI />
+
+        {/* We disabled the default SlashMenu with `slashMenu=false` 
+        and replace it for one with an AI option (defined below). 
+        (See "Suggestion Menus" in docs)
+        */}
+        <SuggestionMenuWithAI editor={editor} />
+      </BlockNoteView>
+    </div>
+  );
+}
+
+// Formatting toolbar with the `AIToolbarButton` added
+function FormattingToolbarWithAI() {
+  return (
+    <FormattingToolbarController
+      formattingToolbar={() => (
+        <FormattingToolbar>
+          {...getFormattingToolbarItems()}
+          {/* Add the AI button */}
+          <AIToolbarButton />
+        </FormattingToolbar>
+      )}
+    />
+  );
+}
+
+// Slash menu with the AI option added
+function SuggestionMenuWithAI(props: {
+  editor: BlockNoteEditor<any, any, any>;
+}) {
+  return (
+    <SuggestionMenuController
+      triggerCharacter="/"
+      getItems={async (query) =>
+        filterSuggestionItems(
+          [
+            ...getDefaultReactSlashMenuItems(props.editor),
+            // add the default AI slash menu items, or define your own
+            ...getAISlashMenuItems(props.editor),
+          ],
+          query,
+        )
+      }
+    />
+  );
+}