diff --git a/core/context/providers/CodeContextProvider.ts b/core/context/providers/CodeContextProvider.ts index b746199b0a..db3b816646 100644 --- a/core/context/providers/CodeContextProvider.ts +++ b/core/context/providers/CodeContextProvider.ts @@ -16,7 +16,7 @@ class CodeContextProvider extends BaseContextProvider { displayTitle: "Code", description: "Type to search", type: "submenu", - dependsOnIndexing: true, + dependsOnIndexing: ["chunk", "codeSnippets"], }; async getContextItems( diff --git a/core/context/providers/CodebaseContextProvider.ts b/core/context/providers/CodebaseContextProvider.ts index 51881f457e..43eaec4335 100644 --- a/core/context/providers/CodebaseContextProvider.ts +++ b/core/context/providers/CodebaseContextProvider.ts @@ -13,6 +13,7 @@ class CodebaseContextProvider extends BaseContextProvider { description: "Automatically find relevant files", type: "normal", renderInlineAs: "", + dependsOnIndexing: ["embeddings", "fullTextSearch", "chunk"], }; async getContextItems( diff --git a/core/context/providers/FolderContextProvider.ts b/core/context/providers/FolderContextProvider.ts index b4d2e0e099..48efcb1210 100644 --- a/core/context/providers/FolderContextProvider.ts +++ b/core/context/providers/FolderContextProvider.ts @@ -19,7 +19,7 @@ class FolderContextProvider extends BaseContextProvider { displayTitle: "Folder", description: "Type to search", type: "submenu", - dependsOnIndexing: true, + dependsOnIndexing: ["embeddings", "fullTextSearch", "chunk"], }; async getContextItems( diff --git a/core/index.d.ts b/core/index.d.ts index 39e7cb9110..6f18e1c32b 100644 --- a/core/index.d.ts +++ b/core/index.d.ts @@ -165,6 +165,11 @@ export interface ModelInstaller { } export type ContextProviderType = "normal" | "query" | "submenu"; +export type ContextIndexingType = + | "chunk" + | "embeddings" + | "fullTextSearch" + | "codeSnippets"; export interface ContextProviderDescription { title: ContextProviderName; @@ -172,7 +177,7 @@ export interface ContextProviderDescription { description: string; renderInlineAs?: string; type: ContextProviderType; - dependsOnIndexing?: boolean; + dependsOnIndexing?: ContextIndexingType[]; } export type FetchFunction = (url: string | URL, init?: any) => Promise; diff --git a/core/indexing/CodebaseIndexer.ts b/core/indexing/CodebaseIndexer.ts index d2de0c5124..2706f0c1ec 100644 --- a/core/indexing/CodebaseIndexer.ts +++ b/core/indexing/CodebaseIndexer.ts @@ -1,7 +1,12 @@ import * as fs from "fs/promises"; import { ConfigHandler } from "../config/ConfigHandler.js"; -import { IDE, IndexingProgressUpdate, IndexTag } from "../index.js"; +import { + ContextIndexingType, + IDE, + IndexingProgressUpdate, + IndexTag, +} from "../index.js"; import type { FromCoreProtocol, ToCoreProtocol } from "../protocol"; import type { IMessenger } from "../protocol/messenger"; import { extractMinimalStackTraceInfo } from "../util/extractMinimalStackTraceInfo.js"; @@ -123,27 +128,44 @@ export class CodebaseIndexer { return []; } - const indexes: CodebaseIndex[] = [ - new ChunkCodebaseIndex( - this.ide.readFile.bind(this.ide), - continueServerClient, - embeddingsModel.maxEmbeddingChunkSize, - ), // Chunking must come first - ]; - - const lanceDbIndex = await LanceDbIndex.create( - embeddingsModel, - this.ide.readFile.bind(this.ide), + const indexTypesToBuild = new Set( // use set to remove duplicates + config.contextProviders + .map((provider) => provider.description.dependsOnIndexing) + .filter((indexType) => Array.isArray(indexType)) // remove undefined indexTypes + .flat(), ); - if (lanceDbIndex) { - indexes.push(lanceDbIndex); - } + const indexTypeToIndexerMapping: Record< + ContextIndexingType, + () => Promise + > = { + chunk: async () => + new ChunkCodebaseIndex( + this.ide.readFile.bind(this.ide), + continueServerClient, + embeddingsModel.maxEmbeddingChunkSize, + ), + codeSnippets: async () => new CodeSnippetsCodebaseIndex(this.ide), + fullTextSearch: async () => new FullTextSearchCodebaseIndex(), + embeddings: async () => { + const lanceDbIndex = await LanceDbIndex.create( + embeddingsModel, + this.ide.readFile.bind(this.ide), + ); + return lanceDbIndex; + }, + }; - indexes.push( - new FullTextSearchCodebaseIndex(), - new CodeSnippetsCodebaseIndex(this.ide), - ); + const indexes: CodebaseIndex[] = []; + // not parallelizing to avoid race conditions in sqlite + for (const indexType of indexTypesToBuild) { + if (indexType && indexType in indexTypeToIndexerMapping) { + const index = await indexTypeToIndexerMapping[indexType](); + if (index) { + indexes.push(index); + } + } + } return indexes; } diff --git a/gui/src/context/SubmenuContextProviders.tsx b/gui/src/context/SubmenuContextProviders.tsx index cf3939c48d..9f6bcd55c0 100644 --- a/gui/src/context/SubmenuContextProviders.tsx +++ b/gui/src/context/SubmenuContextProviders.tsx @@ -115,7 +115,7 @@ export const SubmenuContextProvidersProvider = ({ const interval = setInterval(refreshOpenFiles, 2000); - refreshOpenFiles(); // Initial call + void refreshOpenFiles(); // Initial call return () => { isMounted = false; @@ -237,7 +237,8 @@ export const SubmenuContextProvidersProvider = ({ providers === "all" ? true : providers === "dependsOnIndexing" - ? description.dependsOnIndexing + ? description.dependsOnIndexing && + description.dependsOnIndexing?.length > 0 : providers.includes(description.title); if (!refreshProvider) { @@ -351,7 +352,7 @@ export const SubmenuContextProvidersProvider = ({ useWebviewListener( "refreshSubmenuItems", async (data) => { - loadSubmenuItems(data.providers); + void loadSubmenuItems(data.providers); }, [loadSubmenuItems], ); @@ -360,7 +361,7 @@ export const SubmenuContextProvidersProvider = ({ "indexProgress", async (data) => { if (data.status === "done") { - loadSubmenuItems("dependsOnIndexing"); + void loadSubmenuItems("dependsOnIndexing"); } }, [loadSubmenuItems], @@ -379,7 +380,7 @@ export const SubmenuContextProvidersProvider = ({ ) .map((provider) => provider.title); if (newTitles.length > 0) { - loadSubmenuItems(newTitles); + void loadSubmenuItems(newTitles); } lastProviders.current = submenuContextProviders; }, [loadSubmenuItems, submenuContextProviders]);