diff --git a/core/context/providers/CurrentFileContextProvider.ts b/core/context/providers/CurrentFileContextProvider.ts index e624f539ad..950568cf5b 100644 --- a/core/context/providers/CurrentFileContextProvider.ts +++ b/core/context/providers/CurrentFileContextProvider.ts @@ -4,6 +4,7 @@ import { ContextProviderDescription, ContextProviderExtras, } from "../../"; +import { formatCodeblock } from "../../util/formatCodeblock"; import { getUriDescription } from "../../util/uri"; class CurrentFileContextProvider extends BaseContextProvider { @@ -24,10 +25,9 @@ class CurrentFileContextProvider extends BaseContextProvider { return []; } - const { relativePathOrBasename, last2Parts, baseName } = getUriDescription( - currentFile.path, - await extras.ide.getWorkspaceDirs(), - ); + const workspaceDirs = await extras.ide.getWorkspaceDirs(); + const { relativePathOrBasename, last2Parts, baseName, extension } = + getUriDescription(currentFile.path, workspaceDirs); let prefix = "This is the currently open file:"; let name = baseName; @@ -39,10 +39,15 @@ class CurrentFileContextProvider extends BaseContextProvider { name = "Active file: " + baseName; } + const codeblock = formatCodeblock( + relativePathOrBasename, + currentFile.contents, + extension, + ); return [ { description: last2Parts, - content: `${prefix}\n\n\`\`\`${relativePathOrBasename}\n${currentFile.contents}\n\`\`\``, + content: `${prefix}\n\n${codeblock}`, name, uri: { type: "file", diff --git a/core/context/providers/FileContextProvider.ts b/core/context/providers/FileContextProvider.ts index 5f7861b83d..d0f1b25d0c 100644 --- a/core/context/providers/FileContextProvider.ts +++ b/core/context/providers/FileContextProvider.ts @@ -7,6 +7,7 @@ import { LoadSubmenuItemsArgs, } from "../../"; import { walkDirs } from "../../indexing/walkDir"; +import { formatCodeblock } from "../../util/formatCodeblock"; import { getShortestUniqueRelativeUriPaths, getUriDescription, @@ -30,17 +31,21 @@ class FileContextProvider extends BaseContextProvider { // Assume the query is a filepath const fileUri = query.trim(); const content = await extras.ide.readFile(fileUri); + const workspaceDirs = await extras.ide.getWorkspaceDirs(); - const { relativePathOrBasename, last2Parts, baseName } = getUriDescription( - fileUri, - await extras.ide.getWorkspaceDirs(), + const { relativePathOrBasename, last2Parts, baseName, extension } = + getUriDescription(fileUri, workspaceDirs); + const codeblock = formatCodeblock( + relativePathOrBasename, + content, + extension, ); return [ { name: baseName, description: last2Parts, - content: `\`\`\`${relativePathOrBasename}\n${content}\n\`\`\``, + content: codeblock, uri: { type: "file", value: fileUri, diff --git a/core/context/providers/OpenFilesContextProvider.ts b/core/context/providers/OpenFilesContextProvider.ts index 8267bf1c9b..b874058191 100644 --- a/core/context/providers/OpenFilesContextProvider.ts +++ b/core/context/providers/OpenFilesContextProvider.ts @@ -3,6 +3,7 @@ import { ContextProviderDescription, ContextProviderExtras, } from "../../index.js"; +import { formatCodeblock } from "../../util/formatCodeblock.js"; import { getUriDescription } from "../../util/uri.js"; import { BaseContextProvider } from "../index.js"; @@ -28,12 +29,18 @@ class OpenFilesContextProvider extends BaseContextProvider { return await Promise.all( openFiles.map(async (filepath: string) => { const content = await ide.readFile(filepath); - const { relativePathOrBasename, last2Parts, baseName } = + const { relativePathOrBasename, last2Parts, baseName, extension } = getUriDescription(filepath, workspaceDirs); + const codeblock = formatCodeblock( + relativePathOrBasename, + content, + extension, + ); + return { description: last2Parts, - content: `\`\`\`${relativePathOrBasename}\n${content}\n\`\`\``, + content: codeblock, name: baseName, uri: { type: "file", diff --git a/core/context/providers/ProblemsContextProvider.ts b/core/context/providers/ProblemsContextProvider.ts index d8df5687fc..6b1090b79d 100644 --- a/core/context/providers/ProblemsContextProvider.ts +++ b/core/context/providers/ProblemsContextProvider.ts @@ -3,6 +3,7 @@ import { ContextProviderDescription, ContextProviderExtras, } from "../../index.js"; +import { formatCodeblock } from "../../util/formatCodeblock.js"; import { getUriDescription } from "../../util/uri.js"; import { BaseContextProvider } from "../index.js"; @@ -22,24 +23,44 @@ class ProblemsContextProvider extends BaseContextProvider { const problems = await ide.getProblems(); const workspaceDirs = await ide.getWorkspaceDirs(); + type FileCache = { + lines: string[]; + desc: ReturnType; + }; + const files: Map = new Map(); + const items = await Promise.all( problems.map(async (problem) => { - const { relativePathOrBasename, baseName } = getUriDescription( - problem.filepath, - workspaceDirs, - ); - const content = await ide.readFile(problem.filepath); - const lines = content.split("\n"); - const rangeContent = lines + let cachedFile: FileCache; + if (files.has(problem.filepath)) { + cachedFile = files.get(problem.filepath)!; + } else { + const content = await ide.readFile(problem.filepath); + const desc = getUriDescription(problem.filepath, workspaceDirs); + const lines = content.split("\n"); + cachedFile = { lines, desc }; + files.set(problem.filepath, cachedFile); + } + + const { relativePathOrBasename, baseName, extension } = cachedFile.desc; + + const rangeContent = cachedFile.lines .slice( Math.max(0, problem.range.start.line - 2), problem.range.end.line + 2, ) .join("\n"); + const codeblock = formatCodeblock( + relativePathOrBasename, + rangeContent, + extension, + problem.range, + ); + return { description: "Problems in current file", - content: `\`\`\`${relativePathOrBasename}\n${rangeContent}\n\`\`\`\n${problem.message}\n\n`, + content: `${codeblock}\n${problem.message}\n\n`, name: `Warning in ${baseName}`, }; }), diff --git a/core/context/retrieval/retrieval.ts b/core/context/retrieval/retrieval.ts index 4f29d2ef8b..67fccadfc2 100644 --- a/core/context/retrieval/retrieval.ts +++ b/core/context/retrieval/retrieval.ts @@ -1,4 +1,5 @@ import { BranchAndDir, ContextItem, ContextProviderExtras } from "../../"; +import { formatCodeblock } from "../../util/formatCodeblock"; import { getUriDescription } from "../../util/uri"; import { INSTRUCTIONS_BASE_ITEM } from "../providers/utils"; @@ -105,21 +106,25 @@ export async function retrieveContextItemsFromEmbeddings( ...results .sort((a, b) => a.filepath.localeCompare(b.filepath)) .map((r) => { - const { relativePathOrBasename, last2Parts, baseName } = + const { relativePathOrBasename, last2Parts, baseName, extension } = getUriDescription(r.filepath, workspaceDirs); if (baseName === "package.json") { console.warn("Retrieval pipeline: package.json detected"); } + const rangeString = `(${r.startLine + 1}-${r.endLine + 1})`; + const codeblock = formatCodeblock( + relativePathOrBasename, + r.content, + extension, + rangeString, + ); return { // Don't show line numbers for tool calls where startLine = -1 - name: - r.startLine === -1 - ? baseName - : `${baseName} (${r.startLine + 1}-${r.endLine + 1})`, + name: r.startLine === -1 ? baseName : `${baseName} ${rangeString}`, description: last2Parts, - content: `\`\`\`${relativePathOrBasename}\n${r.content}\n\`\`\``, + content: codeblock, uri: { type: "file" as const, value: r.filepath, diff --git a/core/edit/lazy/applyCodeBlock.ts b/core/edit/lazy/applyCodeBlock.ts index b58b9d4ec2..61c16677c9 100644 --- a/core/edit/lazy/applyCodeBlock.ts +++ b/core/edit/lazy/applyCodeBlock.ts @@ -6,26 +6,26 @@ import { deterministicApplyLazyEdit } from "./deterministic"; import { streamLazyApply } from "./streamLazyApply"; import { applyUnifiedDiff, isUnifiedDiffFormat } from "./unifiedDiffApply"; -function canUseInstantApply(filename: string) { - const fileExtension = getUriFileExtension(filename); - return supportedLanguages[fileExtension] !== undefined; +function canUseInstantApply(fileUri: string) { + const fileExt = getUriFileExtension(fileUri); + return supportedLanguages[fileExt] !== undefined; } export async function applyCodeBlock( oldFile: string, newLazyFile: string, - filename: string, + fileUri: string, llm: ILLM, abortController: AbortController, ): Promise<{ isInstantApply: boolean; diffLinesGenerator: AsyncGenerator; }> { - if (canUseInstantApply(filename)) { + if (canUseInstantApply(fileUri)) { const diffLines = await deterministicApplyLazyEdit({ oldFile, newLazyFile, - filename, + fileUri, onlyFullFileRewrite: true, }); @@ -54,7 +54,7 @@ export async function applyCodeBlock( isInstantApply: false, diffLinesGenerator: streamLazyApply( oldFile, - filename, + fileUri, newLazyFile, llm, abortController, diff --git a/core/edit/lazy/deterministic.test.ts b/core/edit/lazy/deterministic.test.ts index adfaafa5e2..535d75ce25 100644 --- a/core/edit/lazy/deterministic.test.ts +++ b/core/edit/lazy/deterministic.test.ts @@ -6,6 +6,7 @@ import { diff as myersDiff } from "myers-diff"; import { DiffLine } from "../.."; import { myersDiff as continueMyersDiff } from "../../diff/myers"; import { dedent } from "../../util"; +import { localPathToUri } from "../../util/pathToUri"; import { deterministicApplyLazyEdit } from "./deterministic"; const UNIFIED_DIFF_SYMBOLS = { @@ -17,14 +18,14 @@ const UNIFIED_DIFF_SYMBOLS = { async function collectDiffs( oldFile: string, newFile: string, - filename: string, + fileUri: string, ): Promise<{ ourDiffs: DiffLine[]; myersDiffs: any }> { const ourDiffs: DiffLine[] = []; for (const diffLine of (await deterministicApplyLazyEdit({ oldFile, newLazyFile: newFile, - filename, + fileUri, })) ?? []) { ourDiffs.push(diffLine); } @@ -61,7 +62,12 @@ async function expectDiff(file: string) { const [oldFile, newFile, expectedDiff] = testFileContents .split("\n---\n") .map((s) => s.replace(/^\n+/, "").trimEnd()); - const { ourDiffs: streamDiffs } = await collectDiffs(oldFile, newFile, file); + const fileUri = localPathToUri(testFilePath); + const { ourDiffs: streamDiffs } = await collectDiffs( + oldFile, + newFile, + fileUri, + ); const displayedDiff = displayDiff(streamDiffs); if (!expectedDiff || expectedDiff.trim() === "") { diff --git a/core/edit/lazy/deterministic.ts b/core/edit/lazy/deterministic.ts index 87fc7c66d1..ae93989f4d 100644 --- a/core/edit/lazy/deterministic.ts +++ b/core/edit/lazy/deterministic.ts @@ -1,5 +1,3 @@ -import path from "path"; - import { distance } from "fastest-levenshtein"; import Parser from "web-tree-sitter"; @@ -8,6 +6,7 @@ import { LANGUAGES } from "../../autocomplete/constants/AutocompleteLanguageInfo import { myersDiff } from "../../diff/myers"; import { getParserForFile } from "../../util/treeSitter"; +import { getUriFileExtension } from "../../util/uri"; import { findInAst } from "./findInAst"; type AstReplacements = Array<{ @@ -109,12 +108,10 @@ function shouldRejectDiff(diff: DiffLine[]): boolean { function nodeSurroundedInLazyBlocks( parser: Parser, - file: string, - filename: string, + fileExt: string, ): { newTree: Parser.Tree; newFile: string } | undefined { - const ext = path.extname(filename).slice(1); - const language = LANGUAGES[ext]; + const language = LANGUAGES[fileExt]; if (language) { const newFile = `${language.singleLineComment} ... existing code ...\n\n${file}\n\n${language.singleLineComment} ... existing code...`; const newTree = parser.parse(newFile); @@ -129,7 +126,7 @@ function nodeSurroundedInLazyBlocks( export async function deterministicApplyLazyEdit({ oldFile, newLazyFile, - filename, + fileUri, /** * Using this as a flag to slowly reintroduce lazy applies. * With this set, we will only attempt to deterministically apply @@ -140,10 +137,11 @@ export async function deterministicApplyLazyEdit({ }: { oldFile: string; newLazyFile: string; - filename: string; + fileUri: string; onlyFullFileRewrite?: boolean; }): Promise { - const parser = await getParserForFile(filename); + const fileExt = getUriFileExtension(fileUri); + const parser = await getParserForFile(fileUri); if (!parser) { return undefined; } @@ -175,7 +173,7 @@ export async function deterministicApplyLazyEdit({ ); if (firstSimilarNode?.parent?.equals(oldTree.rootNode)) { // If so, we tack lazy blocks to start and end, and run the usual algorithm - const result = nodeSurroundedInLazyBlocks(parser, newLazyFile, filename); + const result = nodeSurroundedInLazyBlocks(parser, newLazyFile, fileExt); if (result) { newLazyFile = result.newFile; newTree = result.newTree; diff --git a/core/edit/lazy/prompts.ts b/core/edit/lazy/prompts.ts index f821a3b077..03dd743cd2 100644 --- a/core/edit/lazy/prompts.ts +++ b/core/edit/lazy/prompts.ts @@ -1,11 +1,12 @@ import { ChatMessage } from "../.."; import { dedent } from "../../util"; +import { getUriFileExtension } from "../../util/uri"; export const UNCHANGED_CODE = "UNCHANGED CODE"; type LazyApplyPrompt = ( oldCode: string, - filename: string, + fileUri: string, newCode: string, ) => ChatMessage[]; @@ -21,18 +22,21 @@ const RULES = [ "The code should always be syntactically valid, even with the comments.", ]; -function claude35SonnetLazyApplyPrompt( - ...args: Parameters -): ReturnType { +const claude35SonnetLazyApplyPrompt: LazyApplyPrompt = ( + oldCode, + fileUri, + newCode, +) => { + const fileExt = getUriFileExtension(fileUri); const userContent = dedent` ORIGINAL CODE: - \`\`\`${args[1]} - ${args[0]} + \`\`\`${fileExt} + ${oldCode} \`\`\` NEW CODE: - \`\`\` - ${args[2]} + \`\`\`${fileExt} + ${newCode} \`\`\` Above is a code block containing the original version of a file (ORIGINAL CODE) and below it is a code snippet (NEW CODE) that was suggested as modification to the original file. Your task is to apply the NEW CODE to the ORIGINAL CODE and show what the entire file would look like after it is applied. @@ -41,14 +45,14 @@ function claude35SonnetLazyApplyPrompt( const assistantContent = dedent` Sure! Here's the modified version of the file after applying the new code: - \`\`\`${args[1]} + \`\`\`${fileExt} `; return [ { role: "user", content: userContent }, { role: "assistant", content: assistantContent }, ]; -} +}; export function lazyApplyPromptForModel( model: string, diff --git a/core/edit/lazy/streamLazyApply.ts b/core/edit/lazy/streamLazyApply.ts index f8ec0905c6..59e8bfea7a 100644 --- a/core/edit/lazy/streamLazyApply.ts +++ b/core/edit/lazy/streamLazyApply.ts @@ -13,7 +13,7 @@ import { BUFFER_LINES_BELOW, getReplacementWithLlm } from "./replace.js"; export async function* streamLazyApply( oldCode: string, - filename: string, + fileUri: string, newCode: string, llm: ILLM, abortController: AbortController, @@ -23,7 +23,7 @@ export async function* streamLazyApply( throw new Error(`Lazy apply not supported for model ${llm.model}`); } - const promptMessages = promptFactory(oldCode, filename, newCode); + const promptMessages = promptFactory(oldCode, fileUri, newCode); const lazyCompletion = llm.streamChat(promptMessages, abortController.signal); // Do find and replace over the lazy edit response diff --git a/core/indexing/CodeSnippetsIndex.ts b/core/indexing/CodeSnippetsIndex.ts index d04eca7eeb..74644e2ba6 100644 --- a/core/indexing/CodeSnippetsIndex.ts +++ b/core/indexing/CodeSnippetsIndex.ts @@ -23,10 +23,12 @@ import type { IndexTag, IndexingProgressUpdate, } from "../"; +import { formatCodeblock } from "../util/formatCodeblock"; import { findUriInDirs, getLastNPathParts, getLastNUriRelativePathParts, + getUriFileExtension, getUriPathBasename, } from "../util/uri"; import { tagToString } from "./utils"; @@ -356,11 +358,20 @@ export class CodeSnippetsCodebaseIndex implements CodebaseIndex { const row = await db.get("SELECT * FROM code_snippets WHERE id = ?", [id]); const last2Parts = getLastNUriRelativePathParts(workspaceDirs, row.path, 2); - const { relativePathOrBasename } = findUriInDirs(row.path, workspaceDirs); + const { relativePathOrBasename, uri } = findUriInDirs( + row.path, + workspaceDirs, + ); + const extension = getUriFileExtension(uri); + const codeblock = formatCodeblock( + relativePathOrBasename, + row.content, + extension, + ); return { name: row.title, description: last2Parts, - content: `\`\`\`${relativePathOrBasename}\n${row.content}\n\`\`\``, + content: codeblock, uri: { type: "file", value: row.path, diff --git a/core/llm/llms/llm.ts b/core/llm/llms/llm.ts index e472b73170..3411d813e2 100644 --- a/core/llm/llms/llm.ts +++ b/core/llm/llms/llm.ts @@ -10,7 +10,7 @@ const RERANK_PROMPT = ( Query: Where is the FastAPI server? Snippet: - \`\`\`/Users/andrew/Desktop/server/main.py + \`\`\`python /Users/andrew/Desktop/server/main.py from fastapi import FastAPI app = FastAPI() @app.get("/") @@ -21,7 +21,7 @@ const RERANK_PROMPT = ( Query: Where in the documentation does it talk about the UI? Snippet: - \`\`\`/Users/andrew/Projects/bubble_sort/src/lib.rs + \`\`\`rust /Users/andrew/Projects/bubble_sort/src/lib.rs fn bubble_sort(arr: &mut [T]) {{ for i in 0..arr.len() {{ for j in 1..arr.len() - i {{ diff --git a/core/tools/implementations/readCurrentlyOpenFile.ts b/core/tools/implementations/readCurrentlyOpenFile.ts index bf3b2c713d..476ccff7c4 100644 --- a/core/tools/implementations/readCurrentlyOpenFile.ts +++ b/core/tools/implementations/readCurrentlyOpenFile.ts @@ -1,6 +1,7 @@ import { getUriDescription } from "../../util/uri"; import { ToolImpl } from "."; +import { formatCodeblock } from "../../util/formatCodeblock"; import { throwIfFileExceedsHalfOfContext } from "./readFileLimit"; export const readCurrentlyOpenFileImpl: ToolImpl = async (args, extras) => { @@ -13,16 +14,22 @@ export const readCurrentlyOpenFileImpl: ToolImpl = async (args, extras) => { extras.config.selectedModelByRole.chat, ); - const { relativePathOrBasename, last2Parts, baseName } = getUriDescription( - result.path, - await extras.ide.getWorkspaceDirs(), + const workspaceDirs = await extras.ide.getWorkspaceDirs(); + + const { relativePathOrBasename, last2Parts, baseName, extension } = + getUriDescription(result.path, workspaceDirs); + + const codeblock = formatCodeblock( + extension, + relativePathOrBasename, + result.contents, ); return [ { name: `Current file: ${baseName}`, description: last2Parts, - content: `\`\`\`${relativePathOrBasename}\n${result.contents}\n\`\`\``, + content: codeblock, uri: { type: "file", value: result.path, diff --git a/core/util/formatCodeblock.ts b/core/util/formatCodeblock.ts new file mode 100644 index 0000000000..a8e696db7a --- /dev/null +++ b/core/util/formatCodeblock.ts @@ -0,0 +1,28 @@ +import { extToLangMap } from "."; +import { Range } from ".."; +import { getFileExtensionFromBasename, getUriPathBasename } from "./uri"; +export function formatCodeblock( + relativePathOrBasename: string, + contents: string, + extension?: string, + range?: Range | string, +) { + const basename = getUriPathBasename(relativePathOrBasename); + const ext = extension || getFileExtensionFromBasename(basename); + const languageTag = extToLangMap[ext] || ext || basename; + + const path = relativePathOrBasename.startsWith("/") + ? relativePathOrBasename + : `/${relativePathOrBasename}`; + + const rangeString = range + ? typeof range === "string" + ? range + : `(${range.start.line + 1}-${range.end.line + 1})` + : ""; + const codeblockHeader = `${languageTag} ${path} ${rangeString}` + .replaceAll(" ", " ") + .trim(); + + return `\`\`\`${codeblockHeader}\n${contents}\n\`\`\``; +} diff --git a/core/util/formatCodeblock.vitest.ts b/core/util/formatCodeblock.vitest.ts new file mode 100644 index 0000000000..17f5c6598d --- /dev/null +++ b/core/util/formatCodeblock.vitest.ts @@ -0,0 +1,98 @@ +import { expect, test } from "vitest"; +import { Range } from ".."; +import { formatCodeblock } from "./formatCodeblock"; + +test("formatCodeblock with JavaScript file", () => { + const result = formatCodeblock("example.js", "const x = 42;"); + expect(result).toBe("```javascript /example.js\nconst x = 42;\n```"); +}); + +test("formatCodeblock with TypeScript file", () => { + const result = formatCodeblock( + "util/helper.ts", + "export const add = (a: number, b: number) => a + b;", + ); + expect(result).toBe( + "```typescript /util/helper.ts\nexport const add = (a: number, b: number) => a + b;\n```", + ); +}); + +test("formatCodeblock with explicit extension override", () => { + const result = formatCodeblock("example", "SELECT * FROM users;", "sql"); + expect(result).toBe("```sql /example\nSELECT * FROM users;\n```"); +}); + +test("formatCodeblock with Range object", () => { + const range: Range = { + start: { line: 4, character: 0 }, + end: { line: 7, character: 10 }, + }; + const result = formatCodeblock( + "app.py", + 'def hello():\n return "world"', + undefined, + range, + ); + expect(result).toBe( + '```python /app.py (5-8)\ndef hello():\n return "world"\n```', + ); +}); + +test("formatCodeblock with string range", () => { + const result = formatCodeblock( + "app.py", + 'def hello():\n return "world"', + undefined, + "5-10", + ); + expect(result).toBe( + '```python /app.py 5-10\ndef hello():\n return "world"\n```', + ); +}); + +test("formatCodeblock with absolute path", () => { + const result = formatCodeblock("/src/main.rs", "fn main() {}\n"); + expect(result).toBe("```rust /src/main.rs\nfn main() {}\n\n```"); +}); + +test("formatCodeblock with unknown extension", () => { + const result = formatCodeblock("config.xyz", "some: data"); + // Should fall back to using the extension as the language tag + expect(result).toBe("```xyz /config.xyz\nsome: data\n```"); +}); + +test("formatCodeblock with empty content", () => { + const result = formatCodeblock("empty.js", ""); + expect(result).toBe("```javascript /empty.js\n\n```"); +}); + +test("formatCodeblock with multiline content", () => { + const code = `function test() { + const a = 1; + const b = 2; + return a + b; +}`; + const result = formatCodeblock("test.js", code); + expect(result).toBe(`\`\`\`javascript /test.js\n${code}\n\`\`\``); +}); + +test("formatCodeblock trims extra spaces in header", () => { + const range: Range = { + start: { line: 0, character: 0 }, + end: { line: 0, character: 0 }, + }; + const result = formatCodeblock("test.js", "const x = 1;", undefined, range); + // There should be no double spaces in the header + expect(result).toBe("```javascript /test.js (1-1)\nconst x = 1;\n```"); + expect(result).not.toContain(" "); +}); + +test("formatCodeblock with no extension", () => { + const result = formatCodeblock("Dockerfile", "FROM node:14"); + expect(result).toBe("```Dockerfile /Dockerfile\nFROM node:14\n```"); +}); + +test("formatCodeblock with no file extension but explicit language", () => { + const result = formatCodeblock("README", "# Title", "md"); + expect(result).toBe("```markdown /README\n# Title\n```"); +}); diff --git a/core/util/index.ts b/core/util/index.ts index 4b3b17535a..8ea2e15308 100644 --- a/core/util/index.ts +++ b/core/util/index.ts @@ -66,37 +66,37 @@ export function dedentAndGetCommonWhitespace(s: string): [string, string] { return [lines.map((x) => x.replace(lcp, "")).join("\n"), lcp]; } -export function getMarkdownLanguageTagForFile(filepath: string): string { - const extToLangMap: { [key: string]: string } = { - py: "python", - js: "javascript", - jsx: "jsx", - tsx: "tsx", - ts: "typescript", - java: "java", - class: "java", //.class files decompile to Java - go: "go", - rb: "ruby", - rs: "rust", - c: "c", - cpp: "cpp", - cs: "csharp", - php: "php", - scala: "scala", - swift: "swift", - kt: "kotlin", - md: "markdown", - json: "json", - html: "html", - css: "css", - sh: "shell", - yaml: "yaml", - toml: "toml", - tex: "latex", - sql: "sql", - ps1: "powershell", - }; +export const extToLangMap: { [key: string]: string } = { + py: "python", + js: "javascript", + jsx: "jsx", + tsx: "tsx", + ts: "typescript", + java: "java", + class: "java", //.class files decompile to Java + go: "go", + rb: "ruby", + rs: "rust", + c: "c", + cpp: "cpp", + cs: "csharp", + php: "php", + scala: "scala", + swift: "swift", + kt: "kotlin", + md: "markdown", + json: "json", + html: "html", + css: "css", + sh: "shell", + yaml: "yaml", + toml: "toml", + tex: "latex", + sql: "sql", + ps1: "powershell", +}; +export function getMarkdownLanguageTagForFile(filepath: string): string { const ext = sanitizeExtension(filepath.split(".").pop()); return ext ? (extToLangMap[ext] ?? ext) : ""; } diff --git a/core/util/treeSitter.ts b/core/util/treeSitter.ts index 61842ba3e8..8b808ed91b 100644 --- a/core/util/treeSitter.ts +++ b/core/util/treeSitter.ts @@ -118,12 +118,13 @@ export const IGNORE_PATH_PATTERNS: Partial> = { [LanguageName.JAVASCRIPT]: [/.*node_modules/], }; -export async function getParserForFile(filepath: string) { +export async function getParserForFile(fileUri: string) { try { await Parser.init(); const parser = new Parser(); + const fileExt = getUriFileExtension(fileUri); - const language = await getLanguageForFile(filepath); + const language = await getLanguageForFileExt(fileExt); if (!language) { return undefined; } @@ -132,7 +133,7 @@ export async function getParserForFile(filepath: string) { return parser; } catch (e) { - console.debug("Unable to load language for file", filepath, e); + console.debug("Unable to load language for file", fileUri, e); return undefined; } } @@ -142,40 +143,39 @@ export async function getParserForFile(filepath: string) { // to Language object const nameToLanguage = new Map(); -export async function getLanguageForFile( - filepath: string, +async function getLanguageForFileExt( + fileExt: string, ): Promise { try { await Parser.init(); - const extension = getUriFileExtension(filepath); - - const languageName = supportedLanguages[extension]; + const languageName = supportedLanguages[fileExt]; if (!languageName) { return undefined; } let language = nameToLanguage.get(languageName); if (!language) { - language = await loadLanguageForFileExt(extension); + language = await loadLanguageForFileExt(fileExt); nameToLanguage.set(languageName, language); } return language; } catch (e) { - console.debug("Unable to load language for file", filepath, e); + console.debug("Unable to load language for file extension", fileExt, e); return undefined; } } -export const getFullLanguageName = (filepath: string) => { - const extension = getUriFileExtension(filepath); +export const getFullLanguageName = (fileUri: string) => { + const extension = getUriFileExtension(fileUri); return supportedLanguages[extension]; }; export async function getQueryForFile( - filepath: string, + fileUri: string, queryPath: string, ): Promise { - const language = await getLanguageForFile(filepath); + const fileExt = getUriFileExtension(fileUri); + const language = await getLanguageForFileExt(fileExt); if (!language) { return undefined; } @@ -197,15 +197,13 @@ export async function getQueryForFile( return query; } -async function loadLanguageForFileExt( - fileExtension: string, -): Promise { +async function loadLanguageForFileExt(fileExt: string): Promise { const wasmPath = path.join( process.env.NODE_ENV === "test" ? process.cwd() : __dirname, ...(process.env.NODE_ENV === "test" ? ["node_modules", "tree-sitter-wasms", "out"] : ["tree-sitter-wasms"]), - `tree-sitter-${supportedLanguages[fileExtension]}.wasm`, + `tree-sitter-${supportedLanguages[fileExt]}.wasm`, ); return await Parser.Language.load(wasmPath); } @@ -225,10 +223,11 @@ const GET_SYMBOLS_FOR_NODE_TYPES: Parser.SyntaxNode["type"][] = [ ]; export async function getSymbolsForFile( - filepath: string, + fileUri: string, contents: string, ): Promise { - const parser = await getParserForFile(filepath); + const fileExt = getUriFileExtension(fileUri); + const parser = await getParserForFile(fileExt); if (!parser) { return; } @@ -237,10 +236,9 @@ export async function getSymbolsForFile( try { tree = parser.parse(contents); } catch (e) { - console.log(`Error parsing file: ${filepath}`); + console.log(`Error parsing file: ${fileUri}`); return; } - // console.log(`file: ${filepath}`); // Function to recursively find all named nodes (classes and functions) const symbols: SymbolWithRange[] = []; @@ -268,7 +266,7 @@ export async function getSymbolsForFile( if (identifier?.text) { symbols.push({ - filepath, + filepath: fileUri, type: node.type, name: identifier.text, range: { diff --git a/extensions/vscode/src/apply/ApplyManager.ts b/extensions/vscode/src/apply/ApplyManager.ts index 1ac84c3681..02fb4663cf 100644 --- a/extensions/vscode/src/apply/ApplyManager.ts +++ b/extensions/vscode/src/apply/ApplyManager.ts @@ -1,6 +1,5 @@ import { ConfigHandler } from "core/config/ConfigHandler"; import { applyCodeBlock } from "core/edit/lazy/applyCodeBlock"; -import { getUriPathBasename } from "core/util/uri"; import * as vscode from "vscode"; import { ApplyToFilePayload } from "core"; @@ -136,7 +135,7 @@ export class ApplyManager { const { isInstantApply, diffLinesGenerator } = await applyCodeBlock( editor.document.getText(), text, - getUriPathBasename(fileUri), + fileUri, llm, abortController, ); diff --git a/gui/src/pages/gui/ToolCallDiv/CreateFile.tsx b/gui/src/pages/gui/ToolCallDiv/CreateFile.tsx index d0a8431042..b9b9538f6d 100644 --- a/gui/src/pages/gui/ToolCallDiv/CreateFile.tsx +++ b/gui/src/pages/gui/ToolCallDiv/CreateFile.tsx @@ -1,4 +1,4 @@ -import { getMarkdownLanguageTagForFile } from "core/util"; +import { formatCodeblock } from "core/util/formatCodeblock"; import StyledMarkdownPreview from "../../../components/StyledMarkdownPreview"; interface CreateFileToolCallProps { @@ -12,7 +12,10 @@ export function CreateFile(props: CreateFileToolCallProps) { return null; } - const src = `\`\`\`${getMarkdownLanguageTagForFile(props.relativeFilepath ?? "output.txt")} ${props.relativeFilepath}\n${props.fileContents ?? ""}\n\`\`\``; + const src = formatCodeblock( + props.relativeFilepath ?? "newFile.txt", + props.fileContents, + ); return props.relativeFilepath ? (