deduplicate code

Author: chezsmithy

core/autocomplete/filtering/streamTransforms/lineStream.ts

@@ -3,6 +3,17 @@ import { distance } from "fastest-levenshtein";
 import { DiffLine } from "../../..";
 import { LineStream } from "../../../diff/util";
 
+import {
+  headerIsMarkdown,
+  isMarkdownFile,
+  MarkdownBlockStateTracker,
+  collectAllLines,
+} from "../../../utils/markdownUtils";
+import {
+  shouldStopAtMarkdownBlock,
+  processBlockNesting as processBlockNestingUtil,
+} from "../../../utils/streamMarkdownUtils";
+
 export { filterCodeBlockLines } from "./filterCodeBlock";
 
 export type LineFilter = (args: {
@@ -18,23 +29,6 @@ export type CharacterFilter = (args: {
   multiline: boolean;
 }) => AsyncGenerator<string>;
 
-// Duplicated from gui/src/components/StyledMarkdownPreview/utils/headerIsMarkdown.ts
-function headerIsMarkdown(header: string): boolean {
-  return (
-    header === "md" ||
-    header === "markdown" ||
-    header === "gfm" ||
-    header === "github-markdown" ||
-    header.includes(" md") ||
-    header.includes(" markdown") ||
-    header.includes(" gfm") ||
-    header.includes(" github-markdown") ||
-    header.split(" ")[0]?.split(".").pop() === "md" ||
-    header.split(" ")[0]?.split(".").pop() === "markdown" ||
-    header.split(" ")[0]?.split(".").pop() === "gfm"
-  );
-}
-
 function isBracketEnding(line: string): boolean {
   return line
     .trim()
@@ -124,19 +118,6 @@ function isUselessLine(line: string): boolean {
   return hasUselessLine || trimmed.startsWith("// end");
 }
 
-/**
- * Determines if a file is a markdown file based on its filepath.
- */
-export function isMarkdownFile(filepath?: string): boolean {
-  if (!filepath) {
-    return false;
-  }
-
-  return ["md", "markdown", "gfm"].includes(
-    filepath.split(".").pop()?.toLowerCase() || "",
-  );
-}
-
 /**
  * Determines if the code block has nested markdown blocks.
  */
@@ -147,116 +128,25 @@ export function hasNestedMarkdownBlocks(
   return (
     (firstLine.startsWith("```") &&
       headerIsMarkdown(firstLine.replace(/`/g, ""))) ||
-    Boolean(filepath && headerIsMarkdown(filepath))
+    Boolean(filepath && isMarkdownFile(filepath))
   );
 }
 
-/**
- * Collects all lines from a LineStream into an array for analysis.
- */
-export async function collectAllLines(rawLines: LineStream): Promise<string[]> {
-  const allLines: string[] = [];
-  for await (const line of rawLines) {
-    allLines.push(line);
-  }
-  return allLines;
-}
-
-/**
- * State tracker for markdown block analysis to avoid recomputing on each call.
- */
-export class MarkdownBlockState {
-  private trimmedLines: string[];
-  private bareBacktickPositions: number[];
-  private markdownNestCount: number = 0;
-  private lastProcessedIndex: number = -1;
-
-  constructor(allLines: string[]) {
-    this.trimmedLines = allLines.map((l) => l.trim());
-    // Pre-compute positions of all bare backtick lines for faster lookup
-    this.bareBacktickPositions = [];
-    for (let i = 0; i < this.trimmedLines.length; i++) {
-      if (this.trimmedLines[i].match(/^`+$/)) {
-        this.bareBacktickPositions.push(i);
-      }
-    }
-  }
-
-  /**
-   * Determines if we should stop at the given markdown block position.
-   * Maintains state across calls to avoid redundant computation.
-   */
-  shouldStopAtPosition(currentIndex: number): boolean {
-    if (this.trimmedLines[currentIndex] !== "```") {
-      return false;
-    }
-
-    // Process any lines we haven't seen yet up to currentIndex
-    for (let j = this.lastProcessedIndex + 1; j <= currentIndex; j++) {
-      const currentLine = this.trimmedLines[j];
-
-      if (this.markdownNestCount > 0) {
-        // Inside a markdown block
-        if (currentLine.match(/^`+$/)) {
-          // Found bare backticks - check if this is the last one
-          if (j === currentIndex) {
-            const remainingBareBackticks = this.bareBacktickPositions.filter(
-              (pos) => pos > j,
-            ).length;
-            if (remainingBareBackticks === 0) {
-              this.markdownNestCount = 0;
-              this.lastProcessedIndex = j;
-              return true;
-            }
-          }
-        } else if (currentLine.startsWith("```")) {
-          // Going into a nested codeblock
-          this.markdownNestCount++;
-        }
-      } else {
-        // Not inside a markdown block yet
-        if (currentLine.startsWith("```")) {
-          const header = currentLine.replaceAll("`", "");
-          if (headerIsMarkdown(header)) {
-            this.markdownNestCount = 1;
-          }
-        }
-      }
-    }
-
-    this.lastProcessedIndex = currentIndex;
-    return false;
-  }
-}
-
-/**
- * Determines if we should stop at a markdown block based on nested markdown logic.
- * This handles the complex case where markdown blocks contain other markdown blocks.
- * Uses optimized state tracking to avoid redundant computation.
- */
-export function shouldStopAtMarkdownBlock(
-  stateTracker: MarkdownBlockState,
-  currentIndex: number,
-): boolean {
-  return stateTracker.shouldStopAtPosition(currentIndex);
-}
+// Re-export shared utilities
+export { collectAllLines, isMarkdownFile };
+export { MarkdownBlockStateTracker as MarkdownBlockState };
+export { shouldStopAtMarkdownBlock };
 
-/**
- * Processes block nesting logic and returns updated state.
- */
+// Wrapper for processBlockNesting with local shouldRemoveLineBeforeStart function
 export function processBlockNesting(
   line: string,
   seenFirstFence: boolean,
 ): { newSeenFirstFence: boolean; shouldSkip: boolean } {
-  if (!seenFirstFence && shouldRemoveLineBeforeStart(line)) {
-    return { newSeenFirstFence: false, shouldSkip: true };
-  }
-
-  if (!seenFirstFence) {
-    return { newSeenFirstFence: true, shouldSkip: false };
-  }
-
-  return { newSeenFirstFence: seenFirstFence, shouldSkip: false };
+  return processBlockNestingUtil(
+    line,
+    seenFirstFence,
+    shouldRemoveLineBeforeStart,
+  );
 }
 
 export const USELESS_LINES = [""];
@@ -326,7 +216,7 @@ export async function* avoidPathLine(
   // Sometimes the model with copy this pattern, which is unwanted
   for await (const line of stream) {
     if (line.startsWith(`${comment} Path: `)) {
-      continue;
+      continue; // continue in the Continue codebase! How meta!
     }
     yield line;
   }

core/autocomplete/filtering/streamTransforms/lineStream.vitest.ts

@@ -846,32 +846,6 @@ describe("lineStream", () => {
   });
 
   // Tests for new helper functions
-  describe("isMarkdownFile", () => {
-    it("should return true for markdown file extensions", () => {
-      expect(lineStream.isMarkdownFile("README.md")).toBe(true);
-      expect(lineStream.isMarkdownFile("docs.markdown")).toBe(true);
-      expect(lineStream.isMarkdownFile("notes.gfm")).toBe(true);
-      expect(lineStream.isMarkdownFile("path/to/file.md")).toBe(true);
-    });
-
-    it("should return false for non-markdown file extensions", () => {
-      expect(lineStream.isMarkdownFile("script.js")).toBe(false);
-      expect(lineStream.isMarkdownFile("styles.css")).toBe(false);
-      expect(lineStream.isMarkdownFile("document.txt")).toBe(false);
-      expect(lineStream.isMarkdownFile("config.json")).toBe(false);
-    });
-
-    it("should return false for undefined or empty filepath", () => {
-      expect(lineStream.isMarkdownFile(undefined)).toBe(false);
-      expect(lineStream.isMarkdownFile("")).toBe(false);
-    });
-
-    it("should handle case insensitive extensions", () => {
-      expect(lineStream.isMarkdownFile("README.MD")).toBe(true);
-      expect(lineStream.isMarkdownFile("docs.MARKDOWN")).toBe(true);
-      expect(lineStream.isMarkdownFile("notes.GFM")).toBe(true);
-    });
-  });
 
   describe("hasNestedMarkdownBlocks", () => {
     it("should detect nested markdown blocks from first line", () => {
@@ -1037,59 +1011,6 @@ describe("lineStream", () => {
     });
   });
 
-  describe("processBlockNesting", () => {
-    it("should skip lines that should be removed before start", () => {
-      const result = lineStream.processBlockNesting("```typescript", false);
-      expect(result).toEqual({
-        newSeenFirstFence: false,
-        shouldSkip: true,
-      });
-    });
-
-    it("should skip [CODE] blocks", () => {
-      const result = lineStream.processBlockNesting("[CODE]", false);
-      expect(result).toEqual({
-        newSeenFirstFence: false,
-        shouldSkip: true,
-      });
-    });
-
-    it("should set first fence when not seen yet", () => {
-      const result = lineStream.processBlockNesting("const x = 5;", false);
-      expect(result).toEqual({
-        newSeenFirstFence: true,
-        shouldSkip: false,
-      });
-    });
-
-    it("should not change state when first fence already seen", () => {
-      const result = lineStream.processBlockNesting("const x = 5;", true);
-      expect(result).toEqual({
-        newSeenFirstFence: true,
-        shouldSkip: false,
-      });
-    });
-
-    it("should handle <COMPLETION> prefix", () => {
-      const result = lineStream.processBlockNesting("<COMPLETION>", false);
-      expect(result).toEqual({
-        newSeenFirstFence: false,
-        shouldSkip: true,
-      });
-    });
-
-    it("should handle START EDITING HERE blocks", () => {
-      const result = lineStream.processBlockNesting(
-        "<START EDITING HERE>",
-        false,
-      );
-      expect(result).toEqual({
-        newSeenFirstFence: false,
-        shouldSkip: true,
-      });
-    });
-  });
-
   describe("MarkdownBlockState", () => {
     it("should initialize with correct state", () => {
       const allLines = ["```markdown", "content", "```", "more content", "```"];