Merge pull request #7342 from continuedev/nate-3

Add tokenizer tests and fix auto-compact calculation to account for max_tokens

Author: sestinj

.github/actions/run-jetbrains-tests/action.yml

@@ -44,6 +44,7 @@ runs:
     - name: Setup FFmpeg
       uses: AnimMouse/setup-ffmpeg@v1
       with:
+        version: 7.1
         token: ${{ inputs.github-token }}
 
     - name: Setup Gradle
@@ -135,14 +136,22 @@ runs:
       shell: bash
       run: |
         cd extensions/intellij
-        mv video build/reports
+        if [ -d "video" ]; then
+          mv video build/reports
+        else
+          echo "No video directory found to move"
+        fi
 
     - name: Copy logs
       if: ${{ failure() }}
       shell: bash
       run: |
         cd extensions/intellij
-        mv build/idea-sandbox/system/log/ build/reports
+        if [ -d "build/idea-sandbox/system/log/" ]; then
+          mv build/idea-sandbox/system/log/ build/reports
+        else
+          echo "No log directory found to copy"
+        fi
 
     - name: Save fails report
       if: ${{ failure() }}

.github/actions/track-rerun/action.yml

@@ -0,0 +1,19 @@
+name: Track Workflow Rerun
+description: Track when GitHub Actions workflows are rerun (attempt > 1)
+
+runs:
+  using: composite
+  steps:
+    # Eventually we can start logging this to monitor codebase health (e.g. to Slack, PostHog, etc.)
+    - name: Track workflow rerun
+      if: github.run_attempt > 1
+      shell: bash
+      run: |
+        echo "🔄 Workflow rerun detected - Attempt #${{ github.run_attempt }}"
+        echo "📊 Workflow: ${{ github.workflow }}"
+        echo "🔀 PR: ${{ github.event.pull_request.number || 'N/A' }}"
+        echo "📝 Commit: ${{ github.sha }}"
+        echo "👤 Actor: ${{ github.actor }}"
+        echo "⏰ Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+        echo ""
+        echo "::notice title=Workflow Rerun Detected::This is attempt #${{ github.run_attempt }} of workflow '${{ github.workflow }}'"

.github/workflows/cli-pr-checks.yml

@@ -12,6 +12,14 @@ permissions:
   issues: write
 
 jobs:
+  track-rerun:
+    runs-on: ubuntu-latest
+    if: github.run_attempt > 1
+    steps:
+      - uses: actions/checkout@v4
+      - name: Track workflow rerun
+        uses: ./.github/actions/track-rerun
+
   lint:
     runs-on: ubuntu-latest
     steps:

.github/workflows/main.yaml

@@ -9,7 +9,7 @@ on:
       publish_build:
         description: "Whether or not to publish the built extension to the VS Code marketplace"
         required: true
-        default: false
+        default: "false"
 
 jobs:
   check_release_name:

.github/workflows/pr-checks.yaml

@@ -11,6 +11,14 @@ on:
       - main
 
 jobs:
+  track-rerun:
+    runs-on: ubuntu-latest
+    if: github.run_attempt > 1
+    steps:
+      - uses: actions/checkout@v5
+      - name: Track workflow rerun
+        uses: ./.github/actions/track-rerun
+
   prettier-check:
     runs-on: ubuntu-latest
     steps:

.github/workflows/preview.yaml

@@ -9,7 +9,7 @@ on:
       publish_build:
         description: "Whether or not to publish the built extension to the VS Code marketplace"
         required: true
-        default: false
+        default: "false"
 
 jobs:
   check_release_name:

extensions/cli/build.mjs

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 
 import * as esbuild from "esbuild";
-import { writeFileSync, chmodSync } from "fs";
-import { resolve, dirname } from "path";
+import { chmodSync, writeFileSync } from "fs";
+import { dirname, resolve } from "path";
 import { fileURLToPath } from "url";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -59,9 +59,15 @@ try {
         __dirname,
         "../../packages/config-types/dist/index.js",
       ),
-      "core": resolve(__dirname, "../../core"),
-      "@continuedev/fetch": resolve(__dirname, "../../packages/fetch/dist/index.js"),
-      "@continuedev/llm-info": resolve(__dirname, "../../packages/llm-info/dist/index.js"),
+      core: resolve(__dirname, "../../core"),
+      "@continuedev/fetch": resolve(
+        __dirname,
+        "../../packages/fetch/dist/index.js",
+      ),
+      "@continuedev/llm-info": resolve(
+        __dirname,
+        "../../packages/llm-info/dist/index.js",
+      ),
     },
 
     // Add banner to create require for CommonJS packages
@@ -76,7 +82,7 @@ const require = createRequire(import.meta.url);`,
 
   // Create wrapper script with shebang
   writeFileSync("dist/cn.js", "#!/usr/bin/env node\nimport('./index.js');");
-  
+
   // Make the wrapper script executable
   chmodSync("dist/cn.js", 0o755);
 

extensions/cli/src/commands/chat.ts

@@ -159,72 +159,79 @@ async function handleManualCompaction(
   }
 }
 
-// Helper function to handle auto-compaction
+// Helper function to handle auto-compaction for headless mode
 async function handleAutoCompaction(
   chatHistory: ChatCompletionMessageParam[],
   model: ModelConfig,
   llmApi: any,
   isHeadless: boolean,
   format?: "json",
 ): Promise<number | null> {
-  logger.info("Auto-compacting triggered due to context limit");
-
-  if (!isHeadless) {
-    console.info(
-      chalk.yellow(
-        "\nApproaching context limit. Auto-compacting chat history...",
-      ),
-    );
-  } else if (format === "json") {
-    safeStdout(
-      JSON.stringify({
-        status: "info",
-        message: "Auto-compacting triggered",
-        contextUsage:
-          calculateContextUsagePercentage(
-            countChatHistoryTokens(chatHistory),
-            model,
-          ) + "%",
-      }) + "\n",
-    );
-  }
+  const { handleAutoCompaction: coreAutoCompaction } = await import(
+    "../streamChatResponse.autoCompaction.js"
+  );
 
-  try {
-    const result = await compactChatHistory(chatHistory, model, llmApi);
-    chatHistory.length = 0;
-    chatHistory.push(...result.compactedHistory);
-    saveSession(chatHistory);
+  // Custom callbacks for headless mode console output
+  const callbacks = {
+    onSystemMessage: (message: string) => {
+      if (
+        message.includes("Auto-compacting") ||
+        message.includes("Approaching")
+      ) {
+        if (!isHeadless) {
+          console.info(chalk.yellow(`\n${message}`));
+        } else if (format === "json") {
+          safeStdout(
+            JSON.stringify({
+              status: "info",
+              message: "Auto-compacting triggered",
+              contextUsage:
+                calculateContextUsagePercentage(
+                  countChatHistoryTokens(chatHistory),
+                  model,
+                ) + "%",
+            }) + "\n",
+          );
+        }
+      } else if (message.includes("✓")) {
+        if (!isHeadless) {
+          console.info(chalk.green(message));
+        } else if (format === "json") {
+          safeStdout(
+            JSON.stringify({
+              status: "success",
+              message: "Auto-compacted successfully",
+              historyLength: chatHistory.length,
+            }) + "\n",
+          );
+        }
+      } else if (message.includes("Warning:")) {
+        if (!isHeadless) {
+          console.error(chalk.red(message));
+          console.info(chalk.yellow("Continuing without compaction..."));
+        } else if (format === "json") {
+          safeStdout(
+            JSON.stringify({
+              status: "warning",
+              message: "Auto-compaction failed, continuing without compaction",
+            }) + "\n",
+          );
+        }
+      }
+    },
+  };
 
-    if (!isHeadless) {
-      console.info(chalk.green("✓ Chat history auto-compacted successfully."));
-    } else if (format === "json") {
-      safeStdout(
-        JSON.stringify({
-          status: "success",
-          message: "Auto-compacted successfully",
-          historyLength: chatHistory.length,
-        }) + "\n",
-      );
-    }
+  const result = await coreAutoCompaction(chatHistory, model, llmApi, {
+    isHeadless,
+    format,
+    callbacks,
+  });
 
-    return result.compactionIndex;
-  } catch (error) {
-    const errorMsg = `Auto-compaction error: ${formatError(error)}`;
-    logger.error(errorMsg);
+  // Update the original array reference for headless mode
+  chatHistory.length = 0;
+  chatHistory.push(...result.chatHistory);
 
-    if (!isHeadless) {
-      console.error(chalk.red(`Warning: ${errorMsg}`));
-      console.info(chalk.yellow("Continuing without compaction..."));
-    } else if (format === "json") {
-      safeStdout(
-        JSON.stringify({
-          status: "warning",
-          message: "Auto-compaction failed, continuing without compaction",
-        }) + "\n",
-      );
-    }
-    return null;
-  }
+  return result.compactionIndex;
 }
 
 interface ProcessMessageOptions {

extensions/cli/src/commands/commands.ts

@@ -4,9 +4,9 @@ import { type AssistantConfig } from "@continuedev/sdk";
 export { chat } from "./chat.js";
 export { login } from "./login.js";
 export { logout } from "./logout.js";
+export { listSessionsCommand } from "./ls.js";
 export { remote } from "./remote.js";
 export { serve } from "./serve.js";
-export { listSessionsCommand } from "./ls.js";
 
 export interface SlashCommand {
   name: string;
@@ -45,6 +45,11 @@ export const SYSTEM_SLASH_COMMANDS: SystemCommand[] = [
     description: "Check who you're currently logged in as",
     category: "system",
   },
+  {
+    name: "info",
+    description: "Show session information",
+    category: "system",
+  },
   {
     name: "model",
     description: "Switch between available chat models",