feat: add global usage & max option

Author: xiaoyu2er

packages/translate/src/index.ts

@@ -33,6 +33,10 @@ program
     '-t, --target-language <language>',
     'Specify the target language code for translation (e.g., "zh-CN", "fr", "es")',
   )
+  .option(
+    '--max <number>',
+    'Maximum number of files to process in one batch (per language) (default: unlimited)',
+  )
   .action(
     async (options: {
       config?: string;
@@ -41,6 +45,7 @@ program
       docsPath?: string;
       listOnly?: boolean;
       targetLanguage?: string;
+      max?: number;
     }) => {
       if (options.verbose) {
         logger.setVerbose(true);
@@ -54,6 +59,7 @@ program
           ...config,
           ...(options.pattern ? { pattern: options.pattern } : {}),
           ...(options.docsPath ? { docsPath: options.docsPath } : {}),
+          ...(options.max ? { max: options.max } : {}),
           listOnly: options.listOnly,
           targetLanguage: options.targetLanguage,
         });

packages/translate/src/main.ts

@@ -4,6 +4,7 @@ import micromatch from 'micromatch';
 import { executeInBatches } from './batch';
 import { logger } from './logger';
 import type { MainConfig } from './types';
+import { GlobalUsage } from './usage';
 import {
   findDocFiles,
   getDocUpdateStatus,
@@ -19,6 +20,7 @@ export async function main({
   outputRoot = 'content',
   docsPath = ['**/*.mdx'],
   listOnly,
+  max = Number.POSITIVE_INFINITY,
   targetLanguage,
 }: MainConfig): Promise<void> {
   // Filter languages based on targetLanguage if specified
@@ -76,6 +78,10 @@ export async function main({
     logger.info(`Using patterns: ${processedPatterns.join(', ')}`);
   }
 
+  if (max !== Number.POSITIVE_INFINITY) {
+    logger.info(`Limiting to ${max} files per language to process`);
+  }
+
   // Process each language
   for (const [lang, langConfig] of Object.entries(filteredLangs)) {
     logger.divider();
@@ -135,7 +141,7 @@ export async function main({
     );
 
     // Build tasks list and document status table
-    const tasks = [];
+    let tasks = [];
     const tableData = [];
 
     for (const docPath of filteredPaths) {
@@ -174,6 +180,12 @@ export async function main({
     logger.info(
       `Found ${tasks.length}/${filteredPaths.length} documents to update`,
     );
+    if (tasks.length > max) {
+      logger.warn(
+        `Limiting to ${max} files to process. You can increase this limit with the --max option.`,
+      );
+      tasks = tasks.slice(0, max);
+    }
 
     // Process tasks if not in list-only mode
     if (!listOnly) {
@@ -207,4 +219,7 @@ export async function main({
   }
 
   logger.divider();
+  logger.info(
+    `Usage statistics for the completion request:\n${JSON.stringify(GlobalUsage, null, 2)}`,
+  );
 }

packages/translate/src/openai.ts

@@ -7,6 +7,7 @@ import {
   splitIntoChunks,
 } from './chunk';
 import { logger } from './logger';
+import { type Usage, addUsage } from './usage';
 
 interface LangConfig {
   name: string;
@@ -95,6 +96,7 @@ HERE IS THE TEXT TO TRANSLATE:
   });
 
   logger.debug(`response.usage:\n${JSON.stringify(response.usage, null, 2)}`);
+  addUsage(response.usage as Usage);
   const message = response.choices?.[0]?.message;
 
   if (!message?.content) {

packages/translate/src/types.ts

@@ -24,6 +24,7 @@ export interface MainConfig {
   docsContext?: string;
   pattern?: string | string[];
   docsPath?: string | string[];
+  max?: number;
   listOnly?: boolean;
   targetLanguage?: string;
 }

packages/translate/src/usage.ts

@@ -0,0 +1,35 @@
+/**
+ * Usage statistics for the completion request.
+ */
+export const GlobalUsage = {
+  completion_tokens: 0, // Number of tokens in the generated completion.
+  prompt_tokens: 0, // Number of tokens in the prompt. It equals prompt_cache_hit_tokens + prompt_cache_miss_tokens.
+  total_tokens: 0, // Total number of tokens used in the request (prompt + completion).
+  prompt_cache_hit_tokens: 0, // Number of tokens in the prompt that hits the context cache.
+  prompt_cache_miss_tokens: 0, // Number of tokens in the prompt that misses the context cache.
+};
+
+export type Usage = typeof GlobalUsage;
+
+export function addUsage(usage: Usage): Usage {
+  if (!usage) {
+    return GlobalUsage;
+  }
+  if (usage.prompt_tokens) {
+    GlobalUsage.prompt_tokens += usage.prompt_tokens;
+  }
+  if (usage.completion_tokens) {
+    GlobalUsage.completion_tokens += usage.completion_tokens;
+  }
+  if (usage.total_tokens) {
+    GlobalUsage.total_tokens += usage.total_tokens;
+  }
+  if (usage.prompt_cache_hit_tokens) {
+    GlobalUsage.prompt_cache_hit_tokens += usage.prompt_cache_hit_tokens;
+  }
+  if (usage.prompt_cache_miss_tokens) {
+    GlobalUsage.prompt_cache_miss_tokens += usage.prompt_cache_miss_tokens;
+  }
+
+  return GlobalUsage;
+}

packages/translate/src/utils.ts

@@ -76,38 +76,14 @@ export async function getDocUpdateStatus({
   const chunks = needsChunking(sourceContent)
     ? splitIntoChunks(sourceContent).length
     : 1;
-  const sourceParsed = matter(sourceContent);
-
-  let sourceLastModifiedDate = getLastModifiedTimeFromGit(sourcePath);
-  if (sourceParsed.data.ref) {
-    try {
-      await fs$.access(sourceParsed.data.ref);
-    } catch (error) {
-      logger.error(
-        `Referenced file not found: ${sourceParsed.data.ref}, don't need updating, consider REMOVING it`,
-      );
-      return {
-        shouldUpdate: false,
-        reason: 'Referenced file not found',
-        chunks,
-      };
-    }
-
-    const refLastModifiedDate = getLastModifiedTimeFromGit(
-      sourceParsed.data.ref,
-    );
-    if (refLastModifiedDate > sourceLastModifiedDate) {
-      sourceLastModifiedDate = refLastModifiedDate;
-    }
-  }
 
   try {
     await fs$.access(targetPath);
   } catch (error) {
     logger.debug(`Target file not found: ${targetPath}, needs updating`);
     return { shouldUpdate: true, reason: 'Target not found.', chunks };
   }
-
+  const sourceLastModifiedDate = getLastModifiedTimeFromGit(sourcePath);
   // Read target file and parse frontmatter
   const targetContent = await fs$.readFile(targetPath, 'utf8');
   const targetParsed = matter(targetContent);
@@ -126,13 +102,13 @@ export async function getDocUpdateStatus({
       );
       return {
         shouldUpdate: true,
-        reason: 'Source has been modified. ',
+        reason: 'Source modified.',
         chunks,
       };
     }
     return {
       shouldUpdate: false,
-      reason: 'Source has not been modified.',
+      reason: 'Source not modified.',
       chunks,
     };
   }
@@ -143,7 +119,7 @@ export async function getDocUpdateStatus({
   );
   return {
     shouldUpdate: true,
-    reason: 'Target no source-updated-at metadata, needs updating.',
+    reason: 'Target has no source-updated-at metadata.',
     chunks,
   };
 }