feat(document): unify binary file parsing and add parseBytes #463

Refactor document parsing to always use parseBytes for binary files, removing legacy string fallback. Add default parseBytes implementation and platform-specific readFileAsBytes support.

Author: phodal

mpp-core/src/commonMain/kotlin/cc/unitmesh/agent/document/DocumentAgent.kt

@@ -202,9 +202,9 @@ class DocumentAgent(
             
             ## Response Workflow
             
-            1. Explain what you plan to query.
-            2. Make **exactly one** DocQL call.
-            3. After tool results, answer normally (no more tool use).
+            1. **Plan**: Analyze the query and identify target documents from filename patterns
+            2. **Query**: Make **exactly one** DocQL call with appropriate query and documentPath
+            3. **Respond**: After tool results, synthesize answer naturally (no more tool use)
             
             ---
             
@@ -257,11 +257,57 @@ class DocumentAgent(
             
             ## Best Practices
             
-            * Prefer `documentPath` when clear.
-            * retry with broader DocQL queries before giving up.
-            * Use TOC early.
-            * Use `heading()` for sections; `chunks()` for full context.
-            * Always retrieve; never speculate.
+            ✅ **DO:**
+            - Always specify `documentPath` when filename clearly matches query keywords
+            - Start with `heading()` for targeted searches, fall back to `chunks()` if empty
+            - Expand keywords BEFORE first query: synonyms, morphology, translations
+            - Try 2-3 different queries before concluding "no information found"
+            - Query multiple related documents for cross-cutting topics
+            
+            ❌ **DON'T:**
+            - Never guess or speculate without actually querying first
+            - Don't use filesystem tools on registered documents
+            - Don't give up after one failed query - retry with broader terms
+            - Don't use `chunks()` as your first choice unless query is very broad
+            
+            ---
+            
+            ## Successful Query Examples
+            
+            **Query: "What colors are used in the design system?"**
+            ```json
+            {
+              "query": "$.content.heading(\"color\")",
+              "documentPath": "design-system-color.md"
+            }
+            ```
+            ✅ Direct filename match + targeted heading search
+            
+            **Query: "How do I use custom icons?"**
+            First attempt:
+            ```json
+            {
+              "query": "$.content.heading(\"custom icons\")",
+              "documentPath": "custom-icons-usage.md"
+            }
+            ```
+            If empty, retry with:
+            ```json
+            {
+              "query": "$.content.chunks(\"icons\")",
+              "documentPath": "custom-icons-usage.md"
+            }
+            ```
+            ✅ Good retry strategy: specific → broader
+            
+            **Query: "Tell me about architecture" (ambiguous)**
+            ```json
+            {
+              "query": "$.toc[*]"
+            }
+            ```
+            Then identify relevant doc(s) from TOC and query specifically
+            ✅ Use TOC when multiple architecture-related files exist
 
         """.trimIndent()
     }

mpp-core/src/commonMain/kotlin/cc/unitmesh/devins/document/DocumentParserService.kt

@@ -10,6 +10,16 @@ interface DocumentParserService {
      */
     suspend fun parse(file: DocumentFile, content: String): DocumentTreeNode
 
+    /**
+     * 解析二进制文档文件（用于 PDF, DOCX 等格式）
+     * 默认实现：将字节转换为字符串后调用 parse
+     */
+    suspend fun parseBytes(file: DocumentFile, bytes: ByteArray): DocumentTreeNode {
+        // Default: decode as UTF-8 and call parse
+        val content = bytes.decodeToString()
+        return parse(file, content)
+    }
+
     /**
      * HeadingQL: 根据关键字或标题文本查找最匹配的标题节点及其内容
      */

mpp-core/src/iosMain/kotlin/cc/unitmesh/devins/filesystem/ProjectFileSystem.ios.kt

@@ -1,6 +1,8 @@
 package cc.unitmesh.devins.filesystem
 
 import kotlinx.cinterop.ExperimentalForeignApi
+import kotlinx.cinterop.get
+import kotlinx.cinterop.reinterpret
 import kotlinx.cinterop.toKString
 import platform.Foundation.*
 import platform.posix.getcwd
@@ -29,6 +31,20 @@ actual class DefaultFileSystem actual constructor(
         }
     }
 
+    actual override fun readFileAsBytes(path: String): ByteArray? {
+        val fullPath = resolvePath(path)
+        return try {
+            val data = NSData.dataWithContentsOfFile(fullPath) ?: return null
+            val length = data.length.toInt()
+            val bytes = data.bytes?.reinterpret<kotlinx.cinterop.ByteVar>() ?: return null
+            ByteArray(length) { i ->
+                bytes[i]
+            }
+        } catch (e: Exception) {
+            null
+        }
+    }
+
     actual override fun writeFile(path: String, content: String): Boolean {
         val fullPath = resolvePath(path)
         return try {

mpp-core/src/jsMain/kotlin/cc/unitmesh/devins/filesystem/DefaultFileSystem.js.kt

@@ -1,5 +1,8 @@
 package cc.unitmesh.devins.filesystem
 
+import org.khronos.webgl.ArrayBuffer
+import org.khronos.webgl.Uint8Array
+
 /**
  * JavaScript 平台的文件系统实现
  * 基于 Node.js fs 模块的高性能实现
@@ -55,8 +58,12 @@ actual class DefaultFileSystem actual constructor(private val projectPath: Strin
             if (exists(resolvedPath) && !isDirectory(resolvedPath)) {
                 val buffer = fs.readFileSync(resolvedPath)
                 // Convert Node.js Buffer to ByteArray
-                val array = Uint8Array(buffer.buffer as ArrayBuffer, buffer.byteOffset as Int, buffer.length as Int)
-                ByteArray(array.length) { array[it] }
+                // Node.js Buffer is already array-like, access elements directly
+                val length = buffer.length.unsafeCast<Int>()
+                ByteArray(length) { i -> 
+                    val value: dynamic = buffer[i]
+                    value.unsafeCast<Byte>()
+                }
             } else {
                 null
             }

mpp-core/src/jvmMain/kotlin/cc/unitmesh/devins/document/TikaDocumentParser.kt

@@ -48,7 +48,7 @@ class TikaDocumentParser : DocumentParserService {
      * Parse document from ByteArray (preferred for binary files)
      * This method properly handles binary data without corruption
      */
-    suspend fun parseBytes(file: DocumentFile, bytes: ByteArray): DocumentTreeNode {
+    override suspend fun parseBytes(file: DocumentFile, bytes: ByteArray): DocumentTreeNode {
         logger.info { "=== Starting Tika Parse (from Bytes) ===" }
         logger.info { "File: ${file.path}, Size: ${bytes.size} bytes" }
 

mpp-ui/src/commonMain/kotlin/cc/unitmesh/devins/service/DocumentIndexService.kt

@@ -167,14 +167,8 @@ class DocumentIndexService(
 
                 // Parse the document based on format type
                 val parsedDoc = if (isBinary && bytes != null) {
-                    // Use TikaDocumentParser's parseBytes for binary files
-                    if (parser is cc.unitmesh.devins.document.TikaDocumentParser) {
-                        parser.parseBytes(docFile, bytes)
-                    } else {
-                        // Fallback: convert bytes to ISO_8859_1 string
-                        val fallbackContent = bytes.toString(Charsets.ISO_8859_1)
-                        parser.parse(docFile, fallbackContent)
-                    }
+                    // Use parseBytes for binary files
+                    parser.parseBytes(docFile, bytes)
                 } else if (content != null) {
                     // Text format
                     parser.parse(docFile, content)

mpp-ui/src/commonMain/kotlin/cc/unitmesh/devins/ui/compose/document/DocumentReaderViewModel.kt

@@ -310,14 +310,8 @@ class DocumentReaderViewModel(private val workspace: Workspace) {
 
                     documentContent = null // Binary files don't have text content to display
 
-                    // Use TikaDocumentParser's parseBytes method for binary files
-                    if (parser is cc.unitmesh.devins.document.TikaDocumentParser) {
-                        parser.parseBytes(doc, bytes)
-                    } else {
-                        // Fallback: convert bytes to ISO_8859_1 string (legacy behavior)
-                        val content = bytes.toString(Charsets.ISO_8859_1)
-                        parser.parse(doc, content)
-                    }
+                    // Use parseBytes method for binary files
+                    parser.parseBytes(doc, bytes)
                 } else {
                     // Text formats (Markdown, TXT) - read as string
                     val content = fileSystem.readFile(doc.path)

mpp-ui/src/commonTest/kotlin/cc/unitmesh/devins/service/DocumentIndexServiceTest.kt

@@ -21,6 +21,10 @@ class DocumentIndexServiceTest {
 
         override fun readFile(path: String): String? = files[path]
 
+        override fun readFileAsBytes(path: String): ByteArray? {
+            return files[path]?.encodeToByteArray()
+        }
+        
         override fun writeFile(path: String, content: String): Boolean {
             files[path] = content
             return true