fix: refine newline handling in lexer and update test snapshots

Author: KeJunMao

.vscode/settings.json

@@ -65,5 +65,8 @@
     "jsonc",
     "yaml",
     "toml"
+  ],
+  "cSpell.words": [
+    "unplugin"
   ]
 }

src/core/context/generator.ts

@@ -8,7 +8,7 @@ export class Generator {
   walk(node: SimpleNode): string | void {
     switch (node.type) {
       case 'Program':
-        return node.body.map(this.walk.bind(this)).filter((n: any) => !!n && n !== '\r\n').join('')
+        return node.body.map(this.walk.bind(this)).filter((n: any) => !!n).join('')
       case 'CodeStatement':
         return node.value
     }

src/core/context/lexer.ts

@@ -13,54 +13,48 @@ export class Lexer {
     scanner:
     while (this.current < code.length) {
       const startIndex = this.current
-      // 查找最近的换行符(\r\n, \n, \r)
-      const nextCR = code.indexOf('\r', startIndex)
-      const nextLF = code.indexOf('\n', startIndex)
+
+      // 使用正则表达式匹配换行符，更优雅地处理各种换行符类型
+      const newlineMatch = code.slice(startIndex).match(/(\r\n|\n|\r)/)
+
       let endIndex: number
+      let nextIndex: number
+      let newlineChar: string | undefined
 
-      if (nextCR === -1 && nextLF === -1) {
-        // 没有找到换行符，说明是最后一行
-        endIndex = code.length
-      }
-      else if (nextCR === -1) {
-        // 只有 \n
-        endIndex = nextLF
-      }
-      else if (nextLF === -1) {
-        // 只有 \r
-        endIndex = nextCR
-      }
-      else if (nextCR < nextLF) {
-        // 如果是 \r\n，跳过 \n
-        endIndex = nextCR
-        if (nextLF === nextCR + 1) {
-          endIndex += 2
-        }
+      if (newlineMatch) {
+        newlineChar = newlineMatch[0]
+        endIndex = startIndex + newlineMatch.index!
+        nextIndex = endIndex + newlineChar.length
       }
       else {
-        // \n
-        endIndex = nextLF
+        // 没有找到换行符，说明是最后一行
+        endIndex = code.length
+        nextIndex = code.length
       }
 
-      const rawLine = code.slice(startIndex, endIndex)
-      const line = rawLine.trim()
-      if (isComment(line)) {
+      // 获取原始行内容，对于 code 类型，我们需要包含换行符
+      const rawLine = code.slice(startIndex, nextIndex)
+      const lineWithoutNewline = code.slice(startIndex, endIndex).trim()
+
+      if (isComment(lineWithoutNewline)) {
         for (const lex of this.lexers) {
-          const comment = parseComment(line)
+          const comment = parseComment(lineWithoutNewline)
 
           const token = lex.bind(this)(comment.content!)
           if (token) {
             this.tokens.push({ comment: comment.type, ...token })
-            this.current = endIndex
+            this.current = nextIndex
             continue scanner
           }
         }
       }
+
+      // 对于 code 类型，保留原始行内容（包括换行符）
       this.tokens.push({
         type: 'code',
         value: rawLine,
       } as CodeToken)
-      this.current = endIndex
+      this.current = nextIndex
     }
     return this.tokens
   }

test/__snapshots__/if.test.ts.snap

@@ -3,12 +3,18 @@
 exports[`if > should parse if.css, dev = false 1`] = `
 "body {
 }
+
+
 body {
   content: "!DEV";
 }
+
+
 body {
   content: "!DEV";
 }
+
+
 body {
 }
 "
@@ -18,44 +24,62 @@ exports[`if > should parse if.css, dev = true 1`] = `
 "body {
   content: "DEV";
 }
+
+
 body {
   content: "!DEV else";
 }
+
+
 body {
   content: "TEST";
 }
+
+
 body {
     content: "else";
 }
 "
 `;
 
 exports[`if > should parse if.html, dev = false 1`] = `
-"<div>!DEV</div>
+"
 <div>!DEV</div>
+
+<div>!DEV</div>
+
 "
 `;
 
 exports[`if > should parse if.html, dev = true 1`] = `
 "<div>DEV</div>
+
 <div>!DEV else</div>
+
 <div>TEST</div>
+
 <div>
   <div>else</div>
 </div>
 "
 `;
 
 exports[`if > should parse if.js, dev = false 1`] = `
-"console.log('!DEV')
+"
+console.log('!DEV')
+
 console.log('!DEV')
+
 "
 `;
 
 exports[`if > should parse if.js, dev = true 1`] = `
 "console.log('DEV')
+
 console.log('!DEV else')
+
 console.log('TEST')
+
 console.log('else')
 "
 `;
@@ -76,23 +100,31 @@ exports[`if > should parse if.jsx, dev = true 1`] = `
 
 exports[`if > should parse if.vue, dev = false 1`] = `
 "<script setup lang="ts">
+
 </script>
+
 <template>
     <pre>{
 }</pre>
 </template>
+
 <style>
+
 </style>"
 `;
 
 exports[`if > should parse if.vue, dev = true 1`] = `
 "<script setup lang="ts">
+
 </script>
+
 <template>
     <pre>{
   "data": 'value' // comment
 }</pre>
 </template>
+
 <style>
+
 </style>"
 `;

test/lexer.test.ts

@@ -9,10 +9,11 @@ describe('lexer', () => {
       // Another comment
       const baz = 'qux';`
     const expectedTokens = [
-      { type: 'code', value: '// This is a comment' },
-      { type: 'code', value: 'const foo = \'bar\';' },
-      { type: 'code', value: '// Another comment' },
-      { type: 'code', value: 'const baz = \'qux\';' },
+      { type: 'code', value: '\n' },
+      { type: 'code', value: '      // This is a comment\n' },
+      { type: 'code', value: '      const foo = \'bar\';\n' },
+      { type: 'code', value: '      // Another comment\n' },
+      { type: 'code', value: '      const baz = \'qux\';' },
     ]
 
     const tokens = Lexer.lex(code)
@@ -25,8 +26,9 @@ describe('lexer', () => {
       const foo = 'bar';
       const baz = 'qux';`
     const expectedTokens = [
-      { type: 'code', value: 'const foo = \'bar\';' },
-      { type: 'code', value: 'const baz = \'qux\';' },
+      { type: 'code', value: '\n' },
+      { type: 'code', value: '      const foo = \'bar\';\n' },
+      { type: 'code', value: '      const baz = \'qux\';' },
     ]
 
     const tokens = Lexer.lex(code)