Backport: Improve multi-byte character handling

packages/core/__tests__/transform/rewrite.test.ts

@@ -33,6 +33,39 @@ describe('Transform: rewriteModule', () => {
       `);
     });
 
+    test('handles the $ character', () => {
+      let script = {
+        filename: 'test.gts',
+        contents: '<template>${{dollarAmount}}</template>;',
+      };
+
+      let transformedModule = rewriteModule(ts, { script }, emberTemplateImportsEnvironment);
+
+      expect(transformedModule?.errors).toEqual([]);
+      expect(transformedModule?.transformedContents).toMatchInlineSnapshot(`
+        "export default ({} as typeof import("@glint/environment-ember-template-imports/-private/dsl")).templateExpression(function(__glintRef__, __glintDSL__: typeof import("@glint/environment-ember-template-imports/-private/dsl")) {
+        __glintDSL__.emitContent(__glintDSL__.resolveOrReturn(dollarAmount)());
+        __glintRef__; __glintDSL__;
+        });"
+      `);
+    });
+
+    test('handles the ` character', () => {
+      let script = {
+        filename: 'test.gts',
+        contents: '<template>`code`</template>;',
+      };
+
+      let transformedModule = rewriteModule(ts, { script }, emberTemplateImportsEnvironment);
+
+      expect(transformedModule?.errors).toEqual([]);
+      expect(transformedModule?.transformedContents).toMatchInlineSnapshot(`
+        "export default ({} as typeof import("@glint/environment-ember-template-imports/-private/dsl")).templateExpression(function(__glintRef__, __glintDSL__: typeof import("@glint/environment-ember-template-imports/-private/dsl")) {
+        __glintRef__; __glintDSL__;
+        });"
+      `);
+    });
+
     test('with a class with type parameters', () => {
       let script = {
         filename: 'test.ts',

packages/environment-ember-template-imports/-private/environment/preprocess.ts

@@ -29,7 +29,7 @@ export const preprocess: GlintExtensionPreprocess<PreprocessData> = (source, pat
   let templates = p.parse(source, { filename: path });
 
   let templateLocations: Array<TemplateLocation> = [];
-  let segments: Array<string> = [];
+  let contents = '';
   let sourceOffsetBytes = 0;
   let deltaBytes = 0;
 
@@ -41,46 +41,48 @@ export const preprocess: GlintExtensionPreprocess<PreprocessData> = (source, pat
     let startTagOffsetBytes = template.startRange.start;
     let endTagOffsetBytes = template.endRange.start;
     let transformedStartBytes = startTagOffsetBytes - deltaBytes;
-
     /**
      * TODO: we want content-tag to manage all this for us, as managing indicies
      *       can be error-prone.
      *
      * SEE: https://github.com/embroider-build/content-tag/issues/39#issuecomment-1832443310
      */
-    let prefixingSegment = sourceBuffer.slice(sourceOffsetBytes, startTagOffsetBytes);
-    segments.push(prefixingSegment.toString());
-    segments.push(TEMPLATE_START);
+    let prefixingSegment = sourceBuffer.subarray(sourceOffsetBytes, startTagOffsetBytes);
+    contents = contents.concat(prefixingSegment.toString());
+    contents = contents.concat(TEMPLATE_START);
 
     // For TEMPLATE_START & TEMPLATE_END, characters === bytes
     deltaBytes += startTagLengthBytes - TEMPLATE_START.length;
 
     let transformedEnd = endTagOffsetBytes - deltaBytes + TEMPLATE_END.length;
-
-    let templateContentSegment = sourceBuffer.slice(
+    let templateContentSegment = sourceBuffer.subarray(
       startTagOffsetBytes + startTagLengthBytes,
       endTagOffsetBytes
     );
-    segments.push(templateContentSegment.toString());
-    segments.push(TEMPLATE_END);
+    let templateContentSegmentString = templateContentSegment.toString();
+    let escapedTemplateContentSegment = templateContentSegmentString
+      .replace(/\$/g, '\\$')
+      .replace(/`/g, '\\`');
+    deltaBytes += templateContentSegmentString.length - escapedTemplateContentSegment.length;
+
+    contents = contents.concat(escapedTemplateContentSegment);
+    contents = contents.concat(TEMPLATE_END);
     deltaBytes += endTagLengthBytes - TEMPLATE_END.length;
 
     sourceOffsetBytes = endTagOffsetBytes + endTagLengthBytes;
-
     templateLocations.push({
       startTagOffset: byteToCharIndex(source, startTagOffsetBytes),
       endTagOffset: byteToCharIndex(source, endTagOffsetBytes),
       startTagLength: byteToCharIndex(source, startTagLengthBytes),
       endTagLength: byteToCharIndex(source, endTagLengthBytes),
-      transformedStart: byteToCharIndex(source, transformedStartBytes),
-      transformedEnd: byteToCharIndex(source, transformedEnd),
+      transformedStart: byteToCharIndex(contents, transformedStartBytes),
+      transformedEnd: byteToCharIndex(contents, transformedEnd),
     });
   }
 
-  segments.push(sourceBuffer.slice(sourceOffsetBytes).toString());
-
+  contents = contents.concat(sourceBuffer.subarray(sourceOffsetBytes).toString());
   return {
-    contents: segments.join(''),
+    contents,
     data: {
       templateLocations,
     },
@@ -89,7 +91,7 @@ export const preprocess: GlintExtensionPreprocess<PreprocessData> = (source, pat
 
 function byteToCharIndex(str: string, byteOffset: number): number {
   const buf = getBuffer(str);
-  return buf.slice(0, byteOffset).toString().length;
+  return buf.subarray(0, byteOffset).toString().length;
 }
 
 const BufferMap = new Map();

packages/environment-ember-template-imports/__tests__/transformation.test.ts

@@ -27,6 +27,40 @@ describe('Environment: ETI', () => {
       });
     });
 
+    test('handles multi-byte characters', () => {
+      let source = [
+        `const a = <template>one 💩</template>;`,
+        `const b = <template>two</template>;`,
+        `const c = "‘foo’";`,
+        `const d = <template>four</template>;`,
+      ].join('\n');
+
+      let result = preprocess(source, 'index.gts');
+
+      expect(result.contents).toMatchInlineSnapshot(`
+        "const a = [___T\`one 💩\`];
+        const b = [___T\`two\`];
+        const c = "‘foo’";
+        const d = [___T\`four\`];"
+      `);
+    });
+
+    test('handles the $ character', () => {
+      let source = '<template>${{dollarAmount}}</template>;';
+
+      let result = preprocess(source, 'index.gts');
+
+      expect(result.contents).toMatchInlineSnapshot('"[___T`\\${{dollarAmount}}`];"');
+    });
+
+    test('handles the ` character', () => {
+      let source = '<template>`code`</template>;';
+
+      let result = preprocess(source, 'index.gts');
+
+      expect(result.contents).toMatchInlineSnapshot('"[___T`\\`code\\``];"');
+    });
+
     test('multiple templates', () => {
       let source = stripIndent`
         <template>
@@ -75,6 +109,45 @@ describe('Environment: ETI', () => {
         ],
       });
     });
+
+    test('handles multi-byte characters', () => {
+      let source = stripIndent`
+        let a = <template></template>;
+        // ‘
+        let b = <template></template>;
+      `;
+
+      let transformed = stripIndent`
+        let a = [___T\`\`];
+        // ‘
+        let b = [___T\`\`];
+      `;
+
+      let result = preprocess(source, 'index.gts');
+
+      expect(result.contents).toEqual(transformed);
+
+      expect(result.data).toEqual({
+        templateLocations: [
+          {
+            startTagOffset: source.indexOf('<template>'),
+            startTagLength: '<template>'.length,
+            endTagOffset: source.indexOf('</template>'),
+            endTagLength: '</template>'.length,
+            transformedStart: transformed.indexOf('[___T'),
+            transformedEnd: transformed.indexOf(']') + 1,
+          },
+          {
+            startTagOffset: source.lastIndexOf('<template>'),
+            startTagLength: '<template>'.length,
+            endTagOffset: source.lastIndexOf('</template>'),
+            endTagLength: '</template>'.length,
+            transformedStart: transformed.lastIndexOf('[___T'),
+            transformedEnd: transformed.lastIndexOf(']') + 1,
+          },
+        ],
+      });
+    });
   });
 
   describe('transform', () => {