fix

Author: patricklx

src/evented-tokenizer.ts

@@ -1,4 +1,4 @@
-import { preprocessInput, isAlpha, isExtendedUnicode, isSpace } from './utils';
+import { preprocessInput, isAlpha, isSpace } from './utils';
 import { EntityParser, TokenizerDelegate, TokenizerState } from './types';
 
 export default class EventedTokenizer {
@@ -170,7 +170,7 @@ export default class EventedTokenizer {
         this.transitionTo(TokenizerState.markupDeclarationOpen);
       } else if (char === '/') {
         this.transitionTo(TokenizerState.endTagOpen);
-      } else if (char === '$' || char === '_' || char === '@' || char === ':' || isAlpha(char) || isExtendedUnicode(char)) {
+      } else if (!isSpace(char)) {
         this.transitionTo(TokenizerState.tagName);
         this.tagNameBuffer = '';
         this.delegate.beginStartTag();
@@ -671,7 +671,7 @@ export default class EventedTokenizer {
     endTagOpen() {
       let char = this.consume();
 
-      if (char === '$' || char === '_' || char === '@' || char === ':' || isAlpha(char) || isUnicode(char)) {
+      if (!isSpace(char)) {
         this.transitionTo(TokenizerState.endTagName);
         this.tagNameBuffer = '';
         this.delegate.beginEndTag();

src/utils.ts

@@ -1,6 +1,5 @@
 const WSP = /[\t\n\f ]/;
 const ALPHA = /[A-Za-z]/;
-const UNICODE = /[\p{Letter}\p{Mark}]/u;
 const CRLF = /\r\n?/g;
 
 export function isSpace(char: string): boolean {
@@ -11,10 +10,6 @@ export function isAlpha(char: string): boolean {
   return ALPHA.test(char);
 }
 
-export function isExtendedUnicode(char: string): boolean {
-  return UNICODE.test(char);
-}
-
 export function preprocessInput(input: string): string {
   return input.replace(CRLF, '\n');
 }

tests/tokenizer-tests.ts

@@ -184,6 +184,9 @@ QUnit.test('A simple tag with leading non alpha chars', function(assert) {
 
   tokens = tokenize('<üdiv>');
   assert.deepEqual(tokens, [startTag('üdiv')]);
+
+  tokens = tokenize('<€div>');
+  assert.deepEqual(tokens, [startTag('€div')]);
 });
 
 QUnit.test('A simple tag with trailing spaces', function(assert) {
@@ -211,6 +214,9 @@ QUnit.test('A simple closing tag with leading non alpha chars', function(assert)
 
   tokens = tokenize('</üdiv>');
   assert.deepEqual(tokens, [endTag('üdiv')]);
+
+  tokens = tokenize('</€div>');
+  assert.deepEqual(tokens, [endTag('€div')]);
 });
 
 QUnit.test('A simple closing tag with trailing spaces', function(assert) {