allow elements to start with underscore

because with gjs we can reference variables that start with underscore

endtag too

add test

add endtag test

also allow leading $

more tests

add unicode check

allow unicode

add unicode test

Author: patricklx

src/evented-tokenizer.ts

@@ -1,4 +1,4 @@
-import { preprocessInput, isAlpha, isSpace } from './utils';
+import { preprocessInput, isAlpha, isUnicode, isSpace } from './utils';
 import { EntityParser, TokenizerDelegate, TokenizerState } from './types';
 
 export default class EventedTokenizer {
@@ -170,7 +170,7 @@ export default class EventedTokenizer {
         this.transitionTo(TokenizerState.markupDeclarationOpen);
       } else if (char === '/') {
         this.transitionTo(TokenizerState.endTagOpen);
-      } else if (char === '@' || char === ':' || isAlpha(char)) {
+      } else if (char === '$' || char === '_' || char === '@' || char === ':' || isAlpha(char) || isUnicode(char)) {
         this.transitionTo(TokenizerState.tagName);
         this.tagNameBuffer = '';
         this.delegate.beginStartTag();
@@ -671,7 +671,7 @@ export default class EventedTokenizer {
     endTagOpen() {
       let char = this.consume();
 
-      if (char === '@' || char === ':' || isAlpha(char)) {
+      if (char === '$' || char === '_' || char === '@' || char === ':' || isAlpha(char) || isUnicode(char)) {
         this.transitionTo(TokenizerState.endTagName);
         this.tagNameBuffer = '';
         this.delegate.beginEndTag();

src/utils.ts

@@ -1,5 +1,6 @@
 const WSP = /[\t\n\f ]/;
 const ALPHA = /[A-Za-z]/;
+const UNICODE = /[^\u0000-\u00ff]/;
 const CRLF = /\r\n?/g;
 
 export function isSpace(char: string): boolean {
@@ -10,6 +11,10 @@ export function isAlpha(char: string): boolean {
   return ALPHA.test(char);
 }
 
+export function isUnicode(char: string): boolean {
+  return UNICODE.test(s);
+}
+
 export function preprocessInput(input: string): string {
   return input.replace(CRLF, '\n');
 }

tests/tokenizer-tests.ts

@@ -169,6 +169,23 @@ QUnit.test('A simple tag', function(assert) {
   assert.deepEqual(tokens, [startTag('div')]);
 });
 
+QUnit.test('A simple tag with leading non alpha chars', function(assert) {
+  let tokens = tokenize('<_div>');
+  assert.deepEqual(tokens, [startTag('_div')]);
+  
+  tokens = tokenize('<$div>');
+  assert.deepEqual(tokens, [startTag('$div')]);
+
+  tokens = tokenize('<:div>');
+  assert.deepEqual(tokens, [startTag(':div')]);
+
+  tokens = tokenize('<@div>');
+  assert.deepEqual(tokens, [startTag('@div')]);
+
+  tokens = tokenize('<üdiv>');
+  assert.deepEqual(tokens, [startTag('üdiv')]);
+});
+
 QUnit.test('A simple tag with trailing spaces', function(assert) {
   let tokens = tokenize('<div   \t\n>');
   assert.deepEqual(tokens, [startTag('div')]);
@@ -179,6 +196,23 @@ QUnit.test('A simple closing tag', function(assert) {
   assert.deepEqual(tokens, [endTag('div')]);
 });
 
+QUnit.test('A simple closing tag with leading non alpha chars', function(assert) {
+  let tokens = tokenize('</_div>');
+  assert.deepEqual(tokens, [endTag('_div')]);
+  
+  tokens = tokenize('</$div>');
+  assert.deepEqual(tokens, [endTag('$div')]);
+
+  tokens = tokenize('</:div>');
+  assert.deepEqual(tokens, [endTag(':div')]);
+
+  tokens = tokenize('</@div>');
+  assert.deepEqual(tokens, [endTag('@div')]);
+
+  tokens = tokenize('</üdiv>');
+  assert.deepEqual(tokens, [endTag('üdiv')]);
+});
+
 QUnit.test('A simple closing tag with trailing spaces', function(assert) {
   let tokens = tokenize('</div   \t\n>');
   assert.deepEqual(tokens, [endTag('div')]);