/**
 * @license
 * Copyright Google LLC All Rights Reserved.
 *
 * Use of this source code is governed by an MIT-style license that can be
 * found in the LICENSE file at https://angular.dev/license
 */

import {Lexer, StringTokenKind, Token} from '../../src/expression_parser/lexer';

function lex(text: string): any[] {
  return new Lexer().tokenize(text);
}

function expectToken(token: any, index: number, end: number) {
  expect(token instanceof Token).toBe(true);
  expect(token.index).toEqual(index);
  expect(token.end).toEqual(end);
}

function expectCharacterToken(token: any, index: number, end: number, character: string) {
  expect(character.length).toBe(1);
  expectToken(token, index, end);
  expect(token.isCharacter(character.charCodeAt(0))).toBe(true);
}

function expectOperatorToken(token: any, index: number, end: number, operator: string) {
  expectToken(token, index, end);
  expect(token.isOperator(operator)).toBe(true);
}

function expectNumberToken(token: any, index: number, end: number, n: number) {
  expectToken(token, index, end);
  expect(token.isNumber()).toBe(true);
  expect(token.toNumber()).toEqual(n);
}

function expectStringToken(
  token: any,
  index: number,
  end: number,
  str: string,
  kind: StringTokenKind,
) {
  expectToken(token, index, end);
  expect(token.isString()).toBe(true);
  expect(token.kind).toBe(kind);
  expect(token.toString()).toEqual(str);
}

function expectIdentifierToken(token: any, index: number, end: number, identifier: string) {
  expectToken(token, index, end);
  expect(token.isIdentifier()).toBe(true);
  expect(token.toString()).toEqual(identifier);
}

function expectPrivateIdentifierToken(token: any, index: number, end: number, identifier: string) {
  expectToken(token, index, end);
  expect(token.isPrivateIdentifier()).toBe(true);
  expect(token.toString()).toEqual(identifier);
}

function expectKeywordToken(token: any, index: number, end: number, keyword: string) {
  expectToken(token, index, end);
  expect(token.isKeyword()).toBe(true);
  expect(token.toString()).toEqual(keyword);
}

function expectErrorToken(token: Token, index: any, end: number, message: string) {
  expectToken(token, index, end);
  expect(token.isError()).toBe(true);
  expect(token.toString()).toEqual(message);
}

function expectRegExpBodyToken(token: any, index: number, end: number, str: string) {
  expectToken(token, index, end);
  expect(token.isRegExpBody()).toBe(true);
  expect(token.toString()).toEqual(str);
}

function expectRegExpFlagsToken(token: any, index: number, end: number, str: string) {
  expectToken(token, index, end);
  expect(token.isRegExpFlags()).toBe(true);
  expect(token.toString()).toEqual(str);
}

describe('lexer', () => {
  describe('token', () => {
    it('should tokenize a simple identifier', () => {
      const tokens: number[] = lex('j');
      expect(tokens.length).toEqual(1);
      expectIdentifierToken(tokens[0], 0, 1, 'j');
    });

    it('should tokenize "this"', () => {
      const tokens: number[] = lex('this');
      expect(tokens.length).toEqual(1);
      expectKeywordToken(tokens[0], 0, 4, 'this');
    });

    it('should tokenize a dotted identifier', () => {
      const tokens: number[] = lex('j.k');
      expect(tokens.length).toEqual(3);
      expectIdentifierToken(tokens[0], 0, 1, 'j');
      expectCharacterToken(tokens[1], 1, 2, '.');
      expectIdentifierToken(tokens[2], 2, 3, 'k');
    });

    it('should tokenize a private identifier', () => {
      const tokens: number[] = lex('#a');
      expect(tokens.length).toEqual(1);
      expectPrivateIdentifierToken(tokens[0], 0, 2, '#a');
    });

    it('should tokenize a property access with private identifier', () => {
      const tokens: number[] = lex('j.#k');
      expect(tokens.length).toEqual(3);
      expectIdentifierToken(tokens[0], 0, 1, 'j');
      expectCharacterToken(tokens[1], 1, 2, '.');
      expectPrivateIdentifierToken(tokens[2], 2, 4, '#k');
    });

    it(
      'should throw an invalid character error when a hash character is discovered but ' +
        'not indicating a private identifier',
      () => {
        expectErrorToken(
          lex('#')[0],
          0,
          1,
          `Lexer Error: Invalid character [#] at column 0 in expression [#]`,
        );
        expectErrorToken(
          lex('#0')[0],
          0,
          1,
          `Lexer Error: Invalid character [#] at column 0 in expression [#0]`,
        );
      },
    );

    it('should tokenize an operator', () => {
      const tokens: number[] = lex('j-k');
      expect(tokens.length).toEqual(3);
      expectOperatorToken(tokens[1], 1, 2, '-');
    });

    it('should tokenize an indexed operator', () => {
      const tokens: number[] = lex('j[k]');
      expect(tokens.length).toEqual(4);
      expectCharacterToken(tokens[1], 1, 2, '[');
      expectCharacterToken(tokens[3], 3, 4, ']');
    });

    it('should tokenize a safe indexed operator', () => {
      const tokens: number[] = lex('j?.[k]');
      expect(tokens.length).toBe(5);
      expectOperatorToken(tokens[1], 1, 3, '?.');
      expectCharacterToken(tokens[2], 3, 4, '[');
      expectCharacterToken(tokens[4], 5, 6, ']');
    });

    it('should tokenize numbers', () => {
      const tokens: number[] = lex('88');
      expect(tokens.length).toEqual(1);
      expectNumberToken(tokens[0], 0, 2, 88);
    });

    it('should tokenize numbers within index ops', () => {
      expectNumberToken(lex('a[22]')[2], 2, 4, 22);
    });

    it('should tokenize simple quoted strings', () => {
      expectStringToken(lex('"a"')[0], 0, 3, 'a', StringTokenKind.Plain);
    });

    it('should tokenize quoted strings with escaped quotes', () => {
      expectStringToken(lex('"a\\""')[0], 0, 5, 'a"', StringTokenKind.Plain);
    });

    it('should tokenize a string', () => {
      const tokens: Token[] = lex('j-a.bc[22]+1.3|f:\'a\\\'c\':"d\\"e"');
      expectIdentifierToken(tokens[0], 0, 1, 'j');
      expectOperatorToken(tokens[1], 1, 2, '-');
      expectIdentifierToken(tokens[2], 2, 3, 'a');
      expectCharacterToken(tokens[3], 3, 4, '.');
      expectIdentifierToken(tokens[4], 4, 6, 'bc');
      expectCharacterToken(tokens[5], 6, 7, '[');
      expectNumberToken(tokens[6], 7, 9, 22);
      expectCharacterToken(tokens[7], 9, 10, ']');
      expectOperatorToken(tokens[8], 10, 11, '+');
      expectNumberToken(tokens[9], 11, 14, 1.3);
      expectOperatorToken(tokens[10], 14, 15, '|');
      expectIdentifierToken(tokens[11], 15, 16, 'f');
      expectCharacterToken(tokens[12], 16, 17, ':');
      expectStringToken(tokens[13], 17, 23, "a'c", StringTokenKind.Plain);
      expectCharacterToken(tokens[14], 23, 24, ':');
      expectStringToken(tokens[15], 24, 30, 'd"e', StringTokenKind.Plain);
    });

    it('should tokenize undefined', () => {
      const tokens: Token[] = lex('undefined');
      expectKeywordToken(tokens[0], 0, 9, 'undefined');
      expect(tokens[0].isKeywordUndefined()).toBe(true);
    });

    it('should tokenize typeof', () => {
      const tokens: Token[] = lex('typeof');
      expectKeywordToken(tokens[0], 0, 6, 'typeof');
      expect(tokens[0].isKeywordTypeof()).toBe(true);
    });

    it('should tokenize void', () => {
      const tokens: Token[] = lex('void');
      expectKeywordToken(tokens[0], 0, 4, 'void');
      expect(tokens[0].isKeywordVoid()).toBe(true);
    });

    it('should tokenize in keyword', () => {
      const tokens: Token[] = lex('in');
      expectKeywordToken(tokens[0], 0, 2, 'in');
      expect(tokens[0].isKeywordIn()).toBe(true);
    });

    it('should tokenize instanceof keyword', () => {
      const tokens: Token[] = lex('instanceof');
      expectKeywordToken(tokens[0], 0, 10, 'instanceof');
      expect(tokens[0].isKeywordInstanceOf()).toBe(true);
    });

    it('should ignore whitespace', () => {
      const tokens: Token[] = lex('a \t \n \r b');
      expectIdentifierToken(tokens[0], 0, 1, 'a');
      expectIdentifierToken(tokens[1], 8, 9, 'b');
    });

    it('should tokenize quoted string', () => {
      const str = '[\'\\\'\', "\\""]';
      const tokens: Token[] = lex(str);
      expectStringToken(tokens[1], 1, 5, "'", StringTokenKind.Plain);
      expectStringToken(tokens[3], 7, 11, '"', StringTokenKind.Plain);
    });

    it('should tokenize escaped quoted string', () => {
      const str = '"\\"\\n\\f\\r\\t\\v\\u00A0"';
      const tokens: Token[] = lex(str);
      expect(tokens.length).toEqual(1);
      expect(tokens[0].toString()).toEqual('"\n\f\r\t\v\u00A0');
    });

    it('should tokenize unicode', () => {
      const tokens: Token[] = lex('"\\u00A0"');
      expect(tokens.length).toEqual(1);
      expect(tokens[0].toString()).toEqual('\u00a0');
    });

    it('should tokenize relation', () => {
      const tokens: Token[] = lex('! == != < > <= >= === !==');
      expectOperatorToken(tokens[0], 0, 1, '!');
      expectOperatorToken(tokens[1], 2, 4, '==');
      expectOperatorToken(tokens[2], 5, 7, '!=');
      expectOperatorToken(tokens[3], 8, 9, '<');
      expectOperatorToken(tokens[4], 10, 11, '>');
      expectOperatorToken(tokens[5], 12, 14, '<=');
      expectOperatorToken(tokens[6], 15, 17, '>=');
      expectOperatorToken(tokens[7], 18, 21, '===');
      expectOperatorToken(tokens[8], 22, 25, '!==');
    });

    it('should tokenize statements', () => {
      const tokens: Token[] = lex('a;b;');
      expectIdentifierToken(tokens[0], 0, 1, 'a');
      expectCharacterToken(tokens[1], 1, 2, ';');
      expectIdentifierToken(tokens[2], 2, 3, 'b');
      expectCharacterToken(tokens[3], 3, 4, ';');
    });

    it('should tokenize function invocation', () => {
      const tokens: Token[] = lex('a()');
      expectIdentifierToken(tokens[0], 0, 1, 'a');
      expectCharacterToken(tokens[1], 1, 2, '(');
      expectCharacterToken(tokens[2], 2, 3, ')');
    });

    it('should tokenize simple method invocations', () => {
      const tokens: Token[] = lex('a.method()');
      expectIdentifierToken(tokens[2], 2, 8, 'method');
    });

    it('should tokenize method invocation', () => {
      const tokens: Token[] = lex('a.b.c (d) - e.f()');
      expectIdentifierToken(tokens[0], 0, 1, 'a');
      expectCharacterToken(tokens[1], 1, 2, '.');
      expectIdentifierToken(tokens[2], 2, 3, 'b');
      expectCharacterToken(tokens[3], 3, 4, '.');
      expectIdentifierToken(tokens[4], 4, 5, 'c');
      expectCharacterToken(tokens[5], 6, 7, '(');
      expectIdentifierToken(tokens[6], 7, 8, 'd');
      expectCharacterToken(tokens[7], 8, 9, ')');
      expectOperatorToken(tokens[8], 10, 11, '-');
      expectIdentifierToken(tokens[9], 12, 13, 'e');
      expectCharacterToken(tokens[10], 13, 14, '.');
      expectIdentifierToken(tokens[11], 14, 15, 'f');
      expectCharacterToken(tokens[12], 15, 16, '(');
      expectCharacterToken(tokens[13], 16, 17, ')');
    });

    it('should tokenize safe function invocation', () => {
      const tokens: Token[] = lex('a?.()');
      expectIdentifierToken(tokens[0], 0, 1, 'a');
      expectOperatorToken(tokens[1], 1, 3, '?.');
      expectCharacterToken(tokens[2], 3, 4, '(');
      expectCharacterToken(tokens[3], 4, 5, ')');
    });

    it('should tokenize a safe method invocations', () => {
      const tokens: Token[] = lex('a.method?.()');
      expectIdentifierToken(tokens[0], 0, 1, 'a');
      expectCharacterToken(tokens[1], 1, 2, '.');
      expectIdentifierToken(tokens[2], 2, 8, 'method');
      expectOperatorToken(tokens[3], 8, 10, '?.');
      expectCharacterToken(tokens[4], 10, 11, '(');
      expectCharacterToken(tokens[5], 11, 12, ')');
    });

    it('should tokenize number', () => {
      expectNumberToken(lex('0.5')[0], 0, 3, 0.5);
    });

    it('should tokenize multiplication and exponentiation', () => {
      const tokens: Token[] = lex('1 * 2 ** 3');
      expectNumberToken(tokens[0], 0, 1, 1);
      expectOperatorToken(tokens[1], 2, 3, '*');
      expectNumberToken(tokens[2], 4, 5, 2);
      expectOperatorToken(tokens[3], 6, 8, '**');
      expectNumberToken(tokens[4], 9, 10, 3);
    });

    it('should tokenize number with exponent', () => {
      let tokens: Token[] = lex('0.5E-10');
      expect(tokens.length).toEqual(1);
      expectNumberToken(tokens[0], 0, 7, 0.5e-10);
      tokens = lex('0.5E+10');
      expectNumberToken(tokens[0], 0, 7, 0.5e10);
    });

    it('should return exception for invalid exponent', () => {
      expectErrorToken(
        lex('0.5E-')[0],
        4,
        5,
        'Lexer Error: Invalid exponent at column 4 in expression [0.5E-]',
      );

      expectErrorToken(
        lex('0.5E-A')[0],
        4,
        5,
        'Lexer Error: Invalid exponent at column 4 in expression [0.5E-A]',
      );
    });

    it('should tokenize number starting with a dot', () => {
      expectNumberToken(lex('.5')[0], 0, 2, 0.5);
    });

    it('should throw error on invalid unicode', () => {
      expectErrorToken(
        lex("'\\u1''bla'")[0],
        2,
        2,
        "Lexer Error: Invalid unicode escape [\\u1''b] at column 2 in expression ['\\u1''bla']",
      );
    });

    it('should tokenize ?. as operator', () => {
      expectOperatorToken(lex('?.')[0], 0, 2, '?.');
    });

    it('should tokenize ?? as operator', () => {
      expectOperatorToken(lex('??')[0], 0, 2, '??');
    });

    it('should tokenize number with separator', () => {
      expectNumberToken(lex('123_456')[0], 0, 7, 123_456);
      expectNumberToken(lex('1_000_000_000')[0], 0, 13, 1_000_000_000);
      expectNumberToken(lex('123_456.78')[0], 0, 10, 123_456.78);
      expectNumberToken(lex('123_456_789.123_456_789')[0], 0, 23, 123_456_789.123_456_789);
      expectNumberToken(lex('1_2_3_4')[0], 0, 7, 1_2_3_4);
      expectNumberToken(lex('1_2_3_4.5_6_7_8')[0], 0, 15, 1_2_3_4.5_6_7_8);
    });

    it('should tokenize number starting with an underscore as an identifier', () => {
      expectIdentifierToken(lex('_123')[0], 0, 4, '_123');
      expectIdentifierToken(lex('_123_')[0], 0, 5, '_123_');
      expectIdentifierToken(lex('_1_2_3_')[0], 0, 7, '_1_2_3_');
    });

    it('should throw error for invalid number separators', () => {
      expectErrorToken(
        lex('123_')[0],
        3,
        3,
        'Lexer Error: Invalid numeric separator at column 3 in expression [123_]',
      );
      expectErrorToken(
        lex('12__3')[0],
        2,
        2,
        'Lexer Error: Invalid numeric separator at column 2 in expression [12__3]',
      );
      expectErrorToken(
        lex('1_2_3_.456')[0],
        5,
        5,
        'Lexer Error: Invalid numeric separator at column 5 in expression [1_2_3_.456]',
      );
      expectErrorToken(
        lex('1_2_3._456')[0],
        6,
        6,
        'Lexer Error: Invalid numeric separator at column 6 in expression [1_2_3._456]',
      );
    });

    it('should tokenize assignment operators', () => {
      expectOperatorToken(lex('=')[0], 0, 1, '=');
      expectOperatorToken(lex('+=')[0], 0, 2, '+=');
      expectOperatorToken(lex('-=')[0], 0, 2, '-=');
      expectOperatorToken(lex('*=')[0], 0, 2, '*=');
      expectOperatorToken(lex('a /= b')[1], 2, 4, '/=');
      expectOperatorToken(lex('%=')[0], 0, 2, '%=');
      expectOperatorToken(lex('**=')[0], 0, 3, '**=');
      expectOperatorToken(lex('&&=')[0], 0, 3, '&&=');
      expectOperatorToken(lex('||=')[0], 0, 3, '||=');
      expectOperatorToken(lex('??=')[0], 0, 3, '??=');
    });

    it('should tokenize a spread operator', () => {
      const tokens = lex('{...foo}');
      expect(tokens.length).toEqual(4);
      expectCharacterToken(tokens[0], 0, 1, '{');
      expectOperatorToken(tokens[1], 1, 4, '...');
      expectIdentifierToken(tokens[2], 4, 7, 'foo');
      expectCharacterToken(tokens[3], 7, 8, '}');
    });

    it('should produce an error for a spread with two dots', () => {
      const tokens = lex('{..foo}');
      expect(tokens.length).toEqual(4);
      expectCharacterToken(tokens[0], 0, 1, '{');
      expectErrorToken(
        tokens[1],
        3,
        3,
        'Lexer Error: Unexpected character [.] at column 3 in expression [{..foo}]',
      );
      expectIdentifierToken(tokens[2], 3, 6, 'foo');
      expectCharacterToken(tokens[3], 6, 7, '}');
    });

    describe('template literals', () => {
      it('should tokenize template literal with no interpolations', () => {
        const tokens: Token[] = lex('`hello world`');
        expect(tokens.length).toBe(1);
        expectStringToken(tokens[0], 0, 13, 'hello world', StringTokenKind.TemplateLiteralEnd);
      });

      it('should tokenize template literal containing strings', () => {
        expectStringToken(lex('`a "b" c`')[0], 0, 9, `a "b" c`, StringTokenKind.TemplateLiteralEnd);
        expectStringToken(lex("`a 'b' c`")[0], 0, 9, `a 'b' c`, StringTokenKind.TemplateLiteralEnd);
        expectStringToken(
          lex('`a \\`b\\` c`')[0],
          0,
          11,
          'a `b` c',
          StringTokenKind.TemplateLiteralEnd,
        );
        expectStringToken(
          lex('`a "\'\\`b\\`\'" c`')[0],
          0,
          15,
          `a "'\`b\`'" c`,
          StringTokenKind.TemplateLiteralEnd,
        );
      });

      it('should tokenize unicode inside a template string', () => {
        const tokens: Token[] = lex('`\\u00A0`');
        expect(tokens.length).toBe(1);
        expect(tokens[0].toString()).toBe('\u00a0');
      });

      it('should tokenize template literal with an interpolation in the end', () => {
        const tokens: Token[] = lex('`hello ${name}`');
        expect(tokens.length).toBe(5);
        expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart);
        expectOperatorToken(tokens[1], 7, 9, '${');
        expectIdentifierToken(tokens[2], 9, 13, 'name');
        expectCharacterToken(tokens[3], 13, 14, '}');
        expectStringToken(tokens[4], 14, 15, '', StringTokenKind.TemplateLiteralEnd);
      });

      it('should tokenize template literal with an interpolation in the beginning', () => {
        const tokens: Token[] = lex('`${name} Johnson`');
        expect(tokens.length).toBe(5);
        expectStringToken(tokens[0], 0, 1, '', StringTokenKind.TemplateLiteralPart);
        expectOperatorToken(tokens[1], 1, 3, '${');
        expectIdentifierToken(tokens[2], 3, 7, 'name');
        expectCharacterToken(tokens[3], 7, 8, '}');
        expectStringToken(tokens[4], 8, 17, ' Johnson', StringTokenKind.TemplateLiteralEnd);
      });

      it('should tokenize template literal with an interpolation in the middle', () => {
        const tokens: Token[] = lex('`foo${bar}baz`');
        expect(tokens.length).toBe(5);
        expectStringToken(tokens[0], 0, 4, 'foo', StringTokenKind.TemplateLiteralPart);
        expectOperatorToken(tokens[1], 4, 6, '${');
        expectIdentifierToken(tokens[2], 6, 9, 'bar');
        expectCharacterToken(tokens[3], 9, 10, '}');
        expectStringToken(tokens[4], 10, 14, 'baz', StringTokenKind.TemplateLiteralEnd);
      });

      it('should be able to use interpolation characters inside template string', () => {
        expectStringToken(lex('`foo $`')[0], 0, 7, 'foo $', StringTokenKind.TemplateLiteralEnd);
        expectStringToken(lex('`foo }`')[0], 0, 7, 'foo }', StringTokenKind.TemplateLiteralEnd);
        expectStringToken(
          lex('`foo $ {}`')[0],
          0,
          10,
          'foo $ {}',
          StringTokenKind.TemplateLiteralEnd,
        );
        expectStringToken(
          lex('`foo \\${bar}`')[0],
          0,
          13,
          'foo ${bar}',
          StringTokenKind.TemplateLiteralEnd,
        );
      });

      it('should tokenize template literal with several interpolations', () => {
        const tokens: Token[] = lex('`${a} - ${b} - ${c}`');
        expect(tokens.length).toBe(13);
        expectStringToken(tokens[0], 0, 1, '', StringTokenKind.TemplateLiteralPart);
        expectOperatorToken(tokens[1], 1, 3, '${');
        expectIdentifierToken(tokens[2], 3, 4, 'a');
        expectCharacterToken(tokens[3], 4, 5, '}');
        expectStringToken(tokens[4], 5, 8, ' - ', StringTokenKind.TemplateLiteralPart);
        expectOperatorToken(tokens[5], 8, 10, '${');
        expectIdentifierToken(tokens[6], 10, 11, 'b');
        expectCharacterToken(tokens[7], 11, 12, '}');
        expectStringToken(tokens[8], 12, 15, ' - ', StringTokenKind.TemplateLiteralPart);
        expectOperatorToken(tokens[9], 15, 17, '${');
        expectIdentifierToken(tokens[10], 17, 18, 'c');
        expectCharacterToken(tokens[11], 18, 19, '}');
      });

      it('should tokenize template literal with an object literal inside the interpolation', () => {
        const tokens: Token[] = lex('`foo ${{$: true}} baz`');
        expect(tokens.length).toBe(9);
        expectStringToken(tokens[0], 0, 5, 'foo ', StringTokenKind.TemplateLiteralPart);
        expectOperatorToken(tokens[1], 5, 7, '${');
        expectCharacterToken(tokens[2], 7, 8, '{');
        expectIdentifierToken(tokens[3], 8, 9, '$');
        expectCharacterToken(tokens[4], 9, 10, ':');
        expectKeywordToken(tokens[5], 11, 15, 'true');
        expectCharacterToken(tokens[6], 15, 16, '}');
        expectCharacterToken(tokens[7], 16, 17, '}');
        expectStringToken(tokens[8], 17, 22, ' baz', StringTokenKind.TemplateLiteralEnd);
      });

      it('should tokenize template literal with template literals inside the interpolation', () => {
        const tokens: Token[] = lex('`foo ${`hello ${`${a} - b`}`} baz`');
        expect(tokens.length).toBe(13);
        expectStringToken(tokens[0], 0, 5, 'foo ', StringTokenKind.TemplateLiteralPart);
        expectOperatorToken(tokens[1], 5, 7, '${');
        expectStringToken(tokens[2], 7, 14, 'hello ', StringTokenKind.TemplateLiteralPart);
        expectOperatorToken(tokens[3], 14, 16, '${');
        expectStringToken(tokens[4], 16, 17, '', StringTokenKind.TemplateLiteralPart);
        expectOperatorToken(tokens[5], 17, 19, '${');
        expectIdentifierToken(tokens[6], 19, 20, 'a');
        expectCharacterToken(tokens[7], 20, 21, '}');
        expectStringToken(tokens[8], 21, 26, ' - b', StringTokenKind.TemplateLiteralEnd);
        expectCharacterToken(tokens[9], 26, 27, '}');
        expectStringToken(tokens[10], 27, 28, '', StringTokenKind.TemplateLiteralEnd);
        expectCharacterToken(tokens[11], 28, 29, '}');
        expectStringToken(tokens[12], 29, 34, ' baz', StringTokenKind.TemplateLiteralEnd);
      });

      it('should tokenize two template literal right after each other', () => {
        const tokens: Token[] = lex('`hello ${name}``see ${name} later`');
        expect(tokens.length).toBe(10);
        expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart);
        expectOperatorToken(tokens[1], 7, 9, '${');
        expectIdentifierToken(tokens[2], 9, 13, 'name');
        expectCharacterToken(tokens[3], 13, 14, '}');
        expectStringToken(tokens[4], 14, 15, '', StringTokenKind.TemplateLiteralEnd);
        expectStringToken(tokens[5], 15, 20, 'see ', StringTokenKind.TemplateLiteralPart);
        expectOperatorToken(tokens[6], 20, 22, '${');
        expectIdentifierToken(tokens[7], 22, 26, 'name');
        expectCharacterToken(tokens[8], 26, 27, '}');
        expectStringToken(tokens[9], 27, 34, ' later', StringTokenKind.TemplateLiteralEnd);
      });

      it('should tokenize a concatenated template literal', () => {
        const tokens: Token[] = lex('`hello ${name}` + 123');
        expect(tokens.length).toBe(7);
        expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart);
        expectOperatorToken(tokens[1], 7, 9, '${');
        expectIdentifierToken(tokens[2], 9, 13, 'name');
        expectCharacterToken(tokens[3], 13, 14, '}');
        expectStringToken(tokens[4], 14, 15, '', StringTokenKind.TemplateLiteralEnd);
        expectOperatorToken(tokens[5], 16, 17, '+');
        expectNumberToken(tokens[6], 18, 21, 123);
      });

      it('should tokenize a template literal with a pipe inside an interpolation', () => {
        const tokens: Token[] = lex('`hello ${name | capitalize}!!!`');
        expect(tokens.length).toBe(7);
        expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart);
        expectOperatorToken(tokens[1], 7, 9, '${');
        expectIdentifierToken(tokens[2], 9, 13, 'name');
        expectOperatorToken(tokens[3], 14, 15, '|');
        expectIdentifierToken(tokens[4], 16, 26, 'capitalize');
        expectCharacterToken(tokens[5], 26, 27, '}');
        expectStringToken(tokens[6], 27, 31, '!!!', StringTokenKind.TemplateLiteralEnd);
      });

      it('should tokenize a template literal with a pipe inside a parenthesized interpolation', () => {
        const tokens: Token[] = lex('`hello ${(name | capitalize)}!!!`');
        expect(tokens.length).toBe(9);
        expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart);
        expectOperatorToken(tokens[1], 7, 9, '${');
        expectCharacterToken(tokens[2], 9, 10, '(');
        expectIdentifierToken(tokens[3], 10, 14, 'name');
        expectOperatorToken(tokens[4], 15, 16, '|');
        expectIdentifierToken(tokens[5], 17, 27, 'capitalize');
        expectCharacterToken(tokens[6], 27, 28, ')');
        expectCharacterToken(tokens[7], 28, 29, '}');
        expectStringToken(tokens[8], 29, 33, '!!!', StringTokenKind.TemplateLiteralEnd);
      });

      it('should tokenize a template literal in an literal object value', () => {
        const tokens: Token[] = lex('{foo: `${name}`}');
        expect(tokens.length).toBe(9);
        expectCharacterToken(tokens[0], 0, 1, '{');
        expectIdentifierToken(tokens[1], 1, 4, 'foo');
        expectCharacterToken(tokens[2], 4, 5, ':');
        expectStringToken(tokens[3], 6, 7, '', StringTokenKind.TemplateLiteralPart);
        expectOperatorToken(tokens[4], 7, 9, '${');
        expectIdentifierToken(tokens[5], 9, 13, 'name');
        expectCharacterToken(tokens[6], 13, 14, '}');
        expectStringToken(tokens[7], 14, 15, '', StringTokenKind.TemplateLiteralEnd);
        expectCharacterToken(tokens[8], 15, 16, '}');
      });

      it('should produce an error if a template literal is not terminated', () => {
        expectErrorToken(
          lex('`hello')[0],
          6,
          6,
          'Lexer Error: Unterminated template literal at column 6 in expression [`hello]',
        );
      });

      it('should produce an error for an unterminated template literal with an interpolation', () => {
        const tokens: Token[] = lex('`hello ${name}!');
        expect(tokens.length).toBe(5);
        expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart);
        expectOperatorToken(tokens[1], 7, 9, '${');
        expectIdentifierToken(tokens[2], 9, 13, 'name');
        expectCharacterToken(tokens[3], 13, 14, '}');
        expectErrorToken(
          tokens[4],
          15,
          15,
          'Lexer Error: Unterminated template literal at column 15 in expression [`hello ${name}!]',
        );
      });

      it('should produce an error for an unterminate template literal interpolation', () => {
        const tokens: Token[] = lex('`hello ${name!`');
        expect(tokens.length).toBe(5);
        expectStringToken(tokens[0], 0, 7, 'hello ', StringTokenKind.TemplateLiteralPart);
        expectOperatorToken(tokens[1], 7, 9, '${');
        expectIdentifierToken(tokens[2], 9, 13, 'name');
        expectOperatorToken(tokens[3], 13, 14, '!');
        expectErrorToken(
          tokens[4],
          15,
          15,
          'Lexer Error: Unterminated template literal at column 15 in expression [`hello ${name!`]',
        );
      });

      it('should tokenize tagged template literal with no interpolations', () => {
        const tokens: Token[] = lex('tag`hello world`');
        expect(tokens.length).toBe(2);
        expectIdentifierToken(tokens[0], 0, 3, 'tag');
        expectStringToken(tokens[1], 3, 16, 'hello world', StringTokenKind.TemplateLiteralEnd);
      });

      it('should tokenize nested tagged template literals', () => {
        const tokens: Token[] = lex('tag`hello ${tag`world`}`');
        expect(tokens.length).toBe(7);
        expectIdentifierToken(tokens[0], 0, 3, 'tag');
        expectStringToken(tokens[1], 3, 10, 'hello ', StringTokenKind.TemplateLiteralPart);
        expectOperatorToken(tokens[2], 10, 12, '${');
        expectIdentifierToken(tokens[3], 12, 15, 'tag');
        expectStringToken(tokens[4], 15, 22, 'world', StringTokenKind.TemplateLiteralEnd);
        expectCharacterToken(tokens[5], 22, 23, '}');
        expectStringToken(tokens[6], 23, 24, '', StringTokenKind.TemplateLiteralEnd);
      });
    });

    describe('regular expressions', () => {
      it('should tokenize a simple regex', () => {
        const tokens: Token[] = lex('/abc/');
        expect(tokens.length).toBe(1);
        expectRegExpBodyToken(tokens[0], 0, 5, 'abc');
      });

      it('should tokenize a regex with flags', () => {
        const tokens: Token[] = lex('/abc/gim');
        expect(tokens.length).toBe(2);
        expectRegExpBodyToken(tokens[0], 0, 5, 'abc');
        expectRegExpFlagsToken(tokens[1], 5, 8, 'gim');
      });

      it('should tokenize an identifier immediately after a regex', () => {
        const tokens: Token[] = lex('/abc/ g');
        expect(tokens.length).toBe(2);
        expectRegExpBodyToken(tokens[0], 0, 5, 'abc');
        expectIdentifierToken(tokens[1], 6, 7, 'g');
      });

      it('should tokenize a regex with an escaped slashes', () => {
        const tokens: Token[] = lex('/^http:\\/\\/foo\\.bar/');
        expect(tokens.length).toBe(1);
        expectRegExpBodyToken(tokens[0], 0, 20, '^http:\\/\\/foo\\.bar');
      });

      it('should tokenize a regex with un-escaped slashes in a character class', () => {
        const tokens: Token[] = lex('/[a/]$/');
        expect(tokens.length).toBe(1);
        expectRegExpBodyToken(tokens[0], 0, 7, '[a/]$');
      });

      it('should tokenize a regex with a backslash', () => {
        const tokens: Token[] = lex('/a\\w+/');
        expect(tokens.length).toBe(1);
        expectRegExpBodyToken(tokens[0], 0, 6, 'a\\w+');
      });

      it('should tokenize a regex after an operator', () => {
        const tokens: Token[] = lex('a = /b/');
        expect(tokens.length).toBe(3);
        expectIdentifierToken(tokens[0], 0, 1, 'a');
        expectOperatorToken(tokens[1], 2, 3, '=');
        expectRegExpBodyToken(tokens[2], 4, 7, 'b');
      });

      it('should tokenize a regex inside parentheses', () => {
        const tokens: Token[] = lex('log(/a/)');
        expect(tokens.length).toBe(4);
        expectIdentifierToken(tokens[0], 0, 3, 'log');
        expectCharacterToken(tokens[1], 3, 4, '(');
        expectRegExpBodyToken(tokens[2], 4, 7, 'a');
        expectCharacterToken(tokens[3], 7, 8, ')');
      });

      it('should tokenize a regex at the beggining of an array', () => {
        const tokens: Token[] = lex('[/a/]');
        expect(tokens.length).toBe(3);
        expectCharacterToken(tokens[0], 0, 1, '[');
        expectRegExpBodyToken(tokens[1], 1, 4, 'a');
        expectCharacterToken(tokens[2], 4, 5, ']');
      });

      it('should tokenize a regex in the middle of an array', () => {
        const tokens: Token[] = lex('[1, /a/, 2]');
        expect(tokens.length).toBe(7);
        expectCharacterToken(tokens[0], 0, 1, '[');
        expectNumberToken(tokens[1], 1, 2, 1);
        expectCharacterToken(tokens[2], 2, 3, ',');
        expectRegExpBodyToken(tokens[3], 4, 7, 'a');
        expectCharacterToken(tokens[4], 7, 8, ',');
        expectNumberToken(tokens[5], 9, 10, 2);
        expectCharacterToken(tokens[6], 10, 11, ']');
      });

      it('should tokenize a regex inside an object literal', () => {
        const tokens: Token[] = lex('{a: /b/}');
        expect(tokens.length).toBe(5);
        expectCharacterToken(tokens[0], 0, 1, '{');
        expectIdentifierToken(tokens[1], 1, 2, 'a');
        expectCharacterToken(tokens[2], 2, 3, ':');
        expectRegExpBodyToken(tokens[3], 4, 7, 'b');
        expectCharacterToken(tokens[4], 7, 8, '}');
      });

      it('should tokenize a regex after a negation operator', () => {
        const tokens: Token[] = lex('log(!/a/.test("1"))');
        expect(tokens.length).toBe(10);
        expectIdentifierToken(tokens[0], 0, 3, 'log');
        expectCharacterToken(tokens[1], 3, 4, '(');
        expectOperatorToken(tokens[2], 4, 5, '!');
        expectRegExpBodyToken(tokens[3], 5, 8, 'a');
        expectCharacterToken(tokens[4], 8, 9, '.');
        expectIdentifierToken(tokens[5], 9, 13, 'test');
        expectCharacterToken(tokens[6], 13, 14, '(');
        expectStringToken(tokens[7], 14, 17, '1', StringTokenKind.Plain);
        expectCharacterToken(tokens[8], 17, 18, ')');
        expectCharacterToken(tokens[9], 18, 19, ')');
      });

      it('should tokenize a regex after several negation operators', () => {
        const tokens: Token[] = lex('log(!!!!!!/a/.test("1"))');
        expect(tokens.length).toBe(15);
        expectIdentifierToken(tokens[0], 0, 3, 'log');
        expectCharacterToken(tokens[1], 3, 4, '(');
        expectOperatorToken(tokens[2], 4, 5, '!');
        expectOperatorToken(tokens[3], 5, 6, '!');
        expectOperatorToken(tokens[4], 6, 7, '!');
        expectOperatorToken(tokens[5], 7, 8, '!');
        expectOperatorToken(tokens[6], 8, 9, '!');
        expectOperatorToken(tokens[7], 9, 10, '!');
        expectRegExpBodyToken(tokens[8], 10, 13, 'a');
        expectCharacterToken(tokens[9], 13, 14, '.');
        expectIdentifierToken(tokens[10], 14, 18, 'test');
        expectCharacterToken(tokens[11], 18, 19, '(');
        expectStringToken(tokens[12], 19, 22, '1', StringTokenKind.Plain);
        expectCharacterToken(tokens[13], 22, 23, ')');
        expectCharacterToken(tokens[14], 23, 24, ')');
      });

      it('should tokenize a method call on a regex', () => {
        const tokens: Token[] = lex('/abc/.test("foo")');
        expect(tokens.length).toBe(6);
        expectRegExpBodyToken(tokens[0], 0, 5, 'abc');
        expectCharacterToken(tokens[1], 5, 6, '.');
        expectIdentifierToken(tokens[2], 6, 10, 'test');
        expectCharacterToken(tokens[3], 10, 11, '(');
        expectStringToken(tokens[4], 11, 16, 'foo', StringTokenKind.Plain);
        expectCharacterToken(tokens[5], 16, 17, ')');
      });

      it('should tokenize a method call with a regex parameter', () => {
        const tokens: Token[] = lex('"foo".match(/abc/)');
        expect(tokens.length).toBe(6);
        expectStringToken(tokens[0], 0, 5, 'foo', StringTokenKind.Plain);
        expectCharacterToken(tokens[1], 5, 6, '.');
        expectIdentifierToken(tokens[2], 6, 11, 'match');
        expectCharacterToken(tokens[3], 11, 12, '(');
        expectRegExpBodyToken(tokens[4], 12, 17, 'abc');
        expectCharacterToken(tokens[5], 17, 18, ')');
      });

      it('should not tokenize a regex preceded by a square bracket', () => {
        const tokens: Token[] = lex('a[0] /= b');
        expect(tokens.length).toBe(6);
        expectIdentifierToken(tokens[0], 0, 1, 'a');
        expectCharacterToken(tokens[1], 1, 2, '[');
        expectNumberToken(tokens[2], 2, 3, 0);
        expectCharacterToken(tokens[3], 3, 4, ']');
        expectOperatorToken(tokens[4], 5, 7, '/=');
        expectIdentifierToken(tokens[5], 8, 9, 'b');
      });

      it('should not tokenize a regex preceded by an identifier', () => {
        const tokens: Token[] = lex('a / b');
        expect(tokens.length).toBe(3);
        expectIdentifierToken(tokens[0], 0, 1, 'a');
        expectOperatorToken(tokens[1], 2, 3, '/');
        expectIdentifierToken(tokens[2], 4, 5, 'b');
      });

      it('should not tokenize a regex preceded by a number', () => {
        const tokens: Token[] = lex('1 / b');
        expect(tokens.length).toBe(3);
        expectNumberToken(tokens[0], 0, 1, 1);
        expectOperatorToken(tokens[1], 2, 3, '/');
        expectIdentifierToken(tokens[2], 4, 5, 'b');
      });

      it('should not tokenize a regex that is preceded by a string', () => {
        const tokens: Token[] = lex('"a" / b');
        expect(tokens.length).toBe(3);
        expectStringToken(tokens[0], 0, 3, 'a', StringTokenKind.Plain);
        expectOperatorToken(tokens[1], 4, 5, '/');
        expectIdentifierToken(tokens[2], 6, 7, 'b');
      });

      it('should not tokenize a regex preceded by a closing parenthesis', () => {
        const tokens: Token[] = lex('(a) / b');
        expect(tokens.length).toBe(5);
        expectCharacterToken(tokens[0], 0, 1, '(');
        expectIdentifierToken(tokens[1], 1, 2, 'a');
        expectCharacterToken(tokens[2], 2, 3, ')');
        expectOperatorToken(tokens[3], 4, 5, '/');
        expectIdentifierToken(tokens[4], 6, 7, 'b');
      });

      it('should not tokenize a regex that is preceded by a keyword', () => {
        const tokens: Token[] = lex('this / b');
        expect(tokens.length).toBe(3);
        expectKeywordToken(tokens[0], 0, 4, 'this');
        expectOperatorToken(tokens[1], 5, 6, '/');
        expectIdentifierToken(tokens[2], 7, 8, 'b');
      });

      it('should not tokenize a regex preceded by a non-null assertion on an identifier', () => {
        const tokens: Token[] = lex('foo! / 2');
        expect(tokens.length).toBe(4);
        expectIdentifierToken(tokens[0], 0, 3, 'foo');
        expectOperatorToken(tokens[1], 3, 4, '!');
        expectOperatorToken(tokens[2], 5, 6, '/');
        expectNumberToken(tokens[3], 7, 8, 2);
      });

      it('should not tokenize a regex preceded by a non-null assertion on a function call', () => {
        const tokens: Token[] = lex('foo()! / 2');
        expect(tokens.length).toBe(6);
        expectIdentifierToken(tokens[0], 0, 3, 'foo');
        expectCharacterToken(tokens[1], 3, 4, '(');
        expectCharacterToken(tokens[2], 4, 5, ')');
        expectOperatorToken(tokens[3], 5, 6, '!');
        expectOperatorToken(tokens[4], 7, 8, '/');
        expectNumberToken(tokens[5], 9, 10, 2);
      });

      it('should not tokenize a regex preceded by a non-null assertion on an array', () => {
        const tokens: Token[] = lex('[1]! / 2');
        expect(tokens.length).toBe(6);
        expectCharacterToken(tokens[0], 0, 1, '[');
        expectNumberToken(tokens[1], 1, 2, 1);
        expectCharacterToken(tokens[2], 2, 3, ']');
        expectOperatorToken(tokens[3], 3, 4, '!');
        expectOperatorToken(tokens[4], 5, 6, '/');
        expectNumberToken(tokens[5], 7, 8, 2);
      });

      it('should not tokenize consecutive regexes', () => {
        const tokens: Token[] = lex('/ 1 / 2 / 3 / 4');
        expect(tokens.length).toBe(6);
        expectRegExpBodyToken(tokens[0], 0, 5, ' 1 ');
        expectNumberToken(tokens[1], 6, 7, 2);
        expectOperatorToken(tokens[2], 8, 9, '/');
        expectNumberToken(tokens[3], 10, 11, 3);
        expectOperatorToken(tokens[4], 12, 13, '/');
        expectNumberToken(tokens[5], 14, 15, 4);
      });

      it('should not tokenize regex-like characters inside of a pipe', () => {
        const tokens: Token[] = lex(`foo / 1000 | date: 'M/d/yy'`);
        expect(tokens.length).toBe(7);
        expectIdentifierToken(tokens[0], 0, 3, 'foo');
        expectOperatorToken(tokens[1], 4, 5, '/');
        expectNumberToken(tokens[2], 6, 10, 1000);
        expectOperatorToken(tokens[3], 11, 12, '|');
        expectIdentifierToken(tokens[4], 13, 17, 'date');
        expectCharacterToken(tokens[5], 17, 18, ':');
        expectStringToken(tokens[6], 19, 27, 'M/d/yy', StringTokenKind.Plain);
      });

      it('should produce an error for an unterminated regex', () => {
        expectErrorToken(
          lex('/a')[0],
          2,
          2,
          'Lexer Error: Unterminated regular expression at column 2 in expression [/a]',
        );
      });

      it('should tokenize an arrow function without parenthesis', () => {
        const tokens = lex('a => a + 1');
        expect(tokens.length).toBe(5);
        expectIdentifierToken(tokens[0], 0, 1, 'a');
        expectOperatorToken(tokens[1], 2, 4, '=>');
        expectIdentifierToken(tokens[2], 5, 6, 'a');
        expectOperatorToken(tokens[3], 7, 8, '+');
        expectNumberToken(tokens[4], 9, 10, 1);
      });

      it('should tokenize an arrow function with parenthesis', () => {
        const tokens = lex('(a, b) => a + b');
        expect(tokens.length).toBe(9);
        expectCharacterToken(tokens[0], 0, 1, '(');
        expectIdentifierToken(tokens[1], 1, 2, 'a');
        expectCharacterToken(tokens[2], 2, 3, ',');
        expectIdentifierToken(tokens[3], 4, 5, 'b');
        expectCharacterToken(tokens[4], 5, 6, ')');
        expectOperatorToken(tokens[5], 7, 9, '=>');
        expectIdentifierToken(tokens[6], 10, 11, 'a');
        expectOperatorToken(tokens[7], 12, 13, '+');
        expectIdentifierToken(tokens[8], 14, 15, 'b');
      });
    });
  });
});