Repository: sql-formatter-org/sql-formatter
Pull request #283 (2022-07-05 14:17:36)

Refactor Tokenizer, update TokenTypes
- split Tokenizer into `tokenizer.ts` main file and `tokenizerEngine.ts` where all of the regex matching logic is contained
- renamed `preprocess` to `postProcess`
- moved regex logic into lexer/ dir, split regex util and regex types off into separate files
- replace PARAMETER token type with NAMED_PARAMETER, QUOTED_PARAMETER, INDEXED_PARAMETER, POSITIONAL_PARAMETER
- renamed IDENT to IDENTIFIER
- added QUOTED_IDENTIFIER token type
- added COMMA token type
- add babel plugin to import sql files during jest tests


Patch:
diff --git a/.babelrc b/.babelrc
index c6b5ca5a..fe12103f 100644
--- a/.babelrc
+++ b/.babelrc
@@ -11,6 +11,7 @@
           "src": "./src"
         }
       }
-    ]
+    ],
+    ["babel-plugin-inline-import", { "extensions": [".sql"] }]
   ]
 }
diff --git a/.eslintrc b/.eslintrc
index 57dd6268..f0622c04 100644
--- a/.eslintrc
+++ b/.eslintrc
@@ -28,6 +28,7 @@
     "no-useless-concat": "off",
     "no-restricted-syntax": "off",
     "no-constant-condition": "off",
+    "no-unused-expressions": "warn",
     "prefer-template": "off",
     "default-case": "off",
     "prettier/prettier": ["error"],
diff --git a/package.json b/package.json
index c150240e..e3f5b6aa 100644
--- a/package.json
+++ b/package.json
@@ -113,6 +113,7 @@
     "babel-jest": "^28.1.2",
     "babel-loader": "^8.1.0",
     "babel-plugin-add-module-exports": "^1.0.2",
+    "babel-plugin-inline-import": "^3.0.0",
     "babel-plugin-module-resolver": "^4.1.0",
     "dedent-js": "^1.0.1",
     "eslint": "^8.14.0",
diff --git a/src/core/Tokenizer.ts b/src/core/Tokenizer.ts
deleted file mode 100644
index 13bc50cc..00000000
--- a/src/core/Tokenizer.ts
+++ /dev/null
@@ -1,339 +0,0 @@
-import { equalizeWhitespace, escapeRegExp, id } from 'src/utils';
-
-import * as regexFactory from './regexFactory';
-import { type Token, TokenType } from './token';
-
-// A note about regular expressions
-//
-// We're using a sticky flag "y" in all tokenizing regexes.
-// This works a bit like ^, anchoring the regex to the start,
-// but when ^ anchores the regex to the start of string (or line),
-// the sticky flag anchors it to search start position, which we
-// can change by setting RegExp.lastIndex.
-//
-// This allows us to avoid slicing off tokens from the start of input string
-// (which we used in the past) and just move the match start position forward,
-// which is much more performant on long strings.
-
-const WHITESPACE_REGEX = /(\s+)/uy;
-const NULL_REGEX = /(?!)/uy; // zero-width negative lookahead, matches nothing
-
-const toCanonicalKeyword = (text: string) => equalizeWhitespace(text.toUpperCase());
-
-/** Struct that defines how a SQL language can be broken into tokens */
-interface TokenizerOptions {
-  // Main clauses that start new block, like: SELECT, FROM, WHERE, ORDER BY
-  reservedCommands: string[];
-  // Logical operator keywords, defaults to: [AND, OR]
-  reservedLogicalOperators?: string[];
-  // Keywords in CASE expressions that begin new line, like: WHEN, ELSE
-  reservedDependentClauses: string[];
-  // Keywords that create newline but no indentaion of their body.
-  // These contain set operations like UNION and INTERSECT
-  reservedBinaryCommands: string[];
-  // Various joins like LEFT OUTER JOIN
-  reservedJoins: string[];
-  // keywords used for JOIN conditions, defaults to: [ON, USING]
-  reservedJoinConditions?: string[];
-  // all other reserved words (not included to any of the above lists)
-  reservedKeywords: string[];
-  // Types of quotes to use for strings
-  stringTypes: regexFactory.QuoteType[];
-  // Types of quotes to use for quoted identifiers
-  identTypes: regexFactory.QuoteType[];
-  // Types of quotes to use for variables
-  variableTypes?: regexFactory.VariableType[];
-  // Open-parenthesis characters
-  openParens?: ('(' | '[' | '{')[];
-  // Close-parenthesis characters
-  closeParens?: (')' | ']' | '}')[];
-  // True to allow for positional "?" parameter placeholders
-  positionalParams?: boolean;
-  // Prefixes for numbered parameter placeholders to support, e.g. :1, :2, :3
-  numberedParamTypes?: ('?' | ':' | '$')[];
-  // Prefixes for named parameter placeholders to support, e.g. :name
-  namedParamTypes?: (':' | '@' | '$')[];
-  // Prefixes for quoted parameter placeholders to support, e.g. :"name"
-  // The type of quotes will depend on `identifierTypes` option.
-  quotedParamTypes?: (':' | '@' | '$')[];
-  // Line comment types to support, defaults to --
-  lineCommentTypes?: string[];
-  // Additional characters to support in identifiers
-  identChars?: regexFactory.IdentChars;
-  // Additional characters to support in named parameters
-  // Use this when parameters allow different characters from identifiers
-  // Defaults to `identChars`.
-  paramChars?: regexFactory.IdentChars;
-  // Additional multi-character operators to support, in addition to <=, >=, <>, !=
-  operators?: string[];
-  // Allows custom modifications on the token array.
-  // Called after the whole input string has been split into tokens.
-  // The result of this will be the output of the tokenizer.
-  preprocess?: (tokens: Token[]) => Token[];
-}
-
-interface ParamPattern {
-  regex: RegExp;
-  parseKey: (s: string) => string;
-}
-
-/** Converts SQL language string into a token stream */
-export default class Tokenizer {
-  private REGEX_MAP: Record<TokenType, RegExp>;
-  private quotedIdentRegex: RegExp;
-  private paramPatterns: ParamPattern[];
-  // The input SQL string to process
-  private input = '';
-  // Current position in string
-  private index = 0;
-
-  private preprocess = (tokens: Token[]) => tokens;
-
-  constructor(cfg: TokenizerOptions) {
-    if (cfg.preprocess) {
-      this.preprocess = cfg.preprocess;
-    }
-
-    this.quotedIdentRegex = regexFactory.createQuoteRegex(cfg.identTypes);
-
-    this.REGEX_MAP = {
-      [TokenType.IDENT]: regexFactory.createIdentRegex(cfg.identChars),
-      [TokenType.STRING]: regexFactory.createQuoteRegex(cfg.stringTypes),
-      [TokenType.VARIABLE]: cfg.variableTypes
-        ? regexFactory.createVariableRegex(cfg.variableTypes)
-        : NULL_REGEX,
-      [TokenType.RESERVED_KEYWORD]: regexFactory.createReservedWordRegex(
-        cfg.reservedKeywords,
-        cfg.identChars
-      ),
-      [TokenType.RESERVED_DEPENDENT_CLAUSE]: regexFactory.createReservedWordRegex(
-        cfg.reservedDependentClauses ?? [],
-        cfg.identChars
-      ),
-      [TokenType.RESERVED_LOGICAL_OPERATOR]: regexFactory.createReservedWordRegex(
-        cfg.reservedLogicalOperators ?? ['AND', 'OR'],
-        cfg.identChars
-      ),
-      [TokenType.RESERVED_COMMAND]: regexFactory.createReservedWordRegex(
-        cfg.reservedCommands,
-        cfg.identChars
-      ),
-      [TokenType.RESERVED_BINARY_COMMAND]: regexFactory.createReservedWordRegex(
-        cfg.reservedBinaryCommands,
-        cfg.identChars
-      ),
-      [TokenType.RESERVED_JOIN]: regexFactory.createReservedWordRegex(
-        cfg.reservedJoins,
-        cfg.identChars
-      ),
-      [TokenType.RESERVED_JOIN_CONDITION]: regexFactory.createReservedWordRegex(
-        cfg.reservedJoinConditions ?? ['ON', 'USING'],
-        cfg.identChars
-      ),
-      [TokenType.OPERATOR]: regexFactory.createOperatorRegex('+-/*%&|^><=.,;[]{}`:$@', [
-        '<>',
-        '<=',
-        '>=',
-        '!=',
-        ...(cfg.operators ?? []),
-      ]),
-      [TokenType.OPEN_PAREN]: regexFactory.createParenRegex(cfg.openParens ?? ['(']),
-      [TokenType.CLOSE_PAREN]: regexFactory.createParenRegex(cfg.closeParens ?? [')']),
-      [TokenType.RESERVED_CASE_START]: /(CASE)\b/iuy,
-      [TokenType.RESERVED_CASE_END]: /(END)\b/iuy,
-      [TokenType.LINE_COMMENT]: regexFactory.createLineCommentRegex(cfg.lineCommentTypes ?? ['--']),
-      [TokenType.BLOCK_COMMENT]: /(\/\*[^]*?(?:\*\/|$))/uy,
-      [TokenType.NUMBER]:
-        /(0x[0-9a-fA-F]+|0b[01]+|(-\s*)?[0-9]+(\.[0-9]*)?([eE][-+]?[0-9]+(\.[0-9]+)?)?)/uy,
-      [TokenType.PARAMETER]: NULL_REGEX, // matches nothing
-      [TokenType.DELIMITER]: /(;)/uy,
-      [TokenType.EOF]: NULL_REGEX, // matches nothing
-    };
-
-    this.paramPatterns = this.excludePatternsWithoutRegexes([
-      {
-        // :name placeholders
-        regex: regexFactory.createParameterRegex(
-          cfg.namedParamTypes ?? [],
-          regexFactory.createIdentPattern(cfg.paramChars || cfg.identChars)
-        ),
-        parseKey: v => v.slice(1),
-      },
-      {
-        // :"name" placeholders
-        regex: regexFactory.createParameterRegex(
-          cfg.quotedParamTypes ?? [],
-          regexFactory.createQuotePattern(cfg.identTypes)
-        ),
-        parseKey: v =>
-          this.getEscapedPlaceholderKey({ key: v.slice(2, -1), quoteChar: v.slice(-1) }),
-      },
-      {
-        // :1, :2, :3 placeholders
-        regex: regexFactory.createParameterRegex(cfg.numberedParamTypes ?? [], '[0-9]+'),
-        parseKey: v => v.slice(1),
-      },
-      {
-        // ? placeholders
-        regex: cfg.positionalParams ? /(\?)/uy : undefined,
-        parseKey: v => v.slice(1),
-      },
-    ]);
-  }
-
-  private excludePatternsWithoutRegexes(
-    patterns: { regex?: RegExp; parseKey: (s: string) => string }[]
-  ) {
-    return patterns.filter((p): p is ParamPattern => p.regex !== undefined);
-  }
-
-  /**
-   * Takes a SQL string and breaks it into tokens.
-   * Each token is an object with type and value.
-   *
-   * @param {string} input - The SQL string
-   * @returns {Token[]} output token stream
-   */
-  public tokenize(input: string): Token[] {
-    this.input = input;
-    this.index = 0;
-    const tokens: Token[] = [];
-    let token: Token | undefined;
-
-    // Keep processing the string until end is reached
-    while (this.index < this.input.length) {
-      // grab any preceding whitespace
-      const whitespaceBefore = this.getWhitespace();
-
-      if (this.index < this.input.length) {
-        // Get the next token and the token type
-        token = this.getNextToken(token);
-        if (!token) {
-          throw new Error(`Parse error: Unexpected "${input.slice(this.index, 100)}"`);
-        }
-
-        tokens.push({ ...token, whitespaceBefore });
-      }
-    }
-    return this.preprocess(tokens);
-  }
-
-  private getWhitespace(): string {
-    WHITESPACE_REGEX.lastIndex = this.index;
-    const matches = this.input.match(WHITESPACE_REGEX);
-    if (matches) {
-      // Advance current position by matched whitespace length
-      this.index += matches[1].length;
-      return matches[1];
-    } else {
-      return '';
-    }
-  }
-
-  private getNextToken(previousToken?: Token): Token | undefined {
-    return (
-      this.matchToken(TokenType.LINE_COMMENT) ||
-      this.matchToken(TokenType.BLOCK_COMMENT) ||
-      this.matchToken(TokenType.STRING) ||
-      this.matchQuotedIdentToken() ||
-      this.matchToken(TokenType.VARIABLE) ||
-      this.matchToken(TokenType.OPEN_PAREN) ||
-      this.matchToken(TokenType.CLOSE_PAREN) ||
-      this.matchPlaceholderToken() ||
-      this.matchToken(TokenType.NUMBER) ||
-      this.matchReservedWordToken(previousToken) ||
-      this.matchToken(TokenType.IDENT) ||
-      this.matchToken(TokenType.DELIMITER) ||
-      this.matchToken(TokenType.OPERATOR)
-    );
-  }
-
-  private matchPlaceholderToken(): Token | undefined {
-    for (const { regex, parseKey } of this.paramPatterns) {
-      const token = this.match({
-        regex,
-        type: TokenType.PARAMETER,
-        transform: id,
-      });
-      if (token) {
-        return { ...token, key: parseKey(token.value) };
-      }
-    }
-    return undefined;
-  }
-
-  private getEscapedPlaceholderKey({ key, quoteChar }: { key: string; quoteChar: string }): string {
-    return key.replace(new RegExp(escapeRegExp('\\' + quoteChar), 'gu'), quoteChar);
-  }
-
-  private matchQuotedIdentToken(): Token | undefined {
-    return this.match({
-      regex: this.quotedIdentRegex,
-      type: TokenType.IDENT,
-      transform: id,
-    });
-  }
-
-  private matchReservedWordToken(previousToken?: Token): Token | undefined {
-    // A reserved word cannot be preceded by a '.'
-    // this makes it so in "mytable.from", "from" is not considered a reserved word
-    if (previousToken?.value === '.') {
-      return undefined;
-    }
-
-    // prioritised list of Reserved token types
-    return (
-      this.matchReservedToken(TokenType.RESERVED_CASE_START) ||
-      this.matchReservedToken(TokenType.RESERVED_CASE_END) ||
-      this.matchReservedToken(TokenType.RESERVED_COMMAND) ||
-      this.matchReservedToken(TokenType.RESERVED_BINARY_COMMAND) ||
-      this.matchReservedToken(TokenType.RESERVED_JOIN) ||
-      this.matchReservedToken(TokenType.RESERVED_DEPENDENT_CLAUSE) ||
-      this.matchReservedToken(TokenType.RESERVED_LOGICAL_OPERATOR) ||
-      this.matchReservedToken(TokenType.RESERVED_KEYWORD) ||
-      this.matchReservedToken(TokenType.RESERVED_JOIN_CONDITION)
-    );
-  }
-
-  // Helper for matching RESERVED_* tokens which need to be transformed to canonical form
-  private matchReservedToken(tokenType: TokenType): Token | undefined {
-    return this.match({
-      type: tokenType,
-      regex: this.REGEX_MAP[tokenType],
-      transform: toCanonicalKeyword,
-    });
-  }
-
-  // Shorthand for `match` that looks up regex from REGEX_MAP
-  private matchToken(tokenType: TokenType): Token | undefined {
-    return this.match({
-      type: tokenType,
-      regex: this.REGEX_MAP[tokenType],
-      transform: id,
-    });
-  }
-
-  // Attempts to match RegExp at current position in input
-  private match({
-    type,
-    regex,
-    transform,
-  }: {
-    type: TokenType;
-    regex: RegExp;
-    transform: (s: string) => string;
-  }): Token | undefined {
-    regex.lastIndex = this.index;
-    const matches = this.input.match(regex);
-    if (matches) {
-      // Advance current position by matched token length
-      this.index += matches[1].length;
-      return {
-        type,
-        text: matches[1],
-        value: transform(matches[1]),
-      };
-    }
-    return undefined;
-  }
-}
diff --git a/src/core/regexFactory.ts b/src/core/regexFactory.ts
deleted file mode 100644
index d91d0f5d..00000000
--- a/src/core/regexFactory.ts
+++ /dev/null
@@ -1,195 +0,0 @@
-import { escapeRegExp, isEmpty, sortByLengthDesc } from 'src/utils';
-
-// This enables the following quote styles:
-// 1. backtick quoted using `` to escape
-// 2. square bracket quoted (SQL Server) using ]] to escape
-// 3. double quoted using "" or \" to escape
-// 4. single quoted using '' or \' to escape
-// 5. PostgreSQL dollar-quoted
-// 6. BigQuery '''triple-quoted'''
-// 7. BigQuery """triple-quoted"""
-// 8. Hive and Spark variables: ${name}
-const quotePatterns = {
-  '``': '(`[^`]*($|`))+',
-  '[]': '(\\[[^\\]]*($|\\]))(\\][^\\]]*($|\\]))*',
-  '""': '("[^"\\\\]*(?:\\\\.[^"\\\\]*)*("|$))+',
-  "''": "('[^'\\\\]*(?:\\\\.[^'\\\\]*)*('|$))+",
-  '$$': '(?<tag>\\$\\w*\\$)[\\s\\S]*?(?:\\k<tag>|$)',
-  "'''..'''": "'''[^\\\\]*?(?:\\\\.[^\\\\]*?)*?('''|$)",
-  '""".."""': '"""[^\\\\]*?(?:\\\\.[^\\\\]*?)*?("""|$)',
-  '{}': '(\\{[^\\}]*($|\\}))',
-};
-export type PlainQuoteType = keyof typeof quotePatterns;
-
-export type PrefixedQuoteType = {
-  quote: PlainQuoteType;
-  prefixes: string[];
-  required?: boolean; // True when prefix is required
-};
-
-export type QuoteType = PlainQuoteType | PrefixedQuoteType;
-
-export interface VariableRegex {
-  regex: string;
-}
-
-export type VariableType = VariableRegex | PrefixedQuoteType;
-
-export interface IdentChars {
-  // Additional characters that can be used as first character of an identifier.
-  // That is: in addition to letters and underscore.
-  first?: string;
-  // Additional characters that can appear after the first character of identifier.
-  // That is: in addition to letters, numbers and underscore.
-  rest?: string;
-  // True to allow single dashes (-) inside identifiers, but not at the beginning or end
-  dashes?: boolean;
-}
-
-/**
- * Builds a RegExp containing all operators for a SQL dialect
- * @param {string} monadOperators - concatenated string of all 1-length operators
- * @param {string[]} polyadOperators - list of strings of all >1-length operators
- */
-export const createOperatorRegex = (monadOperators: string, polyadOperators: string[]): RegExp =>
-  patternToRegex(
-    `${sortByLengthDesc(polyadOperators).map(escapeRegExp).join('|')}|` +
-      `[${monadOperators.split('').map(escapeRegExp).join('')}]`
-  );
-
-/**
- * Builds a RegExp for valid line comments in a SQL dialect
- * @param {string[]} lineCommentTypes - list of character strings that denote line comments
- */
-export const createLineCommentRegex = (lineCommentTypes: string[]): RegExp =>
-  new RegExp(
-    `((?:${lineCommentTypes.map(c => escapeRegExp(c)).join('|')}).*?)(?:\r\n|\r|\n|$)`,
-    'uy'
-  );
-
-/**
- * Builds a RegExp for all Reserved Keywords in a SQL dialect
- */
-export const createReservedWordRegex = (
-  reservedKeywords: string[],
-  identChars: IdentChars = {}
-): RegExp => {
-  if (reservedKeywords.length === 0) {
-    return /\b$/uy;
-  }
-
-  const avoidIdentChars = rejectIdentCharsPattern(identChars);
-
-  const reservedKeywordsPattern = sortByLengthDesc(reservedKeywords)
-    .join('|')
-    .replace(/ /gu, '\\s+');
-
-  return new RegExp(`(${reservedKeywordsPattern})${avoidIdentChars}\\b`, 'iuy');
-};
-
-// Negative lookahead to avoid matching a keyword that's actually part of identifier,
-// which can happen when identifier allows word-boundary characters inside it.
-//
-// For example "SELECT$ME" should be tokenized as:
-// - ["SELECT$ME"] when $ is allowed inside identifiers
-// - ["SELECT", "$", "ME"] when $ can't be part of identifiers.
-const rejectIdentCharsPattern = ({ rest, dashes }: IdentChars): string =>
-  rest || dashes ? `(?![${rest || ''}${dashes ? '-' : ''}])` : '';
-
-/**
- * Builds a RegExp for valid identifiers in a SQL dialect
- */
-export const createIdentRegex = (specialChars: IdentChars = {}): RegExp =>
-  patternToRegex(createIdentPattern(specialChars));
-
-/**
- * Builds a RegExp string for valid identifiers in a SQL dialect
- */
-export const createIdentPattern = ({ first, rest, dashes }: IdentChars = {}): string => {
-  // Unicode letters, diacritical marks and underscore
-  const letter = '\\p{Alphabetic}\\p{Mark}_';
-  // Numbers 0..9, plus various unicode numbers
-  const number = '\\p{Decimal_Number}';
-
-  const firstChars = escapeRegExp(first ?? '');
-  const restChars = escapeRegExp(rest ?? '');
-
-  const pattern = `[${letter}${firstChars}][${letter}${number}${restChars}]*`;
-
-  return dashes ? withDashes(pattern) : pattern;
-};
-
-const withDashes = (pattern: string): string => pattern + '(?:-' + pattern + ')*';
-
-// Converts "ab" to "[Aa][Bb]"
-const toCaseInsensitivePattern = (prefix: string): string =>
-  prefix
-    .split('')
-    .map(char => '[' + char.toUpperCase() + char.toLowerCase() + ']')
-    .join('');
-
-// Converts ["a", "b"] to "(?:[Aa]|[Bb]|)" or "(?:[Aa]|[Bb])" when required = true
-const prefixesPattern = ({ prefixes, required }: PrefixedQuoteType): string =>
-  '(?:' + prefixes.map(toCaseInsensitivePattern).join('|') + (required ? '' : '|') + ')';
-
-const createSingleQuotePattern = (type: QuoteType): string => {
-  if (typeof type === 'string') {
-    return quotePatterns[type];
-  } else {
-    return prefixesPattern(type) + quotePatterns[type.quote];
-  }
-};
-
-/** Builds a quote-delimited pattern for matching all given quote types */
-export const createQuotePattern = (quoteTypes: QuoteType[]): string =>
-  quoteTypes.map(createSingleQuotePattern).join('|');
-
-const createSingleVariablePattern = (type: VariableType): string => {
-  if ('regex' in type) {
-    return type.regex;
-  } else {
-    return createSingleQuotePattern(type);
-  }
-};
-
-/** Builds a RegExp for matching variables */
-export const createVariableRegex = (varTypes: VariableType[]): RegExp =>
-  patternToRegex(varTypes.map(createSingleVariablePattern).join('|'));
-
-/** Builds a RegExp for matching quote-delimited patterns */
-export const createQuoteRegex = (quoteTypes: QuoteType[]): RegExp =>
-  patternToRegex(createQuotePattern(quoteTypes));
-
-/** Escapes paren characters for RegExp patterns */
-const escapeParen = (paren: string): string => {
-  if (paren.length === 1) {
-    // A single punctuation character
-    return escapeRegExp(paren);
-  } else {
-    // longer word
-    return '\\b' + paren + '\\b';
-  }
-};
-
-/**
- * Builds a RegExp for matching parenthesis patterns, escaping them with `escapeParen`
- * @param {string[]} parens - list of strings that denote parenthesis patterns
- */
-export const createParenRegex = (parens: string[]): RegExp =>
-  patternToRegex(parens.map(escapeParen).join('|'));
-
-/**
- * Builds a RegExp for parameter placeholder patterns
- * @param {string[]} types - list of strings that denote placeholder types
- * @param {string} pattern - string that denotes placeholder pattern
- */
-export const createParameterRegex = (types: string[], pattern: string): RegExp | undefined => {
-  if (isEmpty(types)) {
-    return undefined;
-  }
-  const typesRegex = types.map(escapeRegExp).join('|');
-
-  return patternToRegex(`(?:${typesRegex})(?:${pattern})`);
-};
-
-const patternToRegex = (pattern: string): RegExp => new RegExp('(' + pattern + ')', 'uy');
diff --git a/src/core/token.ts b/src/core/token.ts
index d50f1e81..d7385d5a 100644
--- a/src/core/token.ts
+++ b/src/core/token.ts
@@ -1,6 +1,7 @@
 /** Token type enum for all possible Token categories */
 export enum TokenType {
-  IDENT = 'IDENT',
+  QUOTED_IDENTIFIER = 'QUOTED_IDENTIFIER',
+  IDENTIFIER = 'IDENTIFIER',
   STRING = 'STRING',
   VARIABLE = 'VARIABLE',
   RESERVED_KEYWORD = 'RESERVED_KEYWORD',
@@ -13,12 +14,16 @@ export enum TokenType {
   RESERVED_CASE_START = 'RESERVED_CASE_START',
   RESERVED_CASE_END = 'RESERVED_CASE_END',
   OPERATOR = 'OPERATOR',
+  COMMA = 'COMMA',
   OPEN_PAREN = 'OPEN_PAREN',
   CLOSE_PAREN = 'CLOSE_PAREN',
   LINE_COMMENT = 'LINE_COMMENT',
   BLOCK_COMMENT = 'BLOCK_COMMENT',
   NUMBER = 'NUMBER',
-  PARAMETER = 'PARAMETER',
+  NAMED_PARAMETER = 'NAMED_PARAMETER',
+  QUOTED_PARAMETER = 'QUOTED_PARAMETER',
+  INDEXED_PARAMETER = 'INDEXED_PARAMETER',
+  POSITIONAL_PARAMETER = 'POSITIONAL_PARAMETER',
   DELIMITER = 'DELIMITER',
   EOF = 'EOF',
 }
diff --git a/src/formatter/AliasAs.ts b/src/formatter/AliasAs.ts
index 75349c4a..6ffa0aeb 100644
--- a/src/formatter/AliasAs.ts
+++ b/src/formatter/AliasAs.ts
@@ -36,7 +36,7 @@ export default class AliasAs {
           processedTokens.push(token);
         }
       } else if (
-        token.type === TokenType.IDENT ||
+        token.type === TokenType.IDENTIFIER ||
         token.type === TokenType.NUMBER ||
         token.type === TokenType.STRING ||
         token.type === TokenType.VARIABLE
@@ -66,7 +66,9 @@ export default class AliasAs {
   // if table alias is missing and should be added
   private isMissingTableAlias(token: Token): boolean {
     return (
-      this.aliasAs === 'always' && token.type === TokenType.IDENT && this.lookBehind().value === ')'
+      this.aliasAs === 'always' &&
+      token.type === TokenType.IDENTIFIER &&
+      this.lookBehind().value === ')'
     );
   }
 
@@ -77,10 +79,10 @@ export default class AliasAs {
     return (
       (this.aliasAs === 'always' || this.aliasAs === 'select') &&
       this.isWithinSelect() &&
-      token.type === TokenType.IDENT &&
+      token.type === TokenType.IDENTIFIER &&
       (isToken.END(prevToken) ||
-        ((prevToken.type === TokenType.IDENT || prevToken.type === TokenType.NUMBER) &&
-          (nextToken.value === ',' || isCommand(nextToken))))
+        ((prevToken.type === TokenType.IDENTIFIER || prevToken.type === TokenType.NUMBER) &&
+          (nextToken.type === TokenType.COMMA || isCommand(nextToken))))
     );
   }
 
@@ -96,7 +98,7 @@ export default class AliasAs {
       this.isWithinSelect() &&
       isToken.CAST(this.getPreviousReservedToken()) &&
       isToken.AS(this.lookAhead()) &&
-      (this.lookAhead(2).type === TokenType.IDENT ||
+      (this.lookAhead(2).type === TokenType.IDENTIFIER ||
         this.lookAhead(2).type === TokenType.RESERVED_KEYWORD) &&
       this.lookAhead(3).value === ')'
     );
diff --git a/src/formatter/ExpressionFormatter.ts b/src/formatter/ExpressionFormatter.ts
index 3ee8d3e1..c3a6dc24 100644
--- a/src/formatter/ExpressionFormatter.ts
+++ b/src/formatter/ExpressionFormatter.ts
@@ -209,11 +209,17 @@ export default class ExpressionFormatter {
         return this.formatCaseStart(token);
       case TokenType.RESERVED_CASE_END:
         return this.formatCaseEnd(token);
-      case TokenType.PARAMETER:
+      case TokenType.NAMED_PARAMETER:
+      case TokenType.QUOTED_PARAMETER:
+      case TokenType.INDEXED_PARAMETER:
+      case TokenType.POSITIONAL_PARAMETER:
         return this.formatParameter(token);
+      case TokenType.COMMA:
+        return this.formatComma(token);
       case TokenType.OPERATOR:
         return this.formatOperator(token);
-      case TokenType.IDENT:
+      case TokenType.IDENTIFIER:
+      case TokenType.QUOTED_IDENTIFIER:
       case TokenType.STRING:
       case TokenType.NUMBER:
       case TokenType.VARIABLE:
@@ -303,10 +309,7 @@ export default class ExpressionFormatter {
    */
   private formatOperator(token: Token) {
     // special operator
-    if (token.value === ',') {
-      this.formatComma(token);
-      return;
-    } else if (token.value === ':') {
+    if (token.value === ':') {
       this.layout.add(WS.NO_SPACE, this.show(token), WS.SPACE);
       return;
     } else if (token.value === '.') {
diff --git a/src/formatter/Formatter.ts b/src/formatter/Formatter.ts
index 0adc2f8a..481b3f20 100644
--- a/src/formatter/Formatter.ts
+++ b/src/formatter/Formatter.ts
@@ -1,7 +1,7 @@
 import type { FormatOptions } from 'src/types';
 import { indentString } from 'src/core/config';
 import Params from 'src/core/Params';
-import Tokenizer from 'src/core/Tokenizer';
+import Tokenizer from 'src/lexer/Tokenizer';
 
 import Parser from 'src/parser/Parser';
 import { Statement } from 'src/parser/ast';
diff --git a/src/formatter/formatCommaPositions.ts b/src/formatter/formatCommaPositions.ts
index 9e9d4b87..2ed95e32 100644
--- a/src/formatter/formatCommaPositions.ts
+++ b/src/formatter/formatCommaPositions.ts
@@ -1,7 +1,7 @@
 import type { CommaPosition } from 'src/types';
 import { maxLength } from 'src/utils';
 
-const WHITESPACE_REGEX = /^(\s+)/u;
+const PRECEDING_WHITESPACE_REGEX = /^\s+/u;
 
 /**
  * Handles comma placement - either before, after or tabulated
@@ -77,7 +77,7 @@ function formatBefore(commaLines: string[], indent: string): string[] {
     if (i === 0) {
       return line; // do not add comma for first item
     }
-    const [whitespace] = line.match(WHITESPACE_REGEX) || [''];
+    const [whitespace] = line.match(PRECEDING_WHITESPACE_REGEX) || [''];
     return (
       removeLastIndent(whitespace, indent) +
       indent.replace(/ {2}$/, ', ') + // add comma to the end of last indent
diff --git a/src/languages/bigquery.formatter.ts b/src/languages/bigquery.formatter.ts
index 988bd702..090dd219 100644
--- a/src/languages/bigquery.formatter.ts
+++ b/src/languages/bigquery.formatter.ts
@@ -1,5 +1,5 @@
 import Formatter from 'src/formatter/Formatter';
-import Tokenizer from 'src/core/Tokenizer';
+import Tokenizer from 'src/lexer/Tokenizer';
 import { EOF_TOKEN, isToken, TokenType, type Token } from 'src/core/token';
 import { dedupe } from 'src/utils';
 
@@ -857,12 +857,12 @@ export default class BigQueryFormatter extends Formatter {
       quotedParamTypes: ['@'],
       lineCommentTypes: ['--', '#'],
       operators: BigQueryFormatter.operators,
-      preprocess,
+      postProcess,
     });
   }
 }
 
-function preprocess(tokens: Token[]): Token[] {
+function postProcess(tokens: Token[]): Token[] {
   return detectArraySubscripts(combineParameterizedTypes(tokens));
 }
 
@@ -892,7 +892,7 @@ function combineParameterizedTypes(tokens: Token[]) {
       const endIndex = findClosingAngleBracketIndex(tokens, i + 1);
       const typeDefTokens = tokens.slice(i, endIndex + 1);
       processed.push({
-        type: TokenType.IDENT,
+        type: TokenType.IDENTIFIER,
         value: typeDefTokens.map(formatTypeDefToken('value')).join(''),
         text: typeDefTokens.map(formatTypeDefToken('text')).join(''),
       });
@@ -907,7 +907,7 @@ function combineParameterizedTypes(tokens: Token[]) {
 const formatTypeDefToken =
   (key: 'text' | 'value') =>
   (token: Token): string => {
-    if (token.type === TokenType.IDENT || token.value === ',') {
+    if (token.type === TokenType.IDENTIFIER || token.type === TokenType.COMMA) {
       return token[key] + ' ';
     } else {
       return token[key];
diff --git a/src/languages/db2.formatter.ts b/src/languages/db2.formatter.ts
index a5bb5831..03dabd3c 100644
--- a/src/languages/db2.formatter.ts
+++ b/src/languages/db2.formatter.ts
@@ -1,5 +1,5 @@
 import Formatter from 'src/formatter/Formatter';
-import Tokenizer from 'src/core/Tokenizer';
+import Tokenizer from 'src/lexer/Tokenizer';
 import { dedupe } from 'src/utils';
 
 /**
diff --git a/src/languages/hive.formatter.ts b/src/languages/hive.formatter.ts
index d67340bf..306fc0fe 100644
--- a/src/languages/hive.formatter.ts
+++ b/src/languages/hive.formatter.ts
@@ -1,5 +1,5 @@
 import Formatter from 'src/formatter/Formatter';
-import Tokenizer from 'src/core/Tokenizer';
+import Tokenizer from 'src/lexer/Tokenizer';
 import { dedupe } from 'src/utils';
 
 /**
@@ -625,7 +625,7 @@ export default class HiveFormatter extends Formatter {
       closeParens: [')', ']'],
       stringTypes: ['""', "''"],
       identTypes: ['``'],
-      variableTypes: [{ quote: '{}', prefixes: ['$'], required: true }],
+      variableTypes: [{ quote: '{}', prefixes: ['$'], requirePrefix: true }],
       operators: HiveFormatter.operators,
     });
   }
diff --git a/src/languages/mariadb.formatter.ts b/src/languages/mariadb.formatter.ts
index 4c357073..e7d255ac 100644
--- a/src/languages/mariadb.formatter.ts
+++ b/src/languages/mariadb.formatter.ts
@@ -1,5 +1,5 @@
 import Formatter from 'src/formatter/Formatter';
-import Tokenizer from 'src/core/Tokenizer';
+import Tokenizer from 'src/lexer/Tokenizer';
 import { EOF_TOKEN, isToken, type Token, TokenType } from 'src/core/token';
 import { dedupe } from 'src/utils';
 
@@ -1166,19 +1166,19 @@ export default class MariaDbFormatter extends Formatter {
       identChars: { first: '$', rest: '$' },
       variableTypes: [
         { regex: '@[A-Za-z0-9_.$]+' },
-        { quote: '""', prefixes: ['@'], required: true },
-        { quote: "''", prefixes: ['@'], required: true },
-        { quote: '``', prefixes: ['@'], required: true },
+        { quote: '""', prefixes: ['@'], requirePrefix: true },
+        { quote: "''", prefixes: ['@'], requirePrefix: true },
+        { quote: '``', prefixes: ['@'], requirePrefix: true },
       ],
       positionalParams: true,
       lineCommentTypes: ['--', '#'],
       operators: MariaDbFormatter.operators,
-      preprocess,
+      postProcess,
     });
   }
 }
 
-function preprocess(tokens: Token[]) {
+function postProcess(tokens: Token[]) {
   return tokens.map((token, i) => {
     const nextToken = tokens[i + 1] || EOF_TOKEN;
     if (isToken.SET(token) && nextToken.value === '(') {
diff --git a/src/languages/mysql.formatter.ts b/src/languages/mysql.formatter.ts
index 77a35b7c..70cac55e 100644
--- a/src/languages/mysql.formatter.ts
+++ b/src/languages/mysql.formatter.ts
@@ -1,5 +1,5 @@
 import Formatter from 'src/formatter/Formatter';
-import Tokenizer from 'src/core/Tokenizer';
+import Tokenizer from 'src/lexer/Tokenizer';
 import { EOF_TOKEN, isToken, type Token, TokenType } from 'src/core/token';
 import { dedupe } from 'src/utils';
 
@@ -17,7 +17,7 @@ const reservedFunctions = [
   'ADDTIME',
   'AES_DECRYPT',
   'AES_ENCRYPT',
-  'AND',
+  // 'AND',
   'ANY_VALUE',
   'ASCII',
   'ASIN',
@@ -247,7 +247,7 @@ const reservedFunctions = [
   'NULLIF',
   'OCT',
   'OCTET_LENGTH',
-  'OR',
+  // 'OR',
   'ORD',
   'PERCENT_RANK',
   'PERIOD_ADD',
@@ -429,7 +429,7 @@ const reservedFunctions = [
   'WEEKDAY',
   'WEEKOFYEAR',
   'WEIGHT_STRING',
-  'XOR',
+  // 'XOR',
   'YEAR',
   'YEARWEEK',
 ];
@@ -1329,19 +1329,19 @@ export default class MySqlFormatter extends Formatter {
       identChars: { first: '$', rest: '$' },
       variableTypes: [
         { regex: '@[A-Za-z0-9_.$]+' },
-        { quote: '""', prefixes: ['@'], required: true },
-        { quote: "''", prefixes: ['@'], required: true },
-        { quote: '``', prefixes: ['@'], required: true },
+        { quote: '""', prefixes: ['@'], requirePrefix: true },
+        { quote: "''", prefixes: ['@'], requirePrefix: true },
+        { quote: '``', prefixes: ['@'], requirePrefix: true },
       ],
       positionalParams: true,
       lineCommentTypes: ['--', '#'],
       operators: MySqlFormatter.operators,
-      preprocess,
+      postProcess,
     });
   }
 }
 
-function preprocess(tokens: Token[]) {
+function postProcess(tokens: Token[]) {
   return tokens.map((token, i) => {
     const nextToken = tokens[i + 1] || EOF_TOKEN;
     if (isToken.SET(token) && nextToken.value === '(') {
diff --git a/src/languages/n1ql.formatter.ts b/src/languages/n1ql.formatter.ts
index 9eccaa99..a110f7f5 100644
--- a/src/languages/n1ql.formatter.ts
+++ b/src/languages/n1ql.formatter.ts
@@ -1,5 +1,5 @@
 import Formatter from 'src/formatter/Formatter';
-import Tokenizer from 'src/core/Tokenizer';
+import Tokenizer from 'src/lexer/Tokenizer';
 import { dedupe } from 'src/utils';
 
 // TODO: split this into object with function categories
diff --git a/src/languages/plsql.formatter.ts b/src/languages/plsql.formatter.ts
index 463468c1..11b19f53 100644
--- a/src/languages/plsql.formatter.ts
+++ b/src/languages/plsql.formatter.ts
@@ -1,5 +1,5 @@
 import Formatter from 'src/formatter/Formatter';
-import Tokenizer from 'src/core/Tokenizer';
+import Tokenizer from 'src/lexer/Tokenizer';
 import { EOF_TOKEN, isReserved, isToken, type Token, TokenType } from 'src/core/token';
 import { dedupe } from 'src/utils';
 
@@ -701,12 +701,12 @@ export default class PlSqlFormatter extends Formatter {
       namedParamTypes: [':'],
       paramChars: {}, // Empty object used on purpose to not allow $ and # chars as specified in identChars
       operators: PlSqlFormatter.operators,
-      preprocess,
+      postProcess,
     });
   }
 }
 
-function preprocess(tokens: Token[]) {
+function postProcess(tokens: Token[]) {
   let previousReservedToken: Token = EOF_TOKEN;
 
   return tokens.map(token => {
diff --git a/src/languages/postgresql.formatter.ts b/src/languages/postgresql.formatter.ts
index 5b9f6673..191d03bf 100644
--- a/src/languages/postgresql.formatter.ts
+++ b/src/languages/postgresql.formatter.ts
@@ -1,5 +1,5 @@
 import Formatter from 'src/formatter/Formatter';
-import Tokenizer from 'src/core/Tokenizer';
+import Tokenizer from 'src/lexer/Tokenizer';
 import { dedupe } from 'src/utils';
 
 /**
@@ -1674,7 +1674,7 @@ export default class PostgreSqlFormatter extends Formatter {
       reservedBinaryCommands,
       reservedJoins,
       reservedDependentClauses,
-      reservedKeywords: dedupe([...Object.values(reservedFunctions).flat(), ...reservedKeywords]),
+      reservedKeywords: dedupe([...reservedKeywords, ...Object.values(reservedFunctions).flat()]),
       openParens: ['(', '['],
       closeParens: [')', ']'],
       stringTypes: [{ quote: "''", prefixes: ['U&', 'E', 'X', 'B'] }, '$$'],
diff --git a/src/languages/redshift.formatter.ts b/src/languages/redshift.formatter.ts
index ca068c1b..d00495d3 100644
--- a/src/languages/redshift.formatter.ts
+++ b/src/languages/redshift.formatter.ts
@@ -1,5 +1,5 @@
 import Formatter from 'src/formatter/Formatter';
-import Tokenizer from 'src/core/Tokenizer';
+import Tokenizer from 'src/lexer/Tokenizer';
 import { dedupe } from 'src/utils';
 
 /**
@@ -720,8 +720,8 @@ export default class RedshiftFormatter extends Formatter {
       reservedJoins,
       reservedDependentClauses,
       reservedKeywords: dedupe([
-        ...Object.values(reservedFunctions).flat(),
         ...Object.values(reservedKeywords).flat(),
+        ...Object.values(reservedFunctions).flat(),
       ]),
       stringTypes: ["''"],
       identTypes: [`""`],
diff --git a/src/languages/spark.formatter.ts b/src/languages/spark.formatter.ts
index 541145c6..6190be98 100644
--- a/src/languages/spark.formatter.ts
+++ b/src/languages/spark.formatter.ts
@@ -1,5 +1,5 @@
 import Formatter from 'src/formatter/Formatter';
-import Tokenizer from 'src/core/Tokenizer';
+import Tokenizer from 'src/lexer/Tokenizer';
 import { EOF_TOKEN, isToken, type Token, TokenType } from 'src/core/token';
 import { dedupe } from 'src/utils';
 
@@ -364,7 +364,7 @@ const reservedKeywords = [
   'ALL',
   'ALTER',
   'ANALYZE',
-  'AND',
+  // 'AND',
   'ANTI',
   'ANY',
   'ARCHIVE',
@@ -500,7 +500,7 @@ const reservedKeywords = [
   'ONLY',
   'OPTION',
   'OPTIONS',
-  'OR',
+  // 'OR',
   'ORDER',
   'OUT',
   'OUTER',
@@ -781,19 +781,19 @@ export default class SparkFormatter extends Formatter {
       reservedJoins,
       reservedDependentClauses,
       reservedLogicalOperators: ['AND', 'OR', 'XOR'],
-      reservedKeywords: dedupe([...Object.values(reservedFunctions).flat(), ...reservedKeywords]),
+      reservedKeywords: dedupe([...reservedKeywords, ...Object.values(reservedFunctions).flat()]),
       openParens: ['(', '['],
       closeParens: [')', ']'],
       stringTypes: [{ quote: "''", prefixes: ['X'] }],
       identTypes: ['``'],
-      variableTypes: [{ quote: '{}', prefixes: ['$'], required: true }],
+      variableTypes: [{ quote: '{}', prefixes: ['$'], requirePrefix: true }],
       operators: SparkFormatter.operators,
-      preprocess,
+      postProcess,
     });
   }
 }
 
-function preprocess(tokens: Token[]) {
+function postProcess(tokens: Token[]) {
   return tokens.map((token, i) => {
     const prevToken = tokens[i - 1] || EOF_TOKEN;
     const nextToken = tokens[i + 1] || EOF_TOKEN;
@@ -808,7 +808,7 @@ function preprocess(tokens: Token[]) {
     if (token.value === 'ITEMS' && token.type === TokenType.RESERVED_KEYWORD) {
       if (!(prevToken.value === 'COLLECTION' && nextToken.value === 'TERMINATED')) {
         // this is a word and not COLLECTION ITEMS
-        return { type: TokenType.IDENT, text: token.text, value: token.text };
+        return { type: TokenType.IDENTIFIER, text: token.text, value: token.text };
       }
     }
 
diff --git a/src/languages/sql.formatter.ts b/src/languages/sql.formatter.ts
index 0c5e6828..90cfb948 100644
--- a/src/languages/sql.formatter.ts
+++ b/src/languages/sql.formatter.ts
@@ -1,5 +1,5 @@
 import Formatter from 'src/formatter/Formatter';
-import Tokenizer from 'src/core/Tokenizer';
+import Tokenizer from 'src/lexer/Tokenizer';
 import { dedupe } from 'src/utils';
 
 const reservedFunctions = {
diff --git a/src/languages/sqlite.formatter.ts b/src/languages/sqlite.formatter.ts
index bf64e555..8548cc90 100644
--- a/src/languages/sqlite.formatter.ts
+++ b/src/languages/sqlite.formatter.ts
@@ -1,5 +1,5 @@
 import Formatter from 'src/formatter/Formatter';
-import Tokenizer from 'src/core/Tokenizer';
+import Tokenizer from 'src/lexer/Tokenizer';
 import { dedupe } from 'src/utils';
 
 const reservedFunctions = {
@@ -62,9 +62,12 @@ const reservedKeywords = [
   'AFTER',
   'ALL',
   'ALTER',
+  // 'AND',
+  'ANY',
+  'ARE',
+  'ARRAY',
   'ALWAYS',
   'ANALYZE',
-  'AND',
   'AS',
   'ASC',
   'ATTACH',
@@ -151,9 +154,11 @@ const reservedKeywords = [
   'NULLS',
   'OF',
   'OFFSET',
-  'ON UPDATE',
   'ON DELETE',
-  'OR',
+  'ON UPDATE',
+  'ONLY',
+  'OPEN',
+  // 'OR',
   'ORDER',
   'OTHERS',
   'OUTER',
diff --git a/src/languages/tsql.formatter.ts b/src/languages/tsql.formatter.ts
index 9ff584c0..35cef863 100644
--- a/src/languages/tsql.formatter.ts
+++ b/src/languages/tsql.formatter.ts
@@ -1,5 +1,5 @@
 import Formatter from 'src/formatter/Formatter';
-import Tokenizer from 'src/core/Tokenizer';
+import Tokenizer from 'src/lexer/Tokenizer';
 import { dedupe } from 'src/utils';
 
 /**
@@ -319,7 +319,7 @@ const reservedKeywords = {
     'ADD',
     'ALL',
     'ALTER',
-    'AND',
+    // 'AND',
     'ANY',
     'AS',
     'ASC',
@@ -426,7 +426,7 @@ const reservedKeywords = {
     'OPENROWSET',
     'OPENXML',
     'OPTION',
-    'OR',
+    // 'OR',
     'ORDER',
     'OUTER',
     'OVER',
@@ -506,7 +506,7 @@ const reservedKeywords = {
     'ALL',
     'ALLOCATE',
     'ALTER',
-    'AND',
+    // 'AND',
     'ANY',
     'ARE',
     'AS',
@@ -647,7 +647,7 @@ const reservedKeywords = {
     'ONLY',
     'OPEN',
     'OPTION',
-    'OR',
+    // 'OR',
     'ORDER',
     'OUTER',
     'OUTPUT',
@@ -1236,8 +1236,8 @@ export default class TSqlFormatter extends Formatter {
       reservedJoins,
       reservedDependentClauses,
       reservedKeywords: dedupe([
-        ...Object.values(reservedFunctions).flat(),
         ...Object.values(reservedKeywords).flat(),
+        ...Object.values(reservedFunctions).flat(),
       ]),
       stringTypes: [{ quote: "''", prefixes: ['N'] }],
       identTypes: [`""`, '[]'],
diff --git a/src/lexer/Tokenizer.ts b/src/lexer/Tokenizer.ts
new file mode 100644
index 00000000..5592647b
--- /dev/null
+++ b/src/lexer/Tokenizer.ts
@@ -0,0 +1,166 @@
+import { Token, TokenType } from 'src/core/token';
+import * as regex from 'src/lexer/regexFactory';
+import * as regexTypes from 'src/lexer/regexTypes';
+
+import { NULL_REGEX, escapeRegExp } from './regexUtil';
+import TokenizerEngine, { type TokenRule } from './TokenizerEngine';
+
+interface TokenizerOptions {
+  // Main clauses that start new block, like: SELECT, FROM, WHERE, ORDER BY
+  reservedCommands: string[];
+  // Logical operator keywords, defaults to: [AND, OR]
+  reservedLogicalOperators?: string[];
+  // Keywords in CASE expressions that begin new line, like: WHEN, ELSE
+  reservedDependentClauses: string[];
+  // Keywords that create newline but no indentaion of their body.
+  // These contain set operations like UNION and various joins like LEFT OUTER JOIN
+  reservedBinaryCommands: string[];
+  // Various joins like LEFT OUTER JOIN
+  reservedJoins: string[];
+  // keywords used for JOIN conditions, defaults to: [ON, USING]
+  reservedJoinConditions?: string[];
+  // all other reserved words (not included to any of the above lists)
+  reservedKeywords: string[];
+  // Types of quotes to use for strings
+  stringTypes: regexTypes.QuoteType[];
+  // Types of quotes to use for quoted identifiers
+  identTypes: regexTypes.QuoteType[];
+  // Types of quotes to use for variables
+  variableTypes?: regexTypes.VariableType[];
+  // Open-parenthesis characters
+  openParens?: ('(' | '[' | '{')[];
+  // Close-parenthesis characters
+  closeParens?: (')' | ']' | '}')[];
+  // True to allow for positional "?" parameter placeholders
+  positionalParams?: boolean;
+  // Prefixes for numbered parameter placeholders to support, e.g. :1, :2, :3
+  numberedParamTypes?: ('?' | ':' | '$')[];
+  // Prefixes for named parameter placeholders to support, e.g. :name
+  namedParamTypes?: (':' | '@' | '$')[];
+  // Prefixes for quoted parameter placeholders to support, e.g. :"name"
+  // The type of quotes will depend on `identifierTypes` option.
+  quotedParamTypes?: (':' | '@' | '$')[];
+  // Line comment types to support, defaults to --
+  lineCommentTypes?: string[];
+  // Additional characters to support in identifiers
+  identChars?: regexTypes.IdentChars;
+  // Additional characters to support in named parameters
+  // Use this when parameters allow different characters from identifiers
+  // Defaults to `identChars`.
+  paramChars?: regexTypes.IdentChars;
+  // Additional multi-character operators to support, in addition to <=, >=, <>, !=
+  operators?: string[];
+  // Allows custom modifications on the token array.
+  // Called after the whole input string has been split into tokens.
+  // The result of this will be the output of the tokenizer.
+  postProcess?: (tokens: Token[]) => Token[];
+}
+
+export default class Tokenizer {
+  private TOKENIZER_RULES: Partial<Record<TokenType, TokenRule | { regex: undefined }>>;
+  engine: TokenizerEngine;
+  private postProcess?: (tokens: Token[]) => Token[];
+
+  constructor(cfg: TokenizerOptions) {
+    this.TOKENIZER_RULES = {
+      [TokenType.BLOCK_COMMENT]: { regex: /(\/\*[^]*?(?:\*\/|$))/uy },
+      [TokenType.LINE_COMMENT]: {
+        regex: regex.lineComment(cfg.lineCommentTypes ?? ['--']),
+      },
+      [TokenType.COMMA]: { regex: /[,]/y },
+      [TokenType.OPEN_PAREN]: { regex: regex.parenthesis(cfg.openParens ?? ['(']) },
+      [TokenType.CLOSE_PAREN]: { regex: regex.parenthesis(cfg.closeParens ?? [')']) },
+      [TokenType.QUOTED_IDENTIFIER]: { regex: regex.string(cfg.identTypes) },
+      [TokenType.NUMBER]: {
+        regex:
+          /(?:0x[0-9a-fA-F]+|0b[01]+|(?:-\s*)?[0-9]+(?:\.[0-9]*)?(?:[eE][-+]?[0-9]+(?:\.[0-9]+)?)?)/uy,
+      },
+      [TokenType.RESERVED_CASE_START]: {
+        regex: /[Cc][Aa][Ss][Ee]\b/uy,
+        value: v => v.toUpperCase(),
+      },
+      [TokenType.RESERVED_CASE_END]: { regex: /[Ee][Nn][Dd]\b/uy, value: v => v.toUpperCase() },
+      [TokenType.RESERVED_COMMAND]: {
+        regex: regex.reservedWord(cfg.reservedCommands, cfg.identChars),
+        value: v => v.toUpperCase(),
+      },
+      [TokenType.RESERVED_BINARY_COMMAND]: {
+        regex: regex.reservedWord(cfg.reservedBinaryCommands, cfg.identChars),
+        value: v => v.toUpperCase(),
+      },
+      [TokenType.RESERVED_DEPENDENT_CLAUSE]: {
+        regex: regex.reservedWord(cfg.reservedDependentClauses, cfg.identChars),
+        value: v => v.toUpperCase(),
+      },
+      [TokenType.RESERVED_JOIN]: {
+        regex: regex.reservedWord(cfg.reservedJoins, cfg.identChars),
+        value: v => v.toUpperCase(),
+      },
+      [TokenType.RESERVED_KEYWORD]: {
+        regex: regex.reservedWord(cfg.reservedKeywords, cfg.identChars),
+        value: v => v.toUpperCase(),
+      },
+      [TokenType.RESERVED_LOGICAL_OPERATOR]: {
+        regex: regex.reservedWord(cfg.reservedLogicalOperators ?? ['AND', 'OR'], cfg.identChars),
+        value: v => v.toUpperCase(),
+      },
+      [TokenType.RESERVED_JOIN_CONDITION]: {
+        regex: regex.reservedWord(cfg.reservedJoinConditions ?? ['ON', 'USING'], cfg.identChars),
+        value: v => v.toUpperCase(),
+      },
+      [TokenType.NAMED_PARAMETER]: {
+        regex: regex.parameter(
+          cfg.namedParamTypes ?? [],
+          regex.identifierPattern(cfg.paramChars || cfg.identChars)
+        ),
+        key: v => v.slice(1),
+      },
+      [TokenType.QUOTED_PARAMETER]: {
+        regex: regex.parameter(cfg.quotedParamTypes ?? [], regex.stringPattern(cfg.identTypes)),
+        key: v =>
+          (({ tokenKey, quoteChar }) =>
+            tokenKey.replace(new RegExp(escapeRegExp('\\' + quoteChar), 'gu'), quoteChar))({
+            tokenKey: v.slice(2, -1),
+            quoteChar: v.slice(-1),
+          }),
+      },
+      [TokenType.INDEXED_PARAMETER]: {
+        regex: regex.parameter(cfg.numberedParamTypes ?? [], '[0-9]+'),
+        key: v => v.slice(1),
+      },
+      [TokenType.POSITIONAL_PARAMETER]: {
+        regex: cfg.positionalParams ? /[?]/y : undefined,
+      },
+      [TokenType.VARIABLE]: {
+        regex: cfg.variableTypes ? regex.variable(cfg.variableTypes) : NULL_REGEX,
+      },
+      [TokenType.STRING]: { regex: regex.string(cfg.stringTypes) },
+      [TokenType.IDENTIFIER]: {
+        regex: regex.identifier(cfg.identChars),
+      },
+      [TokenType.DELIMITER]: { regex: /[;]/uy },
+      [TokenType.OPERATOR]: {
+        regex: regex.operator('+-/*%&|^><=.[]{}`:$@', [
+          '<>',
+          '<=',
+          '>=',
+          '!=',
+          ...(cfg.operators ?? []),
+        ]),
+      },
+    };
+
+    // filter out unsupported parameter types whose regex resolve to undefined
+    const withoutUnsupportedParameters = Object.fromEntries(
+      Object.entries(this.TOKENIZER_RULES).filter(([_, rule]) => rule.regex)
+    );
+    this.engine = new TokenizerEngine(withoutUnsupportedParameters);
+
+    this.postProcess = cfg.postProcess;
+  }
+
+  tokenize(input: string): Token[] {
+    const tokens = this.engine.tokenize(input);
+    return this.postProcess ? this.postProcess(tokens) : tokens;
+  }
+}
diff --git a/src/lexer/TokenizerEngine.ts b/src/lexer/TokenizerEngine.ts
new file mode 100644
index 00000000..4fd666a4
--- /dev/null
+++ b/src/lexer/TokenizerEngine.ts
@@ -0,0 +1,159 @@
+import { Token, TokenType } from 'src/core/token';
+import { WHITESPACE_REGEX } from './regexUtil';
+
+export interface TokenRule {
+  regex: RegExp;
+  key?: (token: string) => string;
+  value?: (token: string) => string;
+}
+
+export default class TokenizerEngine {
+  private REGEX_MAP: Partial<Record<TokenType, TokenRule>>;
+
+  // The input SQL string to process
+  private input = '';
+  // Current position in string
+  private index = 0;
+
+  constructor(tokenizerRules: Partial<Record<TokenType, TokenRule>>) {
+    this.REGEX_MAP = tokenizerRules;
+  }
+
+  /**
+   * Takes a SQL string and breaks it into tokens.
+   * Each token is an object with type and value.
+   *
+   * @param {string} input - The SQL string
+   * @returns {Token[]} output token stream
+   */
+  public tokenize(input: string): Token[] {
+    this.input = input;
+    this.index = 0;
+    const tokens: Token[] = [];
+    let token: Token | undefined;
+
+    // Keep processing the string until end is reached
+    while (this.index < this.input.length) {
+      // grab any preceding whitespace
+      const whitespaceBefore = this.getWhitespace();
+
+      if (this.index < this.input.length) {
+        // Get the next token and the token type
+        token = this.getNextToken(token);
+        if (!token) {
+          throw new Error(`Parse error: Unexpected "${input.slice(this.index, 100)}"`);
+        }
+
+        tokens.push({ ...token, whitespaceBefore });
+      }
+    }
+    return tokens;
+  }
+
+  private getWhitespace(): string {
+    WHITESPACE_REGEX.lastIndex = this.index;
+    const matches = WHITESPACE_REGEX.exec(this.input);
+    if (matches) {
+      // Advance current position by matched whitespace length
+      this.index += matches[0].length;
+      return matches[0];
+    } else {
+      return '';
+    }
+  }
+
+  private getNextToken(previousToken?: Token): Token | undefined {
+    return (
+      this.matchToken(TokenType.BLOCK_COMMENT) ||
+      this.matchToken(TokenType.LINE_COMMENT) ||
+      this.matchToken(TokenType.COMMA) ||
+      this.matchToken(TokenType.OPEN_PAREN) ||
+      this.matchToken(TokenType.CLOSE_PAREN) ||
+      this.matchToken(TokenType.QUOTED_IDENTIFIER) ||
+      this.matchToken(TokenType.NUMBER) ||
+      this.matchReservedWordToken(previousToken) ||
+      this.matchPlaceholderToken(TokenType.NAMED_PARAMETER) ||
+      this.matchPlaceholderToken(TokenType.QUOTED_PARAMETER) ||
+      this.matchPlaceholderToken(TokenType.INDEXED_PARAMETER) ||
+      this.matchPlaceholderToken(TokenType.POSITIONAL_PARAMETER) ||
+      this.matchToken(TokenType.VARIABLE) ||
+      this.matchToken(TokenType.STRING) ||
+      this.matchToken(TokenType.IDENTIFIER) ||
+      this.matchToken(TokenType.DELIMITER) ||
+      this.matchToken(TokenType.OPERATOR)
+    );
+  }
+
+  private matchPlaceholderToken(tokenType: TokenType): Token | undefined {
+    if (tokenType in this.REGEX_MAP) {
+      const token = this.matchToken(tokenType);
+      const tokenRule = this.REGEX_MAP[tokenType];
+      if (token) {
+        if (tokenRule?.key) {
+          return { ...token, key: tokenRule.key(token.value) };
+        }
+        return token; // POSITIONAL_PARAMETER does not have a key transform function
+      }
+    }
+    return undefined;
+  }
+
+  private matchReservedWordToken(previousToken?: Token): Token | undefined {
+    // A reserved word cannot be preceded by a '.'
+    // this makes it so in "mytable.from", "from" is not considered a reserved word
+    if (previousToken?.value === '.') {
+      return undefined;
+    }
+
+    // prioritised list of Reserved token types
+    return (
+      this.matchToken(TokenType.RESERVED_CASE_START) ||
+      this.matchToken(TokenType.RESERVED_CASE_END) ||
+      this.matchToken(TokenType.RESERVED_COMMAND) ||
+      this.matchToken(TokenType.RESERVED_BINARY_COMMAND) ||
+      this.matchToken(TokenType.RESERVED_DEPENDENT_CLAUSE) ||
+      this.matchToken(TokenType.RESERVED_JOIN) ||
+      this.matchToken(TokenType.RESERVED_KEYWORD) ||
+      this.matchToken(TokenType.RESERVED_LOGICAL_OPERATOR) ||
+      this.matchToken(TokenType.RESERVED_JOIN_CONDITION)
+    );
+  }
+
+  // Shorthand for `match` that looks up regex from REGEX_MAP
+  private matchToken(tokenType: TokenType): Token | undefined {
+    if (!(tokenType in this.REGEX_MAP)) {
+      throw Error(`Unknown token type found: ${tokenType}`);
+    }
+    return this.match({
+      type: tokenType,
+      regex: this.REGEX_MAP[tokenType]!.regex,
+      transform: this.REGEX_MAP[tokenType]!.value,
+    });
+  }
+
+  // Attempts to match RegExp at current position in input
+  private match({
+    type,
+    regex,
+    transform,
+  }: {
+    type: TokenType;
+    regex: RegExp;
+    transform?: (s: string) => string;
+  }): Token | undefined {
+    regex.lastIndex = this.index;
+    const matches = regex.exec(this.input);
+    if (matches) {
+      const matchedToken = matches[0];
+
+      // Advance current position by matched token length
+      this.index += matchedToken.length;
+      return {
+        type,
+        text: matchedToken,
+        value: transform ? transform(matchedToken) : matchedToken,
+      };
+    }
+    return undefined;
+  }
+}
diff --git a/src/lexer/regexFactory.ts b/src/lexer/regexFactory.ts
new file mode 100644
index 00000000..d12a4661
--- /dev/null
+++ b/src/lexer/regexFactory.ts
@@ -0,0 +1,144 @@
+import { sortByLengthDesc } from 'src/utils';
+
+import type { IdentChars, QuoteType, VariableType } from './regexTypes';
+import {
+  escapeParen,
+  escapeRegExp,
+  patternToRegex,
+  prefixesPattern,
+  toCaseInsensitivePattern,
+  withDashes,
+} from './regexUtil';
+
+/**
+ * Builds a RegExp for valid line comments in a SQL dialect
+ * @param {string[]} lineCommentTypes - list of character strings that denote line comments
+ */
+export const lineComment = (lineCommentTypes: string[]) =>
+  new RegExp(`(?:${lineCommentTypes.map(escapeRegExp).join('|')}).*?(?=\r\n|\r|\n|$)`, 'uy');
+
+/**
+ * Builds a RegExp for matching parenthesis patterns, escaping them with `escapeParen`
+ * @param {string[]} parens - list of strings that denote parenthesis patterns
+ */
+export const parenthesis = (parens: string[]): RegExp =>
+  patternToRegex(parens.map(escapeParen).join('|'));
+
+/**
+ * Builds a RegExp containing all operators for a SQL dialect
+ * @param {string} monadOperators - concatenated string of all 1-length operators
+ * @param {string[]} polyadOperators - list of strings of all >1-length operators
+ */
+export const operator = (monadOperators: string, polyadOperators: string[]) =>
+  patternToRegex(
+    `${sortByLengthDesc(polyadOperators).map(escapeRegExp).join('|')}|` +
+      `[${monadOperators.split('').map(escapeRegExp).join('')}]`
+  );
+
+// Negative lookahead to avoid matching a keyword that's actually part of identifier,
+// which can happen when identifier allows word-boundary characters inside it.
+//
+// For example "SELECT$ME" should be tokenized as:
+// - ["SELECT$ME"] when $ is allowed inside identifiers
+// - ["SELECT", "$", "ME"] when $ can't be part of identifiers.
+const rejectIdentCharsPattern = ({ rest, dashes }: IdentChars): string =>
+  rest || dashes ? `(?![${rest || ''}${dashes ? '-' : ''}])` : '';
+
+/**
+ * Builds a RegExp for all Reserved Keywords in a SQL dialect
+ */
+export const reservedWord = (reservedKeywords: string[], identChars: IdentChars = {}): RegExp => {
+  if (reservedKeywords.length === 0) {
+    return /^\b$/u;
+  }
+
+  const avoidIdentChars = rejectIdentCharsPattern(identChars);
+
+  const reservedKeywordsPattern = sortByLengthDesc(reservedKeywords)
+    .map(toCaseInsensitivePattern)
+    .join('|')
+    .replace(/ /gu, '\\s+');
+
+  return new RegExp(`(?:${reservedKeywordsPattern})${avoidIdentChars}\\b`, 'iuy');
+};
+
+/**
+ * Builds a RegExp for parameter placeholder patterns
+ * @param {string[]} paramTypes - list of strings that denote placeholder types
+ * @param {string} pattern - string that denotes placeholder pattern
+ */
+export const parameter = (paramTypes: string[], pattern: string): RegExp | undefined => {
+  if (!paramTypes.length) {
+    return undefined;
+  }
+  const typesRegex = paramTypes.map(escapeRegExp).join('|');
+
+  return patternToRegex(`(?:${typesRegex})(?:${pattern})`);
+};
+
+// This enables the following quote styles:
+// 1. backtick quoted using `` to escape
+// 2. square bracket quoted (SQL Server) using ]] to escape
+// 3. double quoted using "" or \" to escape
+// 4. single quoted using '' or \' to escape
+// 5. PostgreSQL dollar-quoted
+// 6. BigQuery '''triple-quoted'''
+// 7. BigQuery """triple-quoted"""
+// 8. Hive and Spark variables: ${name}
+export const quotePatterns = {
+  '``': '(?:`[^`]*(?:$|`))+',
+  '[]': '(?:\\[[^\\]]*(?:$|\\]))(?:\\][^\\]]*(?:$|\\]))*',
+  '""': '(?:"[^"\\\\]*(?:\\\\.[^"\\\\]*)*(?:"|$))+',
+  "''": "(?:'[^'\\\\]*(?:\\\\.[^'\\\\]*)*(?:'|$))+",
+  '$$': '(?<tag>\\$\\w*\\$)[\\s\\S]*?(?:\\k<tag>|$)',
+  "'''..'''": "'''[^\\\\]*?(?:\\\\.[^\\\\]*?)*?(?:'''|$)",
+  '""".."""': '"""[^\\\\]*?(?:\\\\.[^\\\\]*?)*?(?:"""|$)',
+  '{}': '(?:\\{[^\\}]*(?:$|\\}))',
+};
+
+const singleQuotePattern = (quoteTypes: QuoteType): string => {
+  if (typeof quoteTypes === 'string') {
+    return quotePatterns[quoteTypes];
+  } else {
+    return prefixesPattern(quoteTypes) + quotePatterns[quoteTypes.quote];
+  }
+};
+
+/** Builds a RegExp for matching variables */
+export const variable = (varTypes: VariableType[]): RegExp =>
+  patternToRegex(
+    varTypes
+      .map(varType => ('regex' in varType ? varType.regex : singleQuotePattern(varType)))
+      .join('|')
+  );
+
+/** Builds a quote-delimited pattern for matching all given quote types */
+export const stringPattern = (quoteTypes: QuoteType[]): string =>
+  quoteTypes.map(singleQuotePattern).join('|');
+
+/** Builds a RegExp for matching quote-delimited patterns */
+export const string = (quoteTypes: QuoteType[]): RegExp =>
+  patternToRegex(stringPattern(quoteTypes));
+
+/**
+ * Builds a RegExp for valid identifiers in a SQL dialect
+ */
+export const identifier = (specialChars: IdentChars = {}): RegExp =>
+  patternToRegex(identifierPattern(specialChars));
+
+/**
+ * Builds a RegExp string for valid identifiers in a SQL dialect
+ */
+export const identifierPattern = ({ first, rest, dashes }: IdentChars = {}): string => {
+  // Unicode letters, diacritical marks and underscore
+  const letter = '\\p{Alphabetic}\\p{Mark}_';
+  // Numbers 0..9, plus various unicode numbers
+  const number = '\\p{Decimal_Number}';
+
+  const firstChars = escapeRegExp(first ?? '');
+  const restChars = escapeRegExp(rest ?? '');
+
+  const pattern = `[${letter}${firstChars}][${letter}${number}${restChars}]*`;
+
+  return dashes ? withDashes(pattern) : pattern;
+};
diff --git a/src/lexer/regexTypes.d.ts b/src/lexer/regexTypes.d.ts
new file mode 100644
index 00000000..b7226188
--- /dev/null
+++ b/src/lexer/regexTypes.d.ts
@@ -0,0 +1,28 @@
+import { quotePatterns } from './regexFactory';
+
+export interface IdentChars {
+  // Additional characters that can be used as first character of an identifier.
+  // That is: in addition to letters and underscore.
+  first?: string;
+  // Additional characters that can appear after the first character of identifier.
+  // That is: in addition to letters, numbers and underscore.
+  rest?: string;
+  // True to allow single dashes (-) inside identifiers, but not at the beginning or end
+  dashes?: boolean;
+}
+
+export type PlainQuoteType = keyof typeof quotePatterns;
+
+export type PrefixedQuoteType = {
+  quote: PlainQuoteType;
+  prefixes: string[];
+  requirePrefix?: boolean; // True when prefix is required
+};
+
+export type QuoteType = PlainQuoteType | PrefixedQuoteType;
+
+export interface VariableRegex {
+  regex: string;
+}
+
+export type VariableType = VariableRegex | PrefixedQuoteType;
diff --git a/src/lexer/regexUtil.ts b/src/lexer/regexUtil.ts
new file mode 100644
index 00000000..485b614a
--- /dev/null
+++ b/src/lexer/regexUtil.ts
@@ -0,0 +1,34 @@
+import type { PrefixedQuoteType } from './regexTypes';
+
+// Escapes regex special chars
+export const escapeRegExp = (string: string) => string.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
+
+export const NULL_REGEX = /^(?!)/; // zero-width negative lookahead, matches nothing
+
+export const WHITESPACE_REGEX = /(\s+)/uy;
+
+export const patternToRegex = (pattern: string): RegExp => new RegExp(`(?:${pattern})`, 'uy');
+
+// Converts "ab" to "[Aa][Bb]"
+export const toCaseInsensitivePattern = (prefix: string): string =>
+  prefix
+    .split('')
+    .map(char => (/ /gu.test(char) ? '\\s+' : `[${char.toUpperCase()}${char.toLowerCase()}]`))
+    .join('');
+
+/** Escapes paren characters for RegExp patterns */
+export const escapeParen = (paren: string): string => {
+  if (paren.length === 1) {
+    // A single punctuation character
+    return escapeRegExp(paren);
+  } else {
+    // longer word
+    return '\\b' + paren + '\\b';
+  }
+};
+
+export const withDashes = (pattern: string): string => pattern + '(?:-' + pattern + ')*';
+
+// Converts ["a", "b"] to "(?:[Aa]|[Bb]|)" or "(?:[Aa]|[Bb])" when required = true
+export const prefixesPattern = ({ prefixes, requirePrefix }: PrefixedQuoteType): string =>
+  `(?:${prefixes.map(toCaseInsensitivePattern).join('|')}${requirePrefix ? '' : '|'})`;
diff --git a/src/parser/Parser.ts b/src/parser/Parser.ts
index 5920e2ce..e63140c2 100644
--- a/src/parser/Parser.ts
+++ b/src/parser/Parser.ts
@@ -102,7 +102,8 @@ export default class Parser {
 
   private functionCall(): FunctionCall | undefined {
     if (
-      (this.look().type === TokenType.RESERVED_KEYWORD || this.look().type === TokenType.IDENT) &&
+      (this.look().type === TokenType.RESERVED_KEYWORD ||
+        this.look().type === TokenType.IDENTIFIER) &&
       this.look(1).value === '(' &&
       !this.look(1).whitespaceBefore
     ) {
@@ -117,7 +118,8 @@ export default class Parser {
 
   private arraySubscript(): ArraySubscript | undefined {
     if (
-      (this.look().type === TokenType.RESERVED_KEYWORD || this.look().type === TokenType.IDENT) &&
+      (this.look().type === TokenType.RESERVED_KEYWORD ||
+        this.look().type === TokenType.IDENTIFIER) &&
       this.look(1).value === '['
     ) {
       return {
@@ -160,7 +162,7 @@ export default class Parser {
   }
 
   private limitClause(): LimitClause | undefined {
-    if (isToken.LIMIT(this.look()) && this.look(2).value === ',') {
+    if (isToken.LIMIT(this.look()) && this.look(2).type === TokenType.COMMA) {
       return {
         type: NodeType.limit_clause,
         limitToken: this.next(),
diff --git a/src/utils.ts b/src/utils.ts
index 530a0919..7caa9cd1 100644
--- a/src/utils.ts
+++ b/src/utils.ts
@@ -3,12 +3,6 @@ export const dedupe = (arr: string[]) => [...new Set(arr)];
 // Last element from array
 export const last = <T>(arr: T[]): T | undefined => arr[arr.length - 1];
 
-// True array is empty, or it's not an array at all
-export const isEmpty = (arr: any[]) => !Array.isArray(arr) || arr.length === 0;
-
-// Escapes regex special chars
-export const escapeRegExp = (string: string) => string.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
-
 // Sorts strings by length, so that longer ones are first
 // Also sorts alphabetically after sorting by length.
 export const sortByLengthDesc = (strings: string[]) =>
@@ -18,14 +12,9 @@ export const sortByLengthDesc = (strings: string[]) =>
 export const maxLength = (strings: string[]) =>
   strings.reduce((max, cur) => Math.max(max, cur.length), 0);
 
-export const isNumber = (value: any): value is number => typeof value === 'number';
-
 // replaces long whitespace sequences with just one space
 export const equalizeWhitespace = (s: string) => s.replace(/\s+/gu, ' ');
 
-// identity function
-export const id = <T>(x: T) => x;
-
 // Adds up all values in array
 export const sum = (arr: number[]): number => {
   let total = 0;
diff --git a/yarn.lock b/yarn.lock
index e6c8e92a..5005a791 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2225,6 +2225,13 @@ babel-plugin-dynamic-import-node@^2.3.3:
   dependencies:
     object.assign "^4.1.0"
 
+babel-plugin-inline-import@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/babel-plugin-inline-import/-/babel-plugin-inline-import-3.0.0.tgz#220eb2a52f8e779d8fb89447f950275e1e3f5981"
+  integrity sha512-thnykl4FMb8QjMjVCuZoUmAM7r2mnTn5qJwrryCvDv6rugbJlTHZMctdjDtEgD0WBAXJOLJSGXN3loooEwx7UQ==
+  dependencies:
+    require-resolve "0.0.2"
+
 babel-plugin-istanbul@^6.1.1:
   version "6.1.1"
   resolved "https://registry.yarnpkg.com/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz#fa88ec59232fd9b4e36dbbc540a8ec9a9b47da73"
@@ -5311,6 +5318,11 @@ path-exists@^4.0.0:
   resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3"
   integrity sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==
 
+path-extra@^1.0.2:
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/path-extra/-/path-extra-1.0.3.tgz#7c112189a6e50d595790e7ad2037e44e410c1166"
+  integrity sha512-vYm3+GCkjUlT1rDvZnDVhNLXIRvwFPaN8ebHAFcuMJM/H0RBOPD7JrcldiNLd9AS3dhAyUHLa4Hny5wp1A+Ffw==
+
 path-is-absolute@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f"
@@ -5710,6 +5722,13 @@ require-directory@^2.1.1:
   resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
   integrity sha1-jGStX9MNqxyXbiNE/+f3kqam30I=
 
+require-resolve@0.0.2:
+  version "0.0.2"
+  resolved "https://registry.yarnpkg.com/require-resolve/-/require-resolve-0.0.2.tgz#bab410ab1aee2f3f55b79317451dd3428764e6f3"
+  integrity sha512-eafQVaxdQsWUB8HybwognkdcIdKdQdQBwTxH48FuE6WI0owZGKp63QYr1MRp73PoX0AcyB7MDapZThYUY8FD0A==
+  dependencies:
+    x-path "^0.0.2"
+
 reselect@^4.0.0:
   version "4.1.6"
   resolved "https://registry.yarnpkg.com/reselect/-/reselect-4.1.6.tgz#19ca2d3d0b35373a74dc1c98692cdaffb6602656"
@@ -6736,6 +6755,13 @@ write-file-atomic@^4.0.1:
     imurmurhash "^0.1.4"
     signal-exit "^3.0.7"
 
+x-path@^0.0.2:
+  version "0.0.2"
+  resolved "https://registry.yarnpkg.com/x-path/-/x-path-0.0.2.tgz#294d076bb97a7706cc070bbb2a6fd8c54df67b12"
+  integrity sha1-KU0Ha7l6dwbMBwu7Km/YxU32exI=
+  dependencies:
+    path-extra "^1.0.2"
+
 xdg-basedir@^5.0.1, xdg-basedir@^5.1.0:
   version "5.1.0"
   resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-5.1.0.tgz#1efba19425e73be1bc6f2a6ceb52a3d2c884c0c9"


Test patch:
diff --git a/test/sql/README.md b/test/sql/README.md
new file mode 100644
index 00000000..27eaf492
--- /dev/null
+++ b/test/sql/README.md
@@ -0,0 +1,17 @@
+# README
+
+## Importing SQL files
+
+SQL files can be imported into test files as text files, allowing them to be used as raw strings to check formatting.
+
+Example:
+
+```ts
+import testSql from './test.sql';
+```
+
+or using the `fs` library:
+
+```ts
+const testSql = fs.readFileSync('test/test.sql', 'utf8');
+```
diff --git a/test/sql/test.sql b/test/sql/test.sql
new file mode 100644
index 00000000..b18a6860
--- /dev/null
+++ b/test/sql/test.sql
@@ -0,0 +1,19 @@
+select
+    ?,
+	alpha + 1,
+	beta[0] b,
+	gamma['radiation'] AS g,
+	case WHEN iota THEN i END
+FROM (
+	SELECT
+		MAX(epsilon)
+	FROM europe.greek.zeta
+	GROUP BY $eta, iota HAVING chi, psi, phi
+	UNION
+	SELECT MIN(rho)
+	FROM greek.omega
+)
+JOIN `greek_dev.sigma` ON tau = theta
+WHERE kappa AND lambda OR mu
+;
+SELECT upsilon AS y, omicron AS o FROM xi;
diff --git a/test/unit/Parser.test.ts b/test/unit/Parser.test.ts
index 6656525a..599f90ea 100644
--- a/test/unit/Parser.test.ts
+++ b/test/unit/Parser.test.ts
@@ -1,5 +1,5 @@
 import Parser from 'src/parser/Parser';
-import Tokenizer from 'src/core/Tokenizer';
+import Tokenizer from 'src/lexer/Tokenizer';
 
 describe('Parser', () => {
   const parse = (sql: string) => {
@@ -30,7 +30,7 @@ describe('Parser', () => {
             Object {
               "token": Object {
                 "text": "foo",
-                "type": "IDENT",
+                "type": "IDENTIFIER",
                 "value": "foo",
                 "whitespaceBefore": "",
               },
@@ -45,7 +45,7 @@ describe('Parser', () => {
             Object {
               "token": Object {
                 "text": "bar",
-                "type": "IDENT",
+                "type": "IDENTIFIER",
                 "value": "bar",
                 "whitespaceBefore": " ",
               },
@@ -118,7 +118,7 @@ describe('Parser', () => {
                 Object {
                   "arrayToken": Object {
                     "text": "my_array",
-                    "type": "IDENT",
+                    "type": "IDENTIFIER",
                     "value": "my_array",
                     "whitespaceBefore": " ",
                   },
@@ -127,7 +127,7 @@ describe('Parser', () => {
                       Object {
                         "nameToken": Object {
                           "text": "OFFSET",
-                          "type": "IDENT",
+                          "type": "IDENTIFIER",
                           "value": "OFFSET",
                           "whitespaceBefore": "",
                         },
@@ -185,7 +185,7 @@ describe('Parser', () => {
                     Object {
                       "token": Object {
                         "text": "birth_year",
-                        "type": "IDENT",
+                        "type": "IDENTIFIER",
                         "value": "birth_year",
                         "whitespaceBefore": "",
                       },
@@ -205,7 +205,7 @@ describe('Parser', () => {
                         Object {
                           "token": Object {
                             "text": "CURRENT_DATE",
-                            "type": "IDENT",
+                            "type": "IDENTIFIER",
                             "value": "CURRENT_DATE",
                             "whitespaceBefore": "",
                           },
@@ -266,7 +266,7 @@ describe('Parser', () => {
                 Object {
                   "token": Object {
                     "text": "age",
-                    "type": "IDENT",
+                    "type": "IDENTIFIER",
                     "value": "age",
                     "whitespaceBefore": " ",
                   },