Repository: sql-formatter-org/sql-formatter
Pull request #395 (2022-08-16 12:22:50)

Add more token types
Added the following new token types:

- `LIMIT` (previously one of the `RESERVED_COMMAND` tokens)
- `BETWEEN` (previously one of the `RESERVED_KEYWORD` tokens)
- `AND`, `OR`, `XOR` (previously `RESERVED_LOGICAL_OPERATOR`)
- `CASE`, `END` (previously named `RESERVED_CASE_START`, `RESERVED_CASE_END`)

Opted to eliminate the `RESERVED_` prefix from token types which represent just a single word. Makes the code simpler to read, especially when writing a Nearley grammar.

The purpose of all this is mostly the Nearley integration, where I need to distinguish e.g. `LIMIT` from other `RESERVED_COMMAND` tokens.

Functionality-wise everything works just like before.


Patch:
diff --git a/src/formatter/ExpressionFormatter.ts b/src/formatter/ExpressionFormatter.ts
index 42b2ca9f..5ddcc61b 100644
--- a/src/formatter/ExpressionFormatter.ts
+++ b/src/formatter/ExpressionFormatter.ts
@@ -194,15 +194,17 @@ export default class ExpressionFormatter {
         return this.formatJoin(token);
       case TokenType.RESERVED_DEPENDENT_CLAUSE:
         return this.formatDependentClause(token);
-      case TokenType.RESERVED_LOGICAL_OPERATOR:
+      case TokenType.AND:
+      case TokenType.OR:
+      case TokenType.XOR:
         return this.formatLogicalOperator(token);
       case TokenType.RESERVED_KEYWORD:
       case TokenType.RESERVED_FUNCTION_NAME:
       case TokenType.RESERVED_PHRASE:
         return this.formatKeyword(token);
-      case TokenType.RESERVED_CASE_START:
+      case TokenType.CASE:
         return this.formatCaseStart(token);
-      case TokenType.RESERVED_CASE_END:
+      case TokenType.END:
         return this.formatCaseEnd(token);
       case TokenType.COMMA:
         return this.formatComma(token);
diff --git a/src/formatter/InlineBlock.ts b/src/formatter/InlineBlock.ts
index 9da9a255..d6a959e3 100644
--- a/src/formatter/InlineBlock.ts
+++ b/src/formatter/InlineBlock.ts
@@ -1,5 +1,5 @@
 import { sum } from 'src/utils';
-import { isToken, type Token, TokenType } from 'src/lexer/token';
+import { type Token, TokenType, isLogicalOperator } from 'src/lexer/token';
 import { BetweenPredicate, NodeType, Parenthesis } from 'src/parser/ast';
 
 /**
@@ -70,10 +70,10 @@ export default class InlineBlock {
   // are not allowed inside inline parentheses block
   private isForbiddenToken(token: Token) {
     return (
-      token.type === TokenType.RESERVED_LOGICAL_OPERATOR ||
+      isLogicalOperator(token) ||
       token.type === TokenType.LINE_COMMENT ||
       token.type === TokenType.BLOCK_COMMENT ||
-      isToken.CASE(token) // CASE cannot have inline blocks
+      token.type === TokenType.CASE // CASE cannot have inline blocks
     );
   }
 }
diff --git a/src/formatter/tabularStyle.ts b/src/formatter/tabularStyle.ts
index 2916a3f6..7ad623d2 100644
--- a/src/formatter/tabularStyle.ts
+++ b/src/formatter/tabularStyle.ts
@@ -1,5 +1,5 @@
 import type { IndentStyle } from 'src/FormatOptions';
-import { Token, TokenType } from 'src/lexer/token';
+import { isLogicalOperator, Token, TokenType } from 'src/lexer/token';
 
 /**
  * When tabular style enabled,
@@ -30,7 +30,7 @@ export default function toTabularFormat(tokenText: string, indentStyle: IndentSt
  */
 export function isTabularToken(token: Token): boolean {
   return (
-    token.type === TokenType.RESERVED_LOGICAL_OPERATOR ||
+    isLogicalOperator(token) ||
     token.type === TokenType.RESERVED_DEPENDENT_CLAUSE ||
     token.type === TokenType.RESERVED_COMMAND ||
     token.type === TokenType.RESERVED_SET_OPERATION ||
diff --git a/src/languages/mariadb/mariadb.formatter.ts b/src/languages/mariadb/mariadb.formatter.ts
index 22d5b1d1..7ee5be41 100644
--- a/src/languages/mariadb/mariadb.formatter.ts
+++ b/src/languages/mariadb/mariadb.formatter.ts
@@ -265,7 +265,7 @@ export default class MariaDbFormatter extends Formatter {
       reservedJoins,
       reservedDependentClauses: ['WHEN', 'ELSE', 'ELSEIF', 'ELSIF'],
       reservedPhrases,
-      reservedLogicalOperators: ['AND', 'OR', 'XOR'],
+      supportsXor: true,
       reservedKeywords: keywords,
       reservedFunctionNames: functions,
       // TODO: support _ char set prefixes such as _utf8, _latin1, _binary, _utf8mb4, etc.
diff --git a/src/languages/mysql/mysql.formatter.ts b/src/languages/mysql/mysql.formatter.ts
index 88c36177..4b1af0ee 100644
--- a/src/languages/mysql/mysql.formatter.ts
+++ b/src/languages/mysql/mysql.formatter.ts
@@ -233,7 +233,7 @@ export default class MySqlFormatter extends Formatter {
       reservedJoins,
       reservedDependentClauses: ['WHEN', 'ELSE', 'ELSEIF'],
       reservedPhrases,
-      reservedLogicalOperators: ['AND', 'OR', 'XOR'],
+      supportsXor: true,
       reservedKeywords: keywords,
       reservedFunctionNames: functions,
       // TODO: support _ char set prefixes such as _utf8, _latin1, _binary, _utf8mb4, etc.
diff --git a/src/languages/n1ql/n1ql.formatter.ts b/src/languages/n1ql/n1ql.formatter.ts
index c5555c43..ad6c2c62 100644
--- a/src/languages/n1ql/n1ql.formatter.ts
+++ b/src/languages/n1ql/n1ql.formatter.ts
@@ -84,7 +84,7 @@ export default class N1qlFormatter extends Formatter {
       reservedSetOperations,
       reservedJoins,
       reservedDependentClauses: ['WHEN', 'ELSE'],
-      reservedLogicalOperators: ['AND', 'OR', 'XOR'],
+      supportsXor: true,
       reservedKeywords: keywords,
       reservedFunctionNames: functions,
       // NOTE: single quotes are actually not supported in N1QL,
diff --git a/src/languages/plsql/plsql.formatter.ts b/src/languages/plsql/plsql.formatter.ts
index cb5bbde7..5626097a 100644
--- a/src/languages/plsql/plsql.formatter.ts
+++ b/src/languages/plsql/plsql.formatter.ts
@@ -90,7 +90,7 @@ export default class PlSqlFormatter extends Formatter {
       reservedJoins,
       reservedDependentClauses: ['WHEN', 'ELSE'],
       reservedPhrases,
-      reservedLogicalOperators: ['AND', 'OR', 'XOR'],
+      supportsXor: true,
       reservedKeywords: keywords,
       reservedFunctionNames: functions,
       stringTypes: [
diff --git a/src/languages/singlestoredb/singlestoredb.formatter.ts b/src/languages/singlestoredb/singlestoredb.formatter.ts
index 46539e00..a22002db 100644
--- a/src/languages/singlestoredb/singlestoredb.formatter.ts
+++ b/src/languages/singlestoredb/singlestoredb.formatter.ts
@@ -233,7 +233,6 @@ export default class SingleStoreDbFormatter extends Formatter {
       reservedJoins,
       reservedDependentClauses: ['WHEN', 'ELSE', 'ELSEIF'],
       reservedPhrases,
-      reservedLogicalOperators: ['AND', 'OR'],
       reservedKeywords: keywords,
       reservedFunctionNames: functions,
       // TODO: support _binary"some string" prefix
diff --git a/src/languages/spark/spark.formatter.ts b/src/languages/spark/spark.formatter.ts
index 6ec0ab89..28cd941e 100644
--- a/src/languages/spark/spark.formatter.ts
+++ b/src/languages/spark/spark.formatter.ts
@@ -119,7 +119,7 @@ export default class SparkFormatter extends Formatter {
       reservedJoins,
       reservedDependentClauses: ['WHEN', 'ELSE'],
       reservedPhrases,
-      reservedLogicalOperators: ['AND', 'OR', 'XOR'],
+      supportsXor: true,
       reservedKeywords: keywords,
       reservedFunctionNames: functions,
       openParens: ['(', '['],
diff --git a/src/lexer/Tokenizer.ts b/src/lexer/Tokenizer.ts
index d6d0a211..a19e1002 100644
--- a/src/lexer/Tokenizer.ts
+++ b/src/lexer/Tokenizer.ts
@@ -37,45 +37,61 @@ export default class Tokenizer {
         regex:
           /(?:0x[0-9a-fA-F]+|0b[01]+|(?:-\s*)?[0-9]+(?:\.[0-9]*)?(?:[eE][-+]?[0-9]+(?:\.[0-9]+)?)?)(?!\w)/uy,
       },
-      [TokenType.RESERVED_CASE_START]: {
+      [TokenType.CASE]: {
         regex: /[Cc][Aa][Ss][Ee]\b/uy,
-        value: v => equalizeWhitespace(v.toUpperCase()),
+        value: toCanonical,
       },
-      [TokenType.RESERVED_CASE_END]: {
+      [TokenType.END]: {
         regex: /[Ee][Nn][Dd]\b/uy,
-        value: v => equalizeWhitespace(v.toUpperCase()),
+        value: toCanonical,
+      },
+      [TokenType.BETWEEN]: {
+        regex: /BETWEEN\b/iuy,
+        value: toCanonical,
+      },
+      [TokenType.LIMIT]: {
+        regex: cfg.reservedCommands.includes('LIMIT') ? /LIMIT\b/iuy : undefined,
+        value: toCanonical,
       },
       [TokenType.RESERVED_COMMAND]: {
         regex: regex.reservedWord(cfg.reservedCommands, cfg.identChars),
-        value: v => equalizeWhitespace(v.toUpperCase()),
+        value: toCanonical,
       },
       [TokenType.RESERVED_SET_OPERATION]: {
         regex: regex.reservedWord(cfg.reservedSetOperations, cfg.identChars),
-        value: v => equalizeWhitespace(v.toUpperCase()),
+        value: toCanonical,
       },
       [TokenType.RESERVED_DEPENDENT_CLAUSE]: {
         regex: regex.reservedWord(cfg.reservedDependentClauses, cfg.identChars),
-        value: v => equalizeWhitespace(v.toUpperCase()),
+        value: toCanonical,
       },
       [TokenType.RESERVED_JOIN]: {
         regex: regex.reservedWord(cfg.reservedJoins, cfg.identChars),
-        value: v => equalizeWhitespace(v.toUpperCase()),
+        value: toCanonical,
       },
       [TokenType.RESERVED_PHRASE]: {
         regex: regex.reservedWord(cfg.reservedPhrases ?? [], cfg.identChars),
-        value: v => equalizeWhitespace(v.toUpperCase()),
+        value: toCanonical,
+      },
+      [TokenType.AND]: {
+        regex: /AND\b/iuy,
+        value: toCanonical,
       },
-      [TokenType.RESERVED_LOGICAL_OPERATOR]: {
-        regex: regex.reservedWord(cfg.reservedLogicalOperators ?? ['AND', 'OR'], cfg.identChars),
-        value: v => equalizeWhitespace(v.toUpperCase()),
+      [TokenType.OR]: {
+        regex: /OR\b/iuy,
+        value: toCanonical,
+      },
+      [TokenType.XOR]: {
+        regex: cfg.supportsXor ? /XOR\b/iuy : undefined,
+        value: toCanonical,
       },
       [TokenType.RESERVED_FUNCTION_NAME]: {
         regex: regex.reservedWord(cfg.reservedFunctionNames, cfg.identChars),
-        value: v => equalizeWhitespace(v.toUpperCase()),
+        value: toCanonical,
       },
       [TokenType.RESERVED_KEYWORD]: {
         regex: regex.reservedWord(cfg.reservedKeywords, cfg.identChars),
-        value: v => equalizeWhitespace(v.toUpperCase()),
+        value: toCanonical,
       },
       [TokenType.VARIABLE]: {
         regex: cfg.variableTypes ? regex.variable(cfg.variableTypes) : undefined,
@@ -149,3 +165,10 @@ export default class Tokenizer {
     return Object.fromEntries(Object.entries(rules).filter(([_, rule]) => rule.regex));
   }
 }
+
+/**
+ * Converts keywords (and keyword sequences) to canonical form:
+ * - in uppercase
+ * - single spaces between words
+ */
+const toCanonical = (v: string) => equalizeWhitespace(v.toUpperCase());
diff --git a/src/lexer/TokenizerEngine.ts b/src/lexer/TokenizerEngine.ts
index 0a236cbf..cad781e7 100644
--- a/src/lexer/TokenizerEngine.ts
+++ b/src/lexer/TokenizerEngine.ts
@@ -106,14 +106,18 @@ export default class TokenizerEngine {
 
     // prioritised list of Reserved token types
     return (
-      this.matchToken(TokenType.RESERVED_CASE_START) ||
-      this.matchToken(TokenType.RESERVED_CASE_END) ||
+      this.matchToken(TokenType.CASE) ||
+      this.matchToken(TokenType.END) ||
+      this.matchToken(TokenType.BETWEEN) ||
+      this.matchToken(TokenType.LIMIT) ||
       this.matchToken(TokenType.RESERVED_COMMAND) ||
       this.matchToken(TokenType.RESERVED_SET_OPERATION) ||
       this.matchToken(TokenType.RESERVED_DEPENDENT_CLAUSE) ||
       this.matchToken(TokenType.RESERVED_JOIN) ||
       this.matchToken(TokenType.RESERVED_PHRASE) ||
-      this.matchToken(TokenType.RESERVED_LOGICAL_OPERATOR) ||
+      this.matchToken(TokenType.AND) ||
+      this.matchToken(TokenType.OR) ||
+      this.matchToken(TokenType.XOR) ||
       this.matchToken(TokenType.RESERVED_FUNCTION_NAME) ||
       this.matchToken(TokenType.RESERVED_KEYWORD)
     );
diff --git a/src/lexer/TokenizerOptions.ts b/src/lexer/TokenizerOptions.ts
index f8089e65..bb48e6ff 100644
--- a/src/lexer/TokenizerOptions.ts
+++ b/src/lexer/TokenizerOptions.ts
@@ -45,8 +45,8 @@ export interface ParamTypes {
 export interface TokenizerOptions {
   // Main clauses that start new block, like: SELECT, FROM, WHERE, ORDER BY
   reservedCommands: string[];
-  // Logical operator keywords, defaults to: [AND, OR]
-  reservedLogicalOperators?: string[];
+  // True to support XOR in addition to AND and OR
+  supportsXor?: boolean;
   // Keywords in CASE expressions that begin new line, like: WHEN, ELSE
   reservedDependentClauses: string[];
   // Keywords that create newline but no indentaion of their body.
diff --git a/src/lexer/token.ts b/src/lexer/token.ts
index da05729d..88dc82a8 100644
--- a/src/lexer/token.ts
+++ b/src/lexer/token.ts
@@ -6,14 +6,18 @@ export enum TokenType {
   VARIABLE = 'VARIABLE',
   RESERVED_KEYWORD = 'RESERVED_KEYWORD',
   RESERVED_FUNCTION_NAME = 'RESERVED_FUNCTION_NAME',
-  RESERVED_LOGICAL_OPERATOR = 'RESERVED_LOGICAL_OPERATOR',
   RESERVED_PHRASE = 'RESERVED_PHRASE',
   RESERVED_DEPENDENT_CLAUSE = 'RESERVED_DEPENDENT_CLAUSE',
   RESERVED_SET_OPERATION = 'RESERVED_SET_OPERATION',
   RESERVED_COMMAND = 'RESERVED_COMMAND',
   RESERVED_JOIN = 'RESERVED_JOIN',
-  RESERVED_CASE_START = 'RESERVED_CASE_START',
-  RESERVED_CASE_END = 'RESERVED_CASE_END',
+  CASE = 'CASE',
+  END = 'END',
+  LIMIT = 'LIMIT',
+  BETWEEN = 'BETWEEN',
+  AND = 'AND',
+  OR = 'OR',
+  XOR = 'XOR',
   OPERATOR = 'OPERATOR',
   COMMA = 'COMMA',
   OPEN_PAREN = 'OPEN_PAREN',
@@ -60,37 +64,31 @@ export const testToken =
 
 /** Util object that allows for easy checking of Reserved Keywords */
 export const isToken = {
-  AS: testToken({ text: 'AS', type: TokenType.RESERVED_KEYWORD }),
-  AND: testToken({ text: 'AND', type: TokenType.RESERVED_LOGICAL_OPERATOR }),
   ARRAY: testToken({ text: 'ARRAY', type: TokenType.RESERVED_KEYWORD }),
-  BETWEEN: testToken({ text: 'BETWEEN', type: TokenType.RESERVED_KEYWORD }),
-  CASE: testToken({ text: 'CASE', type: TokenType.RESERVED_CASE_START }),
-  CAST: testToken({ text: 'CAST', type: TokenType.RESERVED_FUNCTION_NAME }),
   BY: testToken({ text: 'BY', type: TokenType.RESERVED_KEYWORD }),
-  END: testToken({ text: 'END', type: TokenType.RESERVED_CASE_END }),
-  FROM: testToken({ text: 'FROM', type: TokenType.RESERVED_COMMAND }),
-  LIMIT: testToken({ text: 'LIMIT', type: TokenType.RESERVED_COMMAND }),
   SELECT: (token: Token) =>
     /^SELECT\b/.test(token.text) && token.type === TokenType.RESERVED_COMMAND,
   SET: testToken({ text: 'SET', type: TokenType.RESERVED_COMMAND }),
   STRUCT: testToken({ text: 'STRUCT', type: TokenType.RESERVED_KEYWORD }),
-  TABLE: testToken({ text: 'TABLE', type: TokenType.RESERVED_KEYWORD }),
   WINDOW: testToken({ text: 'WINDOW', type: TokenType.RESERVED_COMMAND }),
-  WITH: testToken({ text: 'WITH', type: TokenType.RESERVED_COMMAND }),
 };
 
 /** Checks if token is any Reserved Keyword or Command */
 export const isReserved = (token: Token): boolean =>
   token.type === TokenType.RESERVED_KEYWORD ||
   token.type === TokenType.RESERVED_FUNCTION_NAME ||
-  token.type === TokenType.RESERVED_LOGICAL_OPERATOR ||
   token.type === TokenType.RESERVED_PHRASE ||
   token.type === TokenType.RESERVED_DEPENDENT_CLAUSE ||
   token.type === TokenType.RESERVED_COMMAND ||
   token.type === TokenType.RESERVED_SET_OPERATION ||
   token.type === TokenType.RESERVED_JOIN ||
-  token.type === TokenType.RESERVED_CASE_START ||
-  token.type === TokenType.RESERVED_CASE_END;
+  token.type === TokenType.CASE ||
+  token.type === TokenType.END ||
+  token.type === TokenType.LIMIT ||
+  token.type === TokenType.BETWEEN ||
+  token.type === TokenType.AND ||
+  token.type === TokenType.OR ||
+  token.type === TokenType.XOR;
 
 /** checks if token is one of the parameter tokens */
 export const isParameter = (token: Token): boolean =>
@@ -98,3 +96,6 @@ export const isParameter = (token: Token): boolean =>
   token.type === TokenType.NAMED_PARAMETER ||
   token.type === TokenType.POSITIONAL_PARAMETER ||
   token.type === TokenType.QUOTED_PARAMETER;
+
+export const isLogicalOperator = (token: Token): boolean =>
+  token.type === TokenType.AND || token.type === TokenType.OR || token.type === TokenType.XOR;
diff --git a/src/parser/Parser.ts b/src/parser/Parser.ts
index 2a5bb27a..dbafbc27 100644
--- a/src/parser/Parser.ts
+++ b/src/parser/Parser.ts
@@ -126,7 +126,7 @@ export default class Parser {
   }
 
   private betweenPredicate(): BetweenPredicate | undefined {
-    if (isToken.BETWEEN(this.look()) && isToken.AND(this.look(2))) {
+    if (this.look().type === TokenType.BETWEEN && this.look(2).type === TokenType.AND) {
       return {
         type: NodeType.between_predicate,
         betweenToken: this.next(),
@@ -139,7 +139,7 @@ export default class Parser {
   }
 
   private limitClause(): LimitClause | undefined {
-    if (isToken.LIMIT(this.look())) {
+    if (this.look().type === TokenType.LIMIT) {
       const limitToken = this.next();
       const expr1 = this.expressionsUntilClauseEnd(t => t.type === TokenType.COMMA);
       if (this.look().type === TokenType.COMMA) {
@@ -176,6 +176,7 @@ export default class Parser {
     const children: AstNode[] = [];
     while (
       this.look().type !== TokenType.RESERVED_COMMAND &&
+      this.look().type !== TokenType.LIMIT &&
       this.look().type !== TokenType.RESERVED_SET_OPERATION &&
       this.look().type !== TokenType.EOF &&
       this.look().type !== TokenType.CLOSE_PAREN &&


Test patch:
diff --git a/test/unit/Parser.test.ts b/test/unit/Parser.test.ts
index fbf0755d..1318144b 100644
--- a/test/unit/Parser.test.ts
+++ b/test/unit/Parser.test.ts
@@ -311,7 +311,7 @@ describe('Parser', () => {
                     "raw": "and",
                     "start": 21,
                     "text": "AND",
-                    "type": "RESERVED_LOGICAL_OPERATOR",
+                    "type": "AND",
                   },
                   "betweenToken": Object {
                     "end": 17,
@@ -319,7 +319,7 @@ describe('Parser', () => {
                     "raw": "BETWEEN",
                     "start": 10,
                     "text": "BETWEEN",
-                    "type": "RESERVED_KEYWORD",
+                    "type": "BETWEEN",
                   },
                   "expr1": Object {
                     "end": 20,
@@ -383,7 +383,7 @@ describe('Parser', () => {
                 "raw": "LIMIT",
                 "start": 0,
                 "text": "LIMIT",
-                "type": "RESERVED_COMMAND",
+                "type": "LIMIT",
               },
               "type": "limit_clause",
             },
@@ -420,7 +420,7 @@ describe('Parser', () => {
                 "raw": "LIMIT",
                 "start": 0,
                 "text": "LIMIT",
-                "type": "RESERVED_COMMAND",
+                "type": "LIMIT",
               },
               "offset": Array [
                 Object {