Repository: sql-formatter-org/sql-formatter
Pull request #373 (2022-08-03 23:47:24)

Rename token attributes
Find and replace for token attributes to eventually match up with nearley implementation
`text` → `raw`
`value` → `text`


Patch:
diff --git a/src/formatter/ExpressionFormatter.ts b/src/formatter/ExpressionFormatter.ts
index 9289b817..93913b26 100644
--- a/src/formatter/ExpressionFormatter.ts
+++ b/src/formatter/ExpressionFormatter.ts
@@ -236,7 +236,7 @@ export default class ExpressionFormatter {
 
   /** Formats a block comment onto query */
   private formatBlockComment(token: Token) {
-    this.splitBlockComment(token.value).forEach(line => {
+    this.splitBlockComment(token.text).forEach(line => {
       this.layout.add(WS.NEWLINE, WS.INDENT, line);
     });
     this.layout.add(WS.NEWLINE, WS.INDENT);
@@ -297,15 +297,15 @@ export default class ExpressionFormatter {
    */
   private formatOperator(token: Token) {
     // special operator
-    if (token.value === ':') {
+    if (token.text === ':') {
       this.layout.add(WS.NO_SPACE, this.show(token), WS.SPACE);
       return;
-    } else if (token.value === '.' || token.value === '::') {
+    } else if (token.text === '.' || token.text === '::') {
       this.layout.add(WS.NO_SPACE, this.show(token));
       return;
     }
     // special case for PLSQL @ dblink syntax
-    else if (token.value === '@' && this.cfg.language === 'plsql') {
+    else if (token.text === '@' && this.cfg.language === 'plsql') {
       this.layout.add(WS.NO_SPACE, this.show(token));
       return;
     }
@@ -380,16 +380,16 @@ export default class ExpressionFormatter {
     if (isReserved(token)) {
       switch (this.cfg.keywordCase) {
         case 'preserve':
-          return equalizeWhitespace(token.text);
+          return equalizeWhitespace(token.raw);
         case 'upper':
-          return token.value;
+          return token.text;
         case 'lower':
-          return token.value.toLowerCase();
+          return token.text.toLowerCase();
       }
     } else if (isParameter(token)) {
       return this.params.get(token);
     } else {
-      return token.value;
+      return token.text;
     }
   }
 }
diff --git a/src/formatter/InlineBlock.ts b/src/formatter/InlineBlock.ts
index adb7e548..9da9a255 100644
--- a/src/formatter/InlineBlock.ts
+++ b/src/formatter/InlineBlock.ts
@@ -26,10 +26,10 @@ export default class InlineBlock {
     for (const node of parenthesis.children) {
       switch (node.type) {
         case NodeType.function_call:
-          length += node.nameToken.value.length + this.inlineWidth(node.parenthesis);
+          length += node.nameToken.text.length + this.inlineWidth(node.parenthesis);
           break;
         case NodeType.array_subscript:
-          length += node.arrayToken.value.length + this.inlineWidth(node.parenthesis);
+          length += node.arrayToken.text.length + this.inlineWidth(node.parenthesis);
           break;
         case NodeType.parenthesis:
           length += this.inlineWidth(node);
@@ -45,7 +45,7 @@ export default class InlineBlock {
           length += 1;
           break;
         case NodeType.token:
-          length += node.token.value.length;
+          length += node.token.text.length;
           if (this.isForbiddenToken(node.token)) {
             return Infinity;
           }
@@ -62,7 +62,7 @@ export default class InlineBlock {
 
   private betweenWidth(node: BetweenPredicate): number {
     return sum(
-      [node.betweenToken, node.expr1, node.andToken, node.expr2].map(token => token.value.length)
+      [node.betweenToken, node.expr1, node.andToken, node.expr2].map(token => token.text.length)
     );
   }
 
diff --git a/src/formatter/Params.ts b/src/formatter/Params.ts
index 41288f67..9abf40da 100644
--- a/src/formatter/Params.ts
+++ b/src/formatter/Params.ts
@@ -20,11 +20,11 @@ export default class Params {
   /**
    * Returns param value that matches given placeholder with param key.
    * @param {Token} token
-   * @return {string} param or token.value when params are missing
+   * @return {string} param or token.text when params are missing
    */
-  get({ key, value }: Token): string {
+  get({ key, text }: Token): string {
     if (!this.params) {
-      return value;
+      return text;
     }
 
     if (key) {
diff --git a/src/languages/bigquery/bigquery.formatter.ts b/src/languages/bigquery/bigquery.formatter.ts
index 5c86f196..c94b2702 100644
--- a/src/languages/bigquery/bigquery.formatter.ts
+++ b/src/languages/bigquery/bigquery.formatter.ts
@@ -170,7 +170,7 @@ function postProcess(tokens: Token[]): Token[] {
 function detectArraySubscripts(tokens: Token[]) {
   let prevToken = EOF_TOKEN;
   return tokens.map(token => {
-    if (token.value === 'OFFSET' && prevToken.value === '[') {
+    if (token.text === 'OFFSET' && prevToken.text === '[') {
       prevToken = token;
       return { ...token, type: TokenType.RESERVED_FUNCTION_NAME };
     } else {
@@ -185,14 +185,13 @@ function combineParameterizedTypes(tokens: Token[]) {
   const processed: Token[] = [];
   for (let i = 0; i < tokens.length; i++) {
     const token = tokens[i];
-    const nextToken = tokens[i + 1] || EOF_TOKEN;
 
-    if ((isToken.ARRAY(token) || isToken.STRUCT(token)) && nextToken.value === '<') {
+    if ((isToken.ARRAY(token) || isToken.STRUCT(token)) && tokens[i + 1]?.text === '<') {
       const endIndex = findClosingAngleBracketIndex(tokens, i + 1);
       const typeDefTokens = tokens.slice(i, endIndex + 1);
       processed.push({
         type: TokenType.IDENTIFIER,
-        value: typeDefTokens.map(formatTypeDefToken('value')).join(''),
+        raw: typeDefTokens.map(formatTypeDefToken('raw')).join(''),
         text: typeDefTokens.map(formatTypeDefToken('text')).join(''),
       });
       i = endIndex;
@@ -204,7 +203,7 @@ function combineParameterizedTypes(tokens: Token[]) {
 }
 
 const formatTypeDefToken =
-  (key: 'text' | 'value') =>
+  (key: Extract<keyof Token, 'raw' | 'text'>) =>
   (token: Token): string => {
     if (token.type === TokenType.IDENTIFIER || token.type === TokenType.COMMA) {
       return token[key] + ' ';
@@ -217,11 +216,11 @@ function findClosingAngleBracketIndex(tokens: Token[], startIndex: number): numb
   let level = 0;
   for (let i = startIndex; i < tokens.length; i++) {
     const token = tokens[i];
-    if (token.value === '<') {
+    if (token.text === '<') {
       level++;
-    } else if (token.value === '>') {
+    } else if (token.text === '>') {
       level--;
-    } else if (token.value === '>>') {
+    } else if (token.text === '>>') {
       level -= 2;
     }
     if (level === 0) {
diff --git a/src/languages/mariadb/mariadb.formatter.ts b/src/languages/mariadb/mariadb.formatter.ts
index eff6c8d5..805485ab 100644
--- a/src/languages/mariadb/mariadb.formatter.ts
+++ b/src/languages/mariadb/mariadb.formatter.ts
@@ -283,7 +283,7 @@ export default class MariaDbFormatter extends Formatter {
 function postProcess(tokens: Token[]) {
   return tokens.map((token, i) => {
     const nextToken = tokens[i + 1] || EOF_TOKEN;
-    if (isToken.SET(token) && nextToken.value === '(') {
+    if (isToken.SET(token) && nextToken.text === '(') {
       // This is SET datatype, not SET statement
       return { ...token, type: TokenType.RESERVED_FUNCTION_NAME };
     }
diff --git a/src/languages/mysql/mysql.formatter.ts b/src/languages/mysql/mysql.formatter.ts
index 32a42dbf..c9a66713 100644
--- a/src/languages/mysql/mysql.formatter.ts
+++ b/src/languages/mysql/mysql.formatter.ts
@@ -250,7 +250,7 @@ export default class MySqlFormatter extends Formatter {
 function postProcess(tokens: Token[]) {
   return tokens.map((token, i) => {
     const nextToken = tokens[i + 1] || EOF_TOKEN;
-    if (isToken.SET(token) && nextToken.value === '(') {
+    if (isToken.SET(token) && nextToken.text === '(') {
       // This is SET datatype, not SET statement
       return { ...token, type: TokenType.RESERVED_FUNCTION_NAME };
     }
diff --git a/src/languages/spark/spark.formatter.ts b/src/languages/spark/spark.formatter.ts
index ae78b8af..dda082f9 100644
--- a/src/languages/spark/spark.formatter.ts
+++ b/src/languages/spark/spark.formatter.ts
@@ -139,10 +139,10 @@ function postProcess(tokens: Token[]) {
     }
 
     // TODO: deprecate this once ITEMS is merged with COLLECTION
-    if (token.value === 'ITEMS' && token.type === TokenType.RESERVED_KEYWORD) {
-      if (!(prevToken.value === 'COLLECTION' && nextToken.value === 'TERMINATED')) {
+    if (token.text === 'ITEMS' && token.type === TokenType.RESERVED_KEYWORD) {
+      if (!(prevToken.text === 'COLLECTION' && nextToken.text === 'TERMINATED')) {
         // this is a word and not COLLECTION ITEMS
-        return { type: TokenType.IDENTIFIER, text: token.text, value: token.text };
+        return { type: TokenType.IDENTIFIER, raw: token.raw, text: token.raw };
       }
     }
 
diff --git a/src/lexer/TokenizerEngine.ts b/src/lexer/TokenizerEngine.ts
index 19a9a108..a13b4ba1 100644
--- a/src/lexer/TokenizerEngine.ts
+++ b/src/lexer/TokenizerEngine.ts
@@ -87,7 +87,7 @@ export default class TokenizerEngine {
       const tokenRule = this.rules[tokenType];
       if (token) {
         if (tokenRule?.key) {
-          return { ...token, key: tokenRule.key(token.value) };
+          return { ...token, key: tokenRule.key(token.text) };
         }
         return token; // POSITIONAL_PARAMETER does not have a key transform function
       }
@@ -98,7 +98,7 @@ export default class TokenizerEngine {
   private matchReservedWordToken(previousToken?: Token): Token | undefined {
     // A reserved word cannot be preceded by a '.'
     // this makes it so in "mytable.from", "from" is not considered a reserved word
-    if (previousToken?.value === '.') {
+    if (previousToken?.text === '.') {
       return undefined;
     }
 
@@ -149,8 +149,8 @@ export default class TokenizerEngine {
       this.index += matchedToken.length;
       return {
         type,
-        text: matchedToken,
-        value: transform ? transform(matchedToken) : matchedToken,
+        raw: matchedToken,
+        text: transform ? transform(matchedToken) : matchedToken,
       };
     }
     return undefined;
diff --git a/src/lexer/token.ts b/src/lexer/token.ts
index 4d371cbd..02621f88 100644
--- a/src/lexer/token.ts
+++ b/src/lexer/token.ts
@@ -32,8 +32,8 @@ export enum TokenType {
 /** Struct to store the most basic cohesive unit of language grammar */
 export interface Token {
   type: TokenType;
-  text: string; // The raw original text that was matched
-  value: string; // Cleaned up `text` e.g. keyword converted to uppercase and extra spaces removed
+  raw: string; // The raw original text that was matched
+  text: string; // Cleaned up text e.g. keyword converted to uppercase and extra spaces removed
   key?: string;
 }
 
@@ -41,33 +41,33 @@ export interface Token {
  * For use as a "missing token"
  * e.g. in lookAhead and lookBehind to avoid dealing with null values
  */
-export const EOF_TOKEN = { type: TokenType.EOF, text: '«EOF»', value: '«EOF»' };
+export const EOF_TOKEN = { type: TokenType.EOF, raw: '«EOF»', text: '«EOF»' };
 
 /** Checks if two tokens are equivalent */
 export const testToken =
-  (compareToken: { type: TokenType; value: string }) =>
+  (compareToken: { type: TokenType; text: string }) =>
   (token: Token): boolean =>
-    token.type === compareToken.type && token.value === compareToken.value;
+    token.type === compareToken.type && token.text === compareToken.text;
 
 /** Util object that allows for easy checking of Reserved Keywords */
 export const isToken = {
-  AS: testToken({ value: 'AS', type: TokenType.RESERVED_KEYWORD }),
-  AND: testToken({ value: 'AND', type: TokenType.RESERVED_LOGICAL_OPERATOR }),
-  ARRAY: testToken({ value: 'ARRAY', type: TokenType.RESERVED_KEYWORD }),
-  BETWEEN: testToken({ value: 'BETWEEN', type: TokenType.RESERVED_KEYWORD }),
-  CASE: testToken({ value: 'CASE', type: TokenType.RESERVED_CASE_START }),
-  CAST: testToken({ value: 'CAST', type: TokenType.RESERVED_FUNCTION_NAME }),
-  BY: testToken({ value: 'BY', type: TokenType.RESERVED_KEYWORD }),
-  END: testToken({ value: 'END', type: TokenType.RESERVED_CASE_END }),
-  FROM: testToken({ value: 'FROM', type: TokenType.RESERVED_COMMAND }),
-  LIMIT: testToken({ value: 'LIMIT', type: TokenType.RESERVED_COMMAND }),
+  AS: testToken({ text: 'AS', type: TokenType.RESERVED_KEYWORD }),
+  AND: testToken({ text: 'AND', type: TokenType.RESERVED_LOGICAL_OPERATOR }),
+  ARRAY: testToken({ text: 'ARRAY', type: TokenType.RESERVED_KEYWORD }),
+  BETWEEN: testToken({ text: 'BETWEEN', type: TokenType.RESERVED_KEYWORD }),
+  CASE: testToken({ text: 'CASE', type: TokenType.RESERVED_CASE_START }),
+  CAST: testToken({ text: 'CAST', type: TokenType.RESERVED_FUNCTION_NAME }),
+  BY: testToken({ text: 'BY', type: TokenType.RESERVED_KEYWORD }),
+  END: testToken({ text: 'END', type: TokenType.RESERVED_CASE_END }),
+  FROM: testToken({ text: 'FROM', type: TokenType.RESERVED_COMMAND }),
+  LIMIT: testToken({ text: 'LIMIT', type: TokenType.RESERVED_COMMAND }),
   SELECT: (token: Token) =>
-    /^SELECT\b/.test(token.value) && token.type === TokenType.RESERVED_COMMAND,
-  SET: testToken({ value: 'SET', type: TokenType.RESERVED_COMMAND }),
-  STRUCT: testToken({ value: 'STRUCT', type: TokenType.RESERVED_KEYWORD }),
-  TABLE: testToken({ value: 'TABLE', type: TokenType.RESERVED_KEYWORD }),
-  WINDOW: testToken({ value: 'WINDOW', type: TokenType.RESERVED_COMMAND }),
-  WITH: testToken({ value: 'WITH', type: TokenType.RESERVED_COMMAND }),
+    /^SELECT\b/.test(token.text) && token.type === TokenType.RESERVED_COMMAND,
+  SET: testToken({ text: 'SET', type: TokenType.RESERVED_COMMAND }),
+  STRUCT: testToken({ text: 'STRUCT', type: TokenType.RESERVED_KEYWORD }),
+  TABLE: testToken({ text: 'TABLE', type: TokenType.RESERVED_KEYWORD }),
+  WINDOW: testToken({ text: 'WINDOW', type: TokenType.RESERVED_COMMAND }),
+  WITH: testToken({ text: 'WITH', type: TokenType.RESERVED_COMMAND }),
 };
 
 /** Checks if token is any Reserved Keyword or Command */
diff --git a/src/parser/Parser.ts b/src/parser/Parser.ts
index 4a14ef2f..2a5bb27a 100644
--- a/src/parser/Parser.ts
+++ b/src/parser/Parser.ts
@@ -83,7 +83,7 @@ export default class Parser {
   }
 
   private functionCall(): FunctionCall | undefined {
-    if (this.look().type === TokenType.RESERVED_FUNCTION_NAME && this.look(1).value === '(') {
+    if (this.look().type === TokenType.RESERVED_FUNCTION_NAME && this.look(1).text === '(') {
       return {
         type: NodeType.function_call,
         nameToken: this.next(),
@@ -97,7 +97,7 @@ export default class Parser {
     if (
       (this.look().type === TokenType.RESERVED_KEYWORD ||
         this.look().type === TokenType.IDENTIFIER) &&
-      this.look(1).value === '['
+      this.look(1).text === '['
     ) {
       return {
         type: NodeType.array_subscript,
@@ -112,13 +112,13 @@ export default class Parser {
     if (this.look().type === TokenType.OPEN_PAREN) {
       const children: AstNode[] = [];
       const token = this.next();
-      const openParen = token.value;
+      const openParen = token.text;
       let closeParen = '';
       while (this.look().type !== TokenType.CLOSE_PAREN && this.look().type !== TokenType.EOF) {
         children.push(this.expression());
       }
       if (this.look().type === TokenType.CLOSE_PAREN) {
-        closeParen = this.next().value;
+        closeParen = this.next().text;
       }
       return { type: NodeType.parenthesis, children, openParen, closeParen };
     }
@@ -163,7 +163,7 @@ export default class Parser {
   }
 
   private allColumnsAsterisk(): AllColumnsAsterisk | undefined {
-    if (this.look().value === '*' && isToken.SELECT(this.look(-1))) {
+    if (this.look().text === '*' && isToken.SELECT(this.look(-1))) {
       this.next();
       return { type: NodeType.all_columns_asterisk };
     }


Test patch:
diff --git a/test/unit/Parser.test.ts b/test/unit/Parser.test.ts
index 1f3fdeee..ff4e5b4b 100644
--- a/test/unit/Parser.test.ts
+++ b/test/unit/Parser.test.ts
@@ -29,9 +29,9 @@ describe('Parser', () => {
           "children": Array [
             Object {
               "token": Object {
+                "raw": "foo",
                 "text": "foo",
                 "type": "IDENTIFIER",
-                "value": "foo",
               },
               "type": "token",
             },
@@ -43,9 +43,9 @@ describe('Parser', () => {
           "children": Array [
             Object {
               "token": Object {
+                "raw": "bar",
                 "text": "bar",
                 "type": "IDENTIFIER",
-                "value": "bar",
               },
               "type": "token",
             },
@@ -66,17 +66,17 @@ describe('Parser', () => {
               "children": Array [
                 Object {
                   "nameToken": Object {
+                    "raw": "SQRT",
                     "text": "SQRT",
                     "type": "RESERVED_FUNCTION_NAME",
-                    "value": "SQRT",
                   },
                   "parenthesis": Object {
                     "children": Array [
                       Object {
                         "token": Object {
+                          "raw": "2",
                           "text": "2",
                           "type": "NUMBER",
-                          "value": "2",
                         },
                         "type": "token",
                       },
@@ -89,9 +89,9 @@ describe('Parser', () => {
                 },
               ],
               "nameToken": Object {
+                "raw": "SELECT",
                 "text": "SELECT",
                 "type": "RESERVED_COMMAND",
-                "value": "SELECT",
               },
               "type": "clause",
             },
@@ -112,25 +112,25 @@ describe('Parser', () => {
               "children": Array [
                 Object {
                   "arrayToken": Object {
+                    "raw": "my_array",
                     "text": "my_array",
                     "type": "IDENTIFIER",
-                    "value": "my_array",
                   },
                   "parenthesis": Object {
                     "children": Array [
                       Object {
                         "nameToken": Object {
+                          "raw": "OFFSET",
                           "text": "OFFSET",
                           "type": "RESERVED_FUNCTION_NAME",
-                          "value": "OFFSET",
                         },
                         "parenthesis": Object {
                           "children": Array [
                             Object {
                               "token": Object {
+                                "raw": "5",
                                 "text": "5",
                                 "type": "NUMBER",
-                                "value": "5",
                               },
                               "type": "token",
                             },
@@ -150,9 +150,9 @@ describe('Parser', () => {
                 },
               ],
               "nameToken": Object {
+                "raw": "SELECT",
                 "text": "SELECT",
                 "type": "RESERVED_COMMAND",
-                "value": "SELECT",
               },
               "type": "clause",
             },
@@ -175,17 +175,17 @@ describe('Parser', () => {
                   "children": Array [
                     Object {
                       "token": Object {
+                        "raw": "birth_year",
                         "text": "birth_year",
                         "type": "IDENTIFIER",
-                        "value": "birth_year",
                       },
                       "type": "token",
                     },
                     Object {
                       "token": Object {
+                        "raw": "-",
                         "text": "-",
                         "type": "OPERATOR",
-                        "value": "-",
                       },
                       "type": "token",
                     },
@@ -193,25 +193,25 @@ describe('Parser', () => {
                       "children": Array [
                         Object {
                           "token": Object {
+                            "raw": "CURRENT_DATE",
                             "text": "CURRENT_DATE",
                             "type": "IDENTIFIER",
-                            "value": "CURRENT_DATE",
                           },
                           "type": "token",
                         },
                         Object {
                           "token": Object {
+                            "raw": "+",
                             "text": "+",
                             "type": "OPERATOR",
-                            "value": "+",
                           },
                           "type": "token",
                         },
                         Object {
                           "token": Object {
+                            "raw": "1",
                             "text": "1",
                             "type": "NUMBER",
-                            "value": "1",
                           },
                           "type": "token",
                         },
@@ -227,9 +227,9 @@ describe('Parser', () => {
                 },
               ],
               "nameToken": Object {
+                "raw": "SELECT",
                 "text": "SELECT",
                 "type": "RESERVED_COMMAND",
-                "value": "SELECT",
               },
               "type": "clause",
             },
@@ -250,40 +250,40 @@ describe('Parser', () => {
               "children": Array [
                 Object {
                   "token": Object {
+                    "raw": "age",
                     "text": "age",
                     "type": "IDENTIFIER",
-                    "value": "age",
                   },
                   "type": "token",
                 },
                 Object {
                   "andToken": Object {
-                    "text": "and",
+                    "raw": "and",
+                    "text": "AND",
                     "type": "RESERVED_LOGICAL_OPERATOR",
-                    "value": "AND",
                   },
                   "betweenToken": Object {
+                    "raw": "BETWEEN",
                     "text": "BETWEEN",
                     "type": "RESERVED_KEYWORD",
-                    "value": "BETWEEN",
                   },
                   "expr1": Object {
+                    "raw": "10",
                     "text": "10",
                     "type": "NUMBER",
-                    "value": "10",
                   },
                   "expr2": Object {
+                    "raw": "15",
                     "text": "15",
                     "type": "NUMBER",
-                    "value": "15",
                   },
                   "type": "between_predicate",
                 },
               ],
               "nameToken": Object {
+                "raw": "WHERE",
                 "text": "WHERE",
                 "type": "RESERVED_COMMAND",
-                "value": "WHERE",
               },
               "type": "clause",
             },
@@ -304,17 +304,17 @@ describe('Parser', () => {
               "count": Array [
                 Object {
                   "token": Object {
+                    "raw": "10",
                     "text": "10",
                     "type": "NUMBER",
-                    "value": "10",
                   },
                   "type": "token",
                 },
               ],
               "limitToken": Object {
+                "raw": "LIMIT",
                 "text": "LIMIT",
                 "type": "RESERVED_COMMAND",
-                "value": "LIMIT",
               },
               "type": "limit_clause",
             },
@@ -335,24 +335,24 @@ describe('Parser', () => {
               "count": Array [
                 Object {
                   "token": Object {
+                    "raw": "10",
                     "text": "10",
                     "type": "NUMBER",
-                    "value": "10",
                   },
                   "type": "token",
                 },
               ],
               "limitToken": Object {
+                "raw": "LIMIT",
                 "text": "LIMIT",
                 "type": "RESERVED_COMMAND",
-                "value": "LIMIT",
               },
               "offset": Array [
                 Object {
                   "token": Object {
+                    "raw": "200",
                     "text": "200",
                     "type": "NUMBER",
-                    "value": "200",
                   },
                   "type": "token",
                 },
@@ -379,9 +379,9 @@ describe('Parser', () => {
                 },
               ],
               "nameToken": Object {
+                "raw": "SELECT",
                 "text": "SELECT",
                 "type": "RESERVED_COMMAND",
-                "value": "SELECT",
               },
               "type": "clause",
             },
@@ -405,9 +405,9 @@ describe('Parser', () => {
                 },
               ],
               "nameToken": Object {
+                "raw": "SELECT DISTINCT",
                 "text": "SELECT DISTINCT",
                 "type": "RESERVED_COMMAND",
-                "value": "SELECT DISTINCT",
               },
               "type": "clause",
             },