Repository: sql-formatter-org/sql-formatter
Pull request #448 (2022-09-20 12:45:54)

Replace Token.loc object with Token.start number field
As the end-location of token is never used, we don't really need to store this data.

As a follow-up for discussion in #440


Patch:
diff --git a/src/languages/bigquery/bigquery.formatter.ts b/src/languages/bigquery/bigquery.formatter.ts
index da73eadb..cac282f4 100644
--- a/src/languages/bigquery/bigquery.formatter.ts
+++ b/src/languages/bigquery/bigquery.formatter.ts
@@ -204,10 +204,7 @@ function combineParameterizedTypes(tokens: Token[]) {
         type: TokenType.IDENTIFIER,
         raw: typeDefTokens.map(formatTypeDefToken('raw')).join(''),
         text: typeDefTokens.map(formatTypeDefToken('text')).join(''),
-        loc: {
-          start: token.loc.start,
-          end: token.loc.end + typeDefTokens.map(t => t.text.length).reduce((a, b) => a + b),
-        },
+        start: token.start,
       });
       i = endIndex;
     } else {
diff --git a/src/lexer/TokenizerEngine.ts b/src/lexer/TokenizerEngine.ts
index 701617eb..f6234b65 100644
--- a/src/lexer/TokenizerEngine.ts
+++ b/src/lexer/TokenizerEngine.ts
@@ -94,10 +94,7 @@ export default class TokenizerEngine {
         type: rule.type,
         raw: matchedText,
         text: rule.text ? rule.text(matchedText) : matchedText,
-        loc: {
-          start: this.index,
-          end: this.index + matchedText.length,
-        },
+        start: this.index,
       };
 
       if (rule.key) {
diff --git a/src/lexer/token.ts b/src/lexer/token.ts
index 2a13ef2d..ccc06bb3 100644
--- a/src/lexer/token.ts
+++ b/src/lexer/token.ts
@@ -44,22 +44,16 @@ export interface Token {
   raw: string; // The raw original text that was matched
   text: string; // Cleaned up text e.g. keyword converted to uppercase and extra spaces removed
   key?: string;
-  loc: Loc;
+  start: number;
   precedingWhitespace?: string; // Whitespace before this token, if any
 }
 
-/** Describes location in source code */
-export interface Loc {
-  start: number; // 0-based index of the token in the whole query string
-  end: number; // 0-based index of where the token ends in the query string
-}
-
 /** Creates EOF token positioned at given location */
 export const createEofToken = (index: number) => ({
   type: TokenType.EOF,
   raw: '«EOF»',
   text: '«EOF»',
-  loc: { start: index, end: index },
+  start: index,
 });
 
 /**
diff --git a/src/parser/LexerAdapter.ts b/src/parser/LexerAdapter.ts
index 95d8f119..8e17f242 100644
--- a/src/parser/LexerAdapter.ts
+++ b/src/parser/LexerAdapter.ts
@@ -25,7 +25,7 @@ export default class LexerAdapter {
   save(): any {}
 
   formatError(token: NearleyToken) {
-    const { line, col } = lineColFromIndex(this.input, token.loc.start);
+    const { line, col } = lineColFromIndex(this.input, token.start);
     return `Parse error at token: ${token.text} at line ${line} column ${col}`;
   }
 


Test patch:
diff --git a/test/unit/__snapshots__/Tokenizer.test.ts.snap b/test/unit/__snapshots__/Tokenizer.test.ts.snap
index 6fb92ebd..611a4c01 100644
--- a/test/unit/__snapshots__/Tokenizer.test.ts.snap
+++ b/test/unit/__snapshots__/Tokenizer.test.ts.snap
@@ -3,37 +3,28 @@
 exports[`Tokenizer tokenizes multiline SQL tokens 1`] = `
 Array [
   Object {
-    "loc": Object {
-      "end": 6,
-      "start": 0,
-    },
     "precedingWhitespace": undefined,
     "raw": "SELECT",
+    "start": 0,
     "text": "SELECT",
     "type": "RESERVED_SELECT",
   },
   Object {
-    "loc": Object {
-      "end": 17,
-      "start": 7,
-    },
     "precedingWhitespace": " ",
     "raw": "\\"foo
  bar\\"",
+    "start": 7,
     "text": "\\"foo
  bar\\"",
     "type": "QUOTED_IDENTIFIER",
   },
   Object {
-    "loc": Object {
-      "end": 27,
-      "start": 18,
-    },
     "precedingWhitespace": " ",
     "raw": "/* 
 
 
  */",
+    "start": 18,
     "text": "/* 
 
 
@@ -41,12 +32,9 @@ Array [
     "type": "BLOCK_COMMENT",
   },
   Object {
-    "loc": Object {
-      "end": 28,
-      "start": 27,
-    },
     "precedingWhitespace": undefined,
     "raw": ";",
+    "start": 27,
     "text": ";",
     "type": "DELIMITER",
   },
@@ -56,52 +44,37 @@ Array [
 exports[`Tokenizer tokenizes single line SQL tokens 1`] = `
 Array [
   Object {
-    "loc": Object {
-      "end": 6,
-      "start": 0,
-    },
     "precedingWhitespace": undefined,
     "raw": "SELECT",
+    "start": 0,
     "text": "SELECT",
     "type": "RESERVED_SELECT",
   },
   Object {
-    "loc": Object {
-      "end": 8,
-      "start": 7,
-    },
     "precedingWhitespace": " ",
     "raw": "*",
+    "start": 7,
     "text": "*",
     "type": "ASTERISK",
   },
   Object {
-    "loc": Object {
-      "end": 13,
-      "start": 9,
-    },
     "precedingWhitespace": " ",
     "raw": "FROM",
+    "start": 9,
     "text": "FROM",
     "type": "RESERVED_COMMAND",
   },
   Object {
-    "loc": Object {
-      "end": 17,
-      "start": 14,
-    },
     "precedingWhitespace": " ",
     "raw": "foo",
+    "start": 14,
     "text": "foo",
     "type": "IDENTIFIER",
   },
   Object {
-    "loc": Object {
-      "end": 18,
-      "start": 17,
-    },
     "precedingWhitespace": undefined,
     "raw": ";",
+    "start": 17,
     "text": ";",
     "type": "DELIMITER",
   },