Repository: sql-formatter-org/sql-formatter
Pull request #440 (2022-09-08 08:09:48)

Group token location data to loc field
Cherry-picked changes from #436:

- Added basic unit test for Tokenizer. 
- Grouped `start`/`end` fields into single `loc: {start, end}` object. This simplifies processing the location info separately from the whole token. Not used right now, but planning to use in the future.
- Changed the appended EOF token to use actual source code length as location to avoid `Infinity` locations.



Patch:
diff --git a/src/languages/bigquery/bigquery.formatter.ts b/src/languages/bigquery/bigquery.formatter.ts
index 109d5057..da73eadb 100644
--- a/src/languages/bigquery/bigquery.formatter.ts
+++ b/src/languages/bigquery/bigquery.formatter.ts
@@ -204,8 +204,10 @@ function combineParameterizedTypes(tokens: Token[]) {
         type: TokenType.IDENTIFIER,
         raw: typeDefTokens.map(formatTypeDefToken('raw')).join(''),
         text: typeDefTokens.map(formatTypeDefToken('text')).join(''),
-        start: token.start,
-        end: token.end + typeDefTokens.map(t => t.text.length).reduce((a, b) => a + b),
+        loc: {
+          start: token.loc.start,
+          end: token.loc.end + typeDefTokens.map(t => t.text.length).reduce((a, b) => a + b),
+        },
       });
       i = endIndex;
     } else {
diff --git a/src/lexer/TokenizerEngine.ts b/src/lexer/TokenizerEngine.ts
index 1093b2e6..5c94a586 100644
--- a/src/lexer/TokenizerEngine.ts
+++ b/src/lexer/TokenizerEngine.ts
@@ -87,8 +87,10 @@ export default class TokenizerEngine {
         type: rule.type,
         raw: matchedText,
         text: rule.text ? rule.text(matchedText) : matchedText,
-        start: this.index,
-        end: this.index + matchedText.length,
+        loc: {
+          start: this.index,
+          end: this.index + matchedText.length,
+        },
       };
 
       if (rule.key) {
diff --git a/src/lexer/token.ts b/src/lexer/token.ts
index 43222bc2..2a13ef2d 100644
--- a/src/lexer/token.ts
+++ b/src/lexer/token.ts
@@ -44,22 +44,29 @@ export interface Token {
   raw: string; // The raw original text that was matched
   text: string; // Cleaned up text e.g. keyword converted to uppercase and extra spaces removed
   key?: string;
+  loc: Loc;
+  precedingWhitespace?: string; // Whitespace before this token, if any
+}
+
+/** Describes location in source code */
+export interface Loc {
   start: number; // 0-based index of the token in the whole query string
   end: number; // 0-based index of where the token ends in the query string
-  precedingWhitespace?: string; // Whitespace before this token, if any
 }
 
+/** Creates EOF token positioned at given location */
+export const createEofToken = (index: number) => ({
+  type: TokenType.EOF,
+  raw: '«EOF»',
+  text: '«EOF»',
+  loc: { start: index, end: index },
+});
+
 /**
  * For use as a "missing token"
  * e.g. in lookAhead and lookBehind to avoid dealing with null values
  */
-export const EOF_TOKEN: Token = {
-  type: TokenType.EOF,
-  raw: '«EOF»',
-  text: '«EOF»',
-  start: Infinity,
-  end: Infinity,
-};
+export const EOF_TOKEN = createEofToken(Infinity);
 
 /** Checks if two tokens are equivalent */
 export const testToken =
diff --git a/src/parser/createParser.ts b/src/parser/createParser.ts
index cdbb5c4f..e15b23a1 100644
--- a/src/parser/createParser.ts
+++ b/src/parser/createParser.ts
@@ -6,7 +6,7 @@ import { ParamTypes } from 'src/lexer/TokenizerOptions';
 import { StatementNode } from 'src/parser/ast';
 import grammar from 'src/parser/grammar';
 import LexerAdapter from 'src/parser/LexerAdapter';
-import { EOF_TOKEN } from 'src/lexer/token';
+import { createEofToken } from 'src/lexer/token';
 
 export interface Parser {
   parse(sql: string, paramTypesOverrides: ParamTypes): StatementNode[];
@@ -19,7 +19,7 @@ export function createParser(tokenizer: Tokenizer): Parser {
   let paramTypesOverrides: ParamTypes = {};
   const lexer = new LexerAdapter(chunk => [
     ...disambiguateTokens(tokenizer.tokenize(chunk, paramTypesOverrides)),
-    EOF_TOKEN,
+    createEofToken(chunk.length),
   ]);
   const parser = new NearleyParser(Grammar.fromCompiled(grammar), { lexer });
 


Test patch:
diff --git a/test/unit/Tokenizer.test.ts b/test/unit/Tokenizer.test.ts
new file mode 100644
index 00000000..6bab1d6c
--- /dev/null
+++ b/test/unit/Tokenizer.test.ts
@@ -0,0 +1,28 @@
+import Tokenizer from 'src/lexer/Tokenizer';
+
+describe('Tokenizer', () => {
+  const tokenize = (sql: string) =>
+    new Tokenizer({
+      reservedCommands: ['FROM', 'WHERE', 'LIMIT', 'CREATE TABLE'],
+      reservedSelect: ['SELECT'],
+      reservedDependentClauses: ['WHEN', 'ELSE'],
+      reservedSetOperations: ['UNION', 'UNION ALL'],
+      reservedJoins: ['JOIN'],
+      reservedFunctionNames: ['SQRT', 'CURRENT_TIME'],
+      reservedKeywords: ['BETWEEN', 'LIKE', 'ON', 'USING'],
+      stringTypes: ["''-qq"],
+      identTypes: ['""-qq'],
+    }).tokenize(sql, {});
+
+  it('tokenizes whitespace to empty array', () => {
+    expect(tokenize(' \t\n \n\r ')).toEqual([]);
+  });
+
+  it('tokenizes single line SQL tokens', () => {
+    expect(tokenize('SELECT * FROM foo;')).toMatchSnapshot();
+  });
+
+  it('tokenizes multiline SQL tokens', () => {
+    expect(tokenize('SELECT "foo\n bar" /* \n\n\n */;')).toMatchSnapshot();
+  });
+});
diff --git a/test/unit/__snapshots__/Tokenizer.test.ts.snap b/test/unit/__snapshots__/Tokenizer.test.ts.snap
new file mode 100644
index 00000000..6fb92ebd
--- /dev/null
+++ b/test/unit/__snapshots__/Tokenizer.test.ts.snap
@@ -0,0 +1,109 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`Tokenizer tokenizes multiline SQL tokens 1`] = `
+Array [
+  Object {
+    "loc": Object {
+      "end": 6,
+      "start": 0,
+    },
+    "precedingWhitespace": undefined,
+    "raw": "SELECT",
+    "text": "SELECT",
+    "type": "RESERVED_SELECT",
+  },
+  Object {
+    "loc": Object {
+      "end": 17,
+      "start": 7,
+    },
+    "precedingWhitespace": " ",
+    "raw": "\\"foo
+ bar\\"",
+    "text": "\\"foo
+ bar\\"",
+    "type": "QUOTED_IDENTIFIER",
+  },
+  Object {
+    "loc": Object {
+      "end": 27,
+      "start": 18,
+    },
+    "precedingWhitespace": " ",
+    "raw": "/* 
+
+
+ */",
+    "text": "/* 
+
+
+ */",
+    "type": "BLOCK_COMMENT",
+  },
+  Object {
+    "loc": Object {
+      "end": 28,
+      "start": 27,
+    },
+    "precedingWhitespace": undefined,
+    "raw": ";",
+    "text": ";",
+    "type": "DELIMITER",
+  },
+]
+`;
+
+exports[`Tokenizer tokenizes single line SQL tokens 1`] = `
+Array [
+  Object {
+    "loc": Object {
+      "end": 6,
+      "start": 0,
+    },
+    "precedingWhitespace": undefined,
+    "raw": "SELECT",
+    "text": "SELECT",
+    "type": "RESERVED_SELECT",
+  },
+  Object {
+    "loc": Object {
+      "end": 8,
+      "start": 7,
+    },
+    "precedingWhitespace": " ",
+    "raw": "*",
+    "text": "*",
+    "type": "ASTERISK",
+  },
+  Object {
+    "loc": Object {
+      "end": 13,
+      "start": 9,
+    },
+    "precedingWhitespace": " ",
+    "raw": "FROM",
+    "text": "FROM",
+    "type": "RESERVED_COMMAND",
+  },
+  Object {
+    "loc": Object {
+      "end": 17,
+      "start": 14,
+    },
+    "precedingWhitespace": " ",
+    "raw": "foo",
+    "text": "foo",
+    "type": "IDENTIFIER",
+  },
+  Object {
+    "loc": Object {
+      "end": 18,
+      "start": 17,
+    },
+    "precedingWhitespace": undefined,
+    "raw": ";",
+    "text": ";",
+    "type": "DELIMITER",
+  },
+]
+`;