Repository: sql-formatter-org/sql-formatter
Pull request #405 (2022-08-24 13:05:59)

Replace open- and closeParens configs with extraParens
This eliminates the possibility of making several bugs:

- not including '(' and ')', which are always mandatory.
- only specifying one type of the pair, e.g. '[' without ']'.

Plus it also makes the config shorter.

Also eliminated Trino `{-` and `-}` paren types support for now. The previous implementation didn't work anyway as the `{` was always matched before `{-`, so it ended up being tokenized as `{` and `-`, which ended up being formatted correctly, as the contents between `{` and `}` is formatted without spaces surrounding it. The contents of these `{- ... -}` expressions really need to be parsed as a different thing altogether. I'm thinking it might be better to treat them as strings with `{-` and `-}`  delimiters - that would eliminate lots of tricky corner cases with formatting of these and also we would avoid spending too much effort on a Trino-specific syntax that's not supported anywhere else.


Patch:
diff --git a/src/languages/bigquery/bigquery.formatter.ts b/src/languages/bigquery/bigquery.formatter.ts
index e8da2c78..be547531 100644
--- a/src/languages/bigquery/bigquery.formatter.ts
+++ b/src/languages/bigquery/bigquery.formatter.ts
@@ -151,8 +151,7 @@ export default class BigQueryFormatter extends Formatter {
       reservedPhrases,
       reservedKeywords: keywords,
       reservedFunctionNames: functions,
-      openParens: ['(', '['],
-      closeParens: [')', ']'],
+      extraParens: ['[]'],
       stringTypes: [
         // The triple-quoted strings are listed first, so they get matched first.
         // Otherwise the first two quotes of """ will get matched as an empty "" string.
diff --git a/src/languages/hive/hive.formatter.ts b/src/languages/hive/hive.formatter.ts
index 679a6e3b..2623ccf7 100644
--- a/src/languages/hive/hive.formatter.ts
+++ b/src/languages/hive/hive.formatter.ts
@@ -91,8 +91,7 @@ export default class HiveFormatter extends Formatter {
       reservedDependentClauses: ['WHEN', 'ELSE'],
       reservedKeywords: keywords,
       reservedFunctionNames: functions,
-      openParens: ['(', '['],
-      closeParens: [')', ']'],
+      extraParens: ['[]'],
       stringTypes: ['""', "''"],
       identTypes: ['``'],
       variableTypes: [{ quote: '{}', prefixes: ['$'], requirePrefix: true }],
diff --git a/src/languages/n1ql/n1ql.formatter.ts b/src/languages/n1ql/n1ql.formatter.ts
index 99df32e9..d3800bba 100644
--- a/src/languages/n1ql/n1ql.formatter.ts
+++ b/src/languages/n1ql/n1ql.formatter.ts
@@ -94,8 +94,7 @@ export default class N1qlFormatter extends Formatter {
       // which simplifies writing tests that are shared between all dialects.
       stringTypes: ['""', "''"],
       identTypes: ['``'],
-      openParens: ['(', '[', '{'],
-      closeParens: [')', ']', '}'],
+      extraParens: ['[]', '{}'],
       paramTypes: { positional: true, numbered: ['$'], named: ['$'] },
       lineCommentTypes: ['#', '--'],
       operators: N1qlFormatter.operators,
diff --git a/src/languages/postgresql/postgresql.formatter.ts b/src/languages/postgresql/postgresql.formatter.ts
index 5f53af1f..4175ec50 100644
--- a/src/languages/postgresql/postgresql.formatter.ts
+++ b/src/languages/postgresql/postgresql.formatter.ts
@@ -326,8 +326,7 @@ export default class PostgreSqlFormatter extends Formatter {
       reservedPhrases,
       reservedKeywords: keywords,
       reservedFunctionNames: functions,
-      openParens: ['(', '['],
-      closeParens: [')', ']'],
+      extraParens: ['[]'],
       stringTypes: ['$$', { quote: "''", prefixes: ['B', 'E', 'X', 'U&'] }],
       identTypes: [{ quote: '""', prefixes: ['U&'] }],
       identChars: { rest: '$' },
diff --git a/src/languages/spark/spark.formatter.ts b/src/languages/spark/spark.formatter.ts
index 0518dece..46f876a6 100644
--- a/src/languages/spark/spark.formatter.ts
+++ b/src/languages/spark/spark.formatter.ts
@@ -124,8 +124,7 @@ export default class SparkFormatter extends Formatter {
       supportsXor: true,
       reservedKeywords: keywords,
       reservedFunctionNames: functions,
-      openParens: ['(', '['],
-      closeParens: [')', ']'],
+      extraParens: ['[]'],
       stringTypes: [
         { quote: "''", prefixes: ['R', 'X'] },
         { quote: '""', prefixes: ['R', 'X'] },
diff --git a/src/languages/trino/trino.formatter.ts b/src/languages/trino/trino.formatter.ts
index 51845f8b..ac382045 100644
--- a/src/languages/trino/trino.formatter.ts
+++ b/src/languages/trino/trino.formatter.ts
@@ -131,9 +131,11 @@ export default class TrinoFormatter extends Formatter {
       reservedDependentClauses: ['WHEN', 'ELSE'],
       reservedKeywords: keywords,
       reservedFunctionNames: functions,
+      // Trino also supports {- ... -} parenthesis.
+      // The formatting of these currently works out as a result of { and -
+      // not getting a space added in-between.
       // https://trino.io/docs/current/sql/match-recognize.html#row-pattern-syntax
-      openParens: ['(', '[', '{', '{-'],
-      closeParens: [')', ']', '}', '-}'],
+      extraParens: ['[]', '{}'],
       // https://trino.io/docs/current/language/types.html#string
       // https://trino.io/docs/current/language/types.html#varbinary
       stringTypes: [{ quote: "''", prefixes: ['X', 'U&'] }],
diff --git a/src/lexer/Tokenizer.ts b/src/lexer/Tokenizer.ts
index 223aa9e5..36183732 100644
--- a/src/lexer/Tokenizer.ts
+++ b/src/lexer/Tokenizer.ts
@@ -29,9 +29,6 @@ export default class Tokenizer {
       [TokenType.LINE_COMMENT]: {
         regex: regex.lineComment(cfg.lineCommentTypes ?? ['--']),
       },
-      [TokenType.COMMA]: { regex: /[,]/y },
-      [TokenType.OPEN_PAREN]: { regex: regex.parenthesis(cfg.openParens ?? ['(']) },
-      [TokenType.CLOSE_PAREN]: { regex: regex.parenthesis(cfg.closeParens ?? [')']) },
       [TokenType.QUOTED_IDENTIFIER]: { regex: regex.string(cfg.identTypes) },
       [TokenType.NUMBER]: {
         regex:
@@ -105,6 +102,9 @@ export default class Tokenizer {
         regex: regex.identifier(cfg.identChars),
       },
       [TokenType.DELIMITER]: { regex: /[;]/uy },
+      [TokenType.COMMA]: { regex: /[,]/y },
+      [TokenType.OPEN_PAREN]: { regex: regex.parenthesis('open', cfg.extraParens) },
+      [TokenType.CLOSE_PAREN]: { regex: regex.parenthesis('close', cfg.extraParens) },
       [TokenType.OPERATOR]: {
         regex: regex.operator('+-/%&|^><=.:$@#?~!', [
           '<>',
diff --git a/src/lexer/TokenizerEngine.ts b/src/lexer/TokenizerEngine.ts
index 0372daac..12ee9cfc 100644
--- a/src/lexer/TokenizerEngine.ts
+++ b/src/lexer/TokenizerEngine.ts
@@ -65,9 +65,6 @@ export default class TokenizerEngine {
     return (
       this.matchToken(TokenType.BLOCK_COMMENT) ||
       this.matchToken(TokenType.LINE_COMMENT) ||
-      this.matchToken(TokenType.COMMA) ||
-      this.matchToken(TokenType.OPEN_PAREN) ||
-      this.matchToken(TokenType.CLOSE_PAREN) ||
       this.matchToken(TokenType.QUOTED_IDENTIFIER) ||
       this.matchToken(TokenType.NUMBER) ||
       this.matchReservedWordToken(previousToken) ||
@@ -79,6 +76,9 @@ export default class TokenizerEngine {
       this.matchToken(TokenType.STRING) ||
       this.matchToken(TokenType.IDENTIFIER) ||
       this.matchToken(TokenType.DELIMITER) ||
+      this.matchToken(TokenType.COMMA) ||
+      this.matchToken(TokenType.OPEN_PAREN) ||
+      this.matchToken(TokenType.CLOSE_PAREN) ||
       this.matchToken(TokenType.OPERATOR) ||
       this.matchToken(TokenType.ASTERISK)
     );
diff --git a/src/lexer/TokenizerOptions.ts b/src/lexer/TokenizerOptions.ts
index 9c3aa285..2cce0e73 100644
--- a/src/lexer/TokenizerOptions.ts
+++ b/src/lexer/TokenizerOptions.ts
@@ -69,10 +69,8 @@ export interface TokenizerOptions {
   identTypes: QuoteType[];
   // Types of quotes to use for variables
   variableTypes?: VariableType[];
-  // Open-parenthesis characters
-  openParens?: ('(' | '[' | '{' | '{-')[];
-  // Close-parenthesis characters
-  closeParens?: (')' | ']' | '}' | '-}')[];
+  // Types of additional parenthesis types to support
+  extraParens?: ('[]' | '{}')[];
   // Types of parameter placeholders supported with prepared statements
   paramTypes?: ParamTypes;
   // Line comment types to support, defaults to --
diff --git a/src/lexer/regexFactory.ts b/src/lexer/regexFactory.ts
index 06864f14..ff48f5a3 100644
--- a/src/lexer/regexFactory.ts
+++ b/src/lexer/regexFactory.ts
@@ -1,13 +1,7 @@
 import { sortByLengthDesc } from 'src/utils';
 
 import type { IdentChars, QuoteType, VariableType } from './TokenizerOptions';
-import {
-  escapeParen,
-  escapeRegExp,
-  patternToRegex,
-  prefixesPattern,
-  withDashes,
-} from './regexUtil';
+import { escapeRegExp, patternToRegex, prefixesPattern, withDashes } from './regexUtil';
 
 /**
  * Builds a RegExp for valid line comments in a SQL dialect
@@ -17,11 +11,13 @@ export const lineComment = (lineCommentTypes: string[]) =>
   new RegExp(`(?:${lineCommentTypes.map(escapeRegExp).join('|')}).*?(?=\r\n|\r|\n|$)`, 'uy');
 
 /**
- * Builds a RegExp for matching parenthesis patterns, escaping them with `escapeParen`
- * @param {string[]} parens - list of strings that denote parenthesis patterns
+ * Builds a RegExp for matching either open- or close-parenthesis patterns
  */
-export const parenthesis = (parens: string[]): RegExp =>
-  patternToRegex(parens.map(escapeParen).join('|'));
+export const parenthesis = (kind: 'open' | 'close', extraParens: ('[]' | '{}')[] = []): RegExp => {
+  const index = kind === 'open' ? 0 : 1;
+  const parens = ['()', ...extraParens].map(pair => pair[index]);
+  return patternToRegex(parens.map(escapeRegExp).join('|'));
+};
 
 /**
  * Builds a RegExp containing all operators for a SQL dialect
diff --git a/src/lexer/regexUtil.ts b/src/lexer/regexUtil.ts
index 38b7a5cd..460a45e2 100644
--- a/src/lexer/regexUtil.ts
+++ b/src/lexer/regexUtil.ts
@@ -14,17 +14,6 @@ export const toCaseInsensitivePattern = (prefix: string): string =>
     .map(char => (/ /gu.test(char) ? '\\s+' : `[${char.toUpperCase()}${char.toLowerCase()}]`))
     .join('');
 
-/** Escapes paren characters for RegExp patterns */
-export const escapeParen = (paren: string): string => {
-  if (paren.length === 1) {
-    // A single punctuation character
-    return escapeRegExp(paren);
-  } else {
-    // longer word
-    return '\\b' + escapeRegExp(paren) + '\\b';
-  }
-};
-
 export const withDashes = (pattern: string): string => pattern + '(?:-' + pattern + ')*';
 
 // Converts ["a", "b"] to "(?:[Aa]|[Bb]|)" or "(?:[Aa]|[Bb])" when required = true


Test patch:
diff --git a/test/unit/Parser.test.ts b/test/unit/Parser.test.ts
index 76852bd6..262b36a4 100644
--- a/test/unit/Parser.test.ts
+++ b/test/unit/Parser.test.ts
@@ -11,8 +11,7 @@ describe('Parser', () => {
       reservedJoins: ['JOIN'],
       reservedFunctionNames: ['SQRT', 'CURRENT_TIME'],
       reservedKeywords: ['BETWEEN', 'LIKE', 'ON', 'USING'],
-      openParens: ['(', '[', '{'],
-      closeParens: [')', ']', '}'],
+      extraParens: ['[]', '{}'],
       stringTypes: ["''"],
       identTypes: ['""'],
     });