Repository: paustint/soql-parser-js (ts)

Issue:
Improve apex bind variable parsing
- [x] Feature
- [ ] Bug

Carried over from #153

## Description
Apex bind variables only support the most basic syntax, which should be improved to support additional common use-cases.

* `:someVarInstance.foo(bar, baz, ...)` Allow function calls with varying number of arguments
* `:new Map<Id, SObject>(someVar).keySet()` Allow new constructors with generics and arguments with chained method calls



Pull request description:
Improve apex bind variable parsing #155
Improved support for apex expressions

Added test-cases for the following queries:
* `SELECT Id, (SELECT Id FROM Contacts WHERE Id IN :contactMap.keySet()) FROM Account WHERE Id IN :accountMap.keySet()`
* `SELECT Id, (SELECT Id FROM Contacts WHERE Id IN :contact_900Map.keySet()) FROM Account WHERE Id IN :acco INVALID untMap.keySet()`
* `SELECT Id FROM Account WHERE Id IN :new Map<Id, SObject>(someVar).keySet()`
* `SELECT Id FROM Account WHERE Id IN :new Map<Id, SObject>(someVar).getSomeClass().records`
* `SELECT Id FROM SBQQ__QuoteTerm__c WHERE SBQQ__StandardTerm__c = :CPQ_Hard_Coded_Ids__c.getInstance().Standard_Quote_Term_Id__c`
* `SELECT Id FROM Opportunity WHERE SBQQ__StandardTerm__c = :quotes[3].SBQQ__QuoteLine__r[0].Term__c`

resolves #155

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e3ea948..4391a1d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,14 @@
 # Changelog
 
-## 4.1.0
+## 4.2.0
+
+June 8, 2021
+
+#155 - Apex bind variable support is improved to allow parsing of more complex Apex.
+
+Review test cases 112 - 117 for examples of supported apex bind variables.
+
+## 4.1.1
 
 June 6, 2021
 
diff --git a/src/models.ts b/src/models.ts
index 0034742..3363946 100644
--- a/src/models.ts
+++ b/src/models.ts
@@ -201,7 +201,47 @@ export interface ExpressionContext {
 }
 
 export interface ApexBindVariableExpressionContext {
+  apex: CstNode[];
+  COLON: IToken[];
+  DECIMAL?: IToken[];
+}
+
+export interface ApexBindVariableIdentifierContext {
   Identifier: IToken[];
+  apexBindVariableFunctionArrayAccessor?: CstNode[];
+}
+
+export interface ApexBindVariableNewInstantiationContext {
+  new: IToken[];
+  function: IToken[];
+  apexBindVariableGeneric?: CstNode[];
+  apexBindVariableFunctionParams: CstNode[];
+  apexBindVariableFunctionArrayAccessor?: CstNode[];
+}
+
+export interface ApexBindVariableFunctionCallContext {
+  function: IToken[];
+  apexBindVariableFunctionParams: CstNode[];
+  apexBindVariableFunctionArrayAccessor?: CstNode[];
+}
+
+export interface ApexBindVariableGenericContext {
+  COMMA: IToken[];
+  GREATER_THAN: IToken[];
+  LESS_THAN: IToken[];
+  parameter: IToken[];
+}
+
+export interface ApexBindVariableFunctionParamsContext {
+  L_PAREN: IToken[];
+  R_PAREN: IToken[];
+  parameter?: IToken[];
+}
+
+export interface ApexBindVariableFunctionArrayAccessorContext {
+  L_SQUARE_BRACKET: IToken[];
+  R_SQUARE_BRACKET: IToken[];
+  value: IToken[];
 }
 
 export interface ExpressionOperatorContext {
diff --git a/src/parser/lexer.ts b/src/parser/lexer.ts
index 7b8b3ab..46d28b6 100644
--- a/src/parser/lexer.ts
+++ b/src/parser/lexer.ts
@@ -263,6 +263,7 @@ export const AboveOrBelow = createToken({
   longer_alt: Identifier,
   categories: [Keyword, Identifier],
 });
+export const ApexNew = createToken({ name: 'new', pattern: /new/i, longer_alt: Identifier, categories: [Keyword, Identifier] });
 export const At = createToken({ name: 'AT', pattern: /AT/i, longer_alt: Identifier, categories: [Keyword, Identifier] });
 export const Below = createToken({ name: 'BELOW', pattern: /BELOW/i, longer_alt: Identifier, categories: [Keyword, Identifier] });
 export const DataCategory = createToken({
@@ -749,6 +750,8 @@ export const Comma = createToken({ name: 'COMMA', pattern: ',', categories: [Sym
 export const Asterisk = createToken({ name: 'ASTERISK', pattern: '*', categories: [SymbolIdentifier] });
 export const LParen = createToken({ name: 'L_PAREN', pattern: '(', categories: [SymbolIdentifier] });
 export const RParen = createToken({ name: 'R_PAREN', pattern: ')', categories: [SymbolIdentifier] });
+export const LSquareBracket = createToken({ name: 'L_SQUARE_BRACKET', pattern: '[', categories: [SymbolIdentifier] });
+export const RSquareBracket = createToken({ name: 'R_SQUARE_BRACKET', pattern: ']', categories: [SymbolIdentifier] });
 export const Plus = createToken({ name: 'PLUS', pattern: '+', categories: [SymbolIdentifier] });
 export const Minus = createToken({ name: 'MINUS', pattern: '-', categories: [SymbolIdentifier] });
 
@@ -913,6 +916,7 @@ export const allTokens = [
 
   AboveOrBelow,
   Above,
+  ApexNew,
   At,
   Below,
   DataCategory,
@@ -1035,6 +1039,8 @@ export const allTokens = [
   Asterisk,
   LParen,
   RParen,
+  LSquareBracket,
+  RSquareBracket,
   Plus,
   Minus,
 ];
diff --git a/src/parser/parser.ts b/src/parser/parser.ts
index 1d9f7dc..7a0cc25 100644
--- a/src/parser/parser.ts
+++ b/src/parser/parser.ts
@@ -33,6 +33,7 @@ export class SoqlParser extends CstParser {
   private $_aggregateFunction: any = undefined;
   private $_otherFunction: any = undefined;
   private $_atomicExpression: any = undefined;
+  private $_apexBindVariableExpression: any = undefined;
   private $_arrayExpression: any = undefined;
   private $_relationalOperator: any = undefined;
   private $_selectClause: any = undefined;
@@ -617,7 +618,79 @@ export class SoqlParser extends CstParser {
 
   private apexBindVariableExpression = this.RULE('apexBindVariableExpression', () => {
     this.CONSUME(lexer.Colon);
+    // First item in list could optionally be a new instantiation
+    this.OPTION(() => {
+      this.SUBRULE(this.apexBindVariableNewInstantiation, { LABEL: 'apex' });
+      this.OPTION1(() => {
+        this.CONSUME(lexer.Decimal);
+      });
+    });
+    // Chained function calls or nested arguments with function calls at the end
+    this.MANY_SEP({
+      SEP: lexer.Decimal,
+      DEF: () => {
+        this.OR(
+          this.$_apexBindVariableExpression ||
+            (this.$_apexBindVariableExpression = [
+              { ALT: () => this.SUBRULE(this.apexBindVariableFunctionCall, { LABEL: 'apex' }) },
+              { ALT: () => this.SUBRULE(this.apexBindVariableIdentifier, { LABEL: 'apex' }) },
+            ]),
+        );
+      },
+    });
+  });
+
+  private apexBindVariableIdentifier = this.RULE('apexBindVariableIdentifier', () => {
     this.CONSUME(lexer.Identifier);
+    this.OPTION(() => this.SUBRULE(this.apexBindVariableFunctionArrayAccessor));
+  });
+
+  private apexBindVariableNewInstantiation = this.RULE('apexBindVariableNewInstantiation', () => {
+    this.CONSUME(lexer.ApexNew, { LABEL: 'new' });
+    this.CONSUME(lexer.Identifier, { LABEL: 'function' });
+    this.OPTION(() => {
+      this.SUBRULE(this.apexBindVariableGeneric);
+    });
+    this.SUBRULE(this.apexBindVariableFunctionParams);
+    this.OPTION1(() => this.SUBRULE(this.apexBindVariableFunctionArrayAccessor));
+  });
+
+  private apexBindVariableFunctionCall = this.RULE('apexBindVariableFunctionCall', () => {
+    this.CONSUME(lexer.Identifier, { LABEL: 'function' });
+    this.SUBRULE(this.apexBindVariableFunctionParams);
+    this.OPTION(() => this.SUBRULE(this.apexBindVariableFunctionArrayAccessor));
+  });
+
+  private apexBindVariableGeneric = this.RULE('apexBindVariableGeneric', () => {
+    this.CONSUME(lexer.LessThan);
+    this.AT_LEAST_ONE_SEP({
+      SEP: lexer.Comma,
+      DEF: () => {
+        this.CONSUME(lexer.Identifier, { LABEL: 'parameter' });
+      },
+    });
+    this.CONSUME(lexer.GreaterThan);
+  });
+
+  private apexBindVariableFunctionParams = this.RULE('apexBindVariableFunctionParams', () => {
+    this.CONSUME(lexer.LParen);
+    this.MANY_SEP({
+      SEP: lexer.Comma,
+      DEF: () => {
+        this.CONSUME(lexer.Identifier, { LABEL: 'parameter' });
+      },
+    });
+    this.CONSUME(lexer.RParen);
+  });
+
+  // foo[3] or foo[somIntVariable]
+  private apexBindVariableFunctionArrayAccessor = this.RULE('apexBindVariableFunctionArrayAccessor', () => {
+    this.CONSUME(lexer.LSquareBracket);
+    this.OR([
+      { ALT: () => this.CONSUME(lexer.UnsignedInteger, { LABEL: 'value' }) },
+      { ALT: () => this.CONSUME(lexer.Identifier, { LABEL: 'value' }) },
+    ]);
+    this.CONSUME(lexer.RSquareBracket);
   });
 
   private arrayExpression = this.RULE('arrayExpression', () => {
diff --git a/src/parser/visitor.ts b/src/parser/visitor.ts
index ec6199a..097dc2f 100644
--- a/src/parser/visitor.ts
+++ b/src/parser/visitor.ts
@@ -34,6 +34,12 @@ import {
 } from '../api/api-models';
 import {
   ApexBindVariableExpressionContext,
+  ApexBindVariableFunctionArrayAccessorContext,
+  ApexBindVariableFunctionCallContext,
+  ApexBindVariableFunctionParamsContext,
+  ApexBindVariableGenericContext,
+  ApexBindVariableIdentifierContext,
+  ApexBindVariableNewInstantiationContext,
   ArrayExpressionWithType,
   AtomicExpressionContext,
   BooleanContext,
@@ -791,7 +797,48 @@ class SOQLVisitor extends BaseSoqlVisitor {
   }
 
   apexBindVariableExpression(ctx: ApexBindVariableExpressionContext): string {
-    return ctx.Identifier[0].image;
+    return ctx.apex.map(item => this.visit(item)).join('.');
+  }
+
+  apexBindVariableIdentifier(ctx: ApexBindVariableIdentifierContext): string {
+    let output = ctx.Identifier[0].image;
+    if (ctx.apexBindVariableFunctionArrayAccessor) {
+      output += this.visit(ctx.apexBindVariableFunctionArrayAccessor[0]);
+    }
+    return output;
+  }
+
+  apexBindVariableNewInstantiation(ctx: ApexBindVariableNewInstantiationContext): string {
+    let output = `new ${ctx.function[0].image}`;
+    if (ctx.apexBindVariableGeneric) {
+      output += this.visit(ctx.apexBindVariableGeneric[0]);
+    }
+    output += this.visit(ctx.apexBindVariableFunctionParams[0]);
+    if (ctx.apexBindVariableFunctionArrayAccessor) {
+      output += this.visit(ctx.apexBindVariableFunctionArrayAccessor[0]);
+    }
+    return output;
+  }
+
+  apexBindVariableFunctionCall(ctx: ApexBindVariableFunctionCallContext): string {
+    let output = `${ctx.function[0].image}${this.visit(ctx.apexBindVariableFunctionParams[0])}`;
+    if (ctx.apexBindVariableFunctionArrayAccessor) {
+      output += this.visit(ctx.apexBindVariableFunctionArrayAccessor[0]);
+    }
+    return output;
+  }
+
+  apexBindVariableGeneric(ctx: ApexBindVariableGenericContext): string {
+    return `<${ctx.parameter.map(item => item.image).join(', ')}>`;
+  }
+
+  apexBindVariableFunctionParams(ctx: ApexBindVariableFunctionParamsContext): string {
+    const params = Array.isArray(ctx.parameter) ? ctx.parameter : [];
+    return `(${params.map(item => item.image).join(', ')})`;
+  }
+
+  apexBindVariableFunctionArrayAccessor(ctx: ApexBindVariableFunctionArrayAccessorContext): string {
+    return `[${ctx.value[0].image}]`;
   }
 
   arrayExpression(ctx: ValueContext): ArrayExpressionWithType[] {


Test patch:
diff --git a/test/test-cases-for-is-valid.ts b/test/test-cases-for-is-valid.ts
index 74740c2..f75b851 100644
--- a/test/test-cases-for-is-valid.ts
+++ b/test/test-cases-for-is-valid.ts
@@ -466,5 +466,22 @@ export const testCases: TestCaseForFormat[] = [
     soql: `SELECT Id FROM Account WHERE NOT Name = '2'`,
     isValid: true,
   },
+  {
+    testCase: 158,
+    options: { allowApexBindVariables: true },
+    soql: `
+SELECT Id
+FROM Account
+WHERE
+  Id IN :new Map<
+    Id,
+    SObject
+  >
+  (someVar)
+  .getSomeClass()
+  .records
+    `,
+    isValid: true,
+  },
 ];
 export default testCases;
diff --git a/test/test-cases.ts b/test/test-cases.ts
index 6f6d5d3..f7e8015 100644
--- a/test/test-cases.ts
+++ b/test/test-cases.ts
@@ -2265,18 +2265,129 @@ export const testCases: TestCase[] = [
   },
   {
     testCase: 112,
-    options: { allowApexBindVariables: true, ignoreParseErrors: true },
+    options: { allowApexBindVariables: true },
     soql: `SELECT Id, (SELECT Id FROM Contacts WHERE Id IN :contactMap.keySet()) FROM Account WHERE Id IN :accountMap.keySet()`,
-    soqlComposed: `SELECT Id, (SELECT Id FROM Contacts) FROM Account`,
     output: {
       fields: [
         {
           type: 'Field',
           field: 'Id',
         },
-        { type: 'FieldSubquery', subquery: { fields: [{ type: 'Field', field: 'Id' }], relationshipName: 'Contacts' } },
+        {
+          type: 'FieldSubquery',
+          subquery: {
+            fields: [{ type: 'Field', field: 'Id' }],
+            relationshipName: 'Contacts',
+            where: { left: { field: 'Id', literalType: 'APEX_BIND_VARIABLE', operator: 'IN', value: 'contactMap.keySet()' } },
+          },
+        },
       ],
       sObject: 'Account',
+      where: { left: { field: 'Id', literalType: 'APEX_BIND_VARIABLE', operator: 'IN', value: 'accountMap.keySet()' } },
+    },
+  },
+  {
+    testCase: 113,
+    options: { allowApexBindVariables: true, ignoreParseErrors: true },
+    soql: `SELECT Id, (SELECT Id FROM Contacts WHERE Id IN :contact_900Map.keySet()) FROM Account WHERE Id IN :acco INVALID untMap.keySet()`,
+    soqlComposed: `SELECT Id, (SELECT Id FROM Contacts WHERE Id IN :contact_900Map.keySet()) FROM Account`,
+    output: {
+      fields: [
+        {
+          type: 'Field',
+          field: 'Id',
+        },
+        {
+          type: 'FieldSubquery',
+          subquery: {
+            fields: [{ type: 'Field', field: 'Id' }],
+            relationshipName: 'Contacts',
+            where: { left: { field: 'Id', literalType: 'APEX_BIND_VARIABLE', operator: 'IN', value: 'contact_900Map.keySet()' } },
+          },
+        },
+      ],
+      sObject: 'Account',
+    },
+  },
+  {
+    testCase: 114,
+    options: { allowApexBindVariables: true },
+    soql: `SELECT Id FROM Account WHERE Id IN :new Map<Id, SObject>(someVar).keySet()`,
+    output: {
+      fields: [
+        {
+          type: 'Field',
+          field: 'Id',
+        },
+      ],
+      sObject: 'Account',
+      where: { left: { field: 'Id', literalType: 'APEX_BIND_VARIABLE', operator: 'IN', value: 'new Map<Id, SObject>(someVar).keySet()' } },
+    },
+  },
+  {
+    testCase: 115,
+    options: { allowApexBindVariables: true },
+    soql: `SELECT Id FROM Account WHERE Id IN :new Map<Id, SObject>(someVar).getSomeClass().records`,
+    output: {
+      fields: [
+        {
+          type: 'Field',
+          field: 'Id',
+        },
+      ],
+      sObject: 'Account',
+      where: {
+        left: {
+          field: 'Id',
+          literalType: 'APEX_BIND_VARIABLE',
+          operator: 'IN',
+          value: 'new Map<Id, SObject>(someVar).getSomeClass().records',
+        },
+      },
+    },
+  },
+  {
+    testCase: 116,
+    options: { allowApexBindVariables: true },
+    soql: `SELECT Id FROM SBQQ__QuoteTerm__c WHERE SBQQ__StandardTerm__c = :CPQ_Hard_Coded_Ids__c.getInstance().Standard_Quote_Term_Id__c`,
+    output: {
+      fields: [
+        {
+          type: 'Field',
+          field: 'Id',
+        },
+      ],
+      sObject: 'SBQQ__QuoteTerm__c',
+      where: {
+        left: {
+          field: 'SBQQ__StandardTerm__c',
+          literalType: 'APEX_BIND_VARIABLE',
+          operator: '=',
+          value: 'CPQ_Hard_Coded_Ids__c.getInstance().Standard_Quote_Term_Id__c',
+        },
+      },
+    },
+  },
+  {
+    testCase: 117,
+    options: { allowApexBindVariables: true },
+    soql: `SELECT Id FROM Opportunity WHERE SBQQ__StandardTerm__c = :quotes[3].SBQQ__QuoteLine__r[0].Term__c`,
+    output: {
+      fields: [
+        {
+          type: 'Field',
+          field: 'Id',
+        },
+      ],
+      sObject: 'Opportunity',
+      where: {
+        left: {
+          field: 'SBQQ__StandardTerm__c',
+          literalType: 'APEX_BIND_VARIABLE',
+          operator: '=',
+          value: 'quotes[3].SBQQ__QuoteLine__r[0].Term__c',
+        },
+      },
     },
   },
 ];
diff --git a/test/test.spec.ts b/test/test.spec.ts
index 20c9af7..dd41818 100644
--- a/test/test.spec.ts
+++ b/test/test.spec.ts
@@ -12,7 +12,7 @@ const replacements = [{ matching: / last /i, replace: ' LAST ' }];
 // Uncomment these to easily test one specific query - useful for troubleshooting/bugfixing
 
 // describe.only('parse queries', () => {
-//   const testCase = testCases.find(tc => tc.testCase === 109);
+//   const testCase = testCases.find(tc => tc.testCase === 117);
 //   it(`should correctly parse test case ${testCase.testCase} - ${testCase.soql}`, () => {
 //     const soqlQuery = parseQuery(testCase.soql, testCase.options);
 //     console.log(soqlQuery);