Repository: rokucommunity/brighterscript
Pull request #885 (2023-09-06 00:37:01)

Adds Leading Trivia to all tokens
Usefully as a way to get comments from expressions/statements

- Function, class, method, field, interface, namespace statements return valid data from `getLeadingTrivia()` function
- works with annotations as well


Patch:
diff --git a/src/astUtils/creators.ts b/src/astUtils/creators.ts
index 7f8e0518..a7447854 100644
--- a/src/astUtils/creators.ts
+++ b/src/astUtils/creators.ts
@@ -89,7 +89,8 @@ export function createToken<T extends TokenKind>(kind: T, text?: string, range =
         text: text ?? tokenDefaults[kind as string] ?? kind.toString().toLowerCase(),
         isReserved: !text || text === kind.toString(),
         range: range,
-        leadingWhitespace: ''
+        leadingWhitespace: '',
+        leadingTrivia: []
     };
 }
 
@@ -99,7 +100,8 @@ export function createIdentifier(name: string, range?: Range): Identifier {
         text: name,
         isReserved: false,
         range: range,
-        leadingWhitespace: ''
+        leadingWhitespace: '',
+        leadingTrivia: []
     };
 }
 
@@ -162,7 +164,8 @@ export function createCall(callee: Expression, args?: Expression[]) {
         callee,
         createToken(TokenKind.LeftParen, '('),
         createToken(TokenKind.RightParen, ')'),
-        args || []
+        args || [],
+        []
     );
 }
 
diff --git a/src/astUtils/reflection.spec.ts b/src/astUtils/reflection.spec.ts
index 5b849f89..3617aaf7 100644
--- a/src/astUtils/reflection.spec.ts
+++ b/src/astUtils/reflection.spec.ts
@@ -193,7 +193,8 @@ describe('reflection', () => {
             range: range,
             isReserved: false,
             charCode: 0,
-            leadingWhitespace: ''
+            leadingWhitespace: '',
+            leadingTrivia: []
         };
 
         const binary = new BinaryExpression(expr, token, expr);
diff --git a/src/lexer/Lexer.spec.ts b/src/lexer/Lexer.spec.ts
index 721f0925..6b026c93 100644
--- a/src/lexer/Lexer.spec.ts
+++ b/src/lexer/Lexer.spec.ts
@@ -1,8 +1,9 @@
 /* eslint no-template-curly-in-string: 0 */
 import { expect } from '../chai-config.spec';
 
-import { TokenKind } from './TokenKind';
+import { AllowedTriviaTokens, TokenKind } from './TokenKind';
 import { Lexer } from './Lexer';
+import type { Token } from './Token';
 import { isToken } from './Token';
 import { rangeToArray } from '../parser/Parser.spec';
 import { Range } from 'vscode-languageserver';
@@ -1377,6 +1378,63 @@ describe('lexer', () => {
             TokenKind.Eof
         ]);
     });
+
+    describe('trivia', () => {
+        function stringify(tokens: Token[]) {
+            return tokens
+                //exclude the explicit triva tokens since they'll be included in the leading/trailing arrays
+                .filter(x => !AllowedTriviaTokens.includes(x.kind))
+                .flatMap(x => [...x.leadingTrivia, x])
+                .map(x => x.text)
+                .join('');
+        }
+
+        it('combining token text and trivia can reproduce full input', () => {
+            const input = `
+                function   test(  )
+                    'comment
+                    print   alpha   '  blabla
+                end function 'trailing
+                'trailing2
+            `;
+            expect(
+                stringify(
+                    Lexer.scan(input).tokens
+                )
+            ).to.eql(input);
+        });
+
+        function expectTrivia(text: string, expected: Array<{ text: string; leadingTrivia?: string[]; trailingTrivia?: string[] }>) {
+            const tokens = Lexer.scan(text).tokens.filter(x => !AllowedTriviaTokens.includes(x.kind));
+            expect(
+                tokens.map(x => {
+                    return {
+                        text: x.text,
+                        leadingTrivia: x.leadingTrivia.map(x => x.text)
+                    };
+                })
+            ).to.eql(
+                expected.map(x => ({
+                    leadingTrivia: [],
+                    ...x
+                }))
+            );
+        }
+
+        it('associates trailing items on same line with the preceeding token', () => {
+            expectTrivia(
+                `'leading\n` +
+                `alpha = true 'trueComment\n` +
+                `'eof`
+                , [
+                    { leadingTrivia: [`'leading`, `\n`], text: `alpha` },
+                    { leadingTrivia: [` `], text: `=` },
+                    { leadingTrivia: [` `], text: `true` },
+                    //EOF
+                    { leadingTrivia: [` `, `'trueComment`, `\n`, `'eof`], text: `` }
+                ]);
+        });
+    });
 });
 
 function expectKinds(text: string, tokenKinds: TokenKind[]) {
diff --git a/src/lexer/Lexer.ts b/src/lexer/Lexer.ts
index 0493d8b9..08b4997b 100644
--- a/src/lexer/Lexer.ts
+++ b/src/lexer/Lexer.ts
@@ -1,5 +1,5 @@
 /* eslint-disable func-names */
-import { TokenKind, ReservedWords, Keywords, PreceedingRegexTypes } from './TokenKind';
+import { TokenKind, ReservedWords, Keywords, PreceedingRegexTypes, AllowedTriviaTokens } from './TokenKind';
 import type { Token } from './Token';
 import { isAlpha, isDecimalDigit, isAlphaNumeric, isHexDigit } from './Characters';
 import type { Range, Diagnostic } from 'vscode-languageserver';
@@ -62,6 +62,11 @@ export class Lexer {
      */
     private leadingWhitespace = '';
 
+    /**
+     * Contains trivia/comments, etc. before this line
+     */
+    private leadingTrivia: Token[] = [];
+
     /**
      * A convenience function, equivalent to `new Lexer().scan(toScan)`, that converts a string
      * containing BrightScript code to an array of `Token` objects that will later be used to build
@@ -103,12 +108,20 @@ export class Lexer {
             isReserved: false,
             text: '',
             range: util.createRange(this.lineBegin, this.columnBegin, this.lineEnd, this.columnEnd + 1),
-            leadingWhitespace: this.leadingWhitespace
+            leadingWhitespace: this.leadingWhitespace,
+            leadingTrivia: this.leadingTrivia
         });
         this.leadingWhitespace = '';
         return this;
     }
 
+    /**
+    * Pushes a token into the leadingTrivia list
+    */
+    private pushTrivia(token: Token) {
+        this.leadingTrivia.push(token);
+    }
+
     /**
      * Fill in missing/invalid options with defaults
      */
@@ -1031,6 +1044,13 @@ export class Lexer {
         return false;
     }
 
+    /**
+    * Determine if this token is a trivia token
+    */
+    private isTrivia(token: Token) {
+        return AllowedTriviaTokens.includes(token.kind);
+    }
+
     /**
      * Creates a `Token` and adds it to the `tokens` array.
      * @param kind the type of token to produce.
@@ -1042,8 +1062,15 @@ export class Lexer {
             text: text,
             isReserved: ReservedWords.has(text.toLowerCase()),
             range: this.rangeOf(),
-            leadingWhitespace: this.leadingWhitespace
+            leadingWhitespace: this.leadingWhitespace,
+            leadingTrivia: []
         };
+        if (this.isTrivia(token)) {
+            this.pushTrivia(token);
+        } else {
+            token.leadingTrivia.push(...this.leadingTrivia);
+            this.leadingTrivia = [];
+        }
         this.leadingWhitespace = '';
         this.tokens.push(token);
         this.sync();
diff --git a/src/lexer/Token.ts b/src/lexer/Token.ts
index 18acfe6a..8dfa811e 100644
--- a/src/lexer/Token.ts
+++ b/src/lexer/Token.ts
@@ -17,6 +17,11 @@ export interface Token {
      * Any leading whitespace found prior to this token. Excludes newline characters.
      */
     leadingWhitespace?: string;
+
+    /**
+     * Any tokens starting on the next line of the previous token, up to the start of this token
+     */
+    leadingTrivia: Token[];
 }
 
 /**
diff --git a/src/lexer/TokenKind.ts b/src/lexer/TokenKind.ts
index b98ed82a..cd1c16b7 100644
--- a/src/lexer/TokenKind.ts
+++ b/src/lexer/TokenKind.ts
@@ -665,3 +665,13 @@ export const PreceedingRegexTypes = new Set([
     TokenKind.Colon,
     TokenKind.Semicolon
 ]);
+
+/**
+ * The tokens that may be in leading trivia
+ */
+export const AllowedTriviaTokens: ReadonlyArray<TokenKind> = [
+    TokenKind.Newline,
+    TokenKind.Whitespace,
+    TokenKind.Comment,
+    TokenKind.Colon
+];
diff --git a/src/parser/AstNode.ts b/src/parser/AstNode.ts
index cd962e11..c924420d 100644
--- a/src/parser/AstNode.ts
+++ b/src/parser/AstNode.ts
@@ -10,6 +10,7 @@ import type { AnnotationExpression } from './Expression';
 import util from '../util';
 import { DynamicType } from '../types/DynamicType';
 import type { BscType } from '../types/BscType';
+import type { Token } from '../lexer/Token';
 
 /**
  * A BrightScript AST node
@@ -126,6 +127,10 @@ export abstract class AstNode {
             walkMode: WalkMode.visitAllRecursive
         });
     }
+
+    public getLeadingTrivia(): Token[] {
+        return [];
+    }
 }
 
 export abstract class Statement extends AstNode {
diff --git a/src/parser/Expression.ts b/src/parser/Expression.ts
index 64b3c270..ddb7e63e 100644
--- a/src/parser/Expression.ts
+++ b/src/parser/Expression.ts
@@ -187,6 +187,10 @@ export class FunctionExpression extends Expression implements TypedefProvider {
      */
     public functionStatement?: FunctionStatement;
 
+    public getLeadingTrivia(): Token[] {
+        return this.functionType?.leadingTrivia ?? [];
+    }
+
     /**
      * The range of the function, starting at the 'f' in function or 's' in sub (or the open paren if the keyword is missing),
      * and ending with the last n' in 'end function' or 'b' in 'end sub'
@@ -1414,6 +1418,10 @@ export class AnnotationExpression extends Expression {
         return this.call.args.map(e => expressionToValue(e, strict));
     }
 
+    public getLeadingTrivia(): Token[] {
+        return this.atToken.leadingTrivia;
+    }
+
     transpile(state: BrsTranspileState) {
         return [];
     }
diff --git a/src/parser/Parser.spec.ts b/src/parser/Parser.spec.ts
index d05b284d..ea6a16be 100644
--- a/src/parser/Parser.spec.ts
+++ b/src/parser/Parser.spec.ts
@@ -8,7 +8,7 @@ import type { AssignmentStatement, ClassStatement } from './Statement';
 import { PrintStatement, FunctionStatement, NamespaceStatement, ImportStatement } from './Statement';
 import { Range } from 'vscode-languageserver';
 import { DiagnosticMessages } from '../DiagnosticMessages';
-import { isAssignmentStatement, isBlock, isCallExpression, isCommentStatement, isDottedGetExpression, isFunctionStatement, isGroupingExpression, isIfStatement, isIndexedGetExpression, isPrintStatement, isTypeCastExpression } from '../astUtils/reflection';
+import { isAssignmentStatement, isBlock, isCallExpression, isClassStatement, isCommentStatement, isDottedGetExpression, isFunctionStatement, isGroupingExpression, isIfStatement, isIndexedGetExpression, isInterfaceStatement, isNamespaceStatement, isPrintStatement, isTypeCastExpression } from '../astUtils/reflection';
 import { expectDiagnosticsIncludes, expectTypeToBe, expectZeroDiagnostics } from '../testHelpers.spec';
 import { BrsTranspileState } from './BrsTranspileState';
 import { SourceNode } from 'source-map';
@@ -1529,6 +1529,177 @@ describe('parser', () => {
         });
 
     });
+
+    describe('leadingTrivia', () => {
+        it('gets leading trivia from functions', () => {
+            let { statements } = parse(`
+                ' Nice function, bro
+                function foo()
+                    return 1
+                end function
+            `);
+            const funcStatements = statements.filter(isFunctionStatement);
+            const fooTrivia = funcStatements[0].getLeadingTrivia();
+            expect(fooTrivia.length).to.be.greaterThan(0);
+            expect(fooTrivia.filter(t => t.kind === TokenKind.Comment).length).to.eq(1);
+        });
+
+        it('gets multiple lines of leading trivia', () => {
+            let { statements } = parse(`
+                ' Say hello to someone
+                '
+                ' @param {string} name the person you want to say hello to.
+                sub sayHello(name as string = "world")
+                end sub
+            `);
+            const funcStatements = statements.filter(isFunctionStatement);
+            const helloTrivia = funcStatements[0].getLeadingTrivia();
+            expect(helloTrivia.length).to.be.greaterThan(0);
+            expect(helloTrivia.filter(t => t.kind === TokenKind.Comment).length).to.eq(3);
+        });
+
+        it('gets leading trivia from classes', () => {
+            let { statements } = parse(`
+                ' hello
+                ' classes
+                class Hello
+                end class
+            `, ParseMode.BrighterScript);
+            const classStatements = statements.filter(isClassStatement);
+            const trivia = classStatements[0].getLeadingTrivia();
+            expect(trivia.length).to.be.greaterThan(0);
+            expect(trivia.filter(t => t.kind === TokenKind.Comment).length).to.eq(2);
+        });
+
+        it('gets leading trivia from functions with annotations', () => {
+            let { statements } = parse(`
+                ' hello comment 1
+                ' hello comment 2
+                @annotation
+                sub sayHello(name as string = "world")
+                end sub
+            `, ParseMode.BrighterScript);
+            const funcStatements = statements.filter(isFunctionStatement);
+            const helloTrivia = funcStatements[0].getLeadingTrivia();
+            expect(helloTrivia.length).to.be.greaterThan(0);
+            expect(helloTrivia.filter(t => t.kind === TokenKind.Comment).length).to.eq(2);
+        });
+
+
+        it('gets leading trivia from class methods', () => {
+            let { statements } = parse(`
+                ' hello
+                ' classes
+                class Hello
+
+                    ' Gets the value of PI
+                    ' Not a dessert
+                    function getPi() as float
+                        return 3.14
+                    end function
+
+                    ' Gets a dessert
+                    function getPie() as string
+                        return "Apple Pie"
+                    end function
+                end class
+            `, ParseMode.BrighterScript);
+            const classStatement = statements.filter(isClassStatement)[0];
+            const methodStatements = classStatement.methods;
+
+            // function getPi()
+            let trivia = methodStatements[0].getLeadingTrivia();
+            expect(trivia.length).to.be.greaterThan(0);
+            expect(trivia.filter(t => t.kind === TokenKind.Comment).length).to.eq(2);
+
+            // function getPie()
+            trivia = methodStatements[1].getLeadingTrivia();
+            expect(trivia.length).to.be.greaterThan(0);
+            expect(trivia.filter(t => t.kind === TokenKind.Comment).length).to.eq(1);
+        });
+
+        it('gets leading trivia from class fields', () => {
+            let { statements } = parse(`
+                ' hello
+                ' classes
+                class Thing
+                    ' like the sky
+                    ' or a blueberry, evn though that's purple
+                    color = "blue"
+
+                    ' My name
+                    public name as string
+
+                    ' Only I know how old I am
+                    private age = 42
+                end class
+            `, ParseMode.BrighterScript);
+            const classStatement = statements.filter(isClassStatement)[0];
+            const fieldStatements = classStatement.fields;
+
+            // color = "blue"
+            let trivia = fieldStatements[0].getLeadingTrivia();
+            expect(trivia.length).to.be.greaterThan(0);
+            expect(trivia.filter(t => t.kind === TokenKind.Comment).length).to.eq(2);
+
+            // public name as string
+            trivia = fieldStatements[1].getLeadingTrivia();
+            expect(trivia.length).to.be.greaterThan(0);
+            expect(trivia.filter(t => t.kind === TokenKind.Comment).length).to.eq(1);
+
+            // private age = 42
+            trivia = fieldStatements[2].getLeadingTrivia();
+            expect(trivia.length).to.be.greaterThan(0);
+            expect(trivia.filter(t => t.kind === TokenKind.Comment).length).to.eq(1);
+        });
+
+        it('gets leading trivia from interfaces', () => {
+            let { statements } = parse(`
+                ' Description of interface
+                interface myIface
+                    ' comment
+                    someField as integer
+
+                    'comment
+                    function someFunc() as string
+                end interface
+            `, ParseMode.BrighterScript);
+            const ifaceStatement = statements.filter(isInterfaceStatement)[0];
+            const fieldStatements = ifaceStatement.fields;
+            const methodStatements = ifaceStatement.methods;
+
+            // interface myIface
+            let trivia = ifaceStatement.getLeadingTrivia();
+            expect(trivia.length).to.be.greaterThan(0);
+            expect(trivia.filter(t => t.kind === TokenKind.Comment).length).to.eq(1);
+
+            // someField as integer
+            trivia = fieldStatements[0].getLeadingTrivia();
+            expect(trivia.length).to.be.greaterThan(0);
+            expect(trivia.filter(t => t.kind === TokenKind.Comment).length).to.eq(1);
+
+            // function someFunc() as string
+            trivia = methodStatements[0].getLeadingTrivia();
+            expect(trivia.length).to.be.greaterThan(0);
+            expect(trivia.filter(t => t.kind === TokenKind.Comment).length).to.eq(1);
+        });
+
+
+        it('gets leading trivia from namespaces', () => {
+            let { statements } = parse(`
+                ' Description of interface
+                namespace Nested.Name.Space
+
+                end  namespace
+            `, ParseMode.BrighterScript);
+            const nameSpaceStatement = statements.filter(isNamespaceStatement)[0];
+
+            // namespace Nested.Name.Space
+            let trivia = nameSpaceStatement.getLeadingTrivia();
+            expect(trivia.length).to.be.greaterThan(0);
+            expect(trivia.filter(t => t.kind === TokenKind.Comment).length).to.eq(1);
+        });
+    });
 });
 
 export function parse(text: string, mode?: ParseMode) {
diff --git a/src/parser/Parser.ts b/src/parser/Parser.ts
index 40c1486f..ebe5fed8 100644
--- a/src/parser/Parser.ts
+++ b/src/parser/Parser.ts
@@ -790,7 +790,8 @@ export class Parser {
                         start: this.peek().range.start,
                         end: this.peek().range.start
                     },
-                    leadingWhitespace: ''
+                    leadingWhitespace: '',
+                    leadingTrivia: []
                 };
             }
             let isSub = functionType?.kind === TokenKind.Sub;
diff --git a/src/parser/Statement.ts b/src/parser/Statement.ts
index d198e91b..3d74f684 100644
--- a/src/parser/Statement.ts
+++ b/src/parser/Statement.ts
@@ -378,6 +378,10 @@ export class FunctionStatement extends Statement implements TypedefProvider {
         }
     }
 
+    public getLeadingTrivia(): Token[] {
+        return util.concatAnnotationLeadingTrivia(this, this.func.getLeadingTrivia());
+    }
+
     transpile(state: BrsTranspileState) {
         //create a fake token using the full transpiled name
         let nameToken = {
@@ -1234,6 +1238,10 @@ export class NamespaceStatement extends Statement implements TypedefProvider {
         return name;
     }
 
+    public getLeadingTrivia(): Token[] {
+        return util.concatAnnotationLeadingTrivia(this, this.keyword.leadingTrivia);
+    }
+
     public getNameParts() {
         let parts = util.getAllDottedGetParts(this.nameExpression);
 
@@ -1389,6 +1397,11 @@ export class InterfaceStatement extends Statement implements TypedefProvider {
         return !!this.parentInterfaceName;
     }
 
+    public getLeadingTrivia(): Token[] {
+        return util.concatAnnotationLeadingTrivia(this,
+            this.tokens.interface.leadingTrivia);
+    }
+
 
     /**
      * The name of the interface WITH its leading namespace (if applicable)
@@ -1541,6 +1554,10 @@ export class InterfaceFieldStatement extends Statement implements TypedefProvide
         as: Token;
     };
 
+    public getLeadingTrivia(): Token[] {
+        return util.concatAnnotationLeadingTrivia(this, this.tokens.name.leadingTrivia);
+    }
+
     public get name() {
         return this.tokens.name.text;
     }
@@ -1630,6 +1647,10 @@ export class InterfaceMethodStatement extends Statement implements TypedefProvid
         as: Token;
     };
 
+    public getLeadingTrivia(): Token[] {
+        return util.concatAnnotationLeadingTrivia(this, this.tokens.functionType.leadingTrivia);
+    }
+
     walk(visitor: WalkVisitor, options: WalkOptions) {
         if (options.walkMode & InternalWalkMode.walkExpressions) {
             walk(this, 'returnTypeExpression', visitor, options);
@@ -1758,6 +1779,10 @@ export class ClassStatement extends Statement implements TypedefProvider {
         }
     }
 
+    public getLeadingTrivia(): Token[] {
+        return util.concatAnnotationLeadingTrivia(this, this.classKeyword.leadingTrivia);
+    }
+
     public memberMap = {} as Record<string, MemberStatement>;
     public methods = [] as MethodStatement[];
     public fields = [] as FieldStatement[];
@@ -2147,6 +2172,10 @@ export class MethodStatement extends FunctionStatement {
         return this.name.text;
     }
 
+    public getLeadingTrivia(): Token[] {
+        return util.concatAnnotationLeadingTrivia(this, this.func.getLeadingTrivia());
+    }
+
     transpile(state: BrsTranspileState) {
         if (this.name.text.toLowerCase() === 'new') {
             this.ensureSuperConstructorCall(state);
@@ -2236,7 +2265,8 @@ export class MethodStatement extends FunctionStatement {
                         text: 'super',
                         isReserved: false,
                         range: state.classStatement.name.range,
-                        leadingWhitespace: ''
+                        leadingWhitespace: '',
+                        leadingTrivia: []
                     }
                 ),
                 {
@@ -2244,14 +2274,16 @@ export class MethodStatement extends FunctionStatement {
                     text: '(',
                     isReserved: false,
                     range: state.classStatement.name.range,
-                    leadingWhitespace: ''
+                    leadingWhitespace: '',
+                    leadingTrivia: []
                 },
                 {
                     kind: TokenKind.RightParen,
                     text: ')',
                     isReserved: false,
                     range: state.classStatement.name.range,
-                    leadingWhitespace: ''
+                    leadingWhitespace: '',
+                    leadingTrivia: []
                 },
                 []
             )
@@ -2328,6 +2360,10 @@ export class FieldStatement extends Statement implements TypedefProvider {
 
     public readonly range: Range;
 
+    public getLeadingTrivia(): Token[] {
+        return util.concatAnnotationLeadingTrivia(this, this.accessModifier?.leadingTrivia ?? this.name?.leadingTrivia ?? []);
+    }
+
     transpile(state: BrsTranspileState): TranspileResult {
         throw new Error('transpile not implemented for ' + Object.getPrototypeOf(this).constructor.name);
     }
diff --git a/src/util.ts b/src/util.ts
index f13c3b50..88b37372 100644
--- a/src/util.ts
+++ b/src/util.ts
@@ -1334,6 +1334,10 @@ export class Util {
     }
 
 
+    public concatAnnotationLeadingTrivia(stmt: Statement, otherTrivia: Token[]): Token[] {
+        return [...(stmt.annotations?.map(anno => anno.getLeadingTrivia()).flat() ?? []), ...otherTrivia];
+    }
+
     /**
      * Create a SourceNode that maps every line to itself. Useful for creating maps for files
      * that haven't changed at all, but we still need the map


Test patch:
diff --git a/src/parser/tests/Parser.spec.ts b/src/parser/tests/Parser.spec.ts
index f6ec177e..d965d7c6 100644
--- a/src/parser/tests/Parser.spec.ts
+++ b/src/parser/tests/Parser.spec.ts
@@ -14,7 +14,8 @@ export function token(kind: TokenKind, text?: string): Token {
         text: text,
         isReserved: ReservedWords.has((text || '').toLowerCase()),
         range: interpolatedRange,
-        leadingWhitespace: ''
+        leadingWhitespace: '',
+        leadingTrivia: []
     };
 }
 
diff --git a/src/parser/tests/controlFlow/For.spec.ts b/src/parser/tests/controlFlow/For.spec.ts
index 0130b76d..1829bf73 100644
--- a/src/parser/tests/controlFlow/For.spec.ts
+++ b/src/parser/tests/controlFlow/For.spec.ts
@@ -103,25 +103,29 @@ describe('parser for loops', () => {
                 kind: TokenKind.For,
                 text: 'for',
                 isReserved: true,
-                range: Range.create(0, 0, 0, 3)
+                range: Range.create(0, 0, 0, 3),
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.Identifier,
                 text: 'i',
                 isReserved: false,
-                range: Range.create(0, 4, 0, 5)
+                range: Range.create(0, 4, 0, 5),
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.Equal,
                 text: '=',
                 isReserved: false,
-                range: Range.create(0, 6, 0, 7)
+                range: Range.create(0, 6, 0, 7),
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.IntegerLiteral,
                 text: '0',
                 isReserved: false,
-                range: Range.create(0, 8, 0, 9)
+                range: Range.create(0, 8, 0, 9),
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.To,
@@ -130,19 +134,22 @@ describe('parser for loops', () => {
                 range: {
                     start: { line: 0, character: 10 },
                     end: { line: 0, character: 12 }
-                }
+                },
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.IntegerLiteral,
                 text: '10',
                 isReserved: false,
-                range: Range.create(0, 13, 0, 15)
+                range: Range.create(0, 13, 0, 15),
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.Newline,
                 text: '\n',
                 isReserved: false,
-                range: Range.create(0, 15, 0, 16)
+                range: Range.create(0, 15, 0, 16),
+                leadingTrivia: []
             },
             // loop body isn't significant for location tracking, so helper functions are safe
             identifier('Rnd'),
@@ -154,7 +161,8 @@ describe('parser for loops', () => {
                 kind: TokenKind.EndFor,
                 text: 'end for',
                 isReserved: false,
-                range: Range.create(2, 0, 2, 8)
+                range: Range.create(2, 0, 2, 8),
+                leadingTrivia: []
             },
             EOF
         ]);
diff --git a/src/parser/tests/controlFlow/ForEach.spec.ts b/src/parser/tests/controlFlow/ForEach.spec.ts
index a340992f..f143659a 100644
--- a/src/parser/tests/controlFlow/ForEach.spec.ts
+++ b/src/parser/tests/controlFlow/ForEach.spec.ts
@@ -66,31 +66,36 @@ describe('parser foreach loops', () => {
                 kind: TokenKind.ForEach,
                 text: 'for each',
                 isReserved: true,
-                range: Range.create(0, 0, 0, 8)
+                range: Range.create(0, 0, 0, 8),
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.Identifier,
                 text: 'a',
                 isReserved: false,
-                range: Range.create(0, 9, 0, 10)
+                range: Range.create(0, 9, 0, 10),
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.Identifier,
                 text: 'in',
                 isReserved: true,
-                range: Range.create(0, 11, 0, 13)
+                range: Range.create(0, 11, 0, 13),
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.Identifier,
                 text: 'b',
                 isReserved: false,
-                range: Range.create(0, 14, 0, 15)
+                range: Range.create(0, 14, 0, 15),
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.Newline,
                 text: '\n',
                 isReserved: false,
-                range: Range.create(0, 15, 0, 16)
+                range: Range.create(0, 15, 0, 16),
+                leadingTrivia: []
             },
             // loop body isn't significant for location tracking, so helper functions are safe
             identifier('Rnd'),
@@ -102,7 +107,8 @@ describe('parser foreach loops', () => {
                 kind: TokenKind.EndFor,
                 text: 'end for',
                 isReserved: false,
-                range: Range.create(2, 0, 2, 7)
+                range: Range.create(2, 0, 2, 7),
+                leadingTrivia: []
             },
             EOF
         ]);
diff --git a/src/parser/tests/controlFlow/While.spec.ts b/src/parser/tests/controlFlow/While.spec.ts
index c8593028..c9d00247 100644
--- a/src/parser/tests/controlFlow/While.spec.ts
+++ b/src/parser/tests/controlFlow/While.spec.ts
@@ -76,19 +76,22 @@ describe('parser while statements', () => {
                 kind: TokenKind.While,
                 text: 'while',
                 isReserved: true,
-                range: Range.create(0, 0, 0, 5)
+                range: Range.create(0, 0, 0, 5),
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.True,
                 text: 'true',
                 isReserved: true,
-                range: Range.create(0, 6, 0, 10)
+                range: Range.create(0, 6, 0, 10),
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.Newline,
                 text: '\n',
                 isReserved: false,
-                range: Range.create(0, 10, 0, 11)
+                range: Range.create(0, 10, 0, 11),
+                leadingTrivia: []
             },
             // loop body isn't significant for location tracking, so helper functions are safe
             identifier('Rnd'),
@@ -100,7 +103,8 @@ describe('parser while statements', () => {
                 kind: TokenKind.EndWhile,
                 text: 'end while',
                 isReserved: false,
-                range: Range.create(2, 0, 2, 9)
+                range: Range.create(2, 0, 2, 9),
+                leadingTrivia: []
             },
             EOF
         ]);
diff --git a/src/parser/tests/expression/Call.spec.ts b/src/parser/tests/expression/Call.spec.ts
index 3d238dc2..d857ed49 100644
--- a/src/parser/tests/expression/Call.spec.ts
+++ b/src/parser/tests/expression/Call.spec.ts
@@ -14,7 +14,7 @@ describe('parser call expressions', () => {
     it('parses named function calls', () => {
         const { statements, diagnostics } = Parser.parse([
             identifier('RebootSystem'),
-            { kind: TokenKind.LeftParen, text: '(', range: null },
+            { kind: TokenKind.LeftParen, text: '(', range: null, leadingTrivia: [] },
             token(TokenKind.RightParen, ')'),
             EOF
         ]);
@@ -65,7 +65,7 @@ describe('parser call expressions', () => {
     it('allows closing parentheses on separate line', () => {
         const { statements, diagnostics } = Parser.parse([
             identifier('RebootSystem'),
-            { kind: TokenKind.LeftParen, text: '(', range: null },
+            { kind: TokenKind.LeftParen, text: '(', range: null, leadingTrivia: [] },
             token(TokenKind.Newline, '\\n'),
             token(TokenKind.Newline, '\\n'),
             token(TokenKind.RightParen, ')'),
@@ -128,9 +128,9 @@ describe('parser call expressions', () => {
     it('accepts arguments', () => {
         const { statements, diagnostics } = Parser.parse([
             identifier('add'),
-            { kind: TokenKind.LeftParen, text: '(', range: null },
+            { kind: TokenKind.LeftParen, text: '(', range: null, leadingTrivia: [] },
             token(TokenKind.IntegerLiteral, '1'),
-            { kind: TokenKind.Comma, text: ',', range: null },
+            { kind: TokenKind.Comma, text: ',', range: null, leadingTrivia: [] },
             token(TokenKind.IntegerLiteral, '2'),
             token(TokenKind.RightParen, ')'),
             EOF
diff --git a/src/parser/tests/statement/PrintStatement.spec.ts b/src/parser/tests/statement/PrintStatement.spec.ts
index 4e8a172c..4bae099e 100644
--- a/src/parser/tests/statement/PrintStatement.spec.ts
+++ b/src/parser/tests/statement/PrintStatement.spec.ts
@@ -80,21 +80,24 @@ describe('parser print statements', () => {
                 text: 'print',
                 isReserved: true,
                 range: Range.create(0, 0, 1, 5),
-                leadingWhitespace: ''
+                leadingWhitespace: '',
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.StringLiteral,
                 text: `"foo"`,
                 isReserved: false,
                 range: Range.create(0, 6, 0, 11),
-                leadingWhitespace: ''
+                leadingWhitespace: '',
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.Eof,
                 text: '\0',
                 isReserved: false,
                 range: Range.create(0, 11, 0, 12),
-                leadingWhitespace: ''
+                leadingWhitespace: '',
+                leadingTrivia: []
             }
         ]);
 
diff --git a/src/parser/tests/statement/ReturnStatement.spec.ts b/src/parser/tests/statement/ReturnStatement.spec.ts
index fe1563f9..9e0f2911 100644
--- a/src/parser/tests/statement/ReturnStatement.spec.ts
+++ b/src/parser/tests/statement/ReturnStatement.spec.ts
@@ -51,7 +51,7 @@ describe('parser return statements', () => {
             token(TokenKind.Newline, '\\n'),
             token(TokenKind.Return, 'return'),
             identifier('RebootSystem'),
-            { kind: TokenKind.LeftParen, text: '(', range: null },
+            { kind: TokenKind.LeftParen, text: '(', range: null, leadingTrivia: [] },
             token(TokenKind.RightParen, ')'),
             token(TokenKind.Newline, '\\n'),
             token(TokenKind.EndFunction, 'end function'),
@@ -81,13 +81,15 @@ describe('parser return statements', () => {
                 kind: TokenKind.Return,
                 text: 'return',
                 isReserved: true,
-                range: Range.create(1, 2, 1, 8)
+                range: Range.create(1, 2, 1, 8),
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.IntegerLiteral,
                 text: '5',
                 isReserved: false,
-                range: Range.create(1, 9, 1, 10)
+                range: Range.create(1, 9, 1, 10),
+                leadingTrivia: []
             },
             token(TokenKind.Newline, '\\n'),
             token(TokenKind.EndFunction, 'end function'),
diff --git a/src/parser/tests/statement/Set.spec.ts b/src/parser/tests/statement/Set.spec.ts
index e52797d9..4ba6601e 100644
--- a/src/parser/tests/statement/Set.spec.ts
+++ b/src/parser/tests/statement/Set.spec.ts
@@ -132,91 +132,104 @@ describe('parser indexed assignment', () => {
                 text: 'arr',
                 isReserved: false,
                 range: Range.create(0, 0, 0, 3),
-                leadingWhitespace: ''
+                leadingWhitespace: '',
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.LeftSquareBracket,
                 text: '[',
                 isReserved: false,
                 range: Range.create(0, 3, 0, 4),
-                leadingWhitespace: ''
+                leadingWhitespace: '',
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.IntegerLiteral,
                 text: '0',
                 isReserved: false,
                 range: Range.create(0, 4, 0, 5),
-                leadingWhitespace: ''
+                leadingWhitespace: '',
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.RightSquareBracket,
                 text: ']',
                 isReserved: false,
                 range: Range.create(0, 5, 0, 6),
-                leadingWhitespace: ''
+                leadingWhitespace: '',
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.Equal,
                 text: '=',
                 isReserved: false,
                 range: Range.create(0, 7, 0, 8),
-                leadingWhitespace: ''
+                leadingWhitespace: '',
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.IntegerLiteral,
                 text: '1',
                 isReserved: false,
                 range: Range.create(0, 9, 0, 10),
-                leadingWhitespace: ''
+                leadingWhitespace: '',
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.Newline,
                 text: '\n',
                 isReserved: false,
                 range: Range.create(0, 10, 0, 11),
-                leadingWhitespace: ''
+                leadingWhitespace: '',
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.Identifier,
                 text: 'obj',
                 isReserved: false,
                 range: Range.create(1, 0, 1, 3),
-                leadingWhitespace: ''
+                leadingWhitespace: '',
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.Dot,
                 text: '.',
                 isReserved: false,
                 range: Range.create(1, 3, 1, 4),
-                leadingWhitespace: ''
+                leadingWhitespace: '',
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.Identifier,
                 text: 'a',
                 isReserved: false,
                 range: Range.create(1, 4, 1, 5),
-                leadingWhitespace: ''
+                leadingWhitespace: '',
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.Equal,
                 text: '=',
                 isReserved: false,
                 range: Range.create(1, 6, 1, 7),
-                leadingWhitespace: ''
+                leadingWhitespace: '',
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.IntegerLiteral,
                 text: '5',
                 isReserved: false,
                 range: Range.create(1, 8, 1, 9),
-                leadingWhitespace: ''
+                leadingWhitespace: '',
+                leadingTrivia: []
             },
             {
                 kind: TokenKind.Eof,
                 text: '\0',
                 isReserved: false,
                 range: Range.create(1, 10, 1, 11),
-                leadingWhitespace: ''
+                leadingWhitespace: '',
+                leadingTrivia: []
             }
         ]);