Repository: rokucommunity/brighterscript
Pull request #1072 (2024-02-13 19:41:50)

Support when tokens have null ranges
Many plugins use `Parser.parse()` to generate AST used to replace existing code. This causes the generated sourcemaps to be very incorrect. To mitigate this, we should support generating AST without any range information at all. But, lots of our codebase explodes when that happens. 

This PR adds a test that verifies _most_ expressions and statements  can transpile to proper code without needing tokens. There may be some edge cases, but we've tried to hit as many as possible.

Here's how you can leverage this new functionality:

```typescript
const tokens = Parser.parse('print hello', { trackLocations: false });
```

The tokens array will have all the same tokens, but none of them will have location information, making them much safer to inject into AST without messing with sourcemaps.


Patch:
diff --git a/package-lock.json b/package-lock.json
index 5e2746c8..1908dc46 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1645,9 +1645,9 @@
       }
     },
     "node_modules/caniuse-lite": {
-      "version": "1.0.30001518",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001518.tgz",
-      "integrity": "sha512-rup09/e3I0BKjncL+FesTayKtPrdwKhUufQFd3riFw1hHg8JmIFoInYfB102cFcY/pPgGmdyl/iy+jgiDi2vdA==",
+      "version": "1.0.30001585",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001585.tgz",
+      "integrity": "sha512-yr2BWR1yLXQ8fMpdS/4ZZXpseBgE7o4g41x3a6AJOqZuOi+iE/WdJYAuZ6Y95i4Ohd2Y+9MzIWRR+uGABH4s3Q==",
       "dev": true,
       "funding": [
         {
@@ -8522,9 +8522,9 @@
       "dev": true
     },
     "caniuse-lite": {
-      "version": "1.0.30001518",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001518.tgz",
-      "integrity": "sha512-rup09/e3I0BKjncL+FesTayKtPrdwKhUufQFd3riFw1hHg8JmIFoInYfB102cFcY/pPgGmdyl/iy+jgiDi2vdA==",
+      "version": "1.0.30001585",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001585.tgz",
+      "integrity": "sha512-yr2BWR1yLXQ8fMpdS/4ZZXpseBgE7o4g41x3a6AJOqZuOi+iE/WdJYAuZ6Y95i4Ohd2Y+9MzIWRR+uGABH4s3Q==",
       "dev": true
     },
     "caseless": {
diff --git a/src/astUtils/creators.spec.ts b/src/astUtils/creators.spec.ts
index 9068dce2..b98ea05c 100644
--- a/src/astUtils/creators.spec.ts
+++ b/src/astUtils/creators.spec.ts
@@ -1,6 +1,4 @@
 import { expect } from '../chai-config.spec';
-import { interpolatedRange } from '..';
-import { TranspileState } from '../parser/TranspileState';
 import { createStringLiteral } from './creators';
 
 describe('creators', () => {
@@ -20,13 +18,4 @@ describe('creators', () => {
             expect(createStringLiteral('hello world"').token.text).to.equal('hello world"');
         });
     });
-
-    describe('interpolatedRange', () => {
-        it('can be used in sourcemaps', () => {
-            const state = new TranspileState('source/main.brs', {});
-            const node = state.sourceNode({ range: interpolatedRange }, 'code');
-            //should not crash
-            node.toStringWithSourceMap();
-        });
-    });
 });
diff --git a/src/astUtils/creators.ts b/src/astUtils/creators.ts
index 92ae752a..2ca6964e 100644
--- a/src/astUtils/creators.ts
+++ b/src/astUtils/creators.ts
@@ -9,6 +9,7 @@ import { Block, MethodStatement } from '../parser/Statement';
 /**
  * A range that points to the beginning of the file. Used to give non-null ranges to programmatically-added source code.
  * (Hardcoded range to prevent circular dependency issue in `../util.ts`)
+ * @deprecated don't use this, it screws up sourcemaps. Just set range to null
  */
 export const interpolatedRange = {
     start: {
@@ -83,7 +84,7 @@ const tokenDefaults = {
     [TokenKind.Whitespace]: ' '
 };
 
-export function createToken<T extends TokenKind>(kind: T, text?: string, range = interpolatedRange): Token & { kind: T } {
+export function createToken<T extends TokenKind>(kind: T, text?: string, range?: Range): Token & { kind: T } {
     return {
         kind: kind,
         text: text ?? tokenDefaults[kind as string] ?? kind.toString().toLowerCase(),
@@ -140,7 +141,7 @@ export function createBooleanLiteral(value: 'true' | 'false', range?: Range) {
 export function createFunctionExpression(kind: TokenKind.Sub | TokenKind.Function) {
     return new FunctionExpression(
         [],
-        new Block([], interpolatedRange),
+        new Block([]),
         createToken(kind),
         kind === TokenKind.Sub ? createToken(TokenKind.EndSub) : createToken(TokenKind.EndFunction),
         createToken(TokenKind.LeftParen),
diff --git a/src/astUtils/reflection.spec.ts b/src/astUtils/reflection.spec.ts
index 8efbc1b9..c47b849e 100644
--- a/src/astUtils/reflection.spec.ts
+++ b/src/astUtils/reflection.spec.ts
@@ -5,7 +5,7 @@ import { FunctionExpression, NamespacedVariableNameExpression, BinaryExpression,
 import type { Token } from '../lexer/Token';
 import { TokenKind } from '../lexer/TokenKind';
 import { isPrintStatement, isIfStatement, isBody, isAssignmentStatement, isBlock, isExpressionStatement, isCommentStatement, isExitForStatement, isExitWhileStatement, isFunctionStatement, isIncrementStatement, isGotoStatement, isLabelStatement, isReturnStatement, isEndStatement, isStopStatement, isForStatement, isForEachStatement, isWhileStatement, isDottedSetStatement, isIndexedSetStatement, isLibraryStatement, isNamespaceStatement, isImportStatement, isExpression, isBinaryExpression, isCallExpression, isFunctionExpression, isNamespacedVariableNameExpression, isDottedGetExpression, isXmlAttributeGetExpression, isIndexedGetExpression, isGroupingExpression, isLiteralExpression, isEscapedCharCodeLiteralExpression, isArrayLiteralExpression, isAALiteralExpression, isUnaryExpression, isVariableExpression, isSourceLiteralExpression, isNewExpression, isCallfuncExpression, isTemplateStringQuasiExpression, isTemplateStringExpression, isTaggedTemplateStringExpression, isBrsFile, isXmlFile, isClassStatement, isStatement, isAnnotationExpression, isTryCatchStatement, isCatchStatement, isThrowStatement } from './reflection';
-import { createToken, createStringLiteral, interpolatedRange as range } from './creators';
+import { createToken, createStringLiteral } from './creators';
 import { Program } from '../Program';
 import { BrsFile } from '../files/BrsFile';
 import { XmlFile } from '../files/XmlFile';
@@ -25,12 +25,12 @@ describe('reflection', () => {
     });
 
     describe('Statements', () => {
-        const ident = createToken(TokenKind.Identifier, 'a', range);
-        const expr = createStringLiteral('', range);
-        const token = createToken(TokenKind.StringLiteral, '', range);
+        const ident = createToken(TokenKind.Identifier, 'a');
+        const expr = createStringLiteral('');
+        const token = createToken(TokenKind.StringLiteral, '');
         const body = new Body([]);
         const assignment = new AssignmentStatement(undefined, ident, expr);
-        const block = new Block([], range);
+        const block = new Block([]);
         const expression = new ExpressionStatement(expr);
         const comment = new CommentStatement([token]);
         const exitFor = new ExitForStatement({ exitFor: token });
@@ -50,7 +50,7 @@ describe('reflection', () => {
         const dottedSet = new DottedSetStatement(expr, ident, expr);
         const indexedSet = new IndexedSetStatement(expr, expr, expr, token, token);
         const library = new LibraryStatement({ library: token, filePath: token });
-        const namespace = new NamespaceStatement(token, new NamespacedVariableNameExpression(createVariableExpression('a', range)), body, token);
+        const namespace = new NamespaceStatement(token, new NamespacedVariableNameExpression(createVariableExpression('a')), body, token);
         const cls = new ClassStatement(token, ident, [], token);
         const imports = new ImportStatement(token, token);
         const catchStmt = new CatchStatement({ catch: token }, ident, block);
@@ -183,19 +183,19 @@ describe('reflection', () => {
     });
 
     describe('Expressions', () => {
-        const ident = createToken(TokenKind.Identifier, 'a', range);
-        const expr = createStringLiteral('', range);
-        const token = createToken(TokenKind.StringLiteral, '', range);
-        const block = new Block([], range);
+        const ident = createToken(TokenKind.Identifier, 'a');
+        const expr = createStringLiteral('');
+        const token = createToken(TokenKind.StringLiteral, '');
+        const block = new Block([]);
         const charCode: Token & { charCode: number } = {
             kind: TokenKind.EscapedCharCodeLiteral,
             text: '0',
-            range: range,
+            range: undefined,
             isReserved: false,
             charCode: 0,
             leadingWhitespace: ''
         };
-        const nsVar = new NamespacedVariableNameExpression(createVariableExpression('a', range));
+        const nsVar = new NamespacedVariableNameExpression(createVariableExpression('a'));
         const binary = new BinaryExpression(expr, token, expr);
         const call = new CallExpression(expr, token, token, []);
         const fun = new FunctionExpression([], block, token, token, token, token);
diff --git a/src/files/BrsFile.spec.ts b/src/files/BrsFile.spec.ts
index cecc910d..158a8f3d 100644
--- a/src/files/BrsFile.spec.ts
+++ b/src/files/BrsFile.spec.ts
@@ -16,8 +16,8 @@ import { DiagnosticMessages } from '../DiagnosticMessages';
 import type { StandardizedFileEntry } from 'roku-deploy';
 import util, { standardizePath as s } from '../util';
 import PluginInterface from '../PluginInterface';
-import { expectCompletionsIncludes, expectDiagnostics, expectHasDiagnostics, expectZeroDiagnostics, getTestGetTypedef, getTestTranspile, trim } from '../testHelpers.spec';
-import { ParseMode } from '../parser/Parser';
+import { expectCompletionsIncludes, expectDiagnostics, expectHasDiagnostics, expectZeroDiagnostics, getTestGetTypedef, getTestTranspile, trim, trimMap } from '../testHelpers.spec';
+import { ParseMode, Parser } from '../parser/Parser';
 import { Logger } from '../Logger';
 import { ImportStatement } from '../parser/Statement';
 import { createToken } from '../astUtils/creators';
@@ -25,6 +25,7 @@ import * as fsExtra from 'fs-extra';
 import { URI } from 'vscode-uri';
 import undent from 'undent';
 import { tempDir, rootDir } from '../testHelpers.spec';
+import * as fileUrl from 'file-url';
 
 let sinon = sinonImport.createSandbox();
 
@@ -2194,6 +2195,237 @@ describe('BrsFile', () => {
     });
 
     describe('transpile', () => {
+        describe('null tokens', () => {
+            it('succeeds when token locations are omitted', () => {
+                doTest(`
+                    library "something" 'comment before func
+                    sub main(arg0, arg1 as string, arg2 = invalid)
+                        'comment
+                        aa = {
+                            'comment
+                            one: 1
+                            "two": 2
+                        }
+                        arr = [
+                            'comment
+                            1
+                            'comment
+                            2
+                        ]
+                        val = +3
+                        print "hello"
+                        'comment after print
+                        num = 1
+                        num++
+                        num += 2
+                        num = +num
+                        test(num)
+                        for i = 0 to 10 step 1
+                            exit for
+                        end for
+                        while true
+                            exit while
+                        end while
+                        if true then
+                            print 1
+                        else if true
+                            print 1
+                        else
+                            print 1
+                        end if
+                        dim thing[1, 2]
+                        label1:
+                        goto label1
+                        end
+                        stop
+                        stuff = [
+                            1
+                            2
+                            3
+                        ]
+                        for each item in stuff
+                            print item
+                        end for
+                        m.thing = 1
+                        m.thing += 1
+                        m[1] = 1
+                        m[1] += 1
+                        m[1, 2] = 2
+                        try
+                            print m.b.c
+                        catch e
+                            print e
+                        end try
+                        throw "crash"
+                        for i = 0 to 10
+                            continue
+                        end for
+                        print m@name
+                        print (1 + 2)
+                    end sub
+
+                    sub test(p1)
+                        return p1
+                    end sub
+                `);
+            });
+
+            it('works for bs content', () => {
+                program.setFile('source/lib.bs', ``);
+                doTest(`
+                    import "pkg:/source/lib.bs"
+                    @annotation()
+                    sub test()
+                        two = 2
+                        print \`1\${two}\${3}\n\`
+                        print (1 as integer)
+                        print SOURCE_LINE_NUM
+                        print FUNCTION_NAME
+                        print SOURCE_FUNCTION_NAME
+                        print PKG_LOCATION
+                        print PKG_PATH
+                        print LINE_NUM
+                        print new Person()
+                        m@.someCallfunc()
+                        m@.someCallfunc(1, 2)
+                        print tag\`stuff\${LINE_NUM}\${LINE_NUM}\`
+                        print 1 = 1 ? 1 : 2
+                        print 1 = 1 ? m.one : m.two
+                        print 1 ?? 2
+                        print m.one ?? m.two
+                        print /123/gi
+                    end sub
+                    function tag(param1, param2)
+                    end function
+                    const a = 1
+                    namespace alpha
+                        function beta()
+                            throw "An error has occurred"
+                        end function
+                        function charlie()
+                        end function
+                    end namespace
+                    sub test()
+                        ' alpha.charlie()
+                    end sub
+
+                    enum Direction
+                        up = "up"
+                    end enum
+
+                    class Person
+                        name as string
+                        sub new()
+                            print m.name
+                        end sub
+
+                        sub test()
+                        end sub
+                    end class
+
+                    interface Beta
+                        name as string
+                    end interface
+                `, `
+                    'import "pkg:/source/lib.bs"
+
+                    sub test()
+                        two = 2
+                        print ("1" + bslib_toString(two) + bslib_toString(3) + chr(10))
+                        print 1
+                        print -1
+                        print "test"
+                        print "test"
+                        print "pkg:/source/main.brs:" + str(LINE_NUM)
+                        print "pkg:/source/main.brs"
+                        print LINE_NUM
+                        print Person()
+                        m.callfunc("someCallfunc", invalid)
+                        m.callfunc("someCallfunc", 1, 2)
+                        print tag(["stuff", "", ""], [LINE_NUM, LINE_NUM])
+                        print bslib_ternary(1 = 1, 1, 2)
+                        print (function(__bsCondition, m)
+                                if __bsCondition then
+                                    return m.one
+                                else
+                                    return m.two
+                                end if
+                            end function)(1 = 1, m)
+                        print bslib_coalesce(1, 2)
+                        print (function(m)
+                                __bsConsequent = m.one
+                                if __bsConsequent <> invalid then
+                                    return __bsConsequent
+                                else
+                                    return m.two
+                                end if
+                            end function)(m)
+                        print CreateObject("roRegex", "123", "gi")
+                    end sub
+
+                    function tag(param1, param2)
+                    end function
+
+                    function alpha_beta()
+                        throw "An error has occurred"
+                    end function
+
+                    function alpha_charlie()
+                    end function
+
+                    sub test()
+                        ' alpha.charlie()
+                    end sub
+
+                    function __Person_builder()
+                        instance = {}
+                        instance.new = sub()
+                            m.name = invalid
+                            print m.name
+                        end sub
+                        instance.test = sub()
+                        end sub
+                        return instance
+                    end function
+                    function Person()
+                        instance = __Person_builder()
+                        instance.new()
+                        return instance
+                    end function
+                `);
+            });
+
+            it('handles source literals properly', () => {
+                const pathUrl = fileUrl(rootDir);
+                let text = `"${pathUrl.substring(0, 4)}" + "${pathUrl.substring(4)}`;
+                doTest(`
+                    sub test()
+                        print SOURCE_FILE_PATH
+                        print SOURCE_LOCATION
+                    end sub
+                `, `
+                    sub test()
+                        print ${text}/source/main.bs"
+                        print ${text}/source/main.bs:-1"
+                    end sub
+                `);
+            });
+            function doTest(source: string, expected = source) {
+                const file = program.setFile<BrsFile>('source/main.bs', '');
+                //override the parser with our locationless parser
+                file['_parser'] = Parser.parse(source, { mode: ParseMode.BrighterScript, trackLocations: false });
+                program.getScopesForFile(file).forEach(x => x['cache'].clear());
+                program.validate();
+                expectZeroDiagnostics(program);
+                const result = file.transpile();
+                expect(
+                    trimMap(undent(result.code))
+                ).to.eql(
+                    undent(expected)
+                );
+            }
+        });
+
         it('transpilies libpkg:/ paths when encountered', () => {
             program.setFile('source/lib.bs', `
                 import "libpkg:/source/numbers.bs"
diff --git a/src/files/BrsFile.ts b/src/files/BrsFile.ts
index 05fd3590..f1ddd4a9 100644
--- a/src/files/BrsFile.ts
+++ b/src/files/BrsFile.ts
@@ -481,7 +481,7 @@ export class BrsFile {
             for (let param of func.parameters) {
                 scope.variableDeclarations.push({
                     nameRange: param.name.range,
-                    lineIndex: param.name.range.start.line,
+                    lineIndex: param.name.range?.start.line,
                     name: param.name.text,
                     type: param.type
                 });
@@ -492,7 +492,7 @@ export class BrsFile {
                 ForEachStatement: (stmt) => {
                     scope.variableDeclarations.push({
                         nameRange: stmt.item.range,
-                        lineIndex: stmt.item.range.start.line,
+                        lineIndex: stmt.item.range?.start.line,
                         name: stmt.item.text,
                         type: new DynamicType()
                     });
@@ -501,7 +501,7 @@ export class BrsFile {
                     const { identifier } = stmt.tokens;
                     scope.labelStatements.push({
                         nameRange: identifier.range,
-                        lineIndex: identifier.range.start.line,
+                        lineIndex: identifier.range?.start.line,
                         name: identifier.text
                     });
                 }
@@ -527,7 +527,7 @@ export class BrsFile {
             if (scope) {
                 scope.variableDeclarations.push({
                     nameRange: statement.name.range,
-                    lineIndex: statement.name.range.start.line,
+                    lineIndex: statement.name.range?.start.line,
                     name: statement.name.text,
                     type: this.getBscTypeFromAssignment(statement, scope)
                 });
diff --git a/src/lexer/Lexer.ts b/src/lexer/Lexer.ts
index ddae1d35..ac970644 100644
--- a/src/lexer/Lexer.ts
+++ b/src/lexer/Lexer.ts
@@ -102,7 +102,9 @@ export class Lexer {
             kind: TokenKind.Eof,
             isReserved: false,
             text: '',
-            range: util.createRange(this.lineBegin, this.columnBegin, this.lineEnd, this.columnEnd + 1),
+            range: this.options.trackLocations
+                ? util.createRange(this.lineBegin, this.columnBegin, this.lineEnd, this.columnEnd + 1)
+                : undefined,
             leadingWhitespace: this.leadingWhitespace
         });
         this.leadingWhitespace = '';
@@ -113,10 +115,10 @@ export class Lexer {
      * Fill in missing/invalid options with defaults
      */
     private sanitizeOptions(options: ScanOptions) {
-        return {
-            includeWhitespace: false,
-            ...options
-        } as ScanOptions;
+        options ??= {};
+        options.includeWhitespace ??= false;
+        options.trackLocations ??= true;
+        return options;
     }
 
     /**
@@ -1064,17 +1066,27 @@ export class Lexer {
     }
 
     /**
-     * Creates a `TokenLocation` at the lexer's current position for the provided `text`.
-     * @returns the range of `text` as a `TokenLocation`
+     * Creates a `Range` at the lexer's current position
+     * @returns the range of `text`
      */
     private rangeOf(): Range {
-        return util.createRange(this.lineBegin, this.columnBegin, this.lineEnd, this.columnEnd);
+        if (this.options.trackLocations) {
+            return util.createRange(this.lineBegin, this.columnBegin, this.lineEnd, this.columnEnd);
+        } else {
+            return undefined;
+        }
     }
 }
 
 export interface ScanOptions {
     /**
      * If true, the whitespace tokens are included. If false, they are discarded
+     * @default false
+     */
+    includeWhitespace?: boolean;
+    /**
+     * Should locations be tracked. If false, the `range` property will be omitted
+     * @default true
      */
-    includeWhitespace: boolean;
+    trackLocations?: boolean;
 }
diff --git a/src/lexer/Token.ts b/src/lexer/Token.ts
index 18acfe6a..36d515c2 100644
--- a/src/lexer/Token.ts
+++ b/src/lexer/Token.ts
@@ -40,7 +40,7 @@ export interface Identifier extends Token {
  * @returns `true` is `obj` is a `Token`, otherwise `false`
  */
 export function isToken(obj: Record<string, any>): obj is Token {
-    return !!(obj.kind && obj.text && obj.range);
+    return !!(obj.kind && obj.text);
 }
 
 /**
diff --git a/src/parser/Expression.ts b/src/parser/Expression.ts
index cc55bd92..c43931be 100644
--- a/src/parser/Expression.ts
+++ b/src/parser/Expression.ts
@@ -16,6 +16,7 @@ import { VoidType } from '../types/VoidType';
 import { DynamicType } from '../types/DynamicType';
 import type { BscType } from '../types/BscType';
 import { FunctionType } from '../types/FunctionType';
+import type { AstNode } from './AstNode';
 import { Expression } from './AstNode';
 import { SymbolTable } from '../SymbolTable';
 import { SourceNode } from 'source-map';
@@ -29,9 +30,7 @@ export class BinaryExpression extends Expression {
         public right: Expression
     ) {
         super();
-        this.range = this.left.range && this.right.range
-            ? util.createRangeFromPositions(this.left.range.start, this.right.range.end)
-            : undefined;
+        this.range = util.createBoundingRange(this.left, this.operator, this.right);
     }
 
     public readonly range: Range | undefined;
@@ -930,7 +929,7 @@ export class SourceLiteralExpression extends Expression {
     public readonly range: Range;
 
     private getFunctionName(state: BrsTranspileState, parseMode: ParseMode) {
-        let func = state.file.getFunctionScopeAtPosition(this.token.range.start).func;
+        let func = this.findAncestor<FunctionExpression>(isFunctionExpression);
         let nameParts = [] as TranspileResult;
         while (func.parentFunction) {
             let index = func.parentFunction.childFunctionExpressions.indexOf(func);
@@ -944,6 +943,20 @@ export class SourceLiteralExpression extends Expression {
         return nameParts.join('$');
     }
 
+    /**
+     * Get the line number from our token or from the closest ancestor that has a range
+     */
+    private getClosestLineNumber() {
+        let node: AstNode = this;
+        while (node) {
+            if (node.range) {
+                return node.range.start.line + 1;
+            }
+            node = node.parent;
+        }
+        return -1;
+    }
+
     transpile(state: BrsTranspileState) {
         let text: string;
         switch (this.token.kind) {
@@ -952,7 +965,8 @@ export class SourceLiteralExpression extends Expression {
                 text = `"${pathUrl.substring(0, 4)}" + "${pathUrl.substring(4)}"`;
                 break;
             case TokenKind.SourceLineNumLiteral:
-                text = `${this.token.range.start.line + 1}`;
+                //TODO find first parent that has range, or default to -1
+                text = `${this.getClosestLineNumber()}`;
                 break;
             case TokenKind.FunctionNameLiteral:
                 text = `"${this.getFunctionName(state, ParseMode.BrightScript)}"`;
@@ -962,7 +976,8 @@ export class SourceLiteralExpression extends Expression {
                 break;
             case TokenKind.SourceLocationLiteral:
                 const locationUrl = fileUrl(state.srcPath);
-                text = `"${locationUrl.substring(0, 4)}" + "${locationUrl.substring(4)}:${this.token.range.start.line + 1}"`;
+                //TODO find first parent that has range, or default to -1
+                text = `"${locationUrl.substring(0, 4)}" + "${locationUrl.substring(4)}:${this.getClosestLineNumber()}"`;
                 break;
             case TokenKind.PkgPathLiteral:
                 let pkgPath1 = `pkg:/${state.file.pkgPath}`
diff --git a/src/parser/Parser.ts b/src/parser/Parser.ts
index 75bff4cf..6d9f764e 100644
--- a/src/parser/Parser.ts
+++ b/src/parser/Parser.ts
@@ -212,15 +212,17 @@ export class Parser {
      * @returns the same instance of the parser which contains the diagnostics and statements
      */
     public parse(toParse: Token[] | string, options?: ParseOptions) {
+        this.logger = options?.logger ?? new Logger();
+        options = this.sanitizeParseOptions(options);
+        this.options = options;
+
         let tokens: Token[];
         if (typeof toParse === 'string') {
-            tokens = Lexer.scan(toParse).tokens;
+            tokens = Lexer.scan(toParse, { trackLocations: options.trackLocations }).tokens;
         } else {
             tokens = toParse;
         }
-        this.logger = options?.logger ?? new Logger();
         this.tokens = tokens;
-        this.options = this.sanitizeParseOptions(options);
         this.allowedLocalIdentifiers = [
             ...AllowedLocalIdentifiers,
             //when in plain brightscript mode, the BrighterScript source literals can be used as regular variables
@@ -277,10 +279,10 @@ export class Parser {
     }
 
     private sanitizeParseOptions(options: ParseOptions) {
-        return {
-            mode: 'brightscript',
-            ...(options || {})
-        } as ParseOptions;
+        options ??= {};
+        options.mode ??= ParseMode.BrightScript;
+        options.trackLocations ??= true;
+        return options;
     }
 
     /**
@@ -1369,7 +1371,7 @@ export class Parser {
         //if this comment is on the same line as the previous statement,
         //then this comment should be treated as a single-line comment
         let prev = this.previous();
-        if (prev?.range.end.line === this.peek().range.start.line) {
+        if (prev?.range?.end.line === this.peek().range?.start.line) {
             return new CommentStatement([this.advance()]);
         } else {
             let comments = [this.advance()];
@@ -3325,11 +3327,16 @@ export interface ParseOptions {
     /**
      * The parse mode. When in 'BrightScript' mode, no BrighterScript syntax is allowed, and will emit diagnostics.
      */
-    mode: ParseMode;
+    mode?: ParseMode;
     /**
      * A logger that should be used for logging. If omitted, a default logger is used
      */
     logger?: Logger;
+    /**
+     * Should locations be tracked. If false, the `range` property will be omitted
+     * @default true
+     */
+    trackLocations?: boolean;
 }
 
 export class References {
diff --git a/src/parser/Statement.ts b/src/parser/Statement.ts
index 026d3161..173774b9 100644
--- a/src/parser/Statement.ts
+++ b/src/parser/Statement.ts
@@ -11,7 +11,7 @@ import type { WalkVisitor, WalkOptions } from '../astUtils/visitors';
 import { InternalWalkMode, walk, createVisitor, WalkMode, walkArray } from '../astUtils/visitors';
 import { isCallExpression, isCommentStatement, isEnumMemberStatement, isExpression, isExpressionStatement, isFieldStatement, isFunctionExpression, isFunctionStatement, isIfStatement, isInterfaceFieldStatement, isInterfaceMethodStatement, isInvalidType, isLiteralExpression, isMethodStatement, isNamespaceStatement, isTypedefProvider, isUnaryExpression, isVoidType } from '../astUtils/reflection';
 import type { TranspileResult, TypedefProvider } from '../interfaces';
-import { createInvalidLiteral, createMethodStatement, createToken, interpolatedRange } from '../astUtils/creators';
+import { createInvalidLiteral, createMethodStatement, createToken } from '../astUtils/creators';
 import { DynamicType } from '../types/DynamicType';
 import type { BscType } from '../types/BscType';
 import type { TranspileState } from './TranspileState';
@@ -24,7 +24,7 @@ export class EmptyStatement extends Statement {
         /**
          * Create a negative range to indicate this is an interpolated location
          */
-        public range: Range = interpolatedRange
+        public range: Range = undefined
     ) {
         super();
     }
@@ -74,14 +74,14 @@ export class Body extends Statement implements TypedefProvider {
 
                 //add double newline if this is a comment, and next is a function
             } else if (isCommentStatement(statement) && nextStatement && isFunctionStatement(nextStatement)) {
-                result.push('\n\n');
+                result.push(state.newline, state.newline);
 
                 //add double newline if is function not preceeded by a comment
             } else if (isFunctionStatement(statement) && previousStatement && !(isCommentStatement(previousStatement))) {
-                result.push('\n\n');
+                result.push(state.newline, state.newline);
             } else {
                 //separate statements by a single newline
-                result.push('\n');
+                result.push(state.newline);
             }
 
             result.push(...statement.transpile(state));
@@ -156,7 +156,7 @@ export class AssignmentStatement extends Statement {
 export class Block extends Statement {
     constructor(
         readonly statements: Statement[],
-        readonly startingRange: Range
+        readonly startingRange?: Range
     ) {
         super();
         this.range = util.createBoundingRange(
@@ -259,7 +259,7 @@ export class CommentStatement extends Statement implements Expression, TypedefPr
             );
             //add newline for all except final comment
             if (i < this.comments.length - 1) {
-                result.push('\n');
+                result.push(state.newline);
             }
         }
         return result;
@@ -1170,7 +1170,7 @@ export class NamespaceStatement extends Statement implements TypedefProvider {
                 this.nameExpression,
                 this.body,
                 this.endKeyword
-            ) ?? interpolatedRange;
+            );
         }
         return this._range;
     }
@@ -1235,13 +1235,15 @@ export class ImportStatement extends Statement implements TypedefProvider {
         if (this.filePathToken) {
             //remove quotes
             this.filePath = this.filePathToken.text.replace(/"/g, '');
-            //adjust the range to exclude the quotes
-            this.filePathToken.range = util.createRange(
-                this.filePathToken.range.start.line,
-                this.filePathToken.range.start.character + 1,
-                this.filePathToken.range.end.line,
-                this.filePathToken.range.end.character - 1
-            );
+            if (this.filePathToken.range) {
+                //adjust the range to exclude the quotes
+                this.filePathToken.range = util.createRange(
+                    this.filePathToken.range.start.line,
+                    this.filePathToken.range.start.character + 1,
+                    this.filePathToken.range.end.line,
+                    this.filePathToken.range.end.character - 1
+                );
+            }
         }
     }
     public filePath: string | undefined;
diff --git a/src/preprocessor/Chunk.ts b/src/preprocessor/Chunk.ts
index d67360d5..89cac98a 100644
--- a/src/preprocessor/Chunk.ts
+++ b/src/preprocessor/Chunk.ts
@@ -93,10 +93,7 @@ export class ErrorChunk implements Chunk {
         readonly hashError: Token,
         readonly message: Token
     ) {
-        this.range = util.createRangeFromPositions(
-            this.hashError.range.start,
-            (this.message ?? this.hashError).range.end
-        );
+        this.range = util.createBoundingRange(this.hashError, this.message);
     }
     public readonly range: Range;
 
diff --git a/src/preprocessor/PreprocessorParser.ts b/src/preprocessor/PreprocessorParser.ts
index 662cef5b..f5014b9a 100644
--- a/src/preprocessor/PreprocessorParser.ts
+++ b/src/preprocessor/PreprocessorParser.ts
@@ -105,7 +105,7 @@ export class PreprocessorParser {
      */
     private hashIf(): CC.Chunk | undefined {
         if (this.match(TokenKind.HashIf)) {
-            let startingLine = this.previous().range.start.line;
+            let startingLine = this.previous().range?.start?.line;
             let elseChunk: CC.Chunk[] | undefined;
 
             let ifCondition = this.advance();
diff --git a/src/util.ts b/src/util.ts
index a49f9f20..62ff230a 100644
--- a/src/util.ts
+++ b/src/util.ts
@@ -843,7 +843,10 @@ export class Util {
      * Get a location object back by extracting location information from other objects that contain location
      */
     public getRange(startObj: { range: Range }, endObj: { range: Range }): Range {
-        return util.createRangeFromPositions(startObj.range.start, endObj.range.end);
+        if (!startObj?.range || !endObj?.range) {
+            return undefined;
+        }
+        return util.createRangeFromPositions(startObj.range?.start, endObj.range?.end);
     }
 
     /**


Test patch:
diff --git a/src/parser/tests/Parser.spec.ts b/src/parser/tests/Parser.spec.ts
index dafd7586..ca503dcf 100644
--- a/src/parser/tests/Parser.spec.ts
+++ b/src/parser/tests/Parser.spec.ts
@@ -1,6 +1,5 @@
 import type { Token } from '../../lexer/Token';
 import { TokenKind, ReservedWords } from '../../lexer/TokenKind';
-import { interpolatedRange } from '../../astUtils/creators';
 import type { Range } from 'vscode-languageserver';
 
 /* A set of utilities to be used while writing tests for the BRS parser. */
@@ -13,7 +12,7 @@ export function token(kind: TokenKind, text?: string): Token {
         kind: kind,
         text: text!,
         isReserved: ReservedWords.has((text ?? '').toLowerCase()),
-        range: interpolatedRange,
+        range: null,
         leadingWhitespace: ''
     };
 }
diff --git a/src/testHelpers.spec.ts b/src/testHelpers.spec.ts
index 891e59d7..fb6576ea 100644
--- a/src/testHelpers.spec.ts
+++ b/src/testHelpers.spec.ts
@@ -175,9 +175,9 @@ export function expectZeroDiagnostics(arg: DiagnosticCollection) {
         for (const diagnostic of diagnostics) {
             //escape any newlines
             diagnostic.message = diagnostic.message.replace(/\r/g, '\\r').replace(/\n/g, '\\n');
-            message += `\n        • bs${diagnostic.code} "${diagnostic.message}" at ${diagnostic.file?.srcPath ?? ''}#(${diagnostic.range.start.line}:${diagnostic.range.start.character})-(${diagnostic.range.end.line}:${diagnostic.range.end.character})`;
+            message += `\n        • bs${diagnostic.code} "${diagnostic.message}" at ${diagnostic.file?.srcPath ?? ''}#(${diagnostic.range?.start.line}:${diagnostic.range?.start.character})-(${diagnostic.range?.end.line}:${diagnostic.range?.end.character})`;
             //print the line containing the error (if we can find it)srcPath
-            const line = diagnostic.file?.fileContents?.split(/\r?\n/g)?.[diagnostic.range.start.line];
+            const line = diagnostic.file?.fileContents?.split(/\r?\n/g)?.[diagnostic.range?.start.line];
             if (line) {
                 message += '\n' + getDiagnosticLine(diagnostic, line, chalk.red);
             }