Repository: sjbarag/brs
Pull request #654 (2021-05-27 18:53:15)

feat(lex): Emit comments from the lexer
Comment nodes are handy for non-executing tooling, and allow tools like `eslint` to automatically ignore rules based on specific directives. Emit `eslint`-compliant comment nodes from the lexer for consumption by external tools.


Patch:
diff --git a/src/lexer/Lexer.ts b/src/lexer/Lexer.ts
index 1ea9591..ba4e240 100644
--- a/src/lexer/Lexer.ts
+++ b/src/lexer/Lexer.ts
@@ -1,7 +1,7 @@
 import { EventEmitter } from "events";
 
 import { Lexeme } from "./Lexeme";
-import { Token, Location } from "./Token";
+import { Token, Location, Comment } from "./Token";
 import { ReservedWords, KeyWords } from "./ReservedWords";
 import { BrsError } from "../Error";
 import { isAlpha, isDecimalDigit, isAlphaNumeric, isHexDigit } from "./Characters";
@@ -12,6 +12,8 @@ import { BrsType, BrsString, Int32, Int64, Float, Double } from "../brsTypes";
 interface ScanResults {
     /** The tokens produced by the Lexer. */
     tokens: Token[];
+    /** The comments produced by the Lexer. */
+    comments: Comment[];
     /** The errors encountered by the Lexer. */
     errors: BrsError[];
 }
@@ -79,6 +81,9 @@ export class Lexer {
         /** The tokens produced from `source`. */
         let tokens: Token[] = [];
 
+        /** The comments produced from `source`. */
+        let comments: Comment[] = [];
+
         /** The errors produced from `source.` */
         let errors: BrsError[] = [];
 
@@ -109,7 +114,7 @@ export class Lexer {
             },
         });
 
-        return { tokens, errors };
+        return { tokens, comments, errors };
 
         /**
          * Determines whether or not the lexer as reached the end of its input.
@@ -287,10 +292,12 @@ export class Lexer {
                     }
                     break;
                 case "'":
-                    // BrightScript doesn't have block comments; only line
-                    while (peek() !== "\n" && !isAtEnd()) {
-                        advance();
-                    }
+                    // BrightScript doesn't _technically_ have block comments, but some external
+                    // tools expect a distinction between the two.  Consider any comment with
+                    // non-whitespace characters ahead of it a "line" comment.  Everything else
+                    // (comments starting at zero or indented with only whitespace ahead) is a
+                    // "block" comment.
+                    comment("'");
                     break;
                 case " ":
                 case "\r":
@@ -300,7 +307,7 @@ export class Lexer {
                 case "\n":
                     // consecutive newlines aren't significant, because they're just blank lines
                     // so only add blank lines when they're not consecutive
-                    let previous = lastToken();
+                    let previous = peekPrevious();
                     if (previous && previous.kind !== Lexeme.Newline) {
                         addToken(Lexeme.Newline);
                     }
@@ -596,7 +603,8 @@ export class Lexer {
                 advance();
             }
 
-            let text = source.slice(start, current).toLowerCase();
+            let originalText = source.slice(start, current);
+            let text = originalText.toLowerCase();
 
             // some identifiers can be split into two words, so check the "next" word and see what we get
             if (
@@ -635,11 +643,11 @@ export class Lexer {
             // may not. Let the parser figure that part out.
             let nextChar = peek();
             if (["$", "%", "!", "#", "&"].includes(nextChar)) {
-                text += nextChar;
+                originalText += nextChar;
                 advance();
             }
 
-            let textLower = text.toLowerCase();
+            let textLower = originalText.toLowerCase();
             let tokenType = KeyWords.hasOwnProperty(textLower)
                 ? KeyWords[textLower]
                 : Lexeme.Identifier;
@@ -649,18 +657,54 @@ export class Lexer {
                 if (checkPrevious(Lexeme.Dot)) {
                     addToken(Lexeme.Identifier);
                 } else {
-                    // The 'rem' keyword can be used to indicate comments as well, so
-                    // consume the rest of the line, but don't add the token; it's not
-                    // particularly useful.
-                    while (peek() !== "\n" && !isAtEnd()) {
-                        advance();
-                    }
+                    // The 'rem' keyword can be used to indicate comments as well
+                    comment(originalText);
                 }
             } else {
                 addToken(tokenType);
             }
         }
 
+        function comment(starter: string) {
+            // if the previous token was on this line, consider this comment an "inline" comment,
+            // similar to JS-like `//`.  Everything else is a "block" comment, similar to JS-like
+            // `/* */` or `/** */`
+            const commentType: Comment["type"] =
+                peekPrevious()?.location.start.line === line ? "Line" : "Block";
+
+            if (starter === "REM") {
+                console.log({ start, current });
+            }
+
+            let commentText = "";
+            while (peek() !== "\n" && !isAtEnd()) {
+                commentText += advance();
+            }
+
+            const curr: Comment = {
+                type: commentType,
+                starter: starter,
+                value: commentText,
+                loc: locationOf(starter + commentText),
+                range: [start, current],
+            };
+
+            let prev = comments[comments.length - 1];
+            if (
+                prev != null &&
+                prev.type === "Block" &&
+                curr.type === "Block" &&
+                prev.starter === curr.starter &&
+                prev.loc.end.line === curr.loc.start.line - 1
+            ) {
+                prev.loc.end = curr.loc.end;
+                prev.range[1] = curr.range[1];
+                prev.value += `\n${curr.value}`;
+            } else {
+                comments.push(curr);
+            }
+        }
+
         /**
          * Check that the previous token was of the specified type
          * @param kind
@@ -756,7 +800,7 @@ export class Lexer {
          * Retrieves the token that was most recently added.
          * @returns the most recently added token.
          */
-        function lastToken(): Token | undefined {
+        function peekPrevious(): Token | undefined {
             return tokens[tokens.length - 1];
         }
 
diff --git a/src/lexer/Token.ts b/src/lexer/Token.ts
index cb19ed6..ee74c0e 100644
--- a/src/lexer/Token.ts
+++ b/src/lexer/Token.ts
@@ -1,6 +1,5 @@
 import { Lexeme } from "./Lexeme";
 import { BrsType } from "../brsTypes";
-import { ReservedWords } from "./ReservedWords";
 
 /**
  * Represents a chunk of BrightScript scanned by the lexer.
@@ -14,10 +13,24 @@ export interface Token {
     isReserved: boolean;
     /** The literal value (using the BRS type system) associated with this token, if any. */
     literal?: BrsType;
-    /** Where the token was found. */
+    /** The starting and ending line/column pairs where the token was found. */
     location: Location;
 }
 
+/** Represents a BrightScript comment scanned by the lexer. */
+export interface Comment {
+    /** The type of comment (block or line) this node represents. */
+    type: "Block" | "Line";
+    /** The text that started this comment - one of `("'" | "rem" | "REM")`, or any other capitalization. */
+    starter: string;
+    /** The text found in this comment (excludes the `starter`). */
+    value: string;
+    /** The starting and ending line/column pairs where the comment was found. */
+    loc: Location;
+    /** The starting and ending byte offsets in `loc.file` where the comment was found. */
+    range: [number, number];
+}
+
 /** Represents the location at which a `Token` was found. */
 export interface Location {
     /** The line and column at which this token began. */


Test patch:
diff --git a/test/lexer/Lexer.test.js b/test/lexer/Lexer.test.js
index 0aa89df..89673c6 100644
--- a/test/lexer/Lexer.test.js
+++ b/test/lexer/Lexer.test.js
@@ -34,19 +34,150 @@ describe("lexer", () => {
     });
 
     describe("comments", () => {
-        it("ignores everything after `'`", () => {
-            let { tokens } = Lexer.scan("= ' (");
+        it("produces line comments starting with `'`", () => {
+            let { tokens, comments } = Lexer.scan("= ' (");
             expect(tokens.map((t) => t.kind)).toEqual([Lexeme.Equal, Lexeme.Eof]);
+            expect(comments).toEqual([
+                expect.objectContaining({
+                    type: "Line",
+                    starter: "'",
+                    value: " (",
+                }),
+            ]);
+        });
+
+        it("produces block comments starting with `'`", () => {
+            let { tokens, comments } = Lexer.scan("    ' (");
+            expect(tokens.map((t) => t.kind)).toEqual([Lexeme.Eof]);
+            expect(comments).toEqual([
+                expect.objectContaining({
+                    type: "Block",
+                    starter: "'",
+                    value: " (",
+                }),
+            ]);
         });
 
-        it("ignores everything after `REM`", () => {
-            let { tokens } = Lexer.scan("= REM (");
+        it("produces line comments starting with `REM`", () => {
+            let { tokens, comments } = Lexer.scan("= REM (");
             expect(tokens.map((t) => t.kind)).toEqual([Lexeme.Equal, Lexeme.Eof]);
+            expect(comments).toEqual([
+                expect.objectContaining({
+                    type: "Line",
+                    starter: "REM",
+                    value: " (",
+                }),
+            ]);
         });
 
-        it("ignores everything after `rem`", () => {
-            let { tokens } = Lexer.scan("= rem (");
+        it("produces block comments starting with `REM`", () => {
+            let { tokens, comments } = Lexer.scan("REM (");
+            expect(tokens.map((t) => t.kind)).toEqual([Lexeme.Eof]);
+            expect(comments).toEqual([
+                expect.objectContaining({
+                    type: "Block",
+                    starter: "REM",
+                    value: " (",
+                }),
+            ]);
+        });
+
+        it("produces line comments starting with `rem` ", () => {
+            let { tokens, comments } = Lexer.scan("= rem (");
             expect(tokens.map((t) => t.kind)).toEqual([Lexeme.Equal, Lexeme.Eof]);
+            expect(comments).toEqual([
+                expect.objectContaining({
+                    type: "Line",
+                    starter: "rem",
+                    value: " (",
+                }),
+            ]);
+        });
+
+        it("produces block comments starting with `rem` ", () => {
+            let { tokens, comments } = Lexer.scan("rem (");
+            expect(tokens.map((t) => t.kind)).toEqual([Lexeme.Eof]);
+            expect(comments).toEqual([
+                expect.objectContaining({
+                    type: "Block",
+                    starter: "rem",
+                    value: " (",
+                }),
+            ]);
+        });
+
+        it("merges block commments", () => {
+            let { comments } = Lexer.scan(
+                [
+                    "'",
+                    "' merged with above",
+                    "a() ' hello world", // line comments not merged
+                    "    REM foo bar", // block and line comments don't get merged together
+                    "rem asdf", // mixed-case block comments don't get merged together
+                    `UCase("123")`,
+                    "rem foo bar", // not merged with `asdf` since UCase("123") is in the way
+                    "rem also merged",
+                ].join("\n")
+            );
+
+            expect(comments.map((c) => c.value)).toEqual([
+                "\n merged with above",
+                " hello world",
+                " foo bar",
+                " asdf",
+                " foo bar\n also merged",
+            ]);
+        });
+
+        it("tracks comment locations", () => {
+            let { comments } = Lexer.scan(
+                // prettier-ignore
+                [
+                //   0   0   0   1   1
+                //   0   4   8   2   6
+                    "'",
+                    "' merged with above",
+                    "a() ' hello world",
+                    "    REM foo bar",
+                    "rem foo bar",
+                    "rem also merged",
+                ].join("\n")
+            );
+
+            expect(comments.map(({ range, loc }) => ({ range, loc }))).toEqual([
+                {
+                    range: [0, 21],
+                    loc: {
+                        start: { line: 1, column: 0 },
+                        end: { line: 2, column: 19 },
+                        file: "",
+                    },
+                },
+                {
+                    range: [26, 39],
+                    loc: {
+                        start: { line: 3, column: 4 },
+                        end: { line: 3, column: 17 },
+                        file: "",
+                    },
+                },
+                {
+                    range: [44, 55],
+                    loc: {
+                        start: { line: 4, column: 4 },
+                        end: { line: 4, column: 15 },
+                        file: "",
+                    },
+                },
+                {
+                    range: [56, 83],
+                    loc: {
+                        start: { line: 5, column: 0 },
+                        end: { line: 6, column: 15 },
+                        file: "",
+                    },
+                },
+            ]);
         });
     }); // comments