Repository: sjbarag/brs
Pull request #477 (2020-07-27 19:46:41)

feat(perf): Memoize component scope resolution lexing and parsing
Many components can include the same files via `<script/>` tag, and we previously lexed, preprocessed, and parsed those files every time they were encountered in any component.  Even if a component and its extension both import `uri="pkg:/components/foo.brs"`, we'd parse it twice.  Bummer!

We can save a significant amount of time by memoizing that lexing, preprocessing, and parsing process across components.

sjbarag: As far as real-world numbers go, I ran a simple "hello world" script from the root of a large production channel (300+ components), which triggered a full lex/parse cycle across all components.  With some timing logs added to `src/index.ts`, here's the results:


#### Before
```
$ /path/to/brs/bin/cli.js foo.brs
manifest parsing: 3.225ms
component definitions: 194.009ms
script path rewriting: 7.750ms
create lexerParserFn: 0.115ms
create interpreter: 11839.352ms
lex & parse explicit files: 0.838ms
pre-interpreter steps: 12048.050ms
hello, world
```

#### After
```
$ /path/to/brs/bin/cli.js foo.brs
manifest parsing: 2.883ms
component definitions: 194.693ms
script path rewriting: 8.228ms
create lexerParserFn: 0.078ms
create interpreter: 1145.850ms
lex & parse explicit files: 4.493ms
pre-interpreter steps: 1358.864ms
hello, world
```

Patch:
diff --git a/src/parser/ComponentScopeResolver.ts b/src/parser/ComponentScopeResolver.ts
index fc3db71..a22cd85 100644
--- a/src/parser/ComponentScopeResolver.ts
+++ b/src/parser/ComponentScopeResolver.ts
@@ -1,10 +1,17 @@
-import { ComponentDefinition } from "../componentprocessor";
+import { ComponentDefinition, ComponentScript } from "../componentprocessor";
 import * as Stmt from "./Statement";
 import { BrsComponentName } from "../brsTypes";
 
 export class ComponentScopeResolver {
     private readonly excludedNames: string[] = ["init"];
 
+    /**
+     * A map of file URIs to a promise that resolves to an array of that file's statements.
+     * Used primary to avoid re-lexing and re-parsing files that are included
+     * in <script /> tags in multiple components.
+     */
+    private memoizedStatements = new Map<string, Promise<Stmt.Statement[]>>();
+
     /**
      * @param componentMap Component definition map to reference for function resolution.
      * @param parserLexerFn Function used to parse statements out of given components
@@ -63,7 +70,10 @@ export class ComponentScopeResolver {
      * @returns An ordered array of component statement arrays.
      */
     private *getStatements(component: ComponentDefinition) {
-        yield this.parserLexerFn(component.scripts.map((c) => c.uri));
+        for (const statements of this.fetchMemoizedStatements(component.scripts)) {
+            yield statements;
+        }
+
         let currentComponent: ComponentDefinition | undefined = component;
         while (currentComponent.extends) {
             // If this is a built-in component, then no work is needed and we can return.
@@ -82,9 +92,33 @@ export class ComponentScopeResolver {
                 );
                 return Promise.resolve();
             }
-            yield this.parserLexerFn(currentComponent.scripts.map((c) => c.uri));
+            for (const statements of this.fetchMemoizedStatements(currentComponent.scripts)) {
+                yield statements;
+            }
         }
 
         return Promise.resolve();
     }
+
+    /**
+     * Generator function that fetches statements from an array of script files without repeatedly
+     * lexing, preprocessing, and parsing files (via memoization).
+     * @param scripts the array of scripts to get statements for
+     * @yields promises that each resolve to an array of statements; one promise per file in `scripts`.
+     */
+    private *fetchMemoizedStatements(scripts: ComponentScript[]) {
+        for (let { uri } of scripts) {
+            let maybeStatements = this.memoizedStatements.get(uri);
+            if (maybeStatements) {
+                yield maybeStatements;
+            } else {
+                let statementsPromise = this.parserLexerFn([uri]);
+                if (!this.memoizedStatements.has(uri)) {
+                    this.memoizedStatements.set(uri, statementsPromise);
+                }
+
+                yield statementsPromise;
+            }
+        }
+    }
 }


Test patch:
diff --git a/test/parser/ComponentScopeResolver.test.js b/test/parser/ComponentScopeResolver.test.js
index 87e8379..688d7ec 100644
--- a/test/parser/ComponentScopeResolver.test.js
+++ b/test/parser/ComponentScopeResolver.test.js
@@ -55,7 +55,6 @@ describe("ComponentScopeResolver", () => {
         let componentScopeResolver = new brs.parser.ComponentScopeResolver(componentMap, parseFn);
         let statements = await componentScopeResolver.resolve(componentToResolve);
         expect(statements).toBeDefined();
-        expect(statements.length).toBe(2);
 
         let statementNames = statements.map((s) => s.name.text);
         expect(statementNames).toEqual(["test", "test2"]);
@@ -70,7 +69,6 @@ describe("ComponentScopeResolver", () => {
         let componentScopeResolver = new brs.parser.ComponentScopeResolver(componentMap, parseFn);
         let statements = await componentScopeResolver.resolve(componentToResolve);
         expect(statements).toBeDefined();
-        expect(statements.length).toBe(5);
 
         let statementNames = statements.map((s) => s.name.text);
         // Statements are in a specific order