Repository: elm-tooling/elm-language-server
Pull request #385 (2020-12-03 12:31:05)

Add github action to test diagnostics
This should give a "Run Manually" button on the actions tab. We could also set it up on a 24 hour schedule.

Razzeee: The reason to not have this on the normal build is due to it taking too long?
jmbockhorst: Yes. I'm running it now on my personal repo and it's been going for over 2 hours. It might be froze up because I can't see the logs.
https://github.com/jmbockhorst/elm-language-server/runs/1493270378?check_suite_focus=true

We can see, but I would suspect that even if it ran smooth it would take at least 30 minutes to an hour, or possibly more. Didn't think we'd want that on every push, but we can consider.
Razzeee: Yeah, was just wondering what too long is in this case :)
jmbockhorst: Actually, it looks like it just keeps getting stuck. Otherwise it should complete in 11-12 minutes. 
jmbockhorst: This is ready to go now. Currently it doesn’t fail when a library fails so that we can test them. We can enable that once they are all passing or we have a known failures list.  Unfortunately it looks like there are about 150 failures, which is way more than I saw last week. I will be looking at the workspaces invididually over the next week trying to find the issues. 

Patch:
diff --git a/src/elmWorkspace.ts b/src/elmWorkspace.ts
index d1f016d..bb9dc0c 100644
--- a/src/elmWorkspace.ts
+++ b/src/elmWorkspace.ts
@@ -78,7 +78,6 @@ export interface IElmWorkspace {
   hasDocument(uri: URI): boolean;
   hasPath(uri: URI): boolean;
   getPath(uri: URI): string | undefined;
-  getSourceFile(uri: string): ITreeContainer | undefined;
   getForest(synchronize?: boolean): IForest;
   getRootPath(): URI;
   getTypeCache(): TypeCache;
@@ -153,10 +152,6 @@ export class ElmWorkspace implements IElmWorkspace {
     );
   }
 
-  public getSourceFile(uri: string): ITreeContainer | undefined {
-    return this.getForest().getByUri(uri);
-  }
-
   public getForest(synchronize = true): IForest {
     if (this.dirty && synchronize) {
       this.forest.synchronize();
diff --git a/src/forest.ts b/src/forest.ts
index a7e07f7..afdaadf 100644
--- a/src/forest.ts
+++ b/src/forest.ts
@@ -26,7 +26,6 @@ export interface ITreeContainer {
   // Resolved during binding
   exposing?: IExposing;
   symbolLinks?: SyntaxNodeMap<SyntaxNode, SymbolMap>;
-  nonShadowableNames?: Set<string>; // Top level function names
 }
 
 export interface IForest {
diff --git a/src/providers/diagnostics/elmLsDiagnostics.ts b/src/providers/diagnostics/elmLsDiagnostics.ts
index 7d9a94e..13f25ef 100644
--- a/src/providers/diagnostics/elmLsDiagnostics.ts
+++ b/src/providers/diagnostics/elmLsDiagnostics.ts
@@ -972,9 +972,7 @@ export class ElmLsDiagnostics {
             (#eq? @type.reference "${unionVariant.text}")
           )
           ((case_of_branch
-            (pattern
-              (union_pattern
-                (upper_case_qid) @variant.reference)))
+            (pattern) @variant.reference)
             (#eq? @variant.reference "${unionVariant.text}")
           )
           `,
diff --git a/src/util/types/binder.ts b/src/util/types/binder.ts
index 43837f2..e961a97 100644
--- a/src/util/types/binder.ts
+++ b/src/util/types/binder.ts
@@ -24,14 +24,12 @@ export function bindTreeContainer(treeContainer: ITreeContainer): void {
   }
 
   const symbolLinks = new SyntaxNodeMap<SyntaxNode, SymbolMap>();
-  const nonShadowableNames = new Set<string>();
   let container: SymbolMap;
   let parent: SyntaxNode;
   const treeCursor = treeContainer.tree.walk();
 
   bind();
   treeContainer.symbolLinks = symbolLinks;
-  treeContainer.nonShadowableNames = nonShadowableNames;
 
   // Bind exposing must happen after symbolLinks is bound
   bindExposing();
@@ -114,16 +112,10 @@ export function bindTreeContainer(treeContainer: ITreeContainer): void {
     );
 
     if (functionDeclarationLeft && functionDeclarationLeft.firstChild) {
-      const functionName = functionDeclarationLeft.firstChild.text;
-      container.set(functionName, {
+      container.set(functionDeclarationLeft.firstChild.text, {
         node: functionDeclarationLeft,
         type: "Function",
       });
-
-      // Add to nonShadowableNames if it is top level
-      if (node.parent?.type === "file") {
-        nonShadowableNames.add(functionDeclarationLeft.firstChild.text);
-      }
     } else {
       // If there is a pattern, bind it to the parent container
       const pattern = node.childForFieldName("pattern");
diff --git a/src/util/types/typeInference.ts b/src/util/types/typeInference.ts
index 51b45ed..640d734 100644
--- a/src/util/types/typeInference.ts
+++ b/src/util/types/typeInference.ts
@@ -548,18 +548,18 @@ export class InferenceScope {
   public static valueDeclarationInference(
     declaration: EValueDeclaration,
     uri: string,
-    program: IElmWorkspace,
+    elmWorkspace: IElmWorkspace,
     activeScopes: Set<EValueDeclaration>,
     cancellationToken?: ICancellationToken,
   ): InferenceResult {
-    const nonShadowableNames =
-      program.getSourceFile(uri)?.nonShadowableNames ?? new Set<string>();
+    // TODO: Need a good way to get all visible values
+    const shadowableNames = new Set<string>();
 
     const setter = (): InferenceResult =>
       new InferenceScope(
         uri,
-        program,
-        nonShadowableNames,
+        elmWorkspace,
+        shadowableNames,
         new Set(activeScopes.values()),
         /* recursionAllowed */ false,
         cancellationToken,
@@ -567,12 +567,12 @@ export class InferenceScope {
 
     const start = performance.now();
     try {
-      if (!program.getForest().getByUri(uri)?.writeable) {
-        return program
+      if (!elmWorkspace.getForest().getByUri(uri)?.writeable) {
+        return elmWorkspace
           .getTypeCache()
           .getOrSet("PACKAGE_VALUE_DECLARATION", declaration, setter);
       } else {
-        return program
+        return elmWorkspace
           .getTypeCache()
           .getOrSet("PROJECT_VALUE_DECLARATION", declaration, setter);
       }


Test patch:
diff --git a/.github/workflows/test-diagnostics.yaml b/.github/workflows/test-diagnostics.yaml
new file mode 100644
index 0000000..a2306ec
--- /dev/null
+++ b/.github/workflows/test-diagnostics.yaml
@@ -0,0 +1,17 @@
+name: Test Diagnostics
+
+on:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 14
+      - run: npm install -g elm
+      - run: npm ci
+      - run: npm run compile
+      - run: node -r ts-node/register --expose-gc ./test/diagnostics.ts
diff --git a/test/diagnosticTests/elmDiagnostics.test.ts b/test/diagnosticTests/elmDiagnostics.test.ts
index 78faef8..9d27846 100644
--- a/test/diagnosticTests/elmDiagnostics.test.ts
+++ b/test/diagnosticTests/elmDiagnostics.test.ts
@@ -158,23 +158,4 @@ func = 5
       { message: Diagnostics.ImportMissing, args: ["App"] },
     ]);
   });
-
-  test("Shadowing a function with the same name", async () => {
-    const source = `
---@ Test.elm
-module Test exposing (..)
-
-type alias Model =
-    { field : Int
-    }
-
-field : Model -> Int
-field { field } =
-        --^
-    4
-`;
-    await testTypeInference(basicsSources + source, [
-      { message: Diagnostics.Redefinition, args: ["field"] },
-    ]);
-  });
 });
diff --git a/test/diagnosticTests/elmLsDiagnostics.test.ts b/test/diagnosticTests/elmLsDiagnostics.test.ts
index 498e2d6..a34ca75 100644
--- a/test/diagnosticTests/elmLsDiagnostics.test.ts
+++ b/test/diagnosticTests/elmLsDiagnostics.test.ts
@@ -1613,28 +1613,5 @@ text en it language =
 
       await testDiagnostics(source, "unused_value_constructor", []);
     });
-
-    it("used in case expr with pattern", async () => {
-      const source = `
-module Bar exposing (text)
-
-type Language
-    = Italian (List String)
-    | English (List String)
-
-
-text : String -> String -> Html msg
-text en it language =
-    Html.text <|
-        case language of
-            English items ->
-                en
-
-            Italian items ->
-                it
-			`;
-
-      await testDiagnostics(source, "unused_value_constructor", []);
-    });
   });
 });
diff --git a/test/diagnostics.ts b/test/diagnostics.ts
index 8c14c8d..2659038 100644
--- a/test/diagnostics.ts
+++ b/test/diagnostics.ts
@@ -5,16 +5,10 @@ import { ElmWorkspace } from "../src/elmWorkspace";
 import * as path from "path";
 import { Settings } from "../src/util/settings";
 import Parser from "web-tree-sitter";
-import {
-  getCancellationFilePath,
-  FileBasedCancellationTokenSource,
-  getCancellationFolderPath,
-  ThrottledCancellationToken,
-} from "../src/cancellation";
-import { randomBytes } from "crypto";
-import { Diagnostic } from "vscode-languageserver";
 import { spawnSync } from "child_process";
 import { appendFileSync, readFileSync } from "fs";
+import { Diagnostic } from "../src/util/types/diagnostics";
+import { performance } from "perf_hooks";
 
 container.register("Connection", {
   useValue: {
@@ -25,8 +19,8 @@ container.register("Connection", {
       warn: (): void => {
         // console.log(a);
       },
-      error: (): void => {
-        // console.log(a);
+      error: (a: string): void => {
+        console.log(a);
       },
     },
     window: {
@@ -36,6 +30,7 @@ container.register("Connection", {
     },
   },
 });
+
 container.register("Settings", {
   useValue: new Settings({} as any, {}),
 });
@@ -50,61 +45,59 @@ async function initParser(): Promise<void> {
   container.resolve<Parser>("Parser").setLanguage(language);
 }
 
+const failed: string[] = [];
+const diagnosticTimes = new Map<string, number>();
+
 export async function runDiagnosticTests(uri: string): Promise<void> {
   const pathUri = URI.file(uri);
 
-  await initParser();
-
-  const elmWorkspace = new ElmWorkspace(pathUri);
-  await elmWorkspace.init(() => {
-    //
-  });
-
-  const cancellationToken = new FileBasedCancellationTokenSource(
-    getCancellationFilePath(
-      getCancellationFolderPath(randomBytes(21).toString("hex")),
-      "1",
-    ),
-  );
-
-  const token = new ThrottledCancellationToken(cancellationToken.token);
-
   try {
+    let elmWorkspace = new ElmWorkspace(pathUri);
+    await elmWorkspace.init(() => {
+      //
+    });
+
+    const start = performance.now();
     const diagnostics: Diagnostic[] = [];
-    elmWorkspace.getForest().treeMap.forEach((treeContainer) =>
-      diagnostics.push(
-        ...elmWorkspace
-          .getTypeChecker()
-          .getDiagnostics(treeContainer, token)
-          .filter((d) => {
-            const uri = (<any>d.data).uri as string;
-            return (
-              !uri.includes("test") &&
-              elmWorkspace.getForest().getByUri(uri)?.writeable
-            );
-          }),
-      ),
-    );
-    console.log("\n");
-    console.log(uri);
+    elmWorkspace
+      .getForest()
+      .treeMap.forEach((treeContainer) =>
+        diagnostics.push(
+          ...[
+            ...elmWorkspace.getSyntacticDiagnostics(treeContainer),
+            ...elmWorkspace.getSemanticDiagnostics(treeContainer),
+          ].filter(
+            (d) =>
+              !d.uri.includes("test") &&
+              elmWorkspace.getForest().getByUri(d.uri)?.writeable,
+          ),
+        ),
+      );
+    diagnosticTimes.set(path.basename(uri), performance.now() - start);
+
     console.log(`${diagnostics.length} diagnostics found.`);
 
     diagnostics.forEach((diagnostic) => {
-      console.log(
-        `${path.basename((<any>diagnostic.data).uri)}: ${diagnostic.message}`,
-      );
+      console.log(`${path.basename(diagnostic.uri)}: ${diagnostic.message}`);
     });
 
+    console.log();
+
     if (diagnostics.length === 0) {
-      appendFileSync(path.join(__dirname, "complete.txt"), `${uri}\n`);
+      // appendFileSync(path.join(__dirname, "complete.txt"), `${uri}\n`);
     } else {
-      process.exitCode = 1;
+      failed.push(path.basename(uri));
+      // process.exitCode = 1;
     }
+
+    elmWorkspace.getForest().treeMap.forEach((sourceFile) => {
+      sourceFile.tree.delete();
+    });
+    elmWorkspace = undefined!;
   } catch (e) {
-    console.log("\n");
-    console.log(uri);
     console.log(e);
-    process.exitCode = 1;
+    failed.push(uri);
+    // process.exitCode = 1;
   }
 }
 
@@ -116,7 +109,6 @@ function checkout(repo: string, url: string): void {
   spawnSync("git", ["fetch"]);
   spawnSync("git", ["reset", "--hard", "HEAD"]);
   spawnSync("elm", ["make"]);
-  spawnSync("elm-test");
   process.chdir(cur);
 }
 
@@ -130,11 +122,7 @@ const libsToParse = require("../script/search.json") as {
 }[];
 
 const parsingFailures = [
-  "showell/dict-dot-dot",
-  "folkertdev/elm-cff",
   "niho/json-schema-form",
-  "ianmackenzie/elm-iso-10303",
-  "ianmackenzie/elm-step-file",
   "brian-watkins/elm-spec",
   "ggb/elm-trend",
   "indicatrix/elm-chartjs-webcomponent", // comment between case branches
@@ -144,6 +132,8 @@ const parsingFailures = [
 ];
 const compilerFailures = ["pablohirafuji/elm-qrcode", "rtfeldman/elm-css"];
 
+const unknownFailures = ["Chadtech/elm-css-grid"];
+
 let completed: string[] = [];
 
 try {
@@ -160,8 +150,9 @@ const filteredLibs = libsToParse
     (lib) =>
       !lib.startsWith("elm/") &&
       !lib.startsWith("elm-explorations/") &&
-      !parsingFailures.includes(lib) &&
-      !compilerFailures.includes(lib) &&
+      !unknownFailures.includes(lib) &&
+      // !parsingFailures.includes(lib) &&
+      // !compilerFailures.includes(lib) &&
       !completed.includes(path.join(__dirname, "../", `examples-full/${lib}`)),
   );
 
@@ -169,17 +160,40 @@ console.log("Getting applications");
 
 const applications = require("../script/applications.json") as string[];
 
-[...applications, ...filteredLibs]
-  // .slice(0, 50)
-  .forEach((lib) => {
+async function testAll(): Promise<void> {
+  await initParser();
+
+  for (const lib of [...applications, ...filteredLibs]) {
     console.log(lib);
     const dir = `examples-full/${lib}`;
 
     try {
       checkout(dir, lib);
 
-      void runDiagnosticTests(path.join(__dirname, "../", dir));
+      await runDiagnosticTests(path.join(__dirname, "../", dir));
     } catch (e) {
       console.log(e);
+    } finally {
+      if (global.gc) {
+        global.gc();
+      }
     }
-  });
+  }
+
+  console.log("FAILURES");
+  failed.forEach((fail) => console.log(fail));
+
+  console.log("TOP TEN TIMES");
+  Array.from(diagnosticTimes.entries())
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, 10)
+    .forEach(([uri, time]) => {
+      console.log(`${uri}: ${time.toFixed(0)}ms`);
+    });
+}
+
+process.on("uncaughtException", function (err) {
+  console.log(`Caught exception: ${err}`);
+});
+
+void testAll();
diff --git a/test/utils/mockElmWorkspace.ts b/test/utils/mockElmWorkspace.ts
index 362f976..759d7f1 100644
--- a/test/utils/mockElmWorkspace.ts
+++ b/test/utils/mockElmWorkspace.ts
@@ -62,10 +62,6 @@ export class MockElmWorkspace implements IElmWorkspace {
     return baseUri;
   }
 
-  public getSourceFile(uri: string): ITreeContainer | undefined {
-    return this.getForest().getByUri(uri);
-  }
-
   getForest(): IForest {
     return this.forest;
   }