Repository: denoland/deno
Pull request: test(ext/node): run all node tests (unchanged) daily, upload reports to dl.deno.land

towards #28318

This PR adds script `tests/node_compat/run_all_test_unmodified.ts`, which:
- Runs all node native tests unchanged
- Creates json report file at `tests/node_compat/report.json`
  - The json has the form of:
    ```jsonc
    {
      "results": {
        "parallel/test-assert-async.js": [true], // for successful case
        "parallel/test-assert.js": [false, { "code": 1, "stderr": "..." }], // for failed case
      }
    }
    ```

This PR also adds CI workflow `node-compat-test`, which runs the above script daily on 3 platforms, sends the gzipped json file to `dl.deno.land/node-compat-test/YYYY-MM-DD/report-OS.json.gz`

Note: The size of report.json is around 2 MB. The size of gzipped version is around 150 KB

---
ref:
- An example `report.json`: https://gist.github.com/kt3k/704217618fbeabd6f119b740c0662e1b

Changed files:
filename: .github/workflows/node_compat_test.yml
patch: @@ -0,0 +1,44 @@
+name: node-compat-test
+
+on:
+  schedule:
+    - cron: '0 10 * * *'
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: '${{ matrix.runner }}'
+    strategy:
+      matrix:
+        include:
+          - os: linux
+            runner: ubuntu-latest
+          - os: windows
+            runner: windows-latest
+          - os: darwin
+            runner: macos-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Deno
+        uses: denoland/setup-deno@v2
+        with:
+          deno-version: canary
+      - name: Authenticate with Google Cloud
+        uses: google-github-actions/auth@v2
+        with:
+          project_id: denoland
+          credentials_json: '${{ secrets.GCP_SA_KEY }}'
+          export_environment_variables: true
+          create_credentials_file: true
+      - name: Setup gcloud
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          project_id: denoland
+      - name: Run tests
+        run: deno -A tests/node_compat/run_all_test_unmodified.ts
+      - name: Gzip the report
+        run: gzip tests/node_compat/report.json
+      - name: Upload the report to dl.deno.land
+        run: |-
+          gsutil -h "Cache-Control: public, max-age=3600" cp tests/node_compat/report.json.gz gs://dl.deno.land/node-compat-test/$(date +%F)/report-${{matrix.os}}.json.gz

filename: tests/node_compat/.gitignore
patch: @@ -1 +1,3 @@
 test/.tmp.*
+report.json
+report.json.gz

filename: tests/node_compat/run_all_test_unmodified.ts
patch: @@ -0,0 +1,324 @@
+// Copyright 2018-2025 the Deno authors. MIT license.
+// deno-lint-ignore-file no-console
+
+import { deadline } from "@std/async/deadline";
+import { expandGlob } from "@std/fs/expand-glob";
+import { toFileUrl } from "@std/path/to-file-url";
+import { basename } from "@std/path/basename";
+import { pooledMap } from "@std/async/pool";
+import { partition } from "@std/collections/partition";
+import { stripAnsiCode } from "@std/fmt/colors";
+
+// The timeout ms for single test execution. If a single test didn't finish in this timeout milliseconds, the test is considered as failure
+const TIMEOUT = 2000;
+const testDirUrl = new URL("runner/suite/test/", import.meta.url).href;
+
+// from https://github.com/denoland/std/pull/2787#discussion_r1001237016
+const NODE_IGNORED_TEST_DIRS = [
+  "addons",
+  "async-hooks",
+  "benchmark",
+  "cctest",
+  "common",
+  "doctool",
+  "embedding",
+  "fixtures",
+  "fuzzers",
+  "js-native-api",
+  "node-api",
+  "overlapped-checker",
+  "report",
+  "testpy",
+  "tick-processor",
+  "tools",
+  "v8-updates",
+  "wasi",
+  "wpt",
+];
+
+// Category names are usually one word, but there are some exceptions.
+// This list contains the exceptions.
+const multiWordsCategoryNames = [
+  "async-hooks",
+  "async-local-storage",
+  "async-wrap",
+  "child-process",
+  "cpu-prof",
+  "double-tls",
+  "diagnostics-channel",
+  "force-repl",
+  "listen-fd",
+  "memory-usage",
+  "next-tick",
+  "outgoing-message",
+  "shadow-realm",
+  "single-executable",
+  "string-decoder",
+];
+
+const categoryMap = {
+  cjs: "module",
+  cwd: "process",
+  diagnostic: "diagnostics-channel",
+  "double-tls": "net",
+  event: "events",
+  eventsource: "events",
+  eventtarget: "events",
+  esm: "module",
+  file: "fs",
+  filehandle: "fs",
+  "force-repl": "repl",
+  inspect: "util",
+  "listen-fd": "net",
+  "next-tick": "process",
+  "outgoing-message": "http",
+  promises: "promise",
+  readable: "stream",
+  require: "module",
+  socket: "net",
+  stdin: "stdio",
+  stdout: "stdio",
+  stream2: "stream",
+  stream3: "stream",
+  tcp: "net",
+  ttywrap: "tty",
+  webstream: "webstreams",
+} as Record<string, string>;
+
+// These name could appear as category name, but they are actually not.
+// If the category name is one of these, it should be categorized as "others".
+const otherCategories = [
+  "common",
+  "compile",
+  "corepack",
+  "disable",
+  "env",
+  "error",
+  "errors",
+  "eslint",
+  "eval",
+  "exception",
+  "handle",
+  "heap",
+  "heapdump",
+  "heapsnapshot",
+  "internal",
+  "memory",
+  "no",
+  "queue",
+  "release",
+  "set",
+  "source",
+  "startup",
+  "sync",
+  "trace",
+  "tick",
+  "unhandled",
+  "uv",
+  "warn",
+  "windows",
+  "wrap",
+];
+
+/**
+ * The test files in these dirs seem categorized in the form
+ * test-[category-name]-test-case.js
+ */
+const categorizedTestGroups = [
+  "es-module",
+  "parallel",
+  "pummel",
+  "sequential",
+  "internet",
+];
+
+/** The group is the directory name of the test file.
+ * e.g. parallel, internet, pummel, sequential, pseudo-tty, etc */
+function getGroupRelUrl(str: string) {
+  return str.split("/")[0];
+}
+
+/** Gets the category name from the test path
+ * e.g.
+ * - parallel/test-async-hooks-destroyed-context.js -> async-hooks
+ * - sequential/test-child-process-exec-stderr.js -> child-process
+ * - internet/test-http-keep-alive.js -> http
+ * - pseudo-tty/test-stdin.js -> tty
+ * - module-hooks/test-require.js -> module
+ */
+function getCategoryFromPath(str: string) {
+  const group = getGroupRelUrl(str);
+  if (group === "pseudo-tty") {
+    return "tty";
+  } else if (group === "module-hooks") {
+    return "module";
+  } else if (categorizedTestGroups.includes(group)) {
+    const name = basename(str).replace(/\.js/, "");
+    let category = name.split("-")[1];
+    for (const multiWord of multiWordsCategoryNames) {
+      if (name.startsWith("test-" + multiWord)) {
+        category = multiWord;
+      }
+    }
+    category = categoryMap[category] ?? category;
+    if (otherCategories.includes(category)) {
+      return "others";
+    }
+    return category;
+  } else {
+    return "others";
+  }
+}
+
+/** Collect the items that are not categorized into the "others" category. */
+function collectNonCategorizedItems(categories: Record<string, string[]>) {
+  const others = [] as string[];
+  for (const [category, items] of Object.entries(categories)) {
+    if (items.length === 1) {
+      delete categories[category];
+      others.push(...items);
+    }
+  }
+  (categories["others"] ??= []).push(...others);
+}
+
+function truncateTestOutput(output: string): string {
+  output = stripAnsiCode(output);
+  if (output.length > 2000) {
+    return output.slice(0, 2000) + " ...";
+  }
+  return output;
+}
+
+type SingleResult = [
+  pass: boolean,
+  error?: ErrorExit | ErrorTimeout | ErrorUnexpected,
+];
+type ErrorExit = {
+  code: number;
+  stderr: string;
+};
+type ErrorTimeout = {
+  timeout: number;
+};
+type ErrorUnexpected = {
+  message: string;
+};
+
+/**
+ * Run a single node test file. Retries 3 times on WouldBlock error.
+ *
+ * @param testPath Relative path to the test file
+ */
+async function runSingle(testPath: string, retry = 0): Promise<SingleResult> {
+  try {
+    const cmd = new Deno.Command(Deno.execPath(), {
+      args: [
+        "-A",
+        "--quiet",
+        "--unstable-bare-node-builtins",
+        "--unstable-node-globals",
+        "tests/node_compat/runner/suite/test/" + testPath,
+      ],
+      env: {
+        NODE_TEST_KNOWN_GLOBALS: "0",
+        NO_COLOR: "1",
+      },
+      stdout: "piped",
+      stderr: "piped",
+      signal: AbortSignal.timeout(TIMEOUT),
+    });
+    const result = await deadline(cmd.output(), TIMEOUT + 1000);
+    if (result.code === 0) {
+      return [true];
+    } else {
+      return [false, {
+        code: result.code,
+        stderr: truncateTestOutput(new TextDecoder().decode(result.stderr)),
+      }];
+    }
+  } catch (e) {
+    if (e instanceof DOMException && e.name === "TimeoutError") {
+      return [false, { timeout: TIMEOUT }];
+    } else if (e instanceof Deno.errors.WouldBlock && retry < 3) {
+      // retry 2 times on WouldBlock error (Resource temporarily unavailable)
+      return runSingle(testPath, retry + 1);
+    } else {
+      return [false, { message: (e as Error).message }];
+    }
+  }
+}
+
+async function main() {
+  const start = Date.now();
+  const tests = [] as string[];
+  const categories = {} as Record<string, string[]>;
+  for await (
+    const test of expandGlob("tests/node_compat/runner/suite/**/test-*.js")
+  ) {
+    if (!test.isFile) continue;
+    const relUrl = toFileUrl(test.path).href.replace(testDirUrl, "");
+    if (NODE_IGNORED_TEST_DIRS.every((dir) => !relUrl.startsWith(dir))) {
+      tests.push(relUrl);
+      (categories[getCategoryFromPath(relUrl)] ??= []).push(relUrl);
+    }
+  }
+  collectNonCategorizedItems(categories);
+  console.log("Running", tests.length, "tests");
+  const categoryList = Object.entries(categories)
+    .sort(([c0], [c1]) => c0.localeCompare(c1));
+  const results = {} as Record<string, SingleResult>;
+  let i = 0;
+  async function run(testPath: string) {
+    const num = String(++i).padStart(4, " ");
+    const result = await runSingle(testPath);
+    results[testPath] = result;
+    if (result[0]) {
+      console.log(`${num} %cPASS`, "color: green", testPath);
+    } else {
+      console.log(`${num} %cFAIL`, "color: red", testPath);
+    }
+  }
+  const [sequential, parallel] = partition(
+    tests,
+    (test) => getGroupRelUrl(test) === "sequential",
+  );
+  // Runs sequential tests
+  for (const path of sequential) {
+    await run(path);
+  }
+  // Runs parallel tests
+  for await (
+    const _ of pooledMap(navigator.hardwareConcurrency * 2, parallel, run)
+  ) {
+    // pass
+  }
+
+  // Reporting to stdout
+  console.log(`Result by categories (${categoryList.length}):`);
+  for (const [category, tests] of categoryList) {
+    const s = tests.filter((test) => results[test][0]).length;
+    const all = tests.length;
+    console.log(`  ${category} ${s}/${all} (${(s / all * 100).toFixed(2)}%)`);
+    for (const testPath of tests) {
+      if (results[testPath][0]) {
+        console.log(`    %cPASS`, "color: green", testPath);
+      } else {
+        console.log(`    %cFAIL`, "color: red", testPath);
+      }
+    }
+  }
+
+  // Summary
+  const all = tests.length;
+  const s = tests.filter((test) => results[test][0]).length;
+  console.log(`All tests: ${s}/${all} (${(s / all * 100).toFixed(2)}%)`);
+  console.log(`Elapsed time: ${((Date.now() - start) / 1000).toFixed(2)}s`);
+  Deno.writeTextFile(
+    "tests/node_compat/report.json",
+    JSON.stringify(results),
+  );
+  Deno.exit(0);
+}
+
+await main();

filename: tests/node_compat/runner/suite
patch: @@ -1 +1 @@
-Subproject commit dfcae8dc3d7ed7dd4d6d51764d307468c849915d
+Subproject commit 8846b5392f4ebd3d4e995b9bed61c23c0192d806