Repository: denoland/deno
Pull request: test: add ability to mark Node compat tests as flaky

This commit updates test harness for Node.js compat tests to
allow marking certain tests as flaky. 

This is done by adding `flaky = true` to dict entries in
the `tests/node_compat/config.toml` file.


Changed files:
filename: tests/node_compat/config.toml
patch: @@ -132,7 +132,7 @@
 "parallel/test-buffer-zero-fill-cli.js" = {}
 "parallel/test-buffer-zero-fill-reset.js" = {}
 "parallel/test-buffer-zero-fill.js" = {}
-"parallel/test-child-process-can-write-to-stdout.js" = {}
+"parallel/test-child-process-can-write-to-stdout.js" = { flaky = true }
 "parallel/test-child-process-default-options.js" = {}
 "parallel/test-child-process-detached.js" = {}
 "parallel/test-child-process-disconnect.js" = {}

filename: tests/node_compat/run_all_test_unmodified.ts
patch: @@ -161,7 +161,13 @@ let testSerialId = 0;
  */
 export async function runSingle(
   testPath: string,
-  retry = 0,
+  {
+    flaky = false,
+    retry = 0,
+  }: {
+    flaky?: boolean;
+    retry?: number;
+  },
 ): Promise<NodeTestFileReport> {
   testSerialId++;
   let cmd: Deno.ChildProcess | undefined;
@@ -220,7 +226,9 @@ export async function runSingle(
       };
     } else if (e instanceof Deno.errors.WouldBlock && retry < 3) {
       // retry 2 times on WouldBlock error (Resource temporarily unavailable)
-      return runSingle(testPath, retry + 1);
+      return runSingle(testPath, { flaky, retry: retry + 1 });
+    } else if (flaky && retry < 3) {
+      return runSingle(testPath, { flaky, retry: retry + 1 });
     } else {
       return {
         result: NodeTestFileResult.FAIL,
@@ -312,7 +320,7 @@ async function main() {
 
   async function run(testPath: string) {
     const num = String(++i).padStart(4, " ");
-    const result = await runSingle(testPath);
+    const result = await runSingle(testPath, {});
     reports[testPath] = result;
     if (result.result === NodeTestFileResult.PASS) {
       console.log(`${num} %cPASS`, "color: green", testPath);

filename: tests/node_compat/test.ts
patch: @@ -6,9 +6,10 @@ import { assert } from "@std/assert";
 import { partition } from "@std/collections/partition";
 import { pooledMap } from "@std/async/pool";
 
-// This is a placeholder for the future extension of the config file.
-// deno-lint-ignore ban-types
-type SingleFileConfig = {};
+interface SingleFileConfig {
+  flaky?: boolean;
+}
+
 type Config = {
   tests: Record<string, SingleFileConfig>;
 };
@@ -22,8 +23,8 @@ const [sequentialTests, parallelTests] = partition(
   ([testName]) => testName.startsWith("sequential/"),
 );
 
-async function run(name: string) {
-  const result = await runSingle(name);
+async function run(name: string, testConfig: SingleFileConfig) {
+  const result = await runSingle(name, testConfig);
   let msg = "";
   const error = result.error;
   if (error && "message" in error) {
@@ -36,20 +37,20 @@ async function run(name: string) {
   assert(result.result === "pass", `Test "${name}" failed: ${msg}`);
 }
 
-for (const [name, _testConfig] of sequentialTests) {
+for (const [name, testConfig] of sequentialTests) {
   Deno.test("Node compat: " + name, async () => {
-    await run(name);
+    await run(name, testConfig);
   });
 }
 
 Deno.test("Node compat: parallel tests", async (t) => {
   const iter = pooledMap(
     navigator.hardwareConcurrency,
     parallelTests,
-    ([name, _testConfig]) =>
+    ([name, testConfig]) =>
       t.step({
         name,
-        fn: () => run(name),
+        fn: () => run(name, testConfig),
         sanitizeExit: false,
         sanitizeOps: false,
         sanitizeResources: false,