Repository: denoland/deno
Pull request: test: allow WPT tests to be marked as flaky

## Summary
- Add `flaky?: boolean` to `TestExpectation` interface and JSON schema so tests can be marked as flaky
- Retry flaky tests up to 3 times before marking as failed, with yellow log output on retries
- Preserve the `flaky` flag when running `wpt.ts update` so it isn't stripped from expectations

## Test plan
- [ ] Mark a known-passing test as `flaky: true` — should pass on first try with no retries
- [ ] Mark a known-failing test as `flaky: true` — should retry 2 more times, then fail
- [ ] Run `wpt.ts update` on a test with `flaky: true` — verify the flag is preserved
- [ ] Verify `{ flaky: true }` alone is recognized as a leaf expectation node

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Changed files:
filename: tests/wpt/runner/expectations/schema.json
patch: @@ -41,6 +41,10 @@
             "type": "string"
           },
           "description": "Subtest names expected to fail; all others should pass"
+        },
+        "flaky": {
+          "type": "boolean",
+          "description": "If true, retry test up to 3 times before marking as failed"
         }
       },
       "additionalProperties": false

filename: tests/wpt/runner/utils.ts
patch: @@ -118,6 +118,7 @@ export const EXPECTATIONS_DIR = join(
 export interface TestExpectation {
   ignore?: boolean;
   expectedFailures?: string[];
+  flaky?: boolean;
 }
 
 export interface Expectation {

filename: tests/wpt/wpt.ts
patch: @@ -197,7 +197,7 @@ async function setup() {
 function isLeafExpectation(e: unknown): e is boolean | TestExpectation {
   return typeof e === "boolean" ||
     (typeof e === "object" && e !== null && !Array.isArray(e) &&
-      ("expectedFailures" in e || "ignore" in e));
+      ("expectedFailures" in e || "ignore" in e || "flaky" in e));
 }
 
 interface TestToRun {
@@ -269,13 +269,54 @@ Options:
         if (!inParallel) {
           console.log(`${blue("-".repeat(40))}\n${bold(test.path)}\n`);
         }
-        const result = await runSingleTest(
+        let result = await runSingleTest(
           test.url,
           test.options,
           inParallel ? () => {} : createReportTestCase(test.expectation),
           inspectBrk,
           getTestTimeout(test),
         );
+        const isFlaky = typeof test.expectation === "object" &&
+          test.expectation !== null && test.expectation.flaky === true;
+        if (isFlaky) {
+          const isFileLevelFailure = result.status !== 0 ||
+            result.harnessStatus === null;
+          const analysis = analyzeTestResult(result, test.expectation);
+          const hasFailed = isFileLevelFailure || analysis.failedCount > 0;
+          if (hasFailed) {
+            for (let attempt = 2; attempt <= 3; attempt++) {
+              console.log(
+                yellow(
+                  `Retrying flaky test ${test.path} (attempt ${attempt}/3)`,
+                ),
+              );
+              const retryResult = await runSingleTest(
+                test.url,
+                test.options,
+                () => {},
+                inspectBrk,
+                getTestTimeout(test),
+              );
+              const retryFileLevelFailure = retryResult.status !== 0 ||
+                retryResult.harnessStatus === null;
+              const retryAnalysis = analyzeTestResult(
+                retryResult,
+                test.expectation,
+              );
+              const retryFailed = retryFileLevelFailure ||
+                retryAnalysis.failedCount > 0;
+              if (!retryFailed) {
+                console.log(
+                  yellow(
+                    `Flaky test ${test.path} passed on attempt ${attempt}/3`,
+                  ),
+                );
+                result = retryResult;
+                break;
+              }
+            }
+          }
+        }
         results.push({ test, result });
         if (inParallel) {
           console.log(`${blue("-".repeat(40))}\n${bold(test.path)}\n`);
@@ -537,6 +578,17 @@ function newExpectation(
 
   const currentExpectation = getExpectation();
 
+  // Build a map of path -> flaky flag from the original expectations
+  const flakyTests = new Set<string>();
+  for (const { test } of results) {
+    if (
+      typeof test.expectation === "object" && test.expectation !== null &&
+      test.expectation.flaky === true
+    ) {
+      flakyTests.add(test.path);
+    }
+  }
+
   for (const [path, result] of Object.entries(resultTests)) {
     const { passed, failed, testSucceeded } = result;
     let finalExpectation: boolean | TestExpectation;
@@ -548,6 +600,15 @@ function newExpectation(
       finalExpectation = false;
     }
 
+    // Preserve the flaky flag from the original expectation
+    if (flakyTests.has(path)) {
+      if (typeof finalExpectation === "object") {
+        finalExpectation.flaky = true;
+      } else if (finalExpectation === true) {
+        finalExpectation = { flaky: true };
+      }
+    }
+
     insertExpectation(
       path.slice(1).split("/"),
       currentExpectation,