Repository: vercel/next.js
Pull request: [test] Always run all tests without aborting on failure

It's more useful and efficient to run all tests and report all failures instead of aborting on the first failure. This way, developers get a complete picture of what needs to be fixed in a single run, and don't have to go through multiple iterations of fixing one failure at a time.

Changed files:
filename: .github/workflows/integration_tests_reusable.yml
patch: @@ -91,7 +91,6 @@ jobs:
       afterBuild: |
         # e2e and ${{ inputs.test_type }} tests with `node run-tests.js`
 
-        export NEXT_TEST_CONTINUE_ON_ERROR=true
         export NEXT_TEST_MODE=${{
           inputs.test_type == 'development' && 'dev' || 'start'
         }}
@@ -123,8 +122,6 @@ jobs:
       afterBuild: |
         # legacy integration tests with `node run-tests.js`
 
-        export NEXT_TEST_CONTINUE_ON_ERROR=true
-
         # HACK: Despite the name, these environment variables are just used to
         # gate tests, so they're applicable to both turbopack and rspack tests
         export ${{

filename: .github/workflows/test_e2e_deploy_release.yml
patch: @@ -19,10 +19,6 @@ on:
         description: Override the proxy address to use for the test
         default: ''
         type: string
-      continueOnError:
-        description: whether the tests should continue on failure
-        default: false
-        type: boolean
 
 env:
   TURBO_TEAM: 'vercel'
@@ -85,7 +81,6 @@ jobs:
         NEXT_E2E_TEST_TIMEOUT=240000 \
         NEXT_TEST_MODE=deploy \
         IS_WEBPACK_TEST=1 \
-        NEXT_TEST_CONTINUE_ON_ERROR="${{ github.event.inputs.continueOnError || false }}" \
         NEXT_EXTERNAL_TESTS_FILTERS="test/deploy-tests-manifest.json" \
         NEXT_TEST_VERSION="${{ github.event.inputs.nextVersion || needs.setup.outputs.next-version || 'canary' }}" \
         VERCEL_CLI_VERSION="${{ github.event.inputs.vercelCliVersion || 'vercel@latest' }}" \

filename: run-tests.js
patch: @@ -53,10 +53,6 @@ const DEFAULT_NUM_RETRIES = 2
 const DEFAULT_CONCURRENCY = 2
 const RESULTS_EXT = `.results.json`
 const isTestJob = !!process.env.NEXT_TEST_JOB
-// Check env to see if test should continue even if some of test fails
-const shouldContinueTestsOnError =
-  process.env.NEXT_TEST_CONTINUE_ON_ERROR === 'true'
-
 const KV_TIMINGS_KEY = 'test-timings'
 
 const kvClient =
@@ -482,8 +478,6 @@ ${ENDGROUP}`)
     `jest${process.platform === 'win32' ? '.CMD' : ''}`
   )
   let firstError = true
-  const testController = new AbortController()
-  const testSignal = testController.signal
   let hadFailures = false
 
   const runTestOnce = (/** @type {TestFile} */ test, isFinalRun, isRetry) =>
@@ -597,13 +591,10 @@ ${ENDGROUP}`)
         if (isChildExitWithNonZero) {
           if (hideOutput) {
             await outputSema.acquire()
-            const isExpanded =
-              firstError && !testSignal.aborted && !shouldContinueTestsOnError
+            const isExpanded = firstError
             if (isExpanded) {
               firstError = false
               process.stdout.write(`❌ ${test.file} output:\n`)
-            } else if (testSignal.aborted) {
-              process.stdout.write(`${GROUP}${test.file} output (killed)\n`)
             } else {
               process.stdout.write(`${GROUP}❌ ${test.file} output\n`)
             }
@@ -616,7 +607,7 @@ ${ENDGROUP}`)
               output += chunk.toString()
             }
 
-            if (process.env.CI && !testSignal.aborted) {
+            if (process.env.CI) {
               errorsPerTests.set(test.file, output)
             }
 
@@ -700,24 +691,12 @@ ${ENDGROUP}`)
 
     if (!passed) {
       hadFailures = true
-      const error = new Error(
-        // "failed to pass within" is a keyword parsed by next-pr-webhook
-        `${test.file} failed to pass within ${numRetries} retries`
-      )
-      console.error(error.message)
-
-      if (!shouldContinueTestsOnError) {
-        testController.abort(error)
-      } else {
-        console.log(
-          `CONTINUE_ON_ERROR enabled, continuing tests after ${test.file} failed`
-        )
-      }
+      // "failed to pass within" is a keyword parsed by next-pr-webhook
+      console.error(`${test.file} failed to pass within ${numRetries} retries`)
     }
 
-    // Emit test output if test failed or if we're continuing tests on error
-    // This is parsed by the commenter webhook to notify about failing tests
-    if ((!passed || shouldContinueTestsOnError) && isTestJob) {
+    // Emit test output, parsed by the commenter webhook to notify about failing tests
+    if (!passed && isTestJob) {
       try {
         const testsOutput = await fsp.readFile(
           `${test.file}${RESULTS_EXT}`,
@@ -761,13 +740,6 @@ ${ENDGROUP}`)
       await sema.acquire()
 
       try {
-        if (testSignal.aborted) {
-          // We already logged the abort reason. No need to include it in cause.
-          const error = new Error(`Skipped due to abort.`)
-          error.name = test.file
-          throw error
-        }
-
         await runTest(test)
       } finally {
         sema.release()
@@ -783,11 +755,6 @@ ${ENDGROUP}`)
     }
   }
 
-  if (hadFailures && !shouldContinueTestsOnError) {
-    // TODO: Does it make sense to update timings if there were failures if without shouldContinueTestsOnError?
-    return hadFailures
-  }
-
   if (options.timings) {
     const curTimings = {}
     // let junitData = `<testsuites name="jest tests">`