Repository: vercel/next.js
Pull request: Update E2E tests workflow

Adds an option to allow continuing tests on error to find all failures in single run also marks jobs as failed if any test failed while running. 

Changed files:
filename: .github/workflows/integration_tests_reusable.yml
patch: @@ -91,7 +91,7 @@ jobs:
       afterBuild: |
         # e2e and ${{ inputs.test_type }} tests with `node run-tests.js`
 
-        export NEXT_TEST_CONTINUE_ON_ERROR=TRUE
+        export NEXT_TEST_CONTINUE_ON_ERROR=true
         export NEXT_TEST_MODE=${{
           inputs.test_type == 'development' && 'dev' || 'start'
         }}
@@ -123,7 +123,7 @@ jobs:
       afterBuild: |
         # legacy integration tests with `node run-tests.js`
 
-        export NEXT_TEST_CONTINUE_ON_ERROR=TRUE
+        export NEXT_TEST_CONTINUE_ON_ERROR=true
 
         # HACK: Despite the name, these environment variables are just used to
         # gate tests, so they're applicable to both turbopack and rspack tests

filename: .github/workflows/test_e2e_deploy_release.yml
patch: @@ -19,6 +19,10 @@ on:
         description: Override the proxy address to use for the test
         default: ''
         type: string
+      continueOnError:
+        description: whether the tests should continue on failure
+        default: false
+        type: boolean
 
 env:
   TURBO_TEAM: 'vercel'
@@ -81,6 +85,7 @@ jobs:
         NEXT_E2E_TEST_TIMEOUT=240000 \
         NEXT_TEST_MODE=deploy \
         IS_WEBPACK_TEST=1 \
+        NEXT_TEST_CONTINUE_ON_ERROR="${{ github.event.inputs.continueOnError || false }}" \
         NEXT_EXTERNAL_TESTS_FILTERS="test/deploy-tests-manifest.json" \
         NEXT_TEST_VERSION="${{ github.event.inputs.nextVersion || needs.setup.outputs.next-version || 'canary' }}" \
         VERCEL_CLI_VERSION="${{ github.event.inputs.vercelCliVersion || 'vercel@latest' }}" \

filename: run-tests.js
patch: @@ -55,7 +55,9 @@ const DEFAULT_CONCURRENCY = 2
 const RESULTS_EXT = `.results.json`
 const isTestJob = !!process.env.NEXT_TEST_JOB
 // Check env to see if test should continue even if some of test fails
-const shouldContinueTestsOnError = !!process.env.NEXT_TEST_CONTINUE_ON_ERROR
+const shouldContinueTestsOnError =
+  process.env.NEXT_TEST_CONTINUE_ON_ERROR === 'true'
+
 // Check env to load a list of test paths to skip retry. This is to be used in conjunction with NEXT_TEST_CONTINUE_ON_ERROR,
 // When try to run all of the tests regardless of pass / fail and want to skip retrying `known` failed tests.
 // manifest should be a json file with an array of test paths.
@@ -500,6 +502,7 @@ ${ENDGROUP}`)
   )
   let firstError = true
   let killed = false
+  let hadFailures = false
 
   const runTest = (/** @type {TestFile} */ test, isFinalRun, isRetry) =>
     new Promise((resolve, reject) => {
@@ -756,6 +759,7 @@ ${ENDGROUP}`)
           children.forEach((child) => child.kill())
           cleanUpAndExit(1)
         } else {
+          hadFailures = true
           console.log(
             `CONTINUE_ON_ERROR enabled, continuing tests after ${test.file} failed`
           )
@@ -852,10 +856,20 @@ ${ENDGROUP}`)
       }
     }
   }
+
+  // Return whether there were any failures
+  return hadFailures
 }
 
 main()
-  .then(() => cleanUpAndExit(0))
+  .then((hadFailures) => {
+    if (hadFailures) {
+      console.error('Some tests failed')
+      cleanUpAndExit(1)
+    } else {
+      cleanUpAndExit(0)
+    }
+  })
   .catch((err) => {
     console.error(err)
     cleanUpAndExit(1)