Repository: vercel/next.js
Pull request: [turbopack] Run the deployment tests for turbopack and webpack

Fork the set of deployment tests so they run with both turbopack and webpack.  Now that turbopack is the default bundler we need this coverage.

Because we are now running twice as many tests, decrease concurrency, otherwise we hit rate limits on vercel.  There were attempts to improve rate limit recovery in the vercel CLI (https://github.com/vercel/vercel/pull/14443 and https://github.com/vercel/vercel/pull/14407) which helped but did not solve the issue.  Also there was an investigation into the API service where we discovered some suspicious but ultimately correct code (https://github.com/vercel/api/pull/55967).  

The basic issue is that the vercel CLI polls `api-deployments-get` fairly aggressively at the beginning of a deploymnet, and since we start so many deployments in parallel we will always hit the rate limits.  The recovery logic in the CLI is good but has (reasonably) a fixed set of retries it will attempt, inevitably some tasks get unlucky.  So retrying the test does work, but really we should slow down which is what we do here.  😥 

https://github.com/vercel/next.js/actions/runs/20290161194/job/58272593085

Fixes PACK-5613

Changed files:
filename: .github/workflows/build_reusable.yml
patch: @@ -114,7 +114,7 @@ env:
   VERCEL_ADAPTER_TEST_TOKEN: ${{ secrets.VERCEL_ADAPTER_TEST_TOKEN }}
   VERCEL_ADAPTER_TEST_TEAM: vtest314-next-adapter-e2e-tests
   VERCEL_TURBOPACK_TEST_TOKEN: ${{ secrets.VERCEL_TURBOPACK_TEST_TOKEN }}
-  VERCEL_TURBOPACK_TEST_TEAM: vtest314-next-turbopack-e2e-tests
+  VERCEL_TURBOPACK_TEST_TEAM: vtest314-next-turbo-e2e-tests
   NEXT_TEST_PREFER_OFFLINE: 1
   NEXT_CI_RUNNER: ${{ inputs.runs_on_labels }}
   NEXT_TEST_PROXY_ADDRESS: ${{ inputs.overrideProxyAddress || '' }}

filename: .github/workflows/test_e2e_deploy_release.yml
patch: @@ -35,7 +35,7 @@ env:
   VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }}
   VERCEL_ADAPTER_TEST_TEAM: vtest314-next-adapter-e2e-tests
   VERCEL_ADAPTER_TEST_TOKEN: ${{ secrets.VERCEL_ADAPTER_TEST_TOKEN }}
-  VERCEL_TURBOPACK_TEST_TEAM: vtest314-next-turbopack-e2e-tests
+  VERCEL_TURBOPACK_TEST_TEAM: vtest314-next-turbo-e2e-tests
   VERCEL_TURBOPACK_TEST_TOKEN: ${{ secrets.VERCEL_TURBOPACK_TEST_TOKEN }}
 
 run-name: test-e2e-deploy ${{ inputs.nextVersion || (github.event_name == 'release' && github.event.release.tag_name) || 'canary' }}
@@ -76,8 +76,8 @@ jobs:
         name: Extract `next` version
         run: echo 'value=${{ fromJson(steps.nextPackageInfo.outputs.value).version }}' >> "$GITHUB_OUTPUT"
 
-  test-deploy:
-    name: Run Deploy Tests
+  test-deploy-webpack:
+    name: Run Deploy Tests (Webpack)
     needs: setup
     uses: ./.github/workflows/build_reusable.yml
     secrets: inherit
@@ -99,7 +99,49 @@ jobs:
         node run-tests.js --timings -g ${{ matrix.group }} -c 2 --type e2e
       skipNativeBuild: 'yes'
       skipNativeInstall: 'no'
-      stepName: 'test-deploy-${{ matrix.group }}'
+      stepName: 'test-deploy-webpack-${{ matrix.group }}'
+      timeout_minutes: 180
+      runs_on_labels: '["ubuntu-latest"]'
+      overrideProxyAddress: ${{ inputs.overrideProxyAddress || '' }}
+
+  test-deploy-turbopack:
+    name: Run Deploy Tests (Turbopack)
+    needs: [setup]
+    uses: ./.github/workflows/build_reusable.yml
+    secrets: inherit
+    strategy:
+      fail-fast: false
+      matrix:
+        group:
+          [
+            1/12,
+            2/12,
+            3/12,
+            4/12,
+            5/12,
+            6/12,
+            7/12,
+            8/12,
+            9/12,
+            10/12,
+            11/12,
+            12/12,
+          ]
+    with:
+      afterBuild: |
+        npm i -g vercel@latest && \
+        NEXT_E2E_TEST_TIMEOUT=240000 \
+        NEXT_TEST_MODE=deploy \
+        IS_TURBOPACK_TEST=1 \
+        TURBOPACK_BUILD=1 \
+        NEXT_TEST_CONTINUE_ON_ERROR="${{ github.event.inputs.continueOnError || false }}" \
+        NEXT_EXTERNAL_TESTS_FILTERS="test/deploy-tests-manifest.json" \
+        NEXT_TEST_VERSION="${{ github.event.inputs.nextVersion || needs.setup.outputs.next-version || 'canary' }}" \
+        VERCEL_CLI_VERSION="${{ github.event.inputs.vercelCliVersion || 'vercel@latest' }}" \
+        node run-tests.js --timings -g ${{ matrix.group }} -c 2 --type e2e
+      skipNativeBuild: 'yes'
+      skipNativeInstall: 'no'
+      stepName: 'test-deploy-turbopack-${{ matrix.group }}'
       timeout_minutes: 180
       runs_on_labels: '["ubuntu-latest"]'
       overrideProxyAddress: ${{ inputs.overrideProxyAddress || '' }}
@@ -147,7 +189,7 @@ jobs:
       overrideProxyAddress: ${{ inputs.overrideProxyAddress || '' }}
 
   report-test-results-to-datadog:
-    needs: test-deploy
+    needs: [test-deploy-turbopack, test-deploy-webpack]
     if: ${{ always() }}
 
     runs-on: ubuntu-latest
@@ -238,8 +280,8 @@ jobs:
 
   update-prs:
     name: Update prs as ready for review
-    needs: [test-deploy, create-draft-prs]
-    if: ${{ needs.test-deploy.result == 'success' && github.repository_owner == 'vercel' && github.event_name == 'release' }}
+    needs: [test-deploy-turbopack, test-deploy-webpack, create-draft-prs]
+    if: ${{ needs.test-deploy-webpack.result == 'success' &&  needs.test-deploy-turbopack.result == 'success' && github.repository_owner == 'vercel' && github.event_name == 'release' }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false

filename: scripts/test-new-tests.mjs
patch: @@ -11,12 +11,12 @@ import getChangedTests from './get-changed-tests.mjs'
  * --flake-detection: run tests multiple times to detect flaky
  */
 async function main() {
-  let argv = await yargs(process.argv.slice(2))
+  const argv = await yargs(process.argv.slice(2))
     .string('mode')
     .string('group')
     .boolean('flake-detection').argv
 
-  let testMode = argv.mode
+  const testMode = argv.mode
   const isFlakeDetectionMode = argv['flake-detection']
   const attempts = isFlakeDetectionMode ? 3 : 1
 
@@ -149,7 +149,7 @@ async function main() {
     )
   }
 
-  if (isFlakeDetectionMode && testMode !== 'deploy') {
+  if (isFlakeDetectionMode) {
     for (let i = 0; i < attempts; i++) {
       console.log(
         `\n\nRun ${i + 1}/${attempts} for ${testMode} tests (Turbopack)`
@@ -160,8 +160,10 @@ async function main() {
           ...process.env,
           NEXT_TEST_MODE: testMode,
           NEXT_TEST_VERSION: nextTestVersion,
+          NEXT_EXTERNAL_TESTS_FILTERS,
           IS_TURBOPACK_TEST: '1',
-          TURBOPACK_BUILD: testMode === 'start' ? '1' : undefined,
+          TURBOPACK_BUILD:
+            testMode === 'start' || testMode === 'deploy' ? '1' : undefined,
           TURBOPACK_DEV: testMode === 'dev' ? '1' : undefined,
         },
       })

filename: test/lib/e2e-utils/index.ts
patch: @@ -252,15 +252,13 @@ export async function createNext(
         rootSpan.traceChild('init next deploy instance').traceFn(() => {
           nextInstance = new NextDeployInstance({
             ...opts,
-            turbo: false,
           })
         })
       } else {
         // next build + next start
         rootSpan.traceChild('init next start instance').traceFn(() => {
           nextInstance = new NextStartInstance({
             ...opts,
-            turbo: false,
           })
         })
       }

filename: test/lib/next-modes/next-deploy.ts
patch: @@ -238,7 +238,6 @@ export class NextDeployInstance extends NextInstance {
     if (TEST_TEAM_NAME) {
       vercelFlags.push('--scope', TEST_TEAM_NAME)
     }
-
     const vercelEnv = { ...process.env }
 
     // If the token is available in the environment, use it as the token in the