Repository: vercel/next.js
Pull request: [test] Deflake `test/integration/invalid-custom-routes/test/index.test.ts`

More and more checks get deferred since we want "ready" to be a signal for "ready to use" not "ready and correct". Tests need to account for that. 

This PR extracts launching the app out of `getStderr` so that we can actually retry reading stderr without attempting to launch a new app.
Some tests already retried `getStderr`. This PR just expands the automatic retry to every test. Just without restarting the app and instead reusing the existing instance.

Fixes 
> Expected substring: "`destination` is missing for route {\"source\":\"/hello\"}"
    Received string:    ""

-- https://github.com/vercel/next.js/actions/runs/20915455078/job/60087861981

<details>
<summary>
GPT-5.1-Codex-Max prompts
</summary>

> This test should always launch the app. `getStderr` should just read stderr and then we need to retry every stderr assertion like "should error when empty headers array is present on header item" does

> Remove `writeConfigAndStart` and `expectStderr` helpers and inline their impl instead.

</details>


Changed files:
filename: test/integration/invalid-custom-routes/test/index.test.ts
patch: @@ -2,7 +2,7 @@
 
 import fs from 'fs-extra'
 import { join } from 'path'
-import { launchApp, findPort, nextBuild, retry } from 'next-test-utils'
+import { launchApp, findPort, nextBuild, retry, killApp } from 'next-test-utils'
 
 let appDir = join(__dirname, '..')
 const nextConfigPath = join(appDir, 'next.config.js')
@@ -21,6 +21,7 @@ const writeConfig = async (routes, type = 'redirects') => {
 }
 
 let getStderr
+let start
 
 const runTests = () => {
   it('should error when empty headers array is present on header item', async () => {
@@ -33,9 +34,9 @@ const runTests = () => {
       ],
       'headers'
     )
-    await retry(async () => {
-      const stderr = await getStderr()
-
+    await start()
+    await retry(() => {
+      const stderr = getStderr()
       expect(stderr).toContain(
         '`headers` field cannot be empty for route {"source":"/:path*"'
       )
@@ -58,14 +59,16 @@ const runTests = () => {
       ],
       'redirects'
     )
-    const stderr = await getStderr()
-
-    expect(stderr).toContain(
-      '`source` exceeds max built length of 4096 for route {"source":"/aaaaaaaaaaaaaaaaaa'
-    )
-    expect(stderr).toContain(
-      '`destination` exceeds max built length of 4096 for route {"source":"/","destination":"/aaaa'
-    )
+    await start()
+    await retry(() => {
+      const stderr = getStderr()
+      expect(stderr).toContain(
+        '`source` exceeds max built length of 4096 for route {"source":"/aaaaaaaaaaaaaaaaaa'
+      )
+      expect(stderr).toContain(
+        '`destination` exceeds max built length of 4096 for route {"source":"/","destination":"/aaaa'
+      )
+    })
   })
 
   it('should error during next build for invalid redirects', async () => {
@@ -139,10 +142,9 @@ const runTests = () => {
       ],
       'redirects'
     )
-
-    await retry(async () => {
-      const stderr = await getStderr()
-
+    await start()
+    await retry(() => {
+      const stderr = getStderr()
       expect(stderr).toContain(
         `\`destination\` is missing for route {"source":"/hello","permanent":false}`
       )
@@ -273,76 +275,78 @@ const runTests = () => {
       ],
       'rewrites'
     )
-    const stderr = await getStderr()
-
-    expect(stderr).toContain(
-      `\`destination\` is missing for route {"source":"/hello"}`
-    )
+    await start()
+    await retry(() => {
+      const stderr = getStderr()
+      expect(stderr).toContain(
+        `\`destination\` is missing for route {"source":"/hello"}`
+      )
 
-    expect(stderr).toContain(
-      `\`source\` is not a string for route {"source":123,"destination":"/another"}`
-    )
+      expect(stderr).toContain(
+        `\`source\` is not a string for route {"source":123,"destination":"/another"}`
+      )
 
-    expect(stderr).toContain(
-      `invalid field: headers for route {"source":"/hello","destination":"/another","headers":"not-allowed"}`
-    )
+      expect(stderr).toContain(
+        `invalid field: headers for route {"source":"/hello","destination":"/another","headers":"not-allowed"}`
+      )
 
-    expect(stderr).toContain(
-      `\`source\` does not start with / for route {"source":"hello","destination":"/another"}`
-    )
+      expect(stderr).toContain(
+        `\`source\` does not start with / for route {"source":"hello","destination":"/another"}`
+      )
 
-    expect(stderr).toContain(
-      `\`destination\` does not start with \`/\`, \`http://\`, or \`https://\` for route {"source":"/hello","destination":"another"}`
-    )
+      expect(stderr).toContain(
+        `\`destination\` does not start with \`/\`, \`http://\`, or \`https://\` for route {"source":"/hello","destination":"another"}`
+      )
 
-    expect(stderr).toContain(
-      `Error parsing \`/feedback/(?!general)\` https://nextjs.org/docs/messages/invalid-route-source`
-    )
+      expect(stderr).toContain(
+        `Error parsing \`/feedback/(?!general)\` https://nextjs.org/docs/messages/invalid-route-source`
+      )
 
-    expect(stderr).toContain(
-      `\`destination\` has unnamed params :0 for route {"source":"/hello/world/(.*)","destination":"/:0"}`
-    )
+      expect(stderr).toContain(
+        `\`destination\` has unnamed params :0 for route {"source":"/hello/world/(.*)","destination":"/:0"}`
+      )
 
-    expect(stderr).toContain(
-      `The route null is not a valid object with \`source\` and \`destination\``
-    )
+      expect(stderr).toContain(
+        `The route null is not a valid object with \`source\` and \`destination\``
+      )
 
-    expect(stderr).toContain(
-      `The route "string" is not a valid object with \`source\` and \`destination\``
-    )
+      expect(stderr).toContain(
+        `The route "string" is not a valid object with \`source\` and \`destination\``
+      )
 
-    expect(stderr).toContain(`Reason: Pattern cannot start with "?" at 11`)
-    expect(stderr).toContain(`/feedback/(?!general)`)
+      expect(stderr).toContain(`Reason: Pattern cannot start with "?" at 11`)
+      expect(stderr).toContain(`/feedback/(?!general)`)
 
-    expect(stderr).not.toContain(
-      'Valid redirect statusCode values are 301, 302, 303, 307, 308'
-    )
+      expect(stderr).not.toContain(
+        'Valid redirect statusCode values are 301, 302, 303, 307, 308'
+      )
 
-    expect(stderr).toContain(
-      `The route /hello rewrites urls outside of the basePath. Please use a destination that starts with \`http://\` or \`https://\` https://nextjs.org/docs/messages/invalid-external-rewrite`
-    )
+      expect(stderr).toContain(
+        `The route /hello rewrites urls outside of the basePath. Please use a destination that starts with \`http://\` or \`https://\` https://nextjs.org/docs/messages/invalid-external-rewrite`
+      )
 
-    expect(stderr).toContain('Invalid `has` item:')
-    expect(stderr).toContain(
-      `invalid type "cookiee" for {"type":"cookiee","key":"loggedIn"}`
-    )
-    expect(stderr).toContain(
-      `invalid \`has\` item found for route {"source":"/hello","destination":"/another","has":[{"type":"cookiee","key":"loggedIn"}]}`
-    )
+      expect(stderr).toContain('Invalid `has` item:')
+      expect(stderr).toContain(
+        `invalid type "cookiee" for {"type":"cookiee","key":"loggedIn"}`
+      )
+      expect(stderr).toContain(
+        `invalid \`has\` item found for route {"source":"/hello","destination":"/another","has":[{"type":"cookiee","key":"loggedIn"}]}`
+      )
 
-    expect(stderr).toContain('Invalid `has` items:')
-    expect(stderr).toContain(
-      `invalid type "headerr", invalid key "undefined" for {"type":"headerr"}`
-    )
-    expect(stderr).toContain(
-      `invalid type "queryr" for {"type":"queryr","key":"hello"}`
-    )
-    expect(stderr).toContain(
-      `invalid \`has\` items found for route {"source":"/hello","destination":"/another","has":[{"type":"headerr"},{"type":"queryr","key":"hello"}]}`
-    )
-    expect(stderr).toContain(`Valid \`has\` object shape is {`)
+      expect(stderr).toContain('Invalid `has` items:')
+      expect(stderr).toContain(
+        `invalid type "headerr", invalid key "undefined" for {"type":"headerr"}`
+      )
+      expect(stderr).toContain(
+        `invalid type "queryr" for {"type":"queryr","key":"hello"}`
+      )
+      expect(stderr).toContain(
+        `invalid \`has\` items found for route {"source":"/hello","destination":"/another","has":[{"type":"headerr"},{"type":"queryr","key":"hello"}]}`
+      )
+      expect(stderr).toContain(`Valid \`has\` object shape is {`)
 
-    expect(stderr).toContain('Invalid rewrites found')
+      expect(stderr).toContain('Invalid rewrites found')
+    })
   })
 
   it('should error during next build for invalid headers', async () => {
@@ -447,57 +451,59 @@ const runTests = () => {
       ],
       'headers'
     )
-    const stderr = await getStderr()
-
-    expect(stderr).toContain(
-      '`source` is missing, `key` in header item must be string for route {"headers":[{"x-first":"first"}]}'
-    )
+    await start()
+    await retry(() => {
+      const stderr = getStderr()
+      expect(stderr).toContain(
+        '`source` is missing, `key` in header item must be string for route {"headers":[{"x-first":"first"}]}'
+      )
 
-    expect(stderr).toContain(
-      '`headers` field must be an array for route {"source":"/hello","headers":{"x-first":"first"}}'
-    )
+      expect(stderr).toContain(
+        '`headers` field must be an array for route {"source":"/hello","headers":{"x-first":"first"}}'
+      )
 
-    expect(stderr).toContain(
-      '`key` in header item must be string for route {"source":"/again","headers":[{"value":"idk"}]}'
-    )
+      expect(stderr).toContain(
+        '`key` in header item must be string for route {"source":"/again","headers":[{"value":"idk"}]}'
+      )
 
-    expect(stderr).toContain(
-      '`value` in header item must be string for route {"source":"/again","headers":[{"key":"idk"}]}'
-    )
+      expect(stderr).toContain(
+        '`value` in header item must be string for route {"source":"/again","headers":[{"key":"idk"}]}'
+      )
 
-    expect(stderr).toContain(
-      'invalid field: destination for route {"source":"/again","destination":"/another","headers":[{"key":"x-first","value":"idk"}]}'
-    )
+      expect(stderr).toContain(
+        'invalid field: destination for route {"source":"/again","destination":"/another","headers":[{"key":"x-first","value":"idk"}]}'
+      )
 
-    expect(stderr).toContain(
-      `The route null is not a valid object with \`source\` and \`headers\``
-    )
+      expect(stderr).toContain(
+        `The route null is not a valid object with \`source\` and \`headers\``
+      )
 
-    expect(stderr).toContain(
-      `The route "string" is not a valid object with \`source\` and \`headers\``
-    )
+      expect(stderr).toContain(
+        `The route "string" is not a valid object with \`source\` and \`headers\``
+      )
 
-    expect(stderr).toContain('Invalid `has` item:')
-    expect(stderr).toContain(
-      `invalid type "cookiee" for {"type":"cookiee","key":"loggedIn"}`
-    )
-    expect(stderr).toContain(
-      `invalid \`has\` item found for route {"source":"/hello","has":[{"type":"cookiee","key":"loggedIn"}],"headers":[{"key":"x-hello","value":"world"}]}`
-    )
+      expect(stderr).toContain('Invalid `has` item:')
+      expect(stderr).toContain(
+        `invalid type "cookiee" for {"type":"cookiee","key":"loggedIn"}`
+      )
+      expect(stderr).toContain(
+        `invalid \`has\` item found for route {"source":"/hello","has":[{"type":"cookiee","key":"loggedIn"}],"headers":[{"key":"x-hello","value":"world"}]}`
+      )
 
-    expect(stderr).toContain('Invalid `has` items:')
-    expect(stderr).toContain(
-      `invalid type "headerr", invalid key "undefined" for {"type":"headerr"}`
-    )
-    expect(stderr).toContain(
-      `invalid type "queryr" for {"type":"queryr","key":"hello"}`
-    )
-    expect(stderr).toContain(
-      `invalid \`has\` items found for route {"source":"/hello","has":[{"type":"headerr"},{"type":"queryr","key":"hello"}],"headers":[{"key":"x-hello","value":"world"}]}`
-    )
-    expect(stderr).toContain(`Valid \`has\` object shape is {`)
+      expect(stderr).toContain('Invalid `has` items:')
+      expect(stderr).toContain(
+        `invalid type "headerr", invalid key "undefined" for {"type":"headerr"}`
+      )
+      expect(stderr).toContain(
+        `invalid type "queryr" for {"type":"queryr","key":"hello"}`
+      )
+      expect(stderr).toContain(
+        `invalid \`has\` items found for route {"source":"/hello","has":[{"type":"headerr"},{"type":"queryr","key":"hello"}],"headers":[{"key":"x-hello","value":"world"}]}`
+      )
+      expect(stderr).toContain(`Valid \`has\` object shape is {`)
 
-    expect(stderr).not.toContain('/valid-header')
+      expect(stderr).not.toContain('/valid-header')
+    })
   })
 
   it('should show formatted error for redirect source parse fail', async () => {
@@ -516,20 +522,23 @@ const runTests = () => {
       ],
       'redirects'
     )
+    await start()
+    await retry(() => {
+      const stderr = getStderr()
+      expect(stderr).toContain(
+        `Error parsing \`/feedback/(?!general)\` https://nextjs.org/docs/messages/invalid-route-source`
+      )
+      expect(stderr).toContain(`Reason: Pattern cannot start with "?" at 11`)
+      expect(stderr).toContain(`/feedback/(?!general)`)
 
-    const stderr = await getStderr()
-
-    expect(stderr).toContain(
-      `Error parsing \`/feedback/(?!general)\` https://nextjs.org/docs/messages/invalid-route-source`
-    )
-    expect(stderr).toContain(`Reason: Pattern cannot start with "?" at 11`)
-    expect(stderr).toContain(`/feedback/(?!general)`)
-
-    expect(stderr).toContain(
-      `Error parsing \`/learning/?\` https://nextjs.org/docs/messages/invalid-route-source`
-    )
-    expect(stderr).toContain(`Reason: Unexpected MODIFIER at 10, expected END`)
-    expect(stderr).toContain(`/learning/?`)
+      expect(stderr).toContain(
+        `Error parsing \`/learning/?\` https://nextjs.org/docs/messages/invalid-route-source`
+      )
+      expect(stderr).toContain(
+        `Reason: Unexpected MODIFIER at 10, expected END`
+      )
+      expect(stderr).toContain(`/learning/?`)
+    })
   })
 
   it('should show valid error when non-array is returned from rewrites', async () => {
@@ -541,25 +550,35 @@ const runTests = () => {
       'rewrites'
     )
 
-    const stderr = await getStderr()
-
-    expect(stderr).toContain(`rewrites must return an array, received object`)
+    await start()
+    await retry(() => {
+      const stderr = getStderr()
+      expect(stderr).toContain(`rewrites must return an array, received object`)
+    })
   })
 
   it('should show valid error when non-array is returned from redirects', async () => {
     await writeConfig(false, 'redirects')
 
-    const stderr = await getStderr()
-
-    expect(stderr).toContain(`redirects must return an array, received boolean`)
+    await start()
+    await retry(() => {
+      const stderr = getStderr()
+      expect(stderr).toContain(
+        `redirects must return an array, received boolean`
+      )
+    })
   })
 
   it('should show valid error when non-array is returned from headers', async () => {
     await writeConfig(undefined, 'headers')
 
-    const stderr = await getStderr()
-
-    expect(stderr).toContain(`headers must return an array, received undefined`)
+    await start()
+    await retry(() => {
+      const stderr = getStderr()
+      expect(stderr).toContain(
+        `headers must return an array, received undefined`
+      )
+    })
   })
 
   it('should show valid error when segments not in source are used in destination', async () => {
@@ -573,11 +592,13 @@ const runTests = () => {
       'rewrites'
     )
 
-    const stderr = await getStderr()
-
-    expect(stderr).toContain(
-      `\`destination\` has segments not in \`source\` or \`has\` (id) for route {"source":"/feedback/:type","destination":"/feedback/:id"}`
-    )
+    await start()
+    await retry(() => {
+      const stderr = getStderr()
+      expect(stderr).toContain(
+        `\`destination\` has segments not in \`source\` or \`has\` (id) for route {"source":"/feedback/:type","destination":"/feedback/:id"}`
+      )
+    })
   })
 }
 
@@ -587,18 +608,25 @@ describe('Errors on invalid custom routes', () => {
     'development mode',
     () => {
       let stderr = ''
+      let app
+
       beforeAll(() => {
-        getStderr = async () => {
+        getStderr = () => stderr
+        start = async () => {
           const port = await findPort()
-          await launchApp(appDir, port, {
+          stderr = ''
+          app = await launchApp(appDir, port, {
             onStderr: (msg) => {
               stderr += msg
             },
           })
-          return stderr
         }
       })
-      afterEach(() => {
+      afterEach(async () => {
+        if (app) {
+          await killApp(app)
+          app = null
+        }
         stderr = ''
       })
 
@@ -609,9 +637,11 @@ describe('Errors on invalid custom routes', () => {
     'production mode',
     () => {
       beforeAll(() => {
-        getStderr = async () => {
-          const { stderr } = await nextBuild(appDir, [], { stderr: true })
-          return stderr
+        let stderr = ''
+        getStderr = () => stderr
+        start = async () => {
+          const result = await nextBuild(appDir, [], { stderr: true })
+          stderr = result.stderr
         }
       })