Repository: vercel/next.js
Pull request: [metadata] only serve block streaming metadata for html bots

### What

We should only do blocking metadata if we explicitly match a configured bot. Streaming metadata should be the default. This change is especially for cases like prerender or revalidation, where it's happened on background in Node.js process without user agent in request. So when normal users visit the streaming metadata option is aligned on both cases.

Closes NEXT-4491

Changed files:
filename: packages/next/src/server/lib/streaming-metadata.ts
patch: @@ -12,10 +12,11 @@ export function shouldServeStreamingMetadata(
     htmlLimitedBots || HTML_LIMITED_BOT_UA_RE_STRING,
     'i'
   )
-  return (
-    // When it's static generation, userAgents are not available - do not serve streaming metadata
-    !!userAgent && !blockingMetadataUARegex.test(userAgent)
-  )
+  // Only block metadata for HTML-limited bots
+  if (userAgent && blockingMetadataUARegex.test(userAgent)) {
+    return false
+  }
+  return true
 }
 
 // When the request UA is a html-limited bot, we should do a dynamic render.

filename: test/integration/next-image-new/app-dir/app/valid-html-w3c/page.js
patch: @@ -1,15 +1,8 @@
-import Head from 'next/head'
 import Image from 'next/image'
 
 const Page = () => {
   return (
     <div>
-      <Head>
-        <title>Title</title>
-        <meta name="description" content="Description..." />
-        <link rel="icon" type="image/jpeg" href="/test.jpg" />
-      </Head>
-
       <main>
         <Image src="/test.jpg" width="400" height="400" alt="basic image" />
       </main>
@@ -18,3 +11,11 @@ const Page = () => {
 }
 
 export default Page
+
+export const metadata = {
+  title: 'Title',
+  description: 'Description...',
+  icons: {
+    icon: '/test.jpg',
+  },
+}

filename: test/integration/next-image-new/app-dir/test/index.test.ts
patch: @@ -1568,25 +1568,26 @@ function runTests(mode: 'dev' | 'server') {
   })
 
   it('should be valid HTML', async () => {
-    let browser
-    try {
-      browser = await webdriver(appPort, '/valid-html-w3c')
-      await waitFor(1000)
-      expect(await browser.hasElementByCssSelector('img')).toBeTruthy()
-      const url = await browser.url()
-      const result = (await validateHTML({
-        url,
-        format: 'json',
-        isLocal: true,
-        validator: 'whatwg',
-      })) as any
-      expect(result.isValid).toBe(true)
-      expect(result.errors).toEqual([])
-    } finally {
-      if (browser) {
-        await browser.close()
-      }
-    }
+    // `html-validator` underlying API (W3C validator or Nu HTML Checker) ignores or overrides custom headers.
+    // So we need to use a custom User-Agent to avoid streaming metadata.
+    // Otherwise the HTML will failed with validation which requires title to be rendered under the `head` tag.
+    const html = await renderViaHTTP(appPort, '/valid-html-w3c', null, {
+      headers: {
+        'User-Agent': 'Discordbot',
+      },
+    })
+
+    const result = (await validateHTML({
+      data: html,
+      format: 'json',
+      isLocal: true,
+      validator: 'whatwg',
+    })) as any
+    expect(result.isValid).toBe(true)
+    expect(result.errors).toEqual([])
+
+    const $ = cheerio.load(html)
+    expect($('img').length).toBeGreaterThan(0)
   })
 
   it('should call callback ref cleanups when unmounting', async () => {