Repository: vercel/next.js
Pull request: Turbopack: filter NftJsonAsset entries earlier

Improves invalidation (and performance)

1. filter out client asset while discovering client assets, so this filters out the whole subgraph already
2. apply the user-defined excludes while discovering the assets, this is also what the JS implementation does:
https://github.com/vercel/next.js/blob/b77eb3e127986b47cab9eced2253f00de12a5a14/packages/next/src/build/collect-build-traces.ts#L336-L343

Closes PACK-5037



Changed files:
filename: crates/next-api/src/nft_json.rs
patch: @@ -3,12 +3,14 @@ use std::collections::{BTreeSet, VecDeque};
 use anyhow::{Result, bail};
 use serde_json::json;
 use turbo_rcstr::RcStr;
-use turbo_tasks::{ResolvedVc, Vc};
+use turbo_tasks::{
+    ReadRef, ResolvedVc, TryFlatJoinIterExt, Vc,
+    graph::{AdjacencyMap, GraphTraversal},
+};
 use turbo_tasks_fs::{DirectoryEntry, File, FileSystem, FileSystemPath, glob::Glob};
 use turbopack_core::{
     asset::{Asset, AssetContent},
-    output::OutputAsset,
-    reference::all_assets_from_entries,
+    output::{OutputAsset, OutputAssets},
 };
 
 use crate::project::Project;
@@ -19,7 +21,7 @@ use crate::project::Project;
 ///
 /// With this file, users can determine the minimum set of files that are needed alongside
 /// their bundle.
-#[turbo_tasks::value(shared)]
+#[turbo_tasks::value]
 pub struct NftJsonAsset {
     project: ResolvedVc<Project>,
     /// The chunk for which the asset is being generated
@@ -75,7 +77,6 @@ fn get_output_specifier(
     ident_folder_in_project_fs: &FileSystemPath,
     output_root: &FileSystemPath,
     project_root: &FileSystemPath,
-    client_root: &FileSystemPath,
 ) -> Result<Option<RcStr>> {
     // include assets in the outputs such as referenced chunks
     if path_ref.is_inside_ref(output_root) {
@@ -91,13 +92,8 @@ fn get_output_specifier(
         ));
     }
 
-    if path_ref.is_inside_ref(client_root) {
-        // Client assets are never needed on the server, they are served via a CDN
-        return Ok(None);
-    }
-
-    // Make this an error for now, this should effectively be unreachable
-    bail!("NftJsonAsset: cannot handle filepath {}", path_ref);
+    // This should effectively be unreachable
+    bail!("NftJsonAsset: cannot handle filepath {path_ref}");
 }
 
 /// Apply outputFileTracingIncludes patterns to find additional files
@@ -146,18 +142,17 @@ impl Asset for NftJsonAsset {
         let project_root_ref = this.project.project_fs().root().await?;
         let next_config = this.project.next_config();
 
-        // Parse outputFileTracingIncludes and outputFileTracingExcludes from config
         let output_file_tracing_includes = &*next_config.output_file_tracing_includes().await?;
         let output_file_tracing_excludes = &*next_config.output_file_tracing_excludes().await?;
 
         let client_root = this.project.client_fs().root();
-        let client_root_ref = client_root.await?;
-        let project_root_path = this.project.project_root_path().await?.clone_value(); // Example: [project]
+        let client_root = client_root.owned().await?;
 
-        // Example: [output]/apps/my-website/.next/server/app -- without the `.nft.json`
+        // [project]/
+        let project_root_path = this.project.project_root_path().owned().await?;
+        // Example: [output]/apps/my-website/.next/server/app -- without the `page.js.nft.json`
         let ident_folder = self.path().await?.parent();
-        // Example: [project]/apps/my-website/.next/server/app -- without the `.nft.json`
-        // apps/my-website/.next/server/app
+        // Example: [project]/apps/my-website/.next/server/app -- without the `page.js.nft.json`
         let ident_folder_in_project_fs = project_root_path.join(&ident_folder.path)?;
 
         let chunk = this.chunk;
@@ -200,8 +195,7 @@ impl Asset for NftJsonAsset {
                             .join(",")
                     )
                     .into(),
-                )
-                .await?;
+                );
 
                 Some(glob)
             } else {
@@ -212,7 +206,9 @@ impl Asset for NftJsonAsset {
         };
 
         // Collect base assets first
-        for referenced_chunk in all_assets_from_entries(Vc::cell(entries)).await? {
+        for referenced_chunk in
+            all_assets_from_entries_filtered(Vc::cell(entries), client_root, exclude_glob).await?
+        {
             if chunk.eq(referenced_chunk) {
                 continue;
             }
@@ -222,19 +218,12 @@ impl Asset for NftJsonAsset {
                 continue;
             }
 
-            if let Some(ref exclude_glob) = exclude_glob
-                && exclude_glob.matches(referenced_chunk_path.path.as_str())
-            {
-                continue;
-            }
-
             let Some(specifier) = get_output_specifier(
                 &referenced_chunk_path,
                 &ident_folder,
                 &ident_folder_in_project_fs,
                 &output_root_ref,
                 &project_root_ref,
-                &client_root_ref,
             )?
             else {
                 continue;
@@ -294,3 +283,61 @@ impl Asset for NftJsonAsset {
         Ok(AssetContent::file(File::from(json.to_string()).into()))
     }
 }
+
+/// Walks the asset graph from multiple assets and collect all referenced
+/// assets, but filters out all client assets and glob matches.
+#[turbo_tasks::function]
+async fn all_assets_from_entries_filtered(
+    entries: Vc<OutputAssets>,
+    client_root: FileSystemPath,
+    exclude_glob: Option<Vc<Glob>>,
+) -> Result<Vc<OutputAssets>> {
+    let exclude_glob = if let Some(exclude_glob) = exclude_glob {
+        Some(exclude_glob.await?)
+    } else {
+        None
+    };
+    Ok(Vc::cell(
+        AdjacencyMap::new()
+            .skip_duplicates()
+            .visit(
+                entries.await?.iter().copied().map(ResolvedVc::upcast),
+                |asset| get_referenced_server_assets(asset, &client_root, &exclude_glob),
+            )
+            .await
+            .completed()?
+            .into_inner()
+            .into_postorder_topological()
+            .collect(),
+    ))
+}
+
+/// Computes the list of all chunk children of a given chunk, but filters out all client assets and
+/// glob matches.
+async fn get_referenced_server_assets(
+    asset: ResolvedVc<Box<dyn OutputAsset>>,
+    client_root: &FileSystemPath,
+    exclude_glob: &Option<ReadRef<Glob>>,
+) -> Result<Vec<ResolvedVc<Box<dyn OutputAsset>>>> {
+    asset
+        .references()
+        .await?
+        .iter()
+        .map(async |asset| {
+            let asset_path = asset.path().await?;
+            if asset_path.is_inside_ref(client_root) {
+                return Ok(None);
+            }
+
+            if exclude_glob
+                .as_ref()
+                .is_some_and(|g| g.matches(&asset_path.path))
+            {
+                return Ok(None);
+            }
+
+            Ok(Some(*asset))
+        })
+        .try_flat_join()
+        .await
+}