Repository: vercel/next.js
Pull request: Turbopack: make routes hashes mainfest more incremental

### What?

Make computing the hashes manifest more incremental.

Changed files:
filename: crates/next-api/src/routes_hashes_manifest.rs
patch: @@ -6,7 +6,7 @@ use turbo_tasks_fs::{FileContent, FileSystemPath};
 use turbo_tasks_hash::{DeterministicHash, Xxh3Hash64Hasher};
 use turbopack_core::{
     asset::{Asset, AssetContent},
-    module::Module,
+    module::{Module, Modules},
     module_graph::{GraphTraversalAction, ModuleGraph},
     output::{
         ExpandOutputAssetsInput, OutputAsset, OutputAssets, OutputAssetsReference,
@@ -16,7 +16,7 @@ use turbopack_core::{
 
 use crate::{
     project::Project,
-    route::{Endpoint, EndpointGroup},
+    route::{Endpoint, EndpointGroup, Endpoints},
 };
 
 #[turbo_tasks::value(shared)]
@@ -45,11 +45,58 @@ impl EndpointHashes {
 }
 
 #[turbo_tasks::function]
-pub async fn endpoint_hashes(
+pub async fn endpoint_outputs(endpoint: Vc<Box<dyn Endpoint>>) -> Result<Vc<OutputAssets>> {
+    Ok(*endpoint.output().await?.output_assets)
+}
+
+#[turbo_tasks::function]
+pub async fn endpoints_outputs(endpoints: Vc<Endpoints>) -> Result<Vc<OutputAssets>> {
+    let endpoints = endpoints.await?;
+    let all_outputs = endpoints
+        .iter()
+        .map(async |endpoint| Ok(endpoint.output().await?.output_assets.await?))
+        .try_join()
+        .await?;
+    let set = all_outputs
+        .into_iter()
+        .flatten()
+        .copied()
+        .collect::<FxIndexSet<_>>();
+    Ok(Vc::cell(set.into_iter().collect()))
+}
+
+#[turbo_tasks::function]
+pub async fn outputs_hash(outputs: Vc<OutputAssets>) -> Result<Vc<u64>> {
+    let output_assets = expand_output_assets(
+        outputs
+            .await?
+            .into_iter()
+            .map(|asset| ExpandOutputAssetsInput::Asset(*asset)),
+        true,
+    )
+    .await?;
+    let outputs_hashes = output_assets
+        .iter()
+        .map(|asset| asset.content().hash())
+        .try_join()
+        .await?;
+
+    let outputs_hash = {
+        let mut hasher = Xxh3Hash64Hasher::new();
+        for hash in outputs_hashes.iter() {
+            hash.deterministic_hash(&mut hasher);
+        }
+        hasher.finish()
+    };
+
+    Ok(Vc::cell(outputs_hash))
+}
+
+#[turbo_tasks::function]
+pub async fn endpoint_entry_modules(
     base_module_graph: Vc<ModuleGraph>,
-    module_graph: Vc<ModuleGraph>,
     endpoint: Vc<Box<dyn Endpoint>>,
-) -> Result<Vc<EndpointHashes>> {
+) -> Result<Vc<Modules>> {
     let entries = endpoint.entries();
     let additional_entries = endpoint.additional_entries(base_module_graph);
     let modules = entries
@@ -58,14 +105,46 @@ pub async fn endpoint_hashes(
         .chain(additional_entries.await?.into_iter())
         .flat_map(|e| e.entries())
         .collect::<FxIndexSet<_>>();
-    let outputs = endpoint.output();
+    Ok(Vc::cell(modules.into_iter().collect()))
+}
+
+#[turbo_tasks::function]
+pub async fn endpoints_entry_modules(
+    base_module_graph: Vc<ModuleGraph>,
+    endpoints: Vc<Endpoints>,
+) -> Result<Vc<Modules>> {
+    let endpoints = endpoints.await?;
+    let entries_and_additional_entries = endpoints
+        .iter()
+        .map(async |endpoint| {
+            let entries = endpoint.entries();
+            let additional_entries = endpoint.additional_entries(base_module_graph);
+            Ok((entries.await?, additional_entries.await?))
+        })
+        .try_join()
+        .await?;
+    let modules = entries_and_additional_entries
+        .into_iter()
+        .flat_map(|(entries, additional_entries)| {
+            entries
+                .into_iter()
+                .chain(additional_entries.into_iter())
+                .flat_map(|e| e.entries())
+        })
+        .collect::<FxIndexSet<_>>();
+    Ok(Vc::cell(modules.into_iter().collect()))
+}
+
+#[turbo_tasks::function]
+pub async fn sources_hash(module_graph: Vc<ModuleGraph>, modules: Vc<Modules>) -> Result<Vc<u64>> {
+    let modules = modules.await?;
 
     let mut all_modules = FxIndexSet::default();
 
     let module_graph = module_graph.await?;
 
     module_graph.traverse_nodes_dfs(
-        modules,
+        modules.into_iter().copied(),
         &mut all_modules,
         |module, all_modules| {
             all_modules.insert(*module);
@@ -84,42 +163,15 @@ pub async fn endpoint_hashes(
         .try_join()
         .await?;
 
-    let output_assets = expand_output_assets(
-        outputs
-            .await?
-            .output_assets
-            .await?
-            .into_iter()
-            .map(|asset| ExpandOutputAssetsInput::Asset(*asset)),
-        true,
-    )
-    .await?;
-    let outputs_hashes = output_assets
-        .iter()
-        .map(|asset| asset.content().hash())
-        .try_join()
-        .await?;
-
     let sources_hash = {
         let mut hasher = Xxh3Hash64Hasher::new();
         for source in sources.iter() {
             source.deterministic_hash(&mut hasher);
         }
         hasher.finish()
     };
-    let outputs_hash = {
-        let mut hasher = Xxh3Hash64Hasher::new();
-        for hash in outputs_hashes.iter() {
-            hash.deterministic_hash(&mut hasher);
-        }
-        hasher.finish()
-    };
 
-    Ok(EndpointHashes {
-        sources_hash,
-        outputs_hash,
-    }
-    .cell())
+    Ok(Vc::cell(sources_hash))
 }
 
 #[derive(Serialize)]
@@ -161,43 +213,45 @@ impl Asset for RoutesHashesManifestAsset {
         let entrypoint_groups = self.project.get_all_endpoint_groups(false).await?;
 
         for (key, EndpointGroup { primary, .. }) in entrypoint_groups {
-            if let &[entry] = &primary.as_slice() {
-                entrypoint_hashes.insert(
-                    key.as_str(),
-                    endpoint_hashes(base_module_graph, full_module_graph, *entry.endpoint),
-                );
+            let entry = if let &[entry] = &primary.as_slice() {
+                (
+                    sources_hash(
+                        full_module_graph,
+                        endpoint_entry_modules(base_module_graph, *entry.endpoint),
+                    ),
+                    outputs_hash(endpoint_outputs(*entry.endpoint)),
+                )
             } else {
-                let hashes = primary
-                    .iter()
-                    .map(|entry| {
-                        endpoint_hashes(base_module_graph, full_module_graph, *entry.endpoint)
-                    })
-                    .try_join()
-                    .await?;
-                let hashes = EndpointHashes::merge(
-                    primary
-                        .iter()
-                        .map(|page| page.sub_name.clone())
-                        .zip(hashes.iter())
-                        .map(|(k, v)| (k, &**v)),
+                let endpoints = Vc::cell(primary.iter().map(|entry| entry.endpoint).collect());
+                (
+                    sources_hash(
+                        full_module_graph,
+                        endpoints_entry_modules(base_module_graph, endpoints),
+                    ),
+                    outputs_hash(endpoints_outputs(endpoints)),
                 )
-                .cell();
-                entrypoint_hashes.insert(key.as_str(), hashes);
-            }
+            };
+            entrypoint_hashes.insert(key.as_str(), entry);
         }
 
-        let entrypoint_hashes_values = entrypoint_hashes.values().copied().try_join().await?;
+        let entrypoint_hashes_values = entrypoint_hashes
+            .values()
+            .map(async |(sources_hash, outputs_hash)| {
+                Ok((sources_hash.await?, outputs_hash.await?))
+            })
+            .try_join()
+            .await?;
 
         let manifest = serde_json::to_string_pretty(&RoutesHashesManifest {
             routes: entrypoint_hashes
                 .into_keys()
                 .zip(entrypoint_hashes_values.into_iter())
-                .map(|(k, v)| {
+                .map(|(k, (sources_hash, outputs_hash))| {
                     (
                         k,
                         EndpointHashStrings {
-                            sources_hash: format!("{:016x}", v.sources_hash),
-                            outputs_hash: format!("{:016x}", v.outputs_hash),
+                            sources_hash: format!("{:016x}", *sources_hash),
+                            outputs_hash: format!("{:016x}", *outputs_hash),
                         },
                     )
                 })