Repository: vercel/next.js
Pull request: Revert "Turbopack: layout segment optimization for Pages"

Reverts vercel/next.js#74815
 
`test/integration/app-document/test/index.test.js` starts failing since #74815 

x-ref: https://github.com/vercel/next.js/actions/runs/13972070006/job/39119143367?pr=77335

Changed files:
filename: crates/next-api/src/pages.rs
patch: @@ -31,7 +31,7 @@ use tracing::Instrument;
 use turbo_rcstr::RcStr;
 use turbo_tasks::{
     fxindexmap, trace::TraceRawVcs, Completion, FxIndexMap, NonLocalValue, ResolvedVc, TaskInput,
-    Value, ValueToString, Vc,
+    Value, Vc,
 };
 use turbo_tasks_fs::{
     self, File, FileContent, FileSystem, FileSystemPath, FileSystemPathOption, VirtualFileSystem,
@@ -46,15 +46,15 @@ use turbopack_core::{
     asset::AssetContent,
     chunk::{
         availability_info::AvailabilityInfo, ChunkGroupResult, ChunkingContext, ChunkingContextExt,
-        EvaluatableAsset, EvaluatableAssets,
+        EntryChunkGroupResult, EvaluatableAsset, EvaluatableAssets,
     },
     context::AssetContext,
     file_source::FileSource,
     ident::AssetIdent,
     module::Module,
     module_graph::{
         chunk_group_info::{ChunkGroup, ChunkGroupEntry},
-        GraphEntries, ModuleGraph, SingleModuleGraph, VisitedModules,
+        GraphEntries, ModuleGraph,
     },
     output::{OptionOutputAsset, OutputAsset, OutputAssets},
     reference_type::{EcmaScriptModulesReferenceSubType, EntryReferenceSubType, ReferenceType},
@@ -772,45 +772,7 @@ impl PageEndpoint {
         let this = self.await?;
         let project = this.pages_project.project();
         let evaluatable_assets = self.client_evaluatable_assets();
-        Ok(project.module_graph_for_modules(evaluatable_assets))
-    }
-
-    #[turbo_tasks::function]
-    async fn ssr_module_graph(self: Vc<Self>) -> Result<Vc<ModuleGraph>> {
-        let this = self.await?;
-        let project = this.pages_project.project();
-
-        if *project.per_page_module_graph().await? {
-            let ssr_chunk_module = self.internal_ssr_chunk_module().await?;
-            // Implements layout segment optimization to compute a graph "chain" for document, app,
-            // page
-            let mut graphs = vec![];
-            let mut visited_modules = VisitedModules::empty();
-            for module in [
-                ssr_chunk_module.document_module,
-                ssr_chunk_module.app_module,
-            ]
-            .into_iter()
-            .flatten()
-            {
-                let graph = SingleModuleGraph::new_with_entries_visited_intern(
-                    vec![ChunkGroupEntry::Shared(module)],
-                    visited_modules,
-                );
-                graphs.push(graph);
-                visited_modules = visited_modules.concatenate(graph);
-            }
-
-            let graph = SingleModuleGraph::new_with_entries_visited_intern(
-                vec![ChunkGroupEntry::Entry(vec![ssr_chunk_module.ssr_module])],
-                visited_modules,
-            );
-            graphs.push(graph);
-
-            Ok(ModuleGraph::from_graphs(graphs))
-        } else {
-            Ok(*project.whole_app_module_graphs().await?.full)
-        }
+        Ok(project.module_graph_for_entries(evaluatable_assets))
     }
 
     #[turbo_tasks::function]
@@ -890,8 +852,10 @@ impl PageEndpoint {
             .module();
 
         let config = parse_config_from_source(ssr_module, NextRuntime::default()).await?;
-        Ok(if config.runtime == NextRuntime::Edge {
-            let modules = create_page_ssr_entry_module(
+        let is_edge = matches!(config.runtime, NextRuntime::Edge);
+
+        let ssr_module = if is_edge {
+            create_page_ssr_entry_module(
                 *this.pathname,
                 reference_type,
                 project_root,
@@ -902,28 +866,15 @@ impl PageEndpoint {
                 config.runtime,
                 this.pages_project.project().next_config(),
             )
-            .await?;
-
-            InternalSsrChunkModule {
-                ssr_module: modules.ssr_module,
-                app_module: modules.app_module,
-                document_module: modules.document_module,
-                runtime: config.runtime,
-            }
         } else {
             let pathname = &**this.pathname.await?;
 
             // `/_app` and `/_document` never get rendered directly so they don't need to be
             // wrapped in the route module.
             if pathname == "/_app" || pathname == "/_document" {
-                InternalSsrChunkModule {
-                    ssr_module: ssr_module.to_resolved().await?,
-                    app_module: None,
-                    document_module: None,
-                    runtime: config.runtime,
-                }
+                ssr_module
             } else {
-                let modules = create_page_ssr_entry_module(
+                create_page_ssr_entry_module(
                     *this.pathname,
                     reference_type,
                     project_root,
@@ -934,14 +885,12 @@ impl PageEndpoint {
                     config.runtime,
                     this.pages_project.project().next_config(),
                 )
-                .await?;
-                InternalSsrChunkModule {
-                    ssr_module: modules.ssr_module,
-                    app_module: modules.app_module,
-                    document_module: modules.document_module,
-                    runtime: config.runtime,
-                }
             }
+        };
+
+        Ok(InternalSsrChunkModule {
+            ssr_module: ssr_module.to_resolved().await?,
+            runtime: config.runtime,
         }
         .cell())
     }
@@ -951,7 +900,7 @@ impl PageEndpoint {
         self: Vc<Self>,
         ty: SsrChunkType,
         node_path: Vc<FileSystemPath>,
-        node_chunking_context: Vc<NodeJsChunkingContext>,
+        chunking_context: Vc<NodeJsChunkingContext>,
         edge_chunking_context: Vc<Box<dyn ChunkingContext>>,
         runtime_entries: Vc<EvaluatableAssets>,
         edge_runtime_entries: Vc<EvaluatableAssets>,
@@ -961,16 +910,14 @@ impl PageEndpoint {
 
             let InternalSsrChunkModule {
                 ssr_module,
-                app_module,
-                document_module,
                 runtime,
             } = *self.internal_ssr_chunk_module().await?;
 
             let project = this.pages_project.project();
             // The SSR and Client Graphs are not connected in Pages Router.
             // We are only interested in get_next_dynamic_imports_for_endpoint at the
             // moment, which only needs the client graph anyway.
-            let ssr_module_graph = self.ssr_module_graph();
+            let module_graph = project.module_graph(*ssr_module);
 
             let next_dynamic_imports = if let PageEndpointType::Html = this.ty {
                 let client_availability_info = self.client_chunks().await?.availability_info;
@@ -1005,40 +952,6 @@ impl PageEndpoint {
                 DynamicImportedChunks::default().resolved_cell()
             };
 
-            let chunking_context: Vc<Box<dyn ChunkingContext>> = match runtime {
-                NextRuntime::NodeJs => Vc::upcast(node_chunking_context),
-                NextRuntime::Edge => Vc::upcast(edge_chunking_context),
-            };
-
-            let mut current_chunks = OutputAssets::empty();
-            let mut current_availability_info = AvailabilityInfo::Root;
-            for layout in [document_module, app_module].iter().flatten().copied() {
-                let span = tracing::trace_span!(
-                    "layout segment",
-                    name = display(layout.ident().to_string().await?)
-                );
-                async {
-                    let chunk_group = chunking_context
-                        .chunk_group(
-                            layout.ident(),
-                            ChunkGroup::Shared(layout),
-                            ssr_module_graph,
-                            Value::new(current_availability_info),
-                        )
-                        .await?;
-
-                    current_chunks = current_chunks
-                        .concatenate(*chunk_group.assets)
-                        .resolve()
-                        .await?;
-                    current_availability_info = chunk_group.availability_info;
-
-                    anyhow::Ok(())
-                }
-                .instrument(span)
-                .await?;
-            }
-
             let ssr_module_evaluatable = ResolvedVc::try_sidecast(ssr_module)
                 .context("could not process page loader entry module")?;
             let is_edge = matches!(runtime, NextRuntime::Edge);
@@ -1049,15 +962,18 @@ impl PageEndpoint {
                     .map(|m| ResolvedVc::upcast(*m))
                     .chain(std::iter::once(ResolvedVc::upcast(ssr_module_evaluatable)));
 
-                let edge_files = edge_chunking_context.evaluated_chunk_group_assets(
-                    ssr_module.ident(),
-                    ChunkGroup::Entry(evaluatable_assets.collect()),
-                    ssr_module_graph,
-                    Value::new(current_availability_info),
-                );
+                let edge_files = edge_chunking_context
+                    .evaluated_chunk_group_assets(
+                        ssr_module.ident(),
+                        ChunkGroup::Entry(evaluatable_assets.collect()),
+                        module_graph,
+                        Value::new(AvailabilityInfo::Root),
+                    )
+                    .to_resolved()
+                    .await?;
 
                 Ok(SsrChunk::Edge {
-                    files: current_chunks.concatenate(edge_files).to_resolved().await?,
+                    files: edge_files,
                     dynamic_import_entries,
                 }
                 .cell())
@@ -1068,15 +984,17 @@ impl PageEndpoint {
 
                 let ssr_entry_chunk_path_string: RcStr = format!("pages{asset_path}").into();
                 let ssr_entry_chunk_path = node_path.join(ssr_entry_chunk_path_string);
-                let ssr_entry_chunk = node_chunking_context
-                    .entry_chunk_group_asset(
+                let EntryChunkGroupResult {
+                    asset: ssr_entry_chunk,
+                    ..
+                } = *chunking_context
+                    .entry_chunk_group(
                         ssr_entry_chunk_path,
                         runtime_entries.with_entry(*ssr_module_evaluatable),
-                        ssr_module_graph,
-                        current_chunks,
-                        Value::new(current_availability_info),
+                        module_graph,
+                        OutputAssets::empty(),
+                        Value::new(AvailabilityInfo::Root),
                     )
-                    .to_resolved()
                     .await?;
 
                 let server_asset_trace_file = if this
@@ -1446,8 +1364,6 @@ impl PageEndpoint {
 #[turbo_tasks::value]
 pub struct InternalSsrChunkModule {
     pub ssr_module: ResolvedVc<Box<dyn Module>>,
-    pub app_module: Option<ResolvedVc<Box<dyn Module>>>,
-    pub document_module: Option<ResolvedVc<Box<dyn Module>>>,
     pub runtime: NextRuntime,
 }
 
@@ -1551,32 +1467,17 @@ impl Endpoint for PageEndpoint {
         let this = self.await?;
 
         let ssr_chunk_module = self.internal_ssr_chunk_module().await?;
+        let mut modules = vec![ChunkGroupEntry::Entry(vec![ssr_chunk_module.ssr_module])];
 
-        let shared_entries = [
-            ssr_chunk_module.document_module,
-            ssr_chunk_module.app_module,
-        ];
-
-        let modules = shared_entries
-            .into_iter()
-            .flatten()
-            .map(ChunkGroupEntry::Shared)
-            .chain(std::iter::once(ChunkGroupEntry::Entry(vec![
-                ssr_chunk_module.ssr_module,
-            ])))
-            .chain(if this.ty == PageEndpointType::Html {
-                Some(ChunkGroupEntry::Entry(
-                    self.client_evaluatable_assets()
-                        .await?
-                        .iter()
-                        .map(|m| ResolvedVc::upcast(*m))
-                        .collect(),
-                ))
-                .into_iter()
-            } else {
-                None.into_iter()
-            })
-            .collect::<Vec<_>>();
+        if let PageEndpointType::Html = this.ty {
+            modules.push(ChunkGroupEntry::Entry(
+                self.client_evaluatable_assets()
+                    .await?
+                    .iter()
+                    .map(|m| ResolvedVc::upcast(*m))
+                    .collect(),
+            ));
+        }
 
         Ok(Vc::cell(modules))
     }

filename: crates/next-api/src/project.rs
patch: @@ -895,7 +895,7 @@ impl Project {
     }
 
     #[turbo_tasks::function]
-    pub async fn module_graph_for_modules(
+    pub async fn module_graph_for_entries(
         self: Vc<Self>,
         evaluatable_assets: Vc<EvaluatableAssets>,
     ) -> Result<Vc<ModuleGraph>> {
@@ -912,18 +912,6 @@ impl Project {
         })
     }
 
-    #[turbo_tasks::function]
-    pub async fn module_graph_for_entries(
-        self: Vc<Self>,
-        entries: Vc<GraphEntries>,
-    ) -> Result<Vc<ModuleGraph>> {
-        Ok(if *self.per_page_module_graph().await? {
-            ModuleGraph::from_modules(entries)
-        } else {
-            *self.whole_app_module_graphs().await?.full
-        })
-    }
-
     #[turbo_tasks::function]
     pub async fn whole_app_module_graphs(self: ResolvedVc<Self>) -> Result<Vc<ModuleGraphs>> {
         async move {

filename: crates/next-core/src/next_pages/page_entry.rs
patch: @@ -23,13 +23,6 @@ use crate::{
     util::{file_content_rope, load_next_js_template, NextRuntime},
 };
 
-#[turbo_tasks::value]
-pub struct PageSsrEntryModule {
-    pub ssr_module: ResolvedVc<Box<dyn Module>>,
-    pub app_module: Option<ResolvedVc<Box<dyn Module>>>,
-    pub document_module: Option<ResolvedVc<Box<dyn Module>>>,
-}
-
 #[turbo_tasks::function]
 pub async fn create_page_ssr_entry_module(
     pathname: Vc<RcStr>,
@@ -41,7 +34,7 @@ pub async fn create_page_ssr_entry_module(
     pages_structure: Vc<PagesStructure>,
     runtime: NextRuntime,
     next_config: Vc<NextConfig>,
-) -> Result<Vc<PageSsrEntryModule>> {
+) -> Result<Vc<Box<dyn Module>>> {
     let definition_page = &*next_original_name.await?;
     let definition_pathname = &*pathname.await?;
 
@@ -125,28 +118,28 @@ pub async fn create_page_ssr_entry_module(
 
     let pages_structure_ref = pages_structure.await?;
 
-    let (app_module, document_module) =
-        if reference_type == ReferenceType::Entry(EntryReferenceSubType::Page) {
-            let document_module = process_global_item(
+    if reference_type == ReferenceType::Entry(EntryReferenceSubType::Page) {
+        inner_assets.insert(
+            INNER_DOCUMENT.into(),
+            process_global_item(
                 *pages_structure_ref.document,
                 Value::new(reference_type.clone()),
                 ssr_module_context,
             )
             .to_resolved()
-            .await?;
-            let app_module = process_global_item(
+            .await?,
+        );
+        inner_assets.insert(
+            INNER_APP.into(),
+            process_global_item(
                 *pages_structure_ref.app,
                 Value::new(reference_type.clone()),
                 ssr_module_context,
             )
             .to_resolved()
-            .await?;
-            inner_assets.insert(INNER_DOCUMENT.into(), document_module);
-            inner_assets.insert(INNER_APP.into(), app_module);
-            (Some(app_module), Some(document_module))
-        } else {
-            (None, None)
-        };
+            .await?,
+        );
+    }
 
     let mut ssr_module = ssr_module_context
         .process(
@@ -177,12 +170,7 @@ pub async fn create_page_ssr_entry_module(
         }
     }
 
-    Ok(PageSsrEntryModule {
-        ssr_module: ssr_module.to_resolved().await?,
-        app_module,
-        document_module,
-    }
-    .cell())
+    Ok(ssr_module)
 }
 
 #[turbo_tasks::function]

filename: packages/next/src/server/dev/hot-reloader-turbopack.ts
patch: @@ -296,7 +296,7 @@ export async function createHotReloaderTurbopack(
       // Always clear the cache, don't check if files have changed
       force?: boolean
     } = {}
-  ): boolean {
+  ): void {
     if (force) {
       for (const { path, contentHash } of writtenEndpoint.serverPaths) {
         serverPathState.set(path, contentHash)
@@ -328,7 +328,7 @@ export async function createHotReloaderTurbopack(
       }
 
       if (!hasChange) {
-        return false
+        return
       }
     }
 
@@ -351,7 +351,7 @@ export async function createHotReloaderTurbopack(
       deleteCache(file)
     }
 
-    return true
+    return
   }
 
   const buildingIds = new Set()
@@ -601,9 +601,9 @@ export async function createHotReloaderTurbopack(
           serverFields,
 
           hooks: {
-            handleWrittenEndpoint: (id, result, forceDeleteCache) => {
+            handleWrittenEndpoint: (id, result) => {
               currentWrittenEntrypoints.set(id, result)
-              return clearRequireCache(id, result, { force: forceDeleteCache })
+              clearRequireCache(id, result)
             },
             propagateServerField: propagateServerField.bind(null, opts),
             sendHmr,
@@ -997,12 +997,10 @@ export async function createHotReloaderTurbopack(
                 logErrors: true,
                 hooks: {
                   subscribeToChanges,
-                  handleWrittenEndpoint: (id, result, forceDeleteCache) => {
+                  handleWrittenEndpoint: (id, result) => {
+                    clearRequireCache(id, result)
                     currentWrittenEntrypoints.set(id, result)
                     assetMapper.setPathsForKey(id, result.clientPaths)
-                    return clearRequireCache(id, result, {
-                      force: forceDeleteCache,
-                    })
                   },
                 },
               })
@@ -1063,12 +1061,10 @@ export async function createHotReloaderTurbopack(
 
               hooks: {
                 subscribeToChanges,
-                handleWrittenEndpoint: (id, result, forceDeleteCache) => {
+                handleWrittenEndpoint: (id, result) => {
                   currentWrittenEntrypoints.set(id, result)
+                  clearRequireCache(id, result)
                   assetMapper.setPathsForKey(id, result.clientPaths)
-                  return clearRequireCache(id, result, {
-                    force: forceDeleteCache,
-                  })
                 },
               },
             })

filename: packages/next/src/server/dev/turbopack-utils.ts
patch: @@ -98,9 +98,8 @@ export type ChangeSubscriptions = Map<
 
 export type HandleWrittenEndpoint = (
   key: EntryKey,
-  result: TurbopackResult<WrittenEndpoint>,
-  forceDeleteCache: boolean
-) => boolean
+  result: TurbopackResult<WrittenEndpoint>
+) => void
 
 export type StartChangeSubscription = (
   key: EntryKey,
@@ -178,17 +177,11 @@ export async function handleRouteType({
       const serverKey = getEntryKey('pages', 'server', page)
 
       try {
-        // In the best case scenario, Turbopack chunks document, app, page separately in that order,
-        // so it can happen that the chunks of document change, but the chunks of app and page
-        // don't. We still need to reload the page chunks in that case though, otherwise the version
-        // of the document or app component export from the pages template is stale.
-        let documentOrAppChanged = false
         if (entrypoints.global.app) {
           const key = getEntryKey('pages', 'server', '_app')
 
           const writtenEndpoint = await entrypoints.global.app.writeToDisk()
-          documentOrAppChanged ||=
-            hooks?.handleWrittenEndpoint(key, writtenEndpoint, false) ?? false
+          hooks?.handleWrittenEndpoint(key, writtenEndpoint)
           processIssues(
             currentEntryIssues,
             key,
@@ -205,8 +198,7 @@ export async function handleRouteType({
 
           const writtenEndpoint =
             await entrypoints.global.document.writeToDisk()
-          documentOrAppChanged ||=
-            hooks?.handleWrittenEndpoint(key, writtenEndpoint, false) ?? false
+          hooks?.handleWrittenEndpoint(key, writtenEndpoint)
           processIssues(
             currentEntryIssues,
             key,
@@ -218,11 +210,7 @@ export async function handleRouteType({
         await manifestLoader.loadPagesManifest('_document')
 
         const writtenEndpoint = await route.htmlEndpoint.writeToDisk()
-        hooks?.handleWrittenEndpoint(
-          serverKey,
-          writtenEndpoint,
-          documentOrAppChanged
-        )
+        hooks?.handleWrittenEndpoint(serverKey, writtenEndpoint)
 
         const type = writtenEndpoint?.type
 
@@ -320,7 +308,7 @@ export async function handleRouteType({
       const key = getEntryKey('pages', 'server', page)
 
       const writtenEndpoint = await route.endpoint.writeToDisk()
-      hooks?.handleWrittenEndpoint(key, writtenEndpoint, false)
+      hooks?.handleWrittenEndpoint(key, writtenEndpoint)
 
       const type = writtenEndpoint.type
 
@@ -345,7 +333,7 @@ export async function handleRouteType({
       const key = getEntryKey('app', 'server', page)
 
       const writtenEndpoint = await route.htmlEndpoint.writeToDisk()
-      hooks?.handleWrittenEndpoint(key, writtenEndpoint, false)
+      hooks?.handleWrittenEndpoint(key, writtenEndpoint)
 
       if (dev) {
         // TODO subscriptions should only be caused by the WebSocket connections
@@ -408,7 +396,7 @@ export async function handleRouteType({
       const key = getEntryKey('app', 'server', page)
 
       const writtenEndpoint = await route.endpoint.writeToDisk()
-      hooks?.handleWrittenEndpoint(key, writtenEndpoint, false)
+      hooks?.handleWrittenEndpoint(key, writtenEndpoint)
 
       const type = writtenEndpoint.type
 
@@ -666,7 +654,7 @@ export async function handleEntrypoints({
       const key = getEntryKey('root', 'server', name)
 
       const writtenEndpoint = await instrumentation[prop].writeToDisk()
-      dev.hooks.handleWrittenEndpoint(key, writtenEndpoint, false)
+      dev.hooks.handleWrittenEndpoint(key, writtenEndpoint)
       processIssues(currentEntryIssues, key, writtenEndpoint, false, logErrors)
     }
     await processInstrumentation('instrumentation.nodeJs', 'nodeJs')
@@ -701,7 +689,7 @@ export async function handleEntrypoints({
 
     async function processMiddleware() {
       const writtenEndpoint = await endpoint.writeToDisk()
-      dev.hooks.handleWrittenEndpoint(key, writtenEndpoint, false)
+      dev.hooks.handleWrittenEndpoint(key, writtenEndpoint)
       processIssues(currentEntryIssues, key, writtenEndpoint, false, logErrors)
       await manifestLoader.loadMiddlewareManifest('middleware', 'middleware')
       const middlewareConfig =
@@ -852,7 +840,7 @@ export async function handlePagesErrorRoute({
     const key = getEntryKey('pages', 'server', '_app')
 
     const writtenEndpoint = await entrypoints.global.app.writeToDisk()
-    hooks.handleWrittenEndpoint(key, writtenEndpoint, false)
+    hooks.handleWrittenEndpoint(key, writtenEndpoint)
     hooks.subscribeToChanges(
       key,
       false,
@@ -879,7 +867,7 @@ export async function handlePagesErrorRoute({
     const key = getEntryKey('pages', 'server', '_document')
 
     const writtenEndpoint = await entrypoints.global.document.writeToDisk()
-    hooks.handleWrittenEndpoint(key, writtenEndpoint, false)
+    hooks.handleWrittenEndpoint(key, writtenEndpoint)
     hooks.subscribeToChanges(
       key,
       false,
@@ -905,7 +893,7 @@ export async function handlePagesErrorRoute({
     const key = getEntryKey('pages', 'server', '_error')
 
     const writtenEndpoint = await entrypoints.global.error.writeToDisk()
-    hooks.handleWrittenEndpoint(key, writtenEndpoint, false)
+    hooks.handleWrittenEndpoint(key, writtenEndpoint)
     hooks.subscribeToChanges(
       key,
       false,