Repository: vercel/next.js
Pull request: feat(turbopack): Implement side-effect optimization

### What?

Implement side-effect-free optimization for turbopack. This is required to build applications properly, and to reduce the final bundle size. The advanced tree shaking consists of module splitting and module merging. The module splitting is done only once per module, and module merging is done naturally by the chunking functions because split modules are still modules. This PR implements optimization for module merging part - the optimization in this PR allows skipping intermediate modules between the requested module and the resolved module.



### Why?



### How?
 - Closes PACK-3500

 - Closes #72947
 	- This is retry of https://github.com/vercel/next.js/pull/72947. 


Changed files:
filename: turbopack/crates/turbopack-ecmascript/src/references/esm/base.rs
patch: @@ -13,6 +13,7 @@ use turbopack_core::{
         ChunkableModuleReference, ChunkingContext, ChunkingType, ChunkingTypeOption,
         ModuleChunkItemIdExt,
     },
+    context::AssetContext,
     issue::{
         Issue, IssueExt, IssueSeverity, IssueSource, IssueStage, OptionIssueSource,
         OptionStyledString, StyledString,
@@ -23,7 +24,7 @@ use turbopack_core::{
     resolve::{
         origin::{ResolveOrigin, ResolveOriginExt},
         parse::Request,
-        ExternalType, ModulePart, ModuleResolveResult, ModuleResolveResultItem,
+        ExternalType, ModulePart, ModuleResolveResult, ModuleResolveResultItem, RequestKey,
     },
 };
 use turbopack_resolve::ecmascript::esm_resolve;
@@ -38,6 +39,7 @@ use crate::{
     runtime_functions::{TURBOPACK_EXTERNAL_IMPORT, TURBOPACK_EXTERNAL_REQUIRE, TURBOPACK_IMPORT},
     tree_shake::{asset::EcmascriptModulePartAsset, TURBOPACK_PART_IMPORT_SOURCE},
     utils::module_id_to_lit,
+    TreeShakingMode,
 };
 
 #[turbo_tasks::value]
@@ -168,18 +170,44 @@ impl ModuleReference for EsmAssetReference {
             EcmaScriptModulesReferenceSubType::Import
         };
 
+        if let Some(ModulePart::Evaluation) = &self.export_name {
+            let module: ResolvedVc<crate::EcmascriptModuleAsset> =
+                ResolvedVc::try_downcast_type(self.origin)
+                    .expect("EsmAssetReference origin should be a EcmascriptModuleAsset");
+
+            let tree_shaking_mode = module.options().await?.tree_shaking_mode;
+
+            if let Some(TreeShakingMode::ModuleFragments) = tree_shaking_mode {
+                let side_effect_free_packages = module.asset_context().side_effect_free_packages();
+
+                if *module
+                    .is_marked_as_side_effect_free(side_effect_free_packages)
+                    .await?
+                {
+                    return Ok(ModuleResolveResult {
+                        primary: Box::new([(
+                            RequestKey::default(),
+                            ModuleResolveResultItem::Ignore,
+                        )]),
+                        affecting_sources: Default::default(),
+                    }
+                    .cell());
+                }
+            }
+        }
+
         if let Request::Module { module, .. } = &*self.request.await? {
             if module == TURBOPACK_PART_IMPORT_SOURCE {
                 if let Some(part) = &self.export_name {
                     let module: ResolvedVc<crate::EcmascriptModuleAsset> =
                         ResolvedVc::try_downcast_type(self.origin)
                             .expect("EsmAssetReference origin should be a EcmascriptModuleAsset");
 
-                    return Ok(*ModuleResolveResult::module(
+                    return Ok(*ModuleResolveResult::module(ResolvedVc::upcast(
                         EcmascriptModulePartAsset::select_part(*module, part.clone())
                             .to_resolved()
                             .await?,
-                    ));
+                    )));
                 }
 
                 bail!("export_name is required for part import")

filename: turbopack/crates/turbopack-ecmascript/src/references/esm/export.rs
patch: @@ -24,6 +24,7 @@ use turbopack_core::{
     module::Module,
     module_graph::ModuleGraph,
     reference::ModuleReference,
+    resolve::ModulePart,
 };
 
 use super::base::ReferencedAsset;
@@ -33,6 +34,8 @@ use crate::{
     magic_identifier,
     parse::ParseResult,
     runtime_functions::{TURBOPACK_DYNAMIC, TURBOPACK_ESM},
+    tree_shake::asset::EcmascriptModulePartAsset,
+    EcmascriptModuleAsset,
 };
 
 #[derive(Clone, Hash, Debug, PartialEq, Eq, Serialize, Deserialize, TraceRawVcs, NonLocalValue)]
@@ -177,9 +180,9 @@ pub async fn follow_reexports(
 
         // Try to find the export in the star exports
         if !exports_ref.star_exports.is_empty() && &*export_name != "default" {
-            let result = get_all_export_names(*module).await?;
-            if let Some(m) = result.esm_exports.get(&export_name) {
-                module = *m;
+            let result = find_export_from_reexports(*module, export_name.clone()).await?;
+            if let Some(m) = result.esm_export {
+                module = m;
                 continue;
             }
             return match &result.dynamic_exporting_modules[..] {
@@ -269,6 +272,48 @@ async fn handle_declared_export(
     }))
 }
 
+#[turbo_tasks::value]
+struct FindExportFromReexportsResult {
+    esm_export: Option<ResolvedVc<Box<dyn EcmascriptChunkPlaceable>>>,
+    dynamic_exporting_modules: Vec<ResolvedVc<Box<dyn EcmascriptChunkPlaceable>>>,
+}
+
+#[turbo_tasks::function]
+async fn find_export_from_reexports(
+    module: ResolvedVc<Box<dyn EcmascriptChunkPlaceable>>,
+    export_name: RcStr,
+) -> Result<Vc<FindExportFromReexportsResult>> {
+    if let Some(module) =
+        Vc::try_resolve_downcast_type::<EcmascriptModulePartAsset>(*module).await?
+    {
+        if matches!(module.await?.part, ModulePart::Exports) {
+            let module_part = EcmascriptModulePartAsset::select_part(
+                *module.await?.full_module,
+                ModulePart::export(export_name.clone()),
+            );
+
+            // If we apply this logic to EcmascriptModuleAsset, we will resolve everything in the
+            // target module.
+            if (Vc::try_resolve_downcast_type::<EcmascriptModuleAsset>(module_part).await?)
+                .is_none()
+            {
+                return Ok(find_export_from_reexports(
+                    Vc::upcast(module_part),
+                    export_name,
+                ));
+            }
+        }
+    }
+
+    let all_export_names = get_all_export_names(*module).await?;
+    let esm_export = all_export_names.esm_exports.get(&export_name).copied();
+    Ok(FindExportFromReexportsResult {
+        esm_export,
+        dynamic_exporting_modules: all_export_names.dynamic_exporting_modules.clone(),
+    }
+    .cell())
+}
+
 #[turbo_tasks::value]
 struct AllExportNamesResult {
     esm_exports: FxIndexMap<RcStr, ResolvedVc<Box<dyn EcmascriptChunkPlaceable>>>,

filename: turbopack/crates/turbopack-ecmascript/src/tree_shake/asset.rs
patch: @@ -122,7 +122,7 @@ impl EcmascriptModulePartAsset {
     pub async fn select_part(
         module: Vc<EcmascriptModuleAsset>,
         part: ModulePart,
-    ) -> Result<Vc<Box<dyn Module>>> {
+    ) -> Result<Vc<Box<dyn EcmascriptChunkPlaceable>>> {
         let SplitResult::Ok { entrypoints, .. } = &*split_module(module).await? else {
             return Ok(Vc::upcast(module));
         };

filename: turbopack/crates/turbopack-ecmascript/src/tree_shake/graph.rs
patch: @@ -717,6 +717,19 @@ impl DepGraph {
                 },
             )));
 
+        if !star_reexports.is_empty() {
+            let mut module = Module::dummy();
+            outputs.insert(Key::StarExports, modules.len() as u32);
+
+            for star in &star_reexports {
+                module
+                    .body
+                    .push(ModuleItem::ModuleDecl(ModuleDecl::ExportAll(star.clone())));
+            }
+
+            modules.push(module);
+        }
+
         SplitModuleResult {
             entrypoints: outputs,
             part_deps,

filename: turbopack/crates/turbopack-ecmascript/src/tree_shake/mod.rs
patch: @@ -374,6 +374,7 @@ pub(crate) enum Key {
     ModuleEvaluation,
     Export(RcStr),
     Exports,
+    StarExports,
 }
 
 /// Converts [ModulePart] to the index.
@@ -407,7 +408,10 @@ async fn get_part_id(result: &SplitResult, part: &ModulePart) -> Result<u32> {
 
     // This is required to handle `export * from 'foo'`
     if let ModulePart::Export(..) = part {
-        if let Some(&v) = entrypoints.get(&Key::Exports) {
+        if let Some(&v) = entrypoints
+            .get(&Key::StarExports)
+            .or_else(|| entrypoints.get(&Key::Exports))
+        {
             return Ok(v);
         }
     }