Repository: vercel/next.js
Pull request: Turbopack: improve references benchmark

This isn't ideal, but the setup is somewhat negligible

```
Benchmarking references/react-dom-client.development.js/full: Warming up for 1.0000 s
Backend: 1.132083ms Setup: 1.93475ms, Analysis: 96.47525ms
Backend: 1.328792ms Setup: 179.875µs, Analysis: 150.265291ms
Backend: 1.352333ms Setup: 95.375µs, Analysis: 80.448791ms
Backend: 967.916µs Setup: 77.083µs, Analysis: 78.769917ms
Backend: 685.166µs Setup: 66.458µs, Analysis: 76.768042ms
Backend: 718.834µs Setup: 70.292µs, Analysis: 75.807458ms
Backend: 825.125µs Setup: 68.625µs, Analysis: 78.019792ms
Backend: 903.209µs Setup: 68.709µs, Analysis: 78.0815ms
Backend: 1.026958ms Setup: 79.959µs, Analysis: 103.203291ms
Backend: 707.208µs Setup: 64.917µs, Analysis: 79.350292ms
Backend: 1.018291ms Setup: 82µs, Analysis: 79.496292ms
Backend: 832.375µs Setup: 64.875µs, Analysis: 75.658958ms
Backend: 795.833µs Setup: 60.333µs, Analysis: 75.704792ms
Backend: 948.417µs Setup: 68.416µs, Analysis: 75.474584ms
Backend: 725.167µs Setup: 65.542µs, Analysis: 75.63275ms
```

and this certainly makes the benchmark more accurate, as it now always recomputes all tasks on every call.

Closes PACK-5571

Changed files:
filename: turbopack/crates/turbopack-ecmascript/benches/references.rs
patch: @@ -1,11 +1,10 @@
-use std::{path::PathBuf, sync::Arc, time::Duration};
+use std::{path::PathBuf, time::Duration};
 
-use criterion::{Bencher, BenchmarkId, Criterion, criterion_group, criterion_main};
+use anyhow::Result;
+use criterion::{BatchSize, Bencher, BenchmarkId, Criterion, criterion_group, criterion_main};
 use turbo_rcstr::rcstr;
 use turbo_tasks::{ResolvedVc, TurboTasks};
-use turbo_tasks_backend::{
-    BackendOptions, BackingStorage, TurboTasksBackend, noop_backing_storage,
-};
+use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
 use turbo_tasks_fs::{DiskFileSystem, FileSystem};
 use turbopack_core::{
     compile_time_info::CompileTimeInfo,
@@ -14,126 +13,140 @@ use turbopack_core::{
     ident::Layer,
 };
 use turbopack_ecmascript::{
-    EcmascriptInputTransforms, EcmascriptModuleAsset, EcmascriptOptions, TreeShakingMode,
-    references::analyze_ecmascript_module_internal,
+    AnalyzeMode, EcmascriptInputTransforms, EcmascriptModuleAsset, EcmascriptOptions,
+    TreeShakingMode, references::analyze_ecmascript_module,
 };
 use turbopack_test_utils::noop_asset_context::NoopAssetContext;
 
 #[global_allocator]
 static ALLOC: turbo_tasks_malloc::TurboMalloc = turbo_tasks_malloc::TurboMalloc;
 
 pub fn benchmark(c: &mut Criterion) {
-    let rt = tokio::runtime::Builder::new_current_thread()
-        .build()
-        .unwrap();
-
-    let tt = TurboTasks::new(TurboTasksBackend::new(
-        BackendOptions {
-            dependency_tracking: false,
-            storage_mode: None,
-            ..Default::default()
-        },
-        noop_backing_storage(),
-    ));
-
-    let cases = rt
-        .block_on(tt.run_once(async {
-            let root_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/benches/");
-            let fs = DiskFileSystem::new(rcstr!("project"), root_dir.to_str().unwrap().into());
-
-            let environment = Environment::new(ExecutionEnvironment::NodeJsLambda(
-                NodeJsEnvironment::default().resolved_cell(),
-            ));
-            let compile_time_info = CompileTimeInfo::new(environment).to_resolved().await?;
-            let layer = Layer::new(rcstr!("test"));
-            let module_asset_context = NoopAssetContext {
-                compile_time_info,
-                layer,
-            }
-            .resolved_cell();
-
-            let mut cases = vec![];
-            for (file, analyze_mode) in [
-                (r#"packages-bundle.js"#, false),
-                (r#"packages-bundle.js"#, true),
-                (r#"app-page-turbo.runtime.prod.js"#, false),
-                (r#"app-page-turbo.runtime.prod.js"#, true),
-                (r#"react-dom-client.development.js"#, false),
-                (r#"react-dom-client.development.js"#, true),
-            ] {
-                let module = EcmascriptModuleAsset::builder(
-                    ResolvedVc::upcast(
-                        FileSource::new(fs.root().await?.join(file).unwrap())
-                            .to_resolved()
-                            .await?,
-                    ),
-                    ResolvedVc::upcast(module_asset_context),
-                    EcmascriptInputTransforms::empty().to_resolved().await?,
-                    EcmascriptOptions {
-                        tree_shaking_mode: Some(TreeShakingMode::ReexportsOnly),
-                        analyze_mode: if analyze_mode {
-                            turbopack_ecmascript::AnalyzeMode::Tracing
-                        } else {
-                            turbopack_ecmascript::AnalyzeMode::CodeGenerationAndTracing
-                        },
-                        ..Default::default()
-                    }
-                    .resolved_cell(),
-                    compile_time_info,
-                )
-                .build()
-                .to_resolved()
-                .await?;
-
-                cases.push((
-                    file.rsplit("/").next().unwrap(),
-                    if analyze_mode { "tracing" } else { "full" },
-                    module,
-                ));
-            }
-            anyhow::Ok(cases)
-        }))
-        .unwrap();
+    let root_dir = String::leak(
+        PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+            .join("tests/benches/")
+            .to_str()
+            .unwrap()
+            .to_string(),
+    );
 
     let mut group = c.benchmark_group("references");
     group.warm_up_time(Duration::from_secs(1));
     group.measurement_time(Duration::from_secs(10));
 
-    for (file, param, module) in cases {
+    for (file, trace_only) in [
+        (r#"packages-bundle.js"#, false),
+        (r#"packages-bundle.js"#, true),
+        (r#"app-page-turbo.runtime.prod.js"#, false),
+        (r#"app-page-turbo.runtime.prod.js"#, true),
+        (r#"react-dom-client.development.js"#, false),
+        (r#"react-dom-client.development.js"#, true),
+        (r#"jsonwebtoken.js"#, false),
+        (r#"jsonwebtoken.js"#, true),
+    ] {
         group.bench_with_input(
-            BenchmarkId::new(file, param),
+            BenchmarkId::new(file, if trace_only { "tracing" } else { "full" }),
             &BenchInput {
-                module,
-                storage: tt.clone(),
+                root_dir,
+                file,
+                analyze_mode: if trace_only {
+                    AnalyzeMode::Tracing
+                } else {
+                    AnalyzeMode::CodeGenerationAndTracing
+                },
             },
             bench_full,
         );
     }
 }
 
-struct BenchInput<B>
-where
-    B: BackingStorage,
-{
-    storage: Arc<TurboTasks<TurboTasksBackend<B>>>,
-    module: ResolvedVc<EcmascriptModuleAsset>,
+async fn setup(
+    root_dir: &str,
+    file: &str,
+    analyze_mode: AnalyzeMode,
+) -> Result<ResolvedVc<EcmascriptModuleAsset>> {
+    let fs = DiskFileSystem::new(rcstr!("project"), root_dir.into());
+
+    let environment = Environment::new(ExecutionEnvironment::NodeJsLambda(
+        NodeJsEnvironment::default().resolved_cell(),
+    ));
+    let compile_time_info = CompileTimeInfo::new(environment).to_resolved().await?;
+    let layer = Layer::new(rcstr!("test"));
+    let module_asset_context = NoopAssetContext {
+        compile_time_info,
+        layer,
+    }
+    .resolved_cell();
+    let module = EcmascriptModuleAsset::builder(
+        ResolvedVc::upcast(
+            FileSource::new(fs.root().await?.join(file).unwrap())
+                .to_resolved()
+                .await?,
+        ),
+        ResolvedVc::upcast(module_asset_context),
+        EcmascriptInputTransforms::empty().to_resolved().await?,
+        EcmascriptOptions {
+            tree_shaking_mode: Some(TreeShakingMode::ReexportsOnly),
+            analyze_mode,
+            ..Default::default()
+        }
+        .resolved_cell(),
+        compile_time_info,
+    )
+    .build()
+    .to_resolved()
+    .await?;
+    Ok(module)
+}
+
+struct BenchInput {
+    root_dir: &'static str,
+    file: &'static str,
+    analyze_mode: AnalyzeMode,
 }
 
-fn bench_full<B>(b: &mut Bencher, input: &BenchInput<B>)
-where
-    B: BackingStorage,
-{
-    let rt = tokio::runtime::Builder::new_current_thread()
+fn bench_full(b: &mut Bencher, input: &BenchInput) {
+    let rt = tokio::runtime::Builder::new_multi_thread()
+        .worker_threads(2)
+        .enable_all()
         .build()
         .unwrap();
 
-    b.to_async(rt).iter(async || {
-        input
-            .storage
-            .run_once(analyze_ecmascript_module_internal(input.module, None))
+    b.to_async(rt).iter_batched(
+        || {
+            let tt = TurboTasks::new(TurboTasksBackend::new(
+                BackendOptions {
+                    dependency_tracking: false,
+                    storage_mode: None,
+                    ..Default::default()
+                },
+                noop_backing_storage(),
+            ));
+            let BenchInput {
+                root_dir,
+                file,
+                analyze_mode,
+            } = *input;
+            let module = tokio::task::block_in_place(|| {
+                tokio::runtime::Handle::current()
+                    .block_on(tt.run_once(async move {
+                        let module = setup(root_dir, file, analyze_mode).await?;
+                        Ok(module)
+                    }))
+                    .unwrap()
+            });
+            (tt, module)
+        },
+        |(tt, module)| async move {
+            tt.run_once(async move {
+                analyze_ecmascript_module(*module, None).await?;
+                Ok(())
+            })
             .await
             .unwrap()
-    });
+        },
+        BatchSize::SmallInput,
+    );
 }
 
 criterion_group!(references_benches, benchmark);

filename: turbopack/crates/turbopack-ecmascript/src/references/mod.rs
patch: @@ -483,7 +483,7 @@ where
 
 /// Analyse a provided [EcmascriptModuleAsset] and return a [AnalyzeEcmascriptModuleResult].
 #[turbo_tasks::function]
-pub(crate) async fn analyze_ecmascript_module(
+pub async fn analyze_ecmascript_module(
     module: ResolvedVc<EcmascriptModuleAsset>,
     part: Option<ModulePart>,
 ) -> Result<Vc<AnalyzeEcmascriptModuleResult>> {
@@ -504,7 +504,7 @@ pub(crate) async fn analyze_ecmascript_module(
     }
 }
 
-pub async fn analyze_ecmascript_module_internal(
+async fn analyze_ecmascript_module_internal(
     module: ResolvedVc<EcmascriptModuleAsset>,
     part: Option<ModulePart>,
 ) -> Result<Vc<AnalyzeEcmascriptModuleResult>> {