Repository: vercel/next.js
Pull request: [turbopack] Issue 'too many matches' warning for DirAssets also

Same as https://github.com/vercel/next.js/pull/84701 except for DirAssets which are what we create for things like `path.resolve` or `path.join`

Closes PACK-5671

Changed files:
filename: turbopack/crates/turbopack-ecmascript/src/references/mod.rs
patch: @@ -104,8 +104,7 @@ use self::{
         EsmAssetReference, EsmAsyncAssetReference, EsmExports, EsmModuleItem, ImportMetaBinding,
         ImportMetaRef, UrlAssetReference, export::EsmExport,
     },
-    node::DirAssetReference,
-    raw::FileSourceReference,
+    raw::{DirAssetReference, FileSourceReference},
     typescript::{TsConfigReference, TsReferencePathAssetReference, TsReferenceTypeAssetReference},
 };
 use super::{
@@ -1652,6 +1651,8 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
             .await
     };
 
+    let make_issue_source =
+        || IssueSource::from_swc_offsets(source, span.lo.to_u32(), span.hi.to_u32());
     if new {
         match func {
             JsValue::WellKnownFunction(WellKnownFunctionKind::URLConstructor) => {
@@ -1987,11 +1988,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
                             context_dir,
                             Pattern::new(pat),
                             collect_affecting_sources,
-                            IssueSource::from_swc_offsets(
-                                source,
-                                span.lo.to_u32(),
-                                span.hi.to_u32(),
-                            ),
+                            make_issue_source(),
                         )
                         .to_resolved()
                         .await?,
@@ -2043,7 +2040,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
             }
             if let Some(context_dir) = get_traced_project_dir().await? {
                 analysis.add_reference(
-                    DirAssetReference::new(context_dir, Pattern::new(pat))
+                    DirAssetReference::new(context_dir, Pattern::new(pat), make_issue_source())
                         .to_resolved()
                         .await?,
                 );
@@ -2085,7 +2082,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
             }
             if let Some(context_dir) = get_traced_project_dir().await? {
                 analysis.add_reference(
-                    DirAssetReference::new(context_dir, Pattern::new(pat))
+                    DirAssetReference::new(context_dir, Pattern::new(pat), make_issue_source())
                         .to_resolved()
                         .await?,
                 );
@@ -2376,9 +2373,13 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
                             };
                             if let Some(context_dir) = get_traced_project_dir().await? {
                                 analysis.add_reference(
-                                    DirAssetReference::new(context_dir, Pattern::new(abs_pattern))
-                                        .to_resolved()
-                                        .await?,
+                                    DirAssetReference::new(
+                                        context_dir,
+                                        Pattern::new(abs_pattern),
+                                        make_issue_source(),
+                                    )
+                                    .to_resolved()
+                                    .await?,
                                 );
                             }
                             return Ok(());
@@ -2441,9 +2442,13 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
                 };
                 if let Some(context_dir) = get_traced_project_dir().await? {
                     analysis.add_reference(
-                        DirAssetReference::new(context_dir, Pattern::new(abs_pattern))
-                            .to_resolved()
-                            .await?,
+                        DirAssetReference::new(
+                            context_dir,
+                            Pattern::new(abs_pattern),
+                            make_issue_source(),
+                        )
+                        .to_resolved()
+                        .await?,
                     );
                 }
                 return Ok(());
@@ -2510,6 +2515,7 @@ async fn handle_call<G: Fn(Vec<Effect>) + Send + Sync>(
                             DirAssetReference::new(
                                 context_dir.clone(),
                                 Pattern::new(Pattern::Constant(dir.into())),
+                                make_issue_source(),
                             )
                             .to_resolved()
                         })

filename: turbopack/crates/turbopack-ecmascript/src/references/node.rs
patch: @@ -1,17 +1,10 @@
-use anyhow::{Result, bail};
-use tracing::Instrument;
-use turbo_rcstr::{RcStr, rcstr};
+use anyhow::Result;
+use turbo_rcstr::RcStr;
 use turbo_tasks::{ResolvedVc, ValueToString, Vc};
 use turbo_tasks_fs::FileSystemPath;
 use turbopack_core::{
-    chunk::{ChunkableModuleReference, ChunkingType, ChunkingTypeOption},
-    file_source::FileSource,
-    raw_module::RawModule,
-    reference::ModuleReference,
-    resolve::{
-        ModuleResolveResult, RequestKey,
-        pattern::{Pattern, PatternMatch, read_matches},
-    },
+    file_source::FileSource, raw_module::RawModule, reference::ModuleReference,
+    resolve::ModuleResolveResult,
 };
 
 #[turbo_tasks::value]
@@ -53,125 +46,3 @@ impl ValueToString for PackageJsonReference {
         ))
     }
 }
-
-#[turbo_tasks::value]
-#[derive(Hash, Debug)]
-pub struct DirAssetReference {
-    pub context_dir: FileSystemPath,
-    pub path: ResolvedVc<Pattern>,
-}
-
-#[turbo_tasks::value_impl]
-impl DirAssetReference {
-    #[turbo_tasks::function]
-    pub fn new(context_dir: FileSystemPath, path: ResolvedVc<Pattern>) -> Vc<Self> {
-        Self::cell(DirAssetReference { context_dir, path })
-    }
-}
-
-async fn resolve_reference_from_dir(
-    context_dir: FileSystemPath,
-    path: Vc<Pattern>,
-) -> Result<Vc<ModuleResolveResult>> {
-    let path_ref = path.await?;
-    let (abs_path, rel_path) = path_ref.split_could_match("/ROOT/");
-    if abs_path.is_none() && rel_path.is_none() {
-        return Ok(*ModuleResolveResult::unresolvable());
-    }
-
-    let abs_matches = if let Some(abs_path) = &abs_path {
-        Some(
-            read_matches(
-                context_dir.root().owned().await?,
-                rcstr!("/ROOT/"),
-                true,
-                Pattern::new(abs_path.or_any_nested_file()),
-            )
-            .await?,
-        )
-    } else {
-        None
-    };
-    let rel_matches = if let Some(rel_path) = &rel_path {
-        Some(
-            read_matches(
-                context_dir,
-                rcstr!(""),
-                true,
-                Pattern::new(rel_path.or_any_nested_file()),
-            )
-            .await?,
-        )
-    } else {
-        None
-    };
-
-    let matches = abs_matches
-        .into_iter()
-        .flatten()
-        .chain(rel_matches.into_iter().flatten());
-
-    let mut affecting_sources = Vec::new();
-    let mut results = Vec::new();
-    for pat_match in matches {
-        match pat_match {
-            PatternMatch::File(matched_path, file) => {
-                let realpath = file.realpath_with_links().await?;
-                for symlink in &realpath.symlinks {
-                    affecting_sources.push(ResolvedVc::upcast(
-                        FileSource::new(symlink.clone()).to_resolved().await?,
-                    ));
-                }
-                let path: FileSystemPath = match &realpath.path_result {
-                    Ok(path) => path.clone(),
-                    Err(e) => bail!(e.as_error_message(file, &realpath)),
-                };
-                results.push((
-                    RequestKey::new(matched_path.clone()),
-                    ResolvedVc::upcast(
-                        RawModule::new(Vc::upcast(FileSource::new(path)))
-                            .to_resolved()
-                            .await?,
-                    ),
-                ));
-            }
-            PatternMatch::Directory(..) => {}
-        }
-    }
-    Ok(*ModuleResolveResult::modules_with_affecting_sources(
-        results,
-        affecting_sources,
-    ))
-}
-
-#[turbo_tasks::value_impl]
-impl ModuleReference for DirAssetReference {
-    #[turbo_tasks::function]
-    async fn resolve_reference(&self) -> Result<Vc<ModuleResolveResult>> {
-        let span = tracing::info_span!(
-            "trace directory",
-            pattern = display(self.path.to_string().await?)
-        );
-        resolve_reference_from_dir(self.context_dir.clone(), *self.path)
-            .instrument(span)
-            .await
-    }
-}
-
-#[turbo_tasks::value_impl]
-impl ChunkableModuleReference for DirAssetReference {
-    #[turbo_tasks::function]
-    fn chunking_type(&self) -> Vc<ChunkingTypeOption> {
-        Vc::cell(Some(ChunkingType::Traced))
-    }
-}
-
-#[turbo_tasks::value_impl]
-impl ValueToString for DirAssetReference {
-    #[turbo_tasks::function]
-    async fn to_string(&self) -> Result<Vc<RcStr>> {
-        Ok(Vc::cell(
-            format!("directory assets {}", self.path.to_string().await?,).into(),
-        ))
-    }
-}

filename: turbopack/crates/turbopack-ecmascript/src/references/raw.rs
patch: @@ -1,18 +1,23 @@
-use anyhow::Result;
+use anyhow::{Result, bail};
 use tracing::Instrument;
 use turbo_rcstr::{RcStr, rcstr};
 use turbo_tasks::{ResolvedVc, ValueToString, Vc};
 use turbo_tasks_fs::FileSystemPath;
 use turbopack_core::{
     chunk::{ChunkableModuleReference, ChunkingType, ChunkingTypeOption},
+    file_source::FileSource,
     issue::{
         Issue, IssueExt, IssueSeverity, IssueSource, IssueStage, OptionIssueSource,
         OptionStyledString, StyledString,
     },
+    raw_module::RawModule,
     reference::ModuleReference,
-    resolve::{ModuleResolveResult, pattern::Pattern, resolve_raw},
+    resolve::{
+        ModuleResolveResult, RequestKey,
+        pattern::{Pattern, PatternMatch, read_matches},
+        resolve_raw,
+    },
 };
-
 #[turbo_tasks::value]
 #[derive(Hash, Debug)]
 pub struct FileSourceReference {
@@ -58,26 +63,202 @@ impl ModuleReference for FileSourceReference {
             .as_raw_module_result()
             .resolve()
             .await?;
-            let num_matches = result.await?.primary.len();
-            if num_matches > TOO_MANY_MATCHES_LIMIT {
-                TooManyMatchesWarning {
-                    source: self.issue_source,
-                    context_dir: self.context_dir.clone(),
-                    num_matches,
-                    pattern: self.path,
+            check_and_emit_too_many_matches_warning(
+                result,
+                self.issue_source,
+                self.context_dir.clone(),
+                self.path,
+            )
+            .await?;
+
+            Ok(result)
+        }
+        .instrument(span)
+        .await
+    }
+}
+#[turbo_tasks::value_impl]
+impl ChunkableModuleReference for FileSourceReference {
+    #[turbo_tasks::function]
+    fn chunking_type(&self) -> Vc<ChunkingTypeOption> {
+        Vc::cell(Some(ChunkingType::Traced))
+    }
+}
+
+#[turbo_tasks::value_impl]
+impl ValueToString for FileSourceReference {
+    #[turbo_tasks::function]
+    async fn to_string(&self) -> Result<Vc<RcStr>> {
+        Ok(Vc::cell(
+            format!("raw asset {}", self.path.to_string().await?,).into(),
+        ))
+    }
+}
+
+#[turbo_tasks::value]
+#[derive(Hash, Debug)]
+pub struct DirAssetReference {
+    context_dir: FileSystemPath,
+    path: ResolvedVc<Pattern>,
+    issue_source: IssueSource,
+}
+
+#[turbo_tasks::value_impl]
+impl DirAssetReference {
+    #[turbo_tasks::function]
+    pub fn new(
+        context_dir: FileSystemPath,
+        path: ResolvedVc<Pattern>,
+        issue_source: IssueSource,
+    ) -> Vc<Self> {
+        Self::cell(DirAssetReference {
+            context_dir,
+            path,
+            issue_source,
+        })
+    }
+}
+
+async fn resolve_reference_from_dir(
+    context_dir: FileSystemPath,
+    path: Vc<Pattern>,
+) -> Result<Vc<ModuleResolveResult>> {
+    let path_ref = path.await?;
+    let (abs_path, rel_path) = path_ref.split_could_match("/ROOT/");
+    if abs_path.is_none() && rel_path.is_none() {
+        return Ok(*ModuleResolveResult::unresolvable());
+    }
+
+    let abs_matches = if let Some(abs_path) = &abs_path {
+        Some(
+            read_matches(
+                context_dir.root().owned().await?,
+                rcstr!("/ROOT/"),
+                true,
+                Pattern::new(abs_path.or_any_nested_file()),
+            )
+            .await?,
+        )
+    } else {
+        None
+    };
+    let rel_matches = if let Some(rel_path) = &rel_path {
+        Some(
+            read_matches(
+                context_dir,
+                rcstr!(""),
+                true,
+                Pattern::new(rel_path.or_any_nested_file()),
+            )
+            .await?,
+        )
+    } else {
+        None
+    };
+
+    let matches = abs_matches
+        .into_iter()
+        .flatten()
+        .chain(rel_matches.into_iter().flatten());
+
+    let mut affecting_sources = Vec::new();
+    let mut results = Vec::new();
+    for pat_match in matches {
+        match pat_match {
+            PatternMatch::File(matched_path, file) => {
+                let realpath = file.realpath_with_links().await?;
+                for symlink in &realpath.symlinks {
+                    affecting_sources.push(ResolvedVc::upcast(
+                        FileSource::new(symlink.clone()).to_resolved().await?,
+                    ));
                 }
-                .resolved_cell()
-                .emit();
+                let path: FileSystemPath = match &realpath.path_result {
+                    Ok(path) => path.clone(),
+                    Err(e) => bail!(e.as_error_message(file, &realpath)),
+                };
+                results.push((
+                    RequestKey::new(matched_path.clone()),
+                    ResolvedVc::upcast(
+                        RawModule::new(Vc::upcast(FileSource::new(path)))
+                            .to_resolved()
+                            .await?,
+                    ),
+                ));
             }
+            PatternMatch::Directory(..) => {}
+        }
+    }
+    Ok(*ModuleResolveResult::modules_with_affecting_sources(
+        results,
+        affecting_sources,
+    ))
+}
 
+#[turbo_tasks::value_impl]
+impl ModuleReference for DirAssetReference {
+    #[turbo_tasks::function]
+    async fn resolve_reference(&self) -> Result<Vc<ModuleResolveResult>> {
+        let span = tracing::info_span!(
+            "trace directory",
+            pattern = display(self.path.to_string().await?)
+        );
+        async {
+            let result = resolve_reference_from_dir(self.context_dir.clone(), *self.path).await?;
+            check_and_emit_too_many_matches_warning(
+                result,
+                self.issue_source,
+                self.context_dir.clone(),
+                self.path,
+            )
+            .await?;
             Ok(result)
         }
         .instrument(span)
         .await
     }
 }
+
+#[turbo_tasks::value_impl]
+impl ChunkableModuleReference for DirAssetReference {
+    #[turbo_tasks::function]
+    fn chunking_type(&self) -> Vc<ChunkingTypeOption> {
+        Vc::cell(Some(ChunkingType::Traced))
+    }
+}
+
+#[turbo_tasks::value_impl]
+impl ValueToString for DirAssetReference {
+    #[turbo_tasks::function]
+    async fn to_string(&self) -> Result<Vc<RcStr>> {
+        Ok(Vc::cell(
+            format!("directory assets {}", self.path.to_string().await?,).into(),
+        ))
+    }
+}
+
 /// If a pattern resolves to more than 10000 results, it's likely a mistake so issue a warning.
 const TOO_MANY_MATCHES_LIMIT: usize = 10000;
+
+async fn check_and_emit_too_many_matches_warning(
+    result: Vc<ModuleResolveResult>,
+    issue_source: IssueSource,
+    context_dir: FileSystemPath,
+    pattern: ResolvedVc<Pattern>,
+) -> Result<()> {
+    let num_matches = result.await?.primary.len();
+    if num_matches > TOO_MANY_MATCHES_LIMIT {
+        TooManyMatchesWarning {
+            source: issue_source,
+            context_dir,
+            num_matches,
+            pattern,
+        }
+        .resolved_cell()
+        .emit();
+    }
+    Ok(())
+}
+
 #[turbo_tasks::value(shared)]
 struct TooManyMatchesWarning {
     source: IssueSource,
@@ -131,21 +312,3 @@ impl Issue for TooManyMatchesWarning {
         Vc::cell(Some(self.source))
     }
 }
-
-#[turbo_tasks::value_impl]
-impl ChunkableModuleReference for FileSourceReference {
-    #[turbo_tasks::function]
-    fn chunking_type(&self) -> Vc<ChunkingTypeOption> {
-        Vc::cell(Some(ChunkingType::Traced))
-    }
-}
-
-#[turbo_tasks::value_impl]
-impl ValueToString for FileSourceReference {
-    #[turbo_tasks::function]
-    async fn to_string(&self) -> Result<Vc<RcStr>> {
-        Ok(Vc::cell(
-            format!("raw asset {}", self.path.to_string().await?,).into(),
-        ))
-    }
-}