Repository: vercel/next.js
Pull request: [turbopack] Delete the `read_glob` function

Why? This was only used for unit tests and an example.  Also, the API was a little strange, it would end up creating a `Vc` that depended on all the directories with matches and produced a value that modeled the matching directory tree.

Happy to replace the example... I wasn't really sure what those were used for.


Closes PACK-4616

Changed files:
filename: turbopack/crates/turbo-tasks-fs/examples/hash_glob.rs
patch: @@ -1,120 +0,0 @@
-#![feature(trivial_bounds)]
-#![allow(clippy::needless_return)] // clippy false positive
-
-use std::{
-    collections::BTreeMap,
-    env::current_dir,
-    io::Read,
-    time::{Duration, Instant},
-};
-
-use anyhow::Result;
-use sha2::{Digest, Sha256};
-use turbo_rcstr::RcStr;
-use turbo_tasks::{ReadConsistency, TurboTasks, UpdateInfo, Vc, util::FormatDuration};
-use turbo_tasks_fs::{
-    DirectoryEntry, DiskFileSystem, FileContent, FileSystem, FileSystemPath, ReadGlobResult,
-    glob::Glob, register,
-};
-use turbo_tasks_memory::MemoryBackend;
-
-#[tokio::main]
-async fn main() -> Result<()> {
-    register();
-    include!(concat!(env!("OUT_DIR"), "/register_example_hash_glob.rs"));
-
-    let tt = TurboTasks::new(MemoryBackend::default());
-    let start = Instant::now();
-
-    let task = tt.spawn_root_task(|| {
-        Box::pin(async {
-            let root = current_dir().unwrap().to_str().unwrap().into();
-            let disk_fs = DiskFileSystem::new("project".into(), root, vec![]);
-            disk_fs.await?.start_watching(None).await?;
-
-            // Smart Pointer cast
-            let fs: Vc<Box<dyn FileSystem>> = Vc::upcast(disk_fs);
-            let input = fs.root().join("crates".into());
-            let glob = Glob::new("**/*.rs".into());
-            let glob_result = input.read_glob(glob, true);
-            let dir_hash = hash_glob_result(glob_result);
-            print_hash(dir_hash).await?;
-            Ok::<Vc<()>, _>(Default::default())
-        })
-    });
-    tt.wait_task_completion(task, ReadConsistency::Strong)
-        .await
-        .unwrap();
-    println!("done in {}", FormatDuration(start.elapsed()));
-
-    loop {
-        let UpdateInfo {
-            duration, tasks, ..
-        } = tt
-            .get_or_wait_aggregated_update_info(Duration::from_millis(100))
-            .await;
-        println!("updated {} tasks in {}", tasks, FormatDuration(duration));
-    }
-}
-
-#[turbo_tasks::function]
-pub fn empty_string() -> Vc<RcStr> {
-    Vc::cell(Default::default())
-}
-
-#[turbo_tasks::function]
-async fn print_hash(dir_hash: Vc<RcStr>) -> Result<Vc<()>> {
-    println!("DIR HASH: {}", dir_hash.await?.as_str());
-    Ok(Default::default())
-}
-
-#[turbo_tasks::function]
-async fn hash_glob_result(result: Vc<ReadGlobResult>) -> Result<Vc<RcStr>> {
-    let result = result.await?;
-    let mut hashes = BTreeMap::new();
-    for (name, entry) in result.results.iter() {
-        if let DirectoryEntry::File(path) = entry {
-            hashes.insert(name, hash_file(**path).owned().await?);
-        }
-    }
-    for (name, result) in result.inner.iter() {
-        let hash = hash_glob_result(**result).owned().await?;
-        if !hash.is_empty() {
-            hashes.insert(name, hash);
-        }
-    }
-    if hashes.is_empty() {
-        return Ok(empty_string());
-    }
-    let hash = hash_content(
-        &mut hashes
-            .into_values()
-            .collect::<Vec<RcStr>>()
-            .join(",")
-            .as_bytes(),
-    );
-    Ok(hash)
-}
-
-#[turbo_tasks::function]
-async fn hash_file(file_path: Vc<FileSystemPath>) -> Result<Vc<RcStr>> {
-    let content = file_path.read().await?;
-    Ok(match &*content {
-        FileContent::Content(file) => hash_content(&mut file.read()),
-        FileContent::NotFound => {
-            // report error
-            Vc::cell(Default::default())
-        }
-    })
-}
-
-fn hash_content<R: Read>(content: &mut R) -> Vc<RcStr> {
-    let mut hasher = Sha256::new();
-    let mut buf = [0; 1024];
-    while let Ok(size) = content.read(&mut buf) {
-        hasher.update(&buf[0..size]);
-    }
-    let result = format!("{:x}", hasher.finalize());
-
-    Vc::cell(result.into())
-}

filename: turbopack/crates/turbo-tasks-fs/src/lib.rs
patch: @@ -46,8 +46,7 @@ use invalidator_map::InvalidatorMap;
 use jsonc_parser::{ParseOptions, parse_to_serde_value};
 use mime::Mime;
 use rayon::iter::{IntoParallelIterator, ParallelIterator};
-pub use read_glob::ReadGlobResult;
-use read_glob::{read_glob, track_glob};
+use read_glob::track_glob;
 use rustc_hash::FxHashSet;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
@@ -1263,15 +1262,6 @@ impl FileSystemPath {
         Ok(FileSystemPathOption::none())
     }
 
-    #[turbo_tasks::function]
-    pub fn read_glob(
-        self: Vc<Self>,
-        glob: Vc<Glob>,
-        include_dot_files: bool,
-    ) -> Vc<ReadGlobResult> {
-        read_glob(self, glob, include_dot_files)
-    }
-
     // Tracks all files and directories matching the glob
     // Follows symlinks as though they were part of the original hierarchy.
     #[turbo_tasks::function]

filename: turbopack/crates/turbo-tasks-fs/src/read_glob.rs
patch: @@ -1,85 +1,10 @@
 use anyhow::{Result, bail};
 use futures::try_join;
-use rustc_hash::FxHashMap;
 use turbo_rcstr::RcStr;
-use turbo_tasks::{Completion, ResolvedVc, TryJoinIterExt, Vc};
+use turbo_tasks::{Completion, TryJoinIterExt, Vc};
 
 use crate::{DirectoryContent, DirectoryEntry, FileSystem, FileSystemPath, glob::Glob};
 
-#[turbo_tasks::value]
-#[derive(Default, Debug)]
-pub struct ReadGlobResult {
-    pub results: FxHashMap<String, DirectoryEntry>,
-    pub inner: FxHashMap<String, ResolvedVc<ReadGlobResult>>,
-}
-
-/// Reads matches of a glob pattern.
-///
-/// DETERMINISM: Result is in random order. Either sort result or do not depend
-/// on the order.
-#[turbo_tasks::function(fs)]
-pub async fn read_glob(
-    directory: Vc<FileSystemPath>,
-    glob: Vc<Glob>,
-    include_dot_files: bool,
-) -> Result<Vc<ReadGlobResult>> {
-    Ok(*read_glob_internal("", directory, glob, include_dot_files).await?)
-}
-
-#[turbo_tasks::function(fs)]
-async fn read_glob_inner(
-    prefix: RcStr,
-    directory: Vc<FileSystemPath>,
-    glob: Vc<Glob>,
-    include_dot_files: bool,
-) -> Result<Vc<ReadGlobResult>> {
-    Ok(*read_glob_internal(&prefix, directory, glob, include_dot_files).await?)
-}
-
-// The `prefix` represents the relative directory path where symlinks are not resolve.
-async fn read_glob_internal(
-    prefix: &str,
-    directory: Vc<FileSystemPath>,
-    glob: Vc<Glob>,
-    include_dot_files: bool,
-) -> Result<ResolvedVc<ReadGlobResult>> {
-    let dir = directory.read_dir().await?;
-    let mut result = ReadGlobResult::default();
-    let glob_value = glob.await?;
-    match &*dir {
-        DirectoryContent::Entries(entries) => {
-            for (segment, entry) in entries.iter() {
-                if !include_dot_files && segment.starts_with('.') {
-                    continue;
-                }
-                // This is redundant with logic inside of `read_dir` but here we track it separately
-                // so we don't follow symlinks.
-                let entry_path: RcStr = if prefix.is_empty() {
-                    segment.clone()
-                } else {
-                    format!("{prefix}/{segment}").into()
-                };
-                let entry = resolve_symlink_safely(entry).await?;
-                if glob_value.execute(&entry_path) {
-                    result.results.insert(entry_path.to_string(), entry);
-                }
-                if let DirectoryEntry::Directory(path) = entry {
-                    if glob_value.match_in_directory(&entry_path) {
-                        result.inner.insert(
-                            entry_path.to_string(),
-                            read_glob_inner(entry_path, *path, glob, include_dot_files)
-                                .to_resolved()
-                                .await?,
-                        );
-                    }
-                }
-            }
-        }
-        DirectoryContent::NotFound => {}
-    }
-    Ok(ReadGlobResult::resolved_cell(result))
-}
-
 // Resolve a symlink checking for recursion.
 async fn resolve_symlink_safely(entry: &DirectoryEntry) -> Result<DirectoryEntry> {
     let resolved_entry = entry.resolve_symlink().await?;
@@ -204,177 +129,61 @@ pub mod tests {
     use turbo_tasks::{Completion, ReadRef, ResolvedVc, Vc, apply_effects};
     use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
 
-    use crate::{
-        DirectoryEntry, DiskFileSystem, FileContent, FileSystem, FileSystemPath, glob::Glob,
-    };
-
-    #[tokio::test]
-    async fn read_glob_basic() {
-        crate::register();
-        let scratch = tempfile::tempdir().unwrap();
-        {
-            // Create a simple directory with 2 files, a subdirectory and a dotfile
-            let path = scratch.path();
-            File::create_new(path.join("foo"))
-                .unwrap()
-                .write_all(b"foo")
-                .unwrap();
-            create_dir(path.join("sub")).unwrap();
-            File::create_new(path.join("sub/bar"))
-                .unwrap()
-                .write_all(b"bar")
-                .unwrap();
-            // Add a dotfile
-            File::create_new(path.join("sub/.gitignore"))
-                .unwrap()
-                .write_all(b"ignore")
-                .unwrap();
-        }
-        let tt = turbo_tasks::TurboTasks::new(TurboTasksBackend::new(
-            BackendOptions::default(),
-            noop_backing_storage(),
-        ));
-        let path: RcStr = scratch.path().to_str().unwrap().into();
-        tt.run_once(async {
-            let fs = Vc::upcast::<Box<dyn FileSystem>>(DiskFileSystem::new(
-                "temp".into(),
-                path,
-                Vec::new(),
-            ));
-            let read_dir = fs
-                .root()
-                .read_glob(Glob::new("**".into()), false)
-                .await
-                .unwrap();
-            assert_eq!(read_dir.results.len(), 2);
-            assert_eq!(
-                read_dir.results.get("foo"),
-                Some(&DirectoryEntry::File(
-                    fs.root().join("foo".into()).to_resolved().await?
-                ))
-            );
-            assert_eq!(
-                read_dir.results.get("sub"),
-                Some(&DirectoryEntry::Directory(
-                    fs.root().join("sub".into()).to_resolved().await?
-                ))
-            );
-            assert_eq!(read_dir.inner.len(), 1);
-            let inner = &*read_dir.inner.get("sub").unwrap().await?;
-            assert_eq!(inner.results.len(), 1);
-            assert_eq!(
-                inner.results.get("sub/bar"),
-                Some(&DirectoryEntry::File(
-                    fs.root().join("sub/bar".into()).to_resolved().await?
-                ))
-            );
-            assert_eq!(inner.inner.len(), 0);
-
-            // Now with a more specific pattern
-            let read_dir = fs
-                .root()
-                .read_glob(Glob::new("**/bar".into()), false)
-                .await
-                .unwrap();
-            assert_eq!(read_dir.results.len(), 0);
-            assert_eq!(read_dir.inner.len(), 1);
-            let inner = &*read_dir.inner.get("sub").unwrap().await?;
-            assert_eq!(inner.results.len(), 1);
-            assert_eq!(
-                inner.results.get("sub/bar"),
-                Some(&DirectoryEntry::File(
-                    fs.root().join("sub/bar".into()).to_resolved().await?
-                ))
-            );
-            assert_eq!(inner.inner.len(), 0);
-
-            anyhow::Ok(())
-        })
-        .await
-        .unwrap();
-    }
-
-    #[cfg(unix)]
-    #[tokio::test]
-    async fn read_glob_symlinks() {
-        crate::register();
-        let scratch = tempfile::tempdir().unwrap();
-        {
-            use std::os::unix::fs::symlink;
-
-            // Create a simple directory with 1 file and a symlink pointing at at a file in a
-            // subdirectory
-            let path = scratch.path();
-            create_dir(path.join("sub")).unwrap();
-            let foo = path.join("sub/foo.js");
-            File::create_new(&foo).unwrap().write_all(b"foo").unwrap();
-            symlink(&foo, path.join("link.js")).unwrap();
-        }
-        let tt = turbo_tasks::TurboTasks::new(TurboTasksBackend::new(
-            BackendOptions::default(),
-            noop_backing_storage(),
-        ));
-        let path: RcStr = scratch.path().to_str().unwrap().into();
-        tt.run_once(async {
-            let fs = Vc::upcast::<Box<dyn FileSystem>>(DiskFileSystem::new(
-                "temp".into(),
-                path,
-                Vec::new(),
-            ));
-            let read_dir = fs
-                .root()
-                .read_glob(Glob::new("*.js".into()), false)
-                .await
-                .unwrap();
-            assert_eq!(read_dir.results.len(), 1);
-            assert_eq!(
-                read_dir.results.get("link.js"),
-                Some(&DirectoryEntry::File(
-                    fs.root().join("sub/foo.js".into()).to_resolved().await?
-                ))
-            );
-            assert_eq!(read_dir.inner.len(), 0);
-
-            anyhow::Ok(())
-        })
-        .await
-        .unwrap();
-    }
+    use crate::{DiskFileSystem, FileContent, FileSystem, FileSystemPath, glob::Glob};
 
     #[turbo_tasks::function(operation)]
     pub async fn delete(path: ResolvedVc<FileSystemPath>) -> anyhow::Result<()> {
         path.write(FileContent::NotFound.cell()).await?;
         Ok(())
     }
+    #[turbo_tasks::function(operation)]
+    pub async fn write(path: ResolvedVc<FileSystemPath>, contents: RcStr) -> anyhow::Result<()> {
+        path.write(
+            FileContent::Content(crate::File::from_bytes(contents.to_string().into_bytes())).cell(),
+        )
+        .await?;
+        Ok(())
+    }
 
     #[turbo_tasks::function(operation)]
     pub async fn track_star_star_glob(path: ResolvedVc<FileSystemPath>) -> Vc<Completion> {
         path.track_glob(Glob::new("**".into()), false)
     }
 
+    #[cfg(unix)]
     #[tokio::test]
     async fn track_glob_invalidations() {
+        use std::os::unix::fs::symlink;
         crate::register();
         let scratch = tempfile::tempdir().unwrap();
 
         // Create a simple directory with 2 files, a subdirectory and a dotfile
         let path = scratch.path();
-        File::create_new(path.join("foo"))
+        let dir = path.join("dir");
+        create_dir(&dir).unwrap();
+        File::create_new(dir.join("foo"))
             .unwrap()
             .write_all(b"foo")
             .unwrap();
-        create_dir(path.join("sub")).unwrap();
-        File::create_new(path.join("sub/bar"))
+        create_dir(dir.join("sub")).unwrap();
+        File::create_new(dir.join("sub/bar"))
             .unwrap()
             .write_all(b"bar")
             .unwrap();
         // Add a dotfile
-        create_dir(path.join("sub/.vim")).unwrap();
-        let gitignore = path.join("sub/.vim/.gitignore");
+        create_dir(dir.join("sub/.vim")).unwrap();
+        let gitignore = dir.join("sub/.vim/.gitignore");
         File::create_new(&gitignore)
             .unwrap()
             .write_all(b"ignore")
             .unwrap();
+        // put a link in the dir that points at a file in the root.
+        let link_target = path.join("link_target.js");
+        File::create_new(&link_target)
+            .unwrap()
+            .write_all(b"link_target")
+            .unwrap();
+        symlink(&link_target, dir.join("link.js")).unwrap();
 
         let tt = turbo_tasks::TurboTasks::new(TurboTasksBackend::new(
             BackendOptions::default(),
@@ -387,35 +196,45 @@ pub mod tests {
                 path,
                 Vec::new(),
             ));
-            let read_dir = track_star_star_glob(fs.root().to_resolved().await?)
-                .read_strongly_consistent()
-                .await?;
+            let dir = fs.root().join("dir".into()).to_resolved().await?;
+            let read_dir = track_star_star_glob(dir).read_strongly_consistent().await?;
 
             // Delete a file that we shouldn't be tracking
             let delete_result = delete(
                 fs.root()
-                    .join("sub/.vim/.gitignore".into())
+                    .join("dir/sub/.vim/.gitignore".into())
                     .to_resolved()
                     .await?,
             );
             delete_result.read_strongly_consistent().await?;
             apply_effects(delete_result).await?;
 
-            let read_dir2 = track_star_star_glob(fs.root().to_resolved().await?)
-                .read_strongly_consistent()
-                .await?;
+            let read_dir2 = track_star_star_glob(dir).read_strongly_consistent().await?;
             assert!(ReadRef::ptr_eq(&read_dir, &read_dir2));
 
             // Delete a file that we should be tracking
-            let delete_result = delete(fs.root().join("foo".into()).to_resolved().await?);
+            let delete_result = delete(fs.root().join("dir/foo".into()).to_resolved().await?);
             delete_result.read_strongly_consistent().await?;
             apply_effects(delete_result).await?;
 
-            let read_dir2 = track_star_star_glob(fs.root().to_resolved().await?)
-                .read_strongly_consistent()
-                .await?;
+            let read_dir2 = track_star_star_glob(dir).read_strongly_consistent().await?;
 
             assert!(!ReadRef::ptr_eq(&read_dir, &read_dir2));
+
+            // Modify a symlink target file
+            let write_result = write(
+                fs.root()
+                    .join("link_target.js".into())
+                    .to_resolved()
+                    .await?,
+                "new_contents".into(),
+            );
+            write_result.read_strongly_consistent().await?;
+            apply_effects(write_result).await?;
+            let read_dir3 = track_star_star_glob(dir).read_strongly_consistent().await?;
+
+            assert!(!ReadRef::ptr_eq(&read_dir3, &read_dir2));
+
             anyhow::Ok(())
         })
         .await
@@ -424,7 +243,7 @@ pub mod tests {
 
     #[cfg(unix)]
     #[tokio::test]
-    async fn read_glob_symlinks_loop() {
+    async fn track_glob_symlinks_loop() {
         crate::register();
         let scratch = tempfile::tempdir().unwrap();
         {
@@ -453,7 +272,7 @@ pub mod tests {
             ));
             let err = fs
                 .root()
-                .read_glob(Glob::new("**".into()), false)
+                .track_glob(Glob::new("**".into()), false)
                 .await
                 .expect_err("Should have detected an infinite loop");