Repository: pkolaczk/fclones (language: rust)

Issue:
Honor .gitignore or have its own .fcignore
can fclones ignore known directories such as node_modules, .mypy_cache, CMakefile Release/Debug directories,etc, or let the user to create .flignore file for them.

or, just respect .gitignore by default as fd and rg do.

the best feature I got from rust tools is that it's modern in the sense it is .git etc aware.


Discussion / hints:
That would make a nice new feature. For now, you can exclude multiple paths with --exclude.

Fix patch:
diff --git a/Cargo.lock b/Cargo.lock
index 94dc165..5f165dc 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -342,6 +342,7 @@ dependencies = [
  "fallible-iterator",
  "fiemap",
  "filetime",
+ "ignore",
  "indicatif",
  "indoc",
  "itertools",
@@ -393,6 +394,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
 [[package]]
 name = "fs2"
 version = "0.4.3"
@@ -423,6 +430,19 @@ dependencies = [
  "wasi",
 ]
 
+[[package]]
+name = "globset"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10463d9ff00a2a068db14231982f5132edebad0d7660cd956a1c30292dbcbfbd"
+dependencies = [
+ "aho-corasick",
+ "bstr",
+ "fnv",
+ "log",
+ "regex",
+]
+
 [[package]]
 name = "heck"
 version = "0.3.3"
@@ -441,6 +461,24 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "ignore"
+version = "0.4.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "713f1b139373f96a2e0ce3ac931cd01ee973c3c5dd7c40c0c2efe96ad2b6751d"
+dependencies = [
+ "crossbeam-utils",
+ "globset",
+ "lazy_static",
+ "log",
+ "memchr",
+ "regex",
+ "same-file",
+ "thread_local",
+ "walkdir",
+ "winapi-util",
+]
+
 [[package]]
 name = "indicatif"
 version = "0.14.0"
@@ -943,6 +981,15 @@ version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f"
 
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.1.0"
@@ -1206,6 +1253,17 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
+[[package]]
+name = "walkdir"
+version = "2.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56"
+dependencies = [
+ "same-file",
+ "winapi",
+ "winapi-util",
+]
+
 [[package]]
 name = "wasi"
 version = "0.10.2+wasi-snapshot-preview1"
diff --git a/Cargo.toml b/Cargo.toml
index 54ba651..4bc5d25 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -33,6 +33,7 @@ dtparse = "1.2"
 dunce = "1.0"
 fallible-iterator = "0.2"
 filetime = "0.2"
+ignore = "0.4.18"
 indicatif = { version = "0.14", features = ["with_rayon"] }
 indoc = "1.0"
 itertools = "0.10"
diff --git a/README.md b/README.md
index 99a5b36..0f63036 100644
--- a/README.md
+++ b/README.md
@@ -177,6 +177,11 @@ You can search in multiple directories:
 
     fclones group dir1 dir2 dir3
 
+By default, hidden files and files matching patterns listed in `.gitignore` and `.fdignore` are
+ignored. To search all files, use:
+
+    fclones group --no-ignore --hidden dir
+
 Limit the recursion depth:
     
     fclones group . --depth 1   # scan only files in the current dir, skip subdirs
diff --git a/src/config.rs b/src/config.rs
index b20400a..c7de342 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -148,11 +148,15 @@ pub struct GroupConfig {
     #[structopt(short = "d", long)]
     pub depth: Option<usize>,
 
-    /// Skips hidden files
+    /// Includes hidden files.
+    #[structopt(short = ".", long)]
+    pub hidden: bool,
+
+    /// Does not respect .gitignore and .fdignore files.
     #[structopt(short = "A", long)]
-    pub skip_hidden: bool,
+    pub no_ignore: bool,
 
-    /// Follows symbolic links
+    /// Follows symbolic links.
     #[structopt(short = "L", long)]
     pub follow_links: bool,
 
@@ -231,16 +235,16 @@ pub struct GroupConfig {
     #[structopt(long = "path", value_name("pattern"))]
     pub path_patterns: Vec<String>,
 
-    /// Excludes paths matched fully by any of the given patterns.
+    /// Ignores paths matched fully by any of the given patterns.
     #[structopt(long = "exclude", value_name("pattern"))]
     pub exclude_patterns: Vec<String>,
 
     /// Makes pattern matching case-insensitive.
     #[structopt(short = "i", long)]
-    pub caseless: bool,
+    pub ignore_case: bool,
 
     /// Expects patterns as Perl compatible regular expressions instead of Unix globs.
-    #[structopt(short = "g", long)]
+    #[structopt(short = "x", long)]
     pub regex: bool,
 
     /// Enables caching of file hashes.
@@ -317,7 +321,7 @@ impl GroupConfig {
     }
 
     fn compile_pattern(&self, s: &str) -> Result<Pattern, PatternError> {
-        let pattern_opts = if self.caseless {
+        let pattern_opts = if self.ignore_case {
             PatternOpts::case_insensitive()
         } else {
             PatternOpts::default()
diff --git a/src/group.rs b/src/group.rs
index 4645691..2f23d97 100644
--- a/src/group.rs
+++ b/src/group.rs
@@ -549,8 +549,9 @@ fn scan_files(ctx: &GroupCtx<'_>) -> Vec<Vec<FileInfo>> {
 
     let mut walk = Walk::new();
     walk.depth = config.depth.unwrap_or(usize::MAX);
-    walk.skip_hidden = config.skip_hidden;
+    walk.hidden = config.hidden;
     walk.follow_links = config.follow_links;
+    walk.no_ignore = config.no_ignore;
     walk.path_selector = ctx.path_selector.clone();
     walk.log = Some(ctx.log);
     walk.on_visit = spinner_tick;
diff --git a/src/walk.rs b/src/walk.rs
index 5290269..25da456 100644
--- a/src/walk.rs
+++ b/src/walk.rs
@@ -1,16 +1,18 @@
+use std::default::Default;
 use std::env::current_dir;
 use std::fs::{read_link, symlink_metadata, DirEntry, FileType, ReadDir};
 use std::sync::Arc;
 use std::{fs, io};
 
 use dashmap::DashSet;
+use ignore::gitignore::{Gitignore, GitignoreBuilder};
 use rayon::Scope;
 
 use crate::log::Log;
 use crate::path::Path;
 use crate::selector::PathSelector;
 
-#[derive(Debug)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
 enum EntryType {
     File,
     Dir,
@@ -61,13 +63,74 @@ impl Entry {
     }
 }
 
+#[derive(Clone)]
+struct IgnoreStack(Arc<Vec<Gitignore>>);
+
+impl IgnoreStack {
+    /// Returns ignore stack initialized with global gitignore settings.
+    fn new(log: Option<&Log>) -> Self {
+        let gitignore = GitignoreBuilder::new("/").build_global();
+        if let Some(err) = gitignore.1 {
+            if let Some(log) = log {
+                log.warn(format!("Error loading global gitignore rules: {}", err))
+            }
+        }
+        IgnoreStack(Arc::new(vec![gitignore.0]))
+    }
+
+    /// Returns an empty gitignore stack that ignores no files.
+    pub fn empty() -> IgnoreStack {
+        IgnoreStack(Arc::new(vec![]))
+    }
+
+    /// If .gitignore file exists in given dir, creates a `Gitignore` struct for it
+    /// and returns the stack with the new `Gitignore` appended. Otherwise returns a cloned self.
+    pub fn push(&self, dir: &Path, log: Option<&Log>) -> IgnoreStack {
+        let mut path = Arc::new(dir.clone()).resolve(Path::from(".gitignore"));
+        let mut path_buf = path.to_path_buf();
+        if !path_buf.is_file() {
+            path = Arc::new(dir.clone()).resolve(Path::from(".fdignore"));
+            path_buf = path.to_path_buf();
+        }
+        if !path_buf.is_file() {
+            return self.clone();
+        }
+        let gitignore = Gitignore::new(&path_buf);
+        if let Some(err) = gitignore.1 {
+            if let Some(log) = log {
+                log.warn(format!(
+                    "Error while loading ignore file {}: {}",
+                    path.display(),
+                    err
+                ))
+            }
+        }
+        let mut stack = self.0.as_ref().clone();
+        stack.push(gitignore.0);
+        IgnoreStack(Arc::new(stack))
+    }
+
+    /// Returns true if any of the gitignore files in the stack selects given path
+    pub fn matches(&self, path: &Path, is_dir: bool) -> bool {
+        // this is on critical performance path, so avoid unnecessary to_path_buf conversion
+        if self.0.is_empty() {
+            return false;
+        }
+        let path = path.to_path_buf();
+        self.0
+            .iter()
+            .any(|gitignore| gitignore.matched(&path, is_dir).is_ignore())
+    }
+}
+
 /// Describes walk configuration.
 /// Many walks can be initiated from the same instance.
 pub struct Walk<'a> {
     pub base_dir: Arc<Path>,
     pub depth: usize,
-    pub skip_hidden: bool,
+    pub hidden: bool,
     pub follow_links: bool,
+    pub no_ignore: bool,
     pub path_selector: PathSelector,
     pub on_visit: &'a (dyn Fn(&Path) + Sync + Send),
     pub log: Option<&'a Log>,
@@ -86,8 +149,9 @@ impl<'a> Walk<'a> {
         Walk {
             base_dir: Arc::new(base_dir.clone()),
             depth: usize::MAX,
-            skip_hidden: false,
+            hidden: false,
             follow_links: false,
+            no_ignore: false,
             path_selector: PathSelector::new(base_dir),
             on_visit: &|_| {},
             log: None,
@@ -110,14 +174,21 @@ impl<'a> Walk<'a> {
             visited: DashSet::new(),
         };
         rayon::scope(|scope| {
+            let ignore = if self.no_ignore {
+                IgnoreStack::empty()
+            } else {
+                IgnoreStack::new(self.log)
+            };
+
             for p in roots.into_iter() {
                 let p = self.absolute(p);
+                let ignore = ignore.clone();
                 match fs::metadata(&p.to_path_buf()) {
                     Ok(metadata) if metadata.is_dir() && self.depth == 0 => self.log_warn(format!(
                         "Skipping directory {} because recursive scan is disabled.",
                         p.display()
                     )),
-                    _ => scope.spawn(|scope| self.visit_path(p, scope, 0, &state)),
+                    _ => scope.spawn(|scope| self.visit_path(p, scope, 0, ignore, &state)),
                 }
             }
         });
@@ -129,6 +200,7 @@ impl<'a> Walk<'a> {
         path: Path,
         scope: &Scope<'w>,
         level: usize,
+        gitignore: IgnoreStack,
         state: &'w WalkState<F>,
     ) where
         F: Fn(Path) + Sync + Send,
@@ -138,7 +210,7 @@ impl<'a> Walk<'a> {
             Entry::from_path(path.clone())
                 .map_err(|e| self.log_warn(format!("Failed to stat {}: {}", path.display(), e)))
                 .into_iter()
-                .for_each(|entry| self.visit_entry(entry, scope, level, state))
+                .for_each(|entry| self.visit_entry(entry, scope, level, gitignore.clone(), state))
         }
     }
 
@@ -149,6 +221,7 @@ impl<'a> Walk<'a> {
         entry: Entry,
         scope: &Scope<'w>,
         level: usize,
+        gitignore: IgnoreStack,
         state: &'w WalkState<F>,
     ) where
         F: Fn(Path) + Sync + Send,
@@ -158,7 +231,7 @@ impl<'a> Walk<'a> {
         (self.on_visit)(&entry.path);
 
         // Skip hidden files
-        if self.skip_hidden {
+        if !self.hidden {
             if let Some(name) = entry.path.file_name_cstr() {
                 if name.to_string_lossy().starts_with('.') {
                     return;
@@ -172,10 +245,15 @@ impl<'a> Walk<'a> {
             return;
         }
 
+        // Skip entries ignored by .gitignore
+        if !self.no_ignore && gitignore.matches(&entry.path, entry.tpe == EntryType::Dir) {
+            return;
+        }
+
         match entry.tpe {
             EntryType::File => self.visit_file(entry.path, state),
-            EntryType::Dir => self.visit_dir(entry.path, scope, level, state),
-            EntryType::SymLink => self.visit_link(&entry.path, scope, level, state),
+            EntryType::Dir => self.visit_dir(entry.path, scope, level, gitignore, state),
+            EntryType::SymLink => self.visit_link(&entry.path, scope, level, gitignore, state),
             EntryType::Other => {}
         }
     }
@@ -197,6 +275,7 @@ impl<'a> Walk<'a> {
         path: &Path,
         scope: &Scope<'w>,
         level: usize,
+        gitignore: IgnoreStack,
         state: &'w WalkState<F>,
     ) where
         F: Fn(Path) + Sync + Send,
@@ -204,7 +283,7 @@ impl<'a> Walk<'a> {
     {
         if self.follow_links {
             match self.resolve_link(path) {
-                Ok(target) => self.visit_path(target, scope, level, state),
+                Ok(target) => self.visit_path(target, scope, level, gitignore, state),
                 Err(e) => self.log_warn(format!("Failed to read link {}: {}", path.display(), e)),
             }
         }
@@ -217,20 +296,33 @@ impl<'a> Walk<'a> {
         path: Path,
         scope: &Scope<'w>,
         level: usize,
+        gitignore: IgnoreStack,
         state: &'w WalkState<F>,
     ) where
         F: Fn(Path) + Sync + Send,
         's: 'w,
     {
-        if level < self.depth && self.path_selector.matches_dir(&path) {
-            match std::fs::read_dir(path.to_path_buf()) {
-                Ok(rd) => {
-                    for entry in Self::sorted_entries(path, rd) {
-                        scope.spawn(move |s| self.visit_entry(entry, s, level + 1, state))
-                    }
+        if level > self.depth {
+            return;
+        }
+        if !self.path_selector.matches_dir(&path) {
+            return;
+        }
+
+        let gitignore = if self.no_ignore {
+            gitignore
+        } else {
+            gitignore.push(&path, self.log)
+        };
+
+        match fs::read_dir(path.to_path_buf()) {
+            Ok(rd) => {
+                for entry in Self::sorted_entries(path, rd) {
+                    let gitignore = gitignore.clone();
+                    scope.spawn(move |s| self.visit_entry(entry, s, level + 1, gitignore, state))
                 }
-                Err(e) => self.log_warn(format!("Failed to read dir {}: {}", path.display(), e)),
             }
+            Err(e) => self.log_warn(format!("Failed to read dir {}: {}", path.display(), e)),
         }
     }
 
@@ -304,12 +396,13 @@ impl<'a> Default for Walk<'a> {
 
 #[cfg(test)]
 mod test {
+    use std::fs::{create_dir, File};
     use std::path::PathBuf;
     use std::sync::Mutex;
 
-    use super::*;
     use crate::util::test::*;
-    use std::fs::{create_dir, File};
+
+    use super::*;
 
     #[test]
     fn list_files() {
@@ -410,19 +503,59 @@ mod test {
 
     #[test]
     fn skip_hidden() {
-        with_dir("target/test/walk/7/", |test_root| {
+        with_dir("target/test/walk/skip_hidden/", |test_root| {
             let hidden_dir = test_root.join(".dir");
             create_dir(&hidden_dir).unwrap();
             let hidden_file_1 = hidden_dir.join("file.txt");
             let hidden_file_2 = test_root.join(".file.txt");
             File::create(&hidden_file_1).unwrap();
             File::create(&hidden_file_2).unwrap();
+
             let mut walk = Walk::new();
-            walk.skip_hidden = true;
-            assert_eq!(run_walk(walk, test_root.clone()), Vec::<PathBuf>::new());
+            walk.hidden = false;
+            assert!(run_walk(walk, test_root.clone()).is_empty());
+
+            let mut walk = Walk::new();
+            walk.hidden = true;
+            assert_eq!(run_walk(walk, test_root.clone()).len(), 2);
         });
     }
 
+    fn respect_ignore(root: &str, ignore_file: &str) {
+        with_dir(root, |test_root| {
+            use std::io::Write;
+            let mut gitignore = File::create(test_root.join(ignore_file)).unwrap();
+            writeln!(gitignore, "foo/").unwrap();
+            writeln!(gitignore, "*.log").unwrap();
+            writeln!(gitignore, "**/bar").unwrap();
+            drop(gitignore);
+
+            create_dir(&test_root.join("foo")).unwrap();
+            create_file(&test_root.join("foo").join("bar"));
+            create_file(&test_root.join("bar.log"));
+            create_dir(&test_root.join("dir")).unwrap();
+            create_dir(&test_root.join("dir").join("bar")).unwrap();
+            create_file(&test_root.join("dir").join("bar").join("file"));
+
+            let walk = Walk::new();
+            assert!(run_walk(walk, test_root.clone()).is_empty());
+
+            let mut walk = Walk::new();
+            walk.no_ignore = true;
+            assert_eq!(run_walk(walk, test_root.clone()).len(), 3)
+        });
+    }
+
+    #[test]
+    fn respect_gitignore() {
+        respect_ignore("target/test/walk/gitignore/", ".gitignore")
+    }
+
+    #[test]
+    fn respect_fdignore() {
+        respect_ignore("target/test/walk/fdignore/", ".fdignore")
+    }
+
     fn run_walk(walk: Walk, root: PathBuf) -> Vec<PathBuf> {
         let results = Mutex::new(Vec::new());
         walk.run(vec![Path::from(root)], |path| {


Test patch: