Repository: sharkdp/fd (language: rust)

Issue:
Add --glob/-g option for glob-based search
A glob-based search (in addition to the default regex-based search) has been requested a few times (see #97, #157). I think we should consider to add this.

A few things to consider are:

- Smart case, `--case-sensitive` and `--ignore-case` should work as expected
- Interplay with other command line options has to be reviewed (`--fixed-strings`, `--full-path`, `--extension`)

Implementation-wise, we would probably just create a regex from the glob-pattern (glob syntax: https://docs.rs/globset/0.3.0/globset/#syntax). Also, we should enable "dot-all", just like in https://github.com/sharkdp/fd/issues/111. Note that there is a (possible outdated) implementation here: #96. If nothing else, we should definitely looks at the extensive tests.

Note: a `--regex` option could be an optional extension to override `--glob` (if someone wants to `alias fd="fd --glob"`, for example).


Discussion / hints:
Just discovered fd, after years of find. While I found it great initially, when I discovered that globbing isn't supported that was the first "meh" moment.

For the implementation...wouldn't it be easier to tread it more like a fixed string but call fnmatch(3) on the filenames instead of plain string comparison (I know Rust too little to tell)? fnmatch(3) is the standard function that bash and find use for globbing.
Pedantic, but I doubt bash uses `fnmatch` for globbing, since it supports richer matching than `fnmatch` supports.
> For the implementation...wouldn't it be easier to tread it more like a fixed string but call fnmatch(3) on the filenames instead of plain string comparison (I know Rust too little to tell)? fnmatch(3) is the standard function that bash and find use for globbing.

The implementation I suggested is probably the easiest one, given how `fd` currently works. It has a potential downside performance-wise because the matching would be regex-based. But the Rust `regex` library is extremely fast.

I would guess that the glob-matching semantics are the same, so I would hope that there is no drawback in choosing the "globs via regexes" route.

Fix patch:
diff --git a/Cargo.lock b/Cargo.lock
index 9f14a4bff..031294a1e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -104,6 +104,7 @@ dependencies = [
  "ctrlc 3.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "diff 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
  "filetime 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "globset 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "ignore 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
diff --git a/Cargo.toml b/Cargo.toml
index 11ec364fc..9a289020f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -43,6 +43,7 @@ regex-syntax = "0.6"
 ctrlc = "3.1"
 humantime = "1.1.1"
 lscolors = "0.5"
+globset = "0.4"
 
 [dependencies.clap]
 version = "2.31.2"
diff --git a/src/app.rs b/src/app.rs
index 65678f8e3..c0b06e1ba 100644
--- a/src/app.rs
+++ b/src/app.rs
@@ -67,6 +67,18 @@ pub fn build_app() -> App<'static, 'static> {
                 .short("i")
                 .overrides_with("case-sensitive"),
         )
+        .arg(
+            arg("glob")
+                .long("glob")
+                .short("g")
+                .conflicts_with("fixed-strings"),
+        )
+        .arg(
+            arg("regex")
+                .long("regex")
+                .overrides_with("glob")
+                .hidden_short_help(true),
+        )
         .arg(
             arg("fixed-strings")
                 .long("fixed-strings")
@@ -250,6 +262,12 @@ fn usage() -> HashMap<&'static str, Help> {
         , "Case-insensitive search (default: smart case)"
         , "Perform a case-insensitive search. By default, fd uses case-insensitive searches, \
            unless the pattern contains an uppercase character (smart case).");
+    doc!(h, "glob"
+        , "Glob-based search (default: regular expression)"
+        , "Perform a glob-based search instead of a regular expression search.");
+    doc!(h, "regex"
+        , "Perform a regex-based search"
+        , "Perform a regular-expression based seach (default). This can be used to override --glob.");
     doc!(h, "fixed-strings"
         , "Treat the pattern as a literal string"
         , "Treat the pattern as a literal string instead of a regular expression.");
@@ -337,7 +355,7 @@ fn usage() -> HashMap<&'static str, Help> {
         , "Amount of time in milliseconds to buffer, before streaming the search results to \
            the console.");
     doc!(h, "pattern"
-        , "the search pattern, a regular expression (optional)");
+        , "the search pattern: a regular expression unless '--glob' is used (optional)");
     doc!(h, "path"
         , "the root directory for the filesystem search (optional)"
         , "The directory where the filesystem search is rooted (optional). \
diff --git a/src/internal/mod.rs b/src/internal/mod.rs
index 250f419ef..ec7d1fad6 100644
--- a/src/internal/mod.rs
+++ b/src/internal/mod.rs
@@ -6,9 +6,11 @@
 // notice may not be copied, modified, or distributed except
 // according to those terms.
 
+use std::borrow::Cow;
+use std::ffi::{OsStr, OsString};
+
 use regex_syntax::hir::Hir;
-use regex_syntax::Parser;
-use std::ffi::OsString;
+use regex_syntax::ParserBuilder;
 
 pub use self::file_types::FileTypes;
 
@@ -27,9 +29,27 @@ macro_rules! print_error_and_exit {
     };
 }
 
+#[cfg(any(unix, target_os = "redox"))]
+pub fn osstr_to_bytes(input: &OsStr) -> Cow<[u8]> {
+    use std::os::unix::ffi::OsStrExt;
+    Cow::Borrowed(input.as_bytes())
+}
+
+#[cfg(windows)]
+pub fn osstr_to_bytes(input: &OsStr) -> Cow<[u8]> {
+    let string = input.to_string_lossy();
+
+    match string {
+        Cow::Owned(string) => Cow::Owned(string.into_bytes()),
+        Cow::Borrowed(string) => Cow::Borrowed(string.as_bytes()),
+    }
+}
+
 /// Determine if a regex pattern contains a literal uppercase character.
 pub fn pattern_has_uppercase_char(pattern: &str) -> bool {
-    Parser::new()
+    let mut parser = ParserBuilder::new().allow_invalid_utf8(true).build();
+
+    parser
         .parse(pattern)
         .map(|hir| hir_has_uppercase_char(&hir))
         .unwrap_or(false)
@@ -41,9 +61,13 @@ fn hir_has_uppercase_char(hir: &Hir) -> bool {
 
     match *hir.kind() {
         HirKind::Literal(Literal::Unicode(c)) => c.is_uppercase(),
+        HirKind::Literal(Literal::Byte(b)) => char::from(b).is_uppercase(),
         HirKind::Class(Class::Unicode(ref ranges)) => ranges
             .iter()
             .any(|r| r.start().is_uppercase() || r.end().is_uppercase()),
+        HirKind::Class(Class::Bytes(ref ranges)) => ranges
+            .iter()
+            .any(|r| char::from(r.start()).is_uppercase() || char::from(r.end()).is_uppercase()),
         HirKind::Group(Group { ref hir, .. }) | HirKind::Repetition(Repetition { ref hir, .. }) => {
             hir_has_uppercase_char(hir)
         }
diff --git a/src/internal/opts.rs b/src/internal/opts.rs
index 9775c3bbb..1673129a0 100644
--- a/src/internal/opts.rs
+++ b/src/internal/opts.rs
@@ -4,7 +4,7 @@ use crate::internal::{
     FileTypes,
 };
 use lscolors::LsColors;
-use regex::RegexSet;
+use regex::bytes::RegexSet;
 use std::{path::PathBuf, sync::Arc, time::Duration};
 
 /// Configuration options for *fd*.
diff --git a/src/main.rs b/src/main.rs
index eb5566556..c4af1ef81 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -24,8 +24,9 @@ use std::sync::Arc;
 use std::time;
 
 use atty::Stream;
+use globset::Glob;
 use lscolors::LsColors;
-use regex::{RegexBuilder, RegexSetBuilder};
+use regex::bytes::{RegexBuilder, RegexSetBuilder};
 
 use crate::exec::CommandTemplate;
 use crate::internal::{
@@ -96,8 +97,16 @@ fn main() {
         );
     }
 
-    // Treat pattern as literal string if '--fixed-strings' is used
-    let pattern_regex = if matches.is_present("fixed-strings") {
+    let pattern_regex = if matches.is_present("glob") {
+        let glob = match Glob::new(pattern) {
+            Ok(glob) => glob,
+            Err(e) => {
+                print_error_and_exit!("{}", e);
+            }
+        };
+        glob.regex().to_owned()
+    } else if matches.is_present("fixed-strings") {
+        // Treat pattern as literal string if '--fixed-strings' is used
         regex::escape(pattern)
     } else {
         String::from(pattern)
diff --git a/src/walk.rs b/src/walk.rs
index 72101f26e..8c28189df 100644
--- a/src/walk.rs
+++ b/src/walk.rs
@@ -9,10 +9,12 @@
 use crate::exec;
 use crate::exit_codes::ExitCode;
 use crate::fshelper;
-use crate::internal::{opts::FdOptions, MAX_BUFFER_LENGTH};
+use crate::internal::{opts::FdOptions, osstr_to_bytes, MAX_BUFFER_LENGTH};
 use crate::output;
 
+use std::borrow::Cow;
 use std::error::Error;
+use std::ffi::OsStr;
 use std::io;
 use std::path::PathBuf;
 use std::process;
@@ -24,7 +26,7 @@ use std::time;
 
 use ignore::overrides::OverrideBuilder;
 use ignore::{self, WalkBuilder};
-use regex::Regex;
+use regex::bytes::Regex;
 
 /// The receiver thread can either be buffering results or directly streaming to the console.
 enum ReceiverMode {
@@ -279,30 +281,34 @@ fn spawn_senders(
             }
 
             // Check the name first, since it doesn't require metadata
-
             let entry_path = entry.path();
 
-            let search_str_o = if config.search_full_path {
+            let search_str: Cow<OsStr> = if config.search_full_path {
                 match fshelper::path_absolute_form(entry_path) {
-                    Ok(path_abs_buf) => Some(path_abs_buf.to_string_lossy().into_owned().into()),
+                    Ok(path_abs_buf) => Cow::Owned(path_abs_buf.as_os_str().to_os_string()),
                     Err(_) => {
                         print_error_and_exit!("Unable to retrieve absolute path.");
                     }
                 }
             } else {
-                entry_path.file_name().map(|f| f.to_string_lossy())
+                match entry_path.file_name() {
+                    Some(filename) => Cow::Borrowed(filename),
+                    None => unreachable!(
+                        "Encountered file system entry without a file name. This should only \
+                         happen for paths like 'foo/bar/..' or '/' which are not supposed to \
+                         appear in a file system traversal."
+                    ),
+                }
             };
 
-            if let Some(search_str) = search_str_o {
-                if !pattern.is_match(&*search_str) {
-                    return ignore::WalkState::Continue;
-                }
+            if !pattern.is_match(&osstr_to_bytes(search_str.as_ref())) {
+                return ignore::WalkState::Continue;
             }
 
             // Filter out unwanted extensions.
             if let Some(ref exts_regex) = config.extensions {
-                if let Some(path_str) = entry_path.file_name().and_then(|s| s.to_str()) {
-                    if !exts_regex.is_match(path_str) {
+                if let Some(path_str) = entry_path.file_name() {
+                    if !exts_regex.is_match(&osstr_to_bytes(path_str)) {
                         return ignore::WalkState::Continue;
                     }
                 } else {
@@ -311,7 +317,6 @@ fn spawn_senders(
             }
 
             // Filter out unwanted file types.
-
             if let Some(ref file_types) = config.file_types {
                 if let Some(ref entry_type) = entry.file_type() {
                     if (!file_types.files && entry_type.is_file())


Test patch:
diff --git a/tests/tests.rs b/tests/tests.rs
index 86ca719fc..d9c0130a8 100644
--- a/tests/tests.rs
+++ b/tests/tests.rs
@@ -239,6 +239,89 @@ fn test_case_insensitive() {
     );
 }
 
+/// Glob-based searches (--glob)
+#[test]
+fn test_glob_searches() {
+    let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES);
+
+    te.assert_output(
+        &["--glob", "*.foo"],
+        "a.foo
+        one/b.foo
+        one/two/c.foo
+        one/two/three/d.foo",
+    );
+
+    te.assert_output(
+        &["--glob", "[a-c].foo"],
+        "a.foo
+        one/b.foo
+        one/two/c.foo",
+    );
+
+    te.assert_output(
+        &["--glob", "[a-c].foo*"],
+        "a.foo
+        one/b.foo
+        one/two/C.Foo2
+        one/two/c.foo",
+    );
+}
+
+/// Glob-based searches (--glob) in combination with full path searches (--full-path)
+#[cfg(not(windows))] // TODO: make this work on Windows
+#[test]
+fn test_full_path_glob_searches() {
+    let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES);
+
+    te.assert_output(
+        &["--glob", "--full-path", "**/one/**/*.foo"],
+        "one/b.foo
+        one/two/c.foo
+        one/two/three/d.foo",
+    );
+}
+
+#[test]
+fn test_smart_case_glob_searches() {
+    let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES);
+
+    te.assert_output(
+        &["--glob", "c.foo*"],
+        "one/two/C.Foo2
+        one/two/c.foo",
+    );
+
+    te.assert_output(&["--glob", "C.Foo*"], "one/two/C.Foo2");
+}
+
+/// Glob-based searches (--glob) in combination with --case-sensitive
+#[test]
+fn test_case_sensitive_glob_searches() {
+    let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES);
+
+    te.assert_output(&["--glob", "--case-sensitive", "c.foo*"], "one/two/c.foo");
+}
+
+/// Glob-based searches (--glob) in combination with --extension
+#[test]
+fn test_glob_searches_with_extension() {
+    let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES);
+
+    te.assert_output(
+        &["--glob", "--extension", "foo2", "[a-z].*"],
+        "one/two/C.Foo2",
+    );
+}
+
+/// Make sure that --regex overrides --glob
+#[test]
+fn test_regex_overrides_glob() {
+    let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES);
+
+    te.assert_output(&["--glob", "--regex", "Foo2$"], "one/two/C.Foo2");
+}
+
 /// Full path search (--full-path)
 #[test]
 fn test_full_path() {