Repository: pkolaczk/fclones (language: rust)

Issue:
Duplicate search between, but not within, two distinct directories? 
Is it possible to use `fclones` to find duplicates between, but not within, two directory trees? Here's an example:

```
destination/
  2021/
    January/
      A.jpg

source/
  A1.jpg <-- copy of destination/2021/January/A.jpg (also same as A2.jpg)
  A2.jpg <-- copy of destination/2021/January/A.jpg (also same as A1.jpg)  
  B1.jpg <-- same as B2.jpg
  B2.jpg <-- same as B1.jpg
```

I want to identify `A1.jpg` and `A2.jpg` under `source` as duplicates of `A.jpg` in `destination`. 

`B1.jpg` and `B2.jpg` are also duplicates but only under `sources`. They should be excluded from the match list because they don't match anything in `destination`.

FWIW, the use case is a source folder of images that have previously been processed by scripts to rename them and sort them into a destination directory structure (e.g. by year and month, or by other EXIF metadata). Then we come across a new folder of images, some of which may have been processed previously, and we want to know if we can safely delete them because we already have copies in the destination directory.


Discussion / hints:
Unfortunately it is not supported yet. But I think it would be useful. 
I did something similar where I wanted to filter the groups based on if they had certain paths. I was able to use `jq` and the following may be helpful. Note that import of JSON was added since this work was done. The comment is accurate to the best of my current knowledge but it's been awhile. Hope it helps.
```
$ fclones group dir1 dir2 dir3 dir4 --format json > dupes.json

# Include entire groups where only 1 file path has 'dir1' in it
$ jq -s '.[0] | {header: .header, groups: [.groups[] | select([.files[] | contains("dir1") | select(.)] | length == 1)]}' < dupes.json > dupes_filtered.json

$ jq -s '.[0]' < dupes.json > dupes_sorted.json

$ diff dupes_sorted.json dupes_filtered.json | less

$ jq '.groups[].files[]' < dupes_filtered.json | grep 'dir1' | sed 's/"//g' | sort > files_to_delete.txt

$ while read -l f; echo $f; rm "$f"; end < files_to_delete.txt

$ find . -type d -empty -exec -delete
```

Fix patch:
diff --git a/README.md b/README.md
index 78f1864..5dceaa1 100644
--- a/README.md
+++ b/README.md
@@ -162,7 +162,12 @@ Limit the recursion depth:
 
 Caution: Versions up to 0.10 did not descend into directories by default.
 In those old versions, add `-R` flag to enable recursive directory walking.
- 
+
+Finding files that match across two directory trees, without matching identical files
+within each tree:
+
+    fclones group --isolate dir1 dir2
+
 Finding duplicate files of size at least 100 MB: 
 
     fclones group . -s 100M
diff --git a/src/config.rs b/src/config.rs
index d31c55a..a992dae 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -158,6 +158,10 @@ pub struct GroupConfig {
     #[structopt(short = "H", long)]
     pub hard_links: bool,
 
+    /// Don't count matching files found within the same directory argument as duplicates.
+    #[structopt(short("I"), long)]
+    pub isolate: bool,
+
     /// Before matching, transforms each file by the specified program.
     /// The value of this parameter should contain a command: the path to the program
     /// and optionally a list of space-separated arguments.
@@ -265,10 +269,34 @@ pub struct GroupConfig {
     /// By default descends into directories recursively, unless a recursion depth
     /// limit is specified with `--depth`.
     #[structopt(parse(from_os_str), required_unless("stdin"))]
-    pub paths: Vec<PathBuf>,
+    pub paths: Vec<Path>,
 }
 
 impl GroupConfig {
+    fn validate(&self) -> Result<(), String> {
+        if self.isolate && self.paths.len() <= self.rf_over() {
+            return Err(format!(
+                "The --isolate flag requires that the number of input paths ({}) \
+                 is at least as large as the replication factor lower bound ({}). \
+                 No files would be considered duplicate, regardless of their contents.",
+                self.paths.len(),
+                self.rf_over() + 1,
+            ));
+        }
+        if self.isolate && self.paths.len() < self.rf_under() {
+            return Err(format!(
+                "The --isolate flag requires that the number of input paths ({}) \
+                 is larger than the replication factor upper bound ({}). \
+                 All files would be considered unique or under-replicated, \
+                 regardless of their contents.",
+                self.paths.len(),
+                self.rf_under() - 1,
+            ));
+        }
+
+        Ok(())
+    }
+
     fn compile_pattern(&self, s: &str) -> Result<Pattern, PatternError> {
         let pattern_opts = if self.caseless {
             PatternOpts::case_insensitive()
@@ -346,13 +374,7 @@ impl GroupConfig {
                     .map(|s| Path::from(s.unwrap().as_str())),
             )
         } else {
-            Box::new(
-                self.paths
-                    .iter()
-                    .map(Path::from)
-                    .collect::<Vec<_>>()
-                    .into_iter(),
-            )
+            Box::new(self.paths.clone().into_iter())
         }
     }
 
@@ -525,6 +547,15 @@ pub enum Command {
     },
 }
 
+impl Command {
+    pub fn validate(&self) -> Result<(), String> {
+        match self {
+            Command::Group(c) => c.validate(),
+            _ => Ok(()),
+        }
+    }
+}
+
 /// Finds and cleans up redundant files
 #[derive(Debug, StructOpt)]
 #[structopt(
diff --git a/src/lib.rs b/src/lib.rs
index 0ef7907..62f5215 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,7 +1,7 @@
 use core::fmt;
 use std::cell::RefCell;
 use std::cmp::{max, Reverse};
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::env::{args, current_dir};
 use std::ffi::{OsStr, OsString};
 use std::fmt::{Display, Formatter};
@@ -93,6 +93,7 @@ impl From<&str> for Error {
 struct AppCtx<'a> {
     pub config: &'a GroupConfig,
     pub log: &'a Log,
+    input_paths: Vec<Path>,
     devices: DiskDevices,
     transform: Option<Transform>,
     path_selector: PathSelector,
@@ -108,6 +109,12 @@ impl<'a> AppCtx<'a> {
             Some(Err(e)) => return Err(Error::new(format!("Invalid transform: {}", e))),
         };
         let base_dir = Path::from(current_dir().unwrap_or_default());
+        let base_dir_ref = Arc::new(base_dir.clone());
+        let root_paths = config
+            .paths
+            .iter()
+            .map(|p| base_dir_ref.resolve(p))
+            .collect();
         let selector = config
             .path_selector(&base_dir)
             .map_err(|e| format!("Invalid pattern: {}", e))?;
@@ -117,6 +124,7 @@ impl<'a> AppCtx<'a> {
         Ok(AppCtx {
             config,
             log,
+            input_paths: root_paths,
             devices,
             transform,
             path_selector: selector,
@@ -276,13 +284,13 @@ where
 
     let (tx, rx): (Sender<HashedFileInfo>, Receiver<HashedFileInfo>) = channel();
 
-    // There is no point in fclonesing groups containing a single file.
+    // There is no point in processing groups containing a single file.
     // Normally when searching for duplicates such groups are filtered out automatically after
     // each stage, however they are possible when searching for unique files.
     let (groups_to_fclones, groups_to_pass): (Vec<_>, Vec<_>) =
         groups.into_iter().partition(group_pre_filter);
 
-    // This way we can split fclonesing to separate thread-pools, one per device:
+    // This way we can split processing to separate thread-pools, one per device:
     let files = partition_by_devices(groups_to_fclones, devices);
     let mut hash_map =
         GroupMap::new(|f: HashedFileInfo| ((f.file_info.len, f.file_hash), f.file_info));
@@ -299,7 +307,7 @@ where
 
             let tx = tx.clone();
 
-            // Launch a separate thread for each device, so we can fclones
+            // Launch a separate thread for each device, so we can process
             // files on each device independently
             s.spawn(move |_| {
                 // Sort files by their physical location, to reduce disk seek latency
@@ -321,7 +329,7 @@ where
                 // Each task creates a heap allocation and reserves memory in the queue.
                 // It is more memory efficient to keep these tasks as long as possible
                 // in our vector. Without this limit we observed over 20% more memory use
-                // when fclonesing 1M of files.
+                // when processing 1M of files.
                 let semaphore = Arc::new(Semaphore::new(8 * thread_count));
 
                 // Run hashing on the thread-pool dedicated to the device
@@ -332,7 +340,7 @@ where
                     // Spawning a task into a thread-pool requires a static lifetime,
                     // because generally the task could outlive caller's stack frame.
                     // However, this is not the case for rehash function, because
-                    // we don't exit before all tasks are fclonesed.
+                    // we don't exit before all tasks are closed.
                     // In the perfect world we should use scopes for that. Unfortunately
                     // the current implementation of rayon scopes runs the scope body
                     // on one of the thread-pool worker threads, so it is not possible
@@ -421,6 +429,33 @@ fn scan_files(ctx: &AppCtx<'_>) -> Vec<Vec<FileInfo>> {
     files
 }
 
+/// Finds the path root that is a prefix of the given path.
+/// If no match is found, returns the original path.
+fn root_of<'a>(path: &'a Path, roots: &'a [Path]) -> &'a Path {
+    for p in roots {
+        if p.is_prefix_of(path) {
+            return p;
+        }
+    }
+    path
+}
+
+/// Returns `true` if a group of files is big enough to meet the desired grouping criteria.
+/// The number of distinct files in a group must be greater than `rf-over`.
+/// If `--isolate` flag is set, file paths that start with a common input path are counted as one.
+/// Called on each result group after each stage.
+fn should_forward<P: AsPath>(ctx: &AppCtx<'_>, group: &[P]) -> bool {
+    if !ctx.config.isolate {
+        group.len() > ctx.config.rf_over()
+    } else {
+        let s: HashSet<_> = group
+            .iter()
+            .map(|p| root_of(p.path(), &ctx.input_paths))
+            .collect();
+        s.len() > ctx.config.rf_over()
+    }
+}
+
 fn group_by_size(ctx: &AppCtx<'_>, files: Vec<Vec<FileInfo>>) -> Vec<FileGroup<FileInfo>> {
     let file_count: usize = files.iter().map(|v| v.len()).sum();
     let progress = ctx.log.progress_bar("Grouping by size", file_count as u64);
@@ -437,7 +472,7 @@ fn group_by_size(ctx: &AppCtx<'_>, files: Vec<Vec<FileInfo>>) -> Vec<FileGroup<F
 
     let groups: Vec<_> = groups
         .into_iter()
-        .filter(|(_, files)| files.len() > rf_over)
+        .filter(|(_, files)| should_forward(ctx, files.as_slice()))
         .map(|(l, files)| FileGroup {
             file_len: l,
             file_hash: FileHash(0),
@@ -507,7 +542,7 @@ fn remove_same_files(
     let groups: Vec<_> = groups
         .into_par_iter()
         .update(|g| deduplicate(ctx, &mut g.files, |_| progress.tick()))
-        .filter(|g| g.files.len() > rf_over)
+        .filter(|g| should_forward(ctx, g.files.as_slice()))
         .collect();
 
     let count: usize = groups.selected_count(rf_over, rf_under);
@@ -587,7 +622,7 @@ fn group_transformed(
     let groups = rehash(
         groups,
         |_| true,
-        |g| g.files.len() > rf_over,
+        |g| should_forward(ctx, g.files.as_slice()),
         &ctx.devices,
         AccessType::Sequential,
         |(fi, _)| {
@@ -655,7 +690,7 @@ fn group_by_prefix(
     let groups = rehash(
         groups,
         pre_filter,
-        |g| g.files.len() > rf_over,
+        |g| should_forward(ctx, g.files.as_slice()),
         &ctx.devices,
         AccessType::Random,
         |(fi, _)| {
@@ -721,7 +756,7 @@ fn group_by_suffix(ctx: &AppCtx<'_>, groups: Vec<FileGroup<FileInfo>>) -> Vec<Fi
     let groups = rehash(
         groups,
         pre_filter,
-        |g| g.files.len() > rf_over,
+        |g| should_forward(ctx, g.files.as_slice()),
         &ctx.devices,
         AccessType::Random,
         |(fi, old_hash)| {
@@ -767,7 +802,7 @@ fn group_by_contents(
     let groups = rehash(
         groups,
         pre_filter,
-        |g| g.files.len() > rf_over,
+        |g| should_forward(ctx, g.files.as_slice()),
         &ctx.devices,
         AccessType::Sequential,
         |(fi, _)| {
@@ -848,12 +883,12 @@ fn group_by_contents(
 /// ```
 /// use fclones::log::Log;
 /// use fclones::config::GroupConfig;
+/// use fclones::path::Path;
 /// use fclones::{group_files, write_report};
-/// use std::path::PathBuf;
 ///
 /// let log = Log::new();
 /// let mut config = GroupConfig::default();
-/// config.paths = vec![PathBuf::from("/path/to/a/dir")];
+/// config.paths = vec![Path::from("/path/to/a/dir")];
 ///
 /// let groups = group_files(&config, &log).unwrap();
 /// println!("Found {} groups: ", groups.len());
@@ -937,7 +972,7 @@ pub fn write_report(config: &GroupConfig, log: &Log, groups: &[FileGroup<Path>])
 
 #[cfg(test)]
 mod test {
-    use std::fs::{hard_link, File, OpenOptions};
+    use std::fs::{create_dir, hard_link, File, OpenOptions};
     use std::io::{Read, Write};
     use std::path::PathBuf;
     use std::sync::atomic::{AtomicBool, Ordering};
@@ -1143,7 +1178,7 @@ mod test {
 
             let log = test_log();
             let mut config = GroupConfig::default();
-            config.paths = vec![file1, file2];
+            config.paths = vec![file1.into(), file2.into()];
             let results = group_files(&config, &log).unwrap();
             assert_eq!(results.len(), 1);
             assert_eq!(results[0].file_len, FileLen(3));
@@ -1161,7 +1196,7 @@ mod test {
 
             let log = test_log();
             let mut config = GroupConfig::default();
-            config.paths = vec![file1, file2];
+            config.paths = vec![file1.into(), file2.into()];
 
             let results = group_files(&config, &log).unwrap();
             assert_eq!(results.len(), 1);
@@ -1177,6 +1212,9 @@ mod test {
             write_test_file(&file1, b"aaaa", b"", b"");
             write_test_file(&file2, b"aaa", b"", b"");
 
+            let file1 = Path::from(file1);
+            let file2 = Path::from(file2);
+
             let log = test_log();
             let mut config = GroupConfig::default();
             config.paths = vec![file1.clone(), file2.clone()];
@@ -1184,14 +1222,8 @@ mod test {
 
             let results = group_files(&config, &log).unwrap();
             assert_eq!(results.len(), 2);
-            assert_eq!(
-                results[0].files,
-                vec![Path::from(file1.canonicalize().unwrap())]
-            );
-            assert_eq!(
-                results[1].files,
-                vec![Path::from(file2.canonicalize().unwrap())]
-            );
+            assert_eq!(results[0].files, vec![Path::from(file1.canonicalize())]);
+            assert_eq!(results[1].files, vec![Path::from(file2.canonicalize())]);
         });
     }
 
@@ -1205,7 +1237,7 @@ mod test {
 
             let log = test_log();
             let mut config = GroupConfig::default();
-            config.paths = vec![file1.clone(), file2.clone()];
+            config.paths = vec![file1.into(), file2.into()];
             config.unique = true;
 
             let results = group_files(&config, &log).unwrap();
@@ -1227,7 +1259,7 @@ mod test {
 
             let log = test_log();
             let mut config = GroupConfig::default();
-            config.paths = vec![file1.clone(), file2.clone()];
+            config.paths = vec![file1.into(), file2.into()];
             config.unique = true;
 
             let results = group_files(&config, &log).unwrap();
@@ -1249,7 +1281,7 @@ mod test {
 
             let log = test_log();
             let mut config = GroupConfig::default();
-            config.paths = vec![file1.clone(), file2.clone()];
+            config.paths = vec![file1.into(), file2.into()];
             config.unique = true;
 
             let results = group_files(&config, &log).unwrap();
@@ -1269,7 +1301,7 @@ mod test {
 
             let log = test_log();
             let mut config = GroupConfig::default();
-            config.paths = vec![file1.clone(), file2.clone()];
+            config.paths = vec![file1.into(), file2.into()];
             config.unique = true;
 
             let results = group_files(&config, &log).unwrap();
@@ -1285,6 +1317,7 @@ mod test {
             write_test_file(&file1, b"foo", b"", b"");
             let log = test_log();
             let mut config = GroupConfig::default();
+            let file1 = Path::from(file1);
             config.paths = vec![file1.clone(), file1.clone(), file1.clone()];
             config.unique = true;
             config.hard_links = true;
@@ -1310,7 +1343,7 @@ mod test {
 
             let log = test_log();
             let mut config = GroupConfig::default();
-            config.paths = vec![file1.clone(), file2.clone()];
+            config.paths = vec![file1.into(), file2.into()];
             config.unique = true;
             config.hard_links = true;
 
@@ -1320,6 +1353,35 @@ mod test {
         });
     }
 
+    #[test]
+    fn duplicate_files_different_roots() {
+        with_dir("main/duplicate_files_different_roots", |root| {
+            let root1 = root.join("root1");
+            let root2 = root.join("root2");
+            create_dir(&root1).unwrap();
+            create_dir(&root2).unwrap();
+
+            let file1 = root1.join("file1");
+            let file2 = root1.join("file2");
+
+            write_test_file(&file1, b"foo", b"", b"");
+            write_test_file(&file2, b"foo", b"", b"");
+
+            let log = test_log();
+            let mut config = GroupConfig::default();
+            config.paths = vec![root1.into(), root2.into()];
+            config.isolate = true;
+
+            let results = group_files(&config, &log).unwrap();
+            assert_eq!(results.len(), 0);
+
+            config.isolate = false;
+            let results = group_files(&config, &log).unwrap();
+            assert_eq!(results.len(), 1);
+            assert_eq!(results[0].files.len(), 2);
+        });
+    }
+
     #[test]
     fn report() {
         with_dir("main/report", |root| {
@@ -1329,7 +1391,7 @@ mod test {
             let report_file = root.join("report.txt");
             let log = test_log();
             let mut config = GroupConfig::default();
-            config.paths = vec![file.clone()];
+            config.paths = vec![file.into()];
             config.unique = true;
             config.output = Some(report_file.clone());
 
diff --git a/src/main.rs b/src/main.rs
index 4d189ba..75a7c77 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -12,6 +12,7 @@ use rayon::iter::ParallelBridge;
 use regex::Regex;
 use structopt::StructOpt;
 
+use console::style;
 use fclones::config::{Command, Config, DedupeConfig, GroupConfig, Parallelism};
 use fclones::log::Log;
 use fclones::report::open_report;
@@ -64,21 +65,23 @@ fn run_group(mut config: GroupConfig, log: &mut Log) -> Result<(), Error> {
         // problems and report as warnings.
         let mut access_error = false;
         let depth = config.depth;
-        config.paths.retain(|p| match fs::metadata(&p) {
-            Ok(m) if m.is_dir() && depth == Some(0) => {
-                log.warn(format!(
-                    "Skipping directory {} because recursive scan is disabled.",
-                    p.display()
-                ));
-                false
-            }
-            Err(e) => {
-                log.err(format!("Can't access {}: {}", p.display(), e));
-                access_error = true;
-                false
-            }
-            Ok(_) => true,
-        });
+        config
+            .paths
+            .retain(|p| match fs::metadata(&p.to_path_buf()) {
+                Ok(m) if m.is_dir() && depth == Some(0) => {
+                    log.warn(format!(
+                        "Skipping directory {} because recursive scan is disabled.",
+                        p.display()
+                    ));
+                    false
+                }
+                Err(e) => {
+                    log.err(format!("Can't access {}: {}", p.display(), e));
+                    access_error = true;
+                    false
+                }
+                Ok(_) => true,
+            });
         if access_error {
             return Err(Error::from("Some input paths could not be accessed."));
         }
@@ -196,7 +199,12 @@ pub fn run_dedupe(op: DedupeOp, config: DedupeConfig, log: &mut Log) -> Result<(
 }
 
 fn main() {
-    let config = Config::from_args();
+    let config: Config = Config::from_args();
+    if let Err(e) = config.command.validate() {
+        eprintln!("{} {}", style("error:").for_stderr().bold().red(), e);
+        exit(1);
+    }
+
     let mut log = Log::new();
     if config.quiet {
         log.no_progress = true;


Test patch: