Repository: pkolaczk/fclones (language: rust)

Issue:
Make fclones usable as a library
... in case someone wanted to create a GUI or just use file deduplication in their own programs.


Discussion / hints:

Fix patch:
diff --git a/src/config.rs b/src/config.rs
index 3a8ccc7..b5ea3c2 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -1,20 +1,20 @@
+//! Main program configuration.
+
 use std::collections::HashMap;
-use std::ffi::{OsStr, OsString};
+use std::ffi::OsString;
 use std::io::{stdin, BufRead, BufReader};
 use std::path::PathBuf;
-use std::process::exit;
 
 use clap::arg_enum;
 use clap::AppSettings;
 use structopt::StructOpt;
 
-use crate::device::Parallelism;
 use crate::files::FileLen;
-use crate::log::Log;
 use crate::path::Path;
-use crate::pattern::{Pattern, PatternOpts};
+use crate::pattern::{Pattern, PatternError, PatternOpts};
 use crate::selector::PathSelector;
 use crate::transform::Transform;
+use std::io;
 
 arg_enum! {
     #[derive(Debug, StructOpt)]
@@ -54,6 +54,12 @@ fn parse_thread_count_option(str: &str) -> Result<(OsString, Parallelism), Strin
     Ok((key, Parallelism { sequential, random }))
 }
 
+#[derive(Clone, Copy, Debug)]
+pub struct Parallelism {
+    pub random: usize,
+    pub sequential: usize,
+}
+
 /// Finds duplicate, unique, under- or over-replicated files
 #[derive(Debug, StructOpt, Default)]
 #[structopt(
@@ -208,29 +214,40 @@ pub struct Config {
 }
 
 impl Config {
-    pub fn path_selector(&self, log: &Log, base_dir: &Path) -> PathSelector {
+    fn compile_pattern(&self, s: &str) -> Result<Pattern, PatternError> {
         let pattern_opts = if self.caseless {
             PatternOpts::case_insensitive()
         } else {
             PatternOpts::default()
         };
-        let pattern = |s: &String| {
-            let pattern = if self.regex {
-                Pattern::regex_with(s.as_str(), &pattern_opts)
-            } else {
-                Pattern::glob_with(s.as_str(), &pattern_opts)
-            };
-
-            pattern.unwrap_or_else(|e| {
-                log.err(e);
-                exit(1);
-            })
-        };
+        if self.regex {
+            Pattern::regex_with(s, &pattern_opts)
+        } else {
+            Pattern::glob_with(s, &pattern_opts)
+        }
+    }
 
-        PathSelector::new(base_dir.clone())
-            .include_names(self.name_patterns.iter().map(pattern).collect())
-            .include_paths(self.path_patterns.iter().map(pattern).collect())
-            .exclude_paths(self.exclude_patterns.iter().map(pattern).collect())
+    pub fn path_selector(&self, base_dir: &Path) -> Result<PathSelector, PatternError> {
+        let include_names: Result<Vec<Pattern>, PatternError> = self
+            .name_patterns
+            .iter()
+            .map(|p| self.compile_pattern(p))
+            .collect();
+        let include_paths: Result<Vec<Pattern>, PatternError> = self
+            .path_patterns
+            .iter()
+            .map(|p| self.compile_pattern(p))
+            .collect();
+        let exclude_paths: Result<Vec<Pattern>, PatternError> = self
+            .exclude_patterns
+            .iter()
+            .map(|p| self.compile_pattern(p))
+            .collect();
+
+        Ok(PathSelector::new(base_dir.clone())
+            .include_names(include_names?)
+            .include_paths(include_paths?)
+            .exclude_paths(exclude_paths?))
     }
 
     pub fn rf_over(&self) -> usize {
@@ -286,33 +303,20 @@ impl Config {
         }
     }
 
-    /// Validates the transform command if defined; logs error and exits if broken
-    pub fn check_transform(&self, log: &Log) {
-        self.transform
-            .as_ref()
-            .map(|command| self.build_transform(command, log));
+    fn build_transform(&self, command: &str) -> io::Result<Transform> {
+        let mut tr = Transform::new(command.to_string(), self.in_place)?;
+        if self.no_copy {
+            tr.copy = false
+        };
+        Ok(tr)
     }
 
-    /// Constructs the transform object and clears the transform command from this object.
-    pub fn transform(&self, log: &Log) -> Option<Transform> {
+    /// Constructs the transform object.
+    /// Returns None if the transform was not set
+    pub fn transform(&self) -> Option<io::Result<Transform>> {
         self.transform
             .as_ref()
-            .map(|command| self.build_transform(command, log))
-    }
-
-    fn build_transform(&self, command: &str, log: &Log) -> Transform {
-        match Transform::new(command.to_string(), self.in_place) {
-            Ok(mut tr) => {
-                if self.no_copy {
-                    tr.copy = false
-                };
-                tr
-            }
-            Err(e) => {
-                log.err(e);
-                exit(1);
-            }
-        }
+            .map(|command| self.build_transform(command))
     }
 
     pub fn thread_pool_sizes(&self) -> HashMap<OsString, Parallelism> {
@@ -322,27 +326,4 @@ impl Config {
         }
         map
     }
-
-    /// Checks if `thread_pool_sizes` map contains unknown thread pool names.
-    /// If unrecognized keys are found, prints an error message and terminates the program.
-    /// The only allowed key name is: "default".
-    /// This method should be called after thread pools have been configured and their entries
-    /// removed from the map.
-    pub fn check_thread_pools(
-        &self,
-        log: &Log,
-        thread_pool_sizes: &mut HashMap<OsString, Parallelism>,
-    ) {
-        thread_pool_sizes.remove(OsStr::new("default"));
-        if !thread_pool_sizes.is_empty() {
-            for (name, _) in thread_pool_sizes.iter() {
-                let name = name.to_string_lossy();
-                match name.strip_prefix("dev:") {
-                    Some(name) => log.err(format!("Unknown device: {}", name)),
-                    None => log.err(format!("Unknown thread pool or device type: {}", name)),
-                }
-            }
-            exit(1);
-        }
-    }
 }
diff --git a/src/device.rs b/src/device.rs
index 12a2150..267574a 100644
--- a/src/device.rs
+++ b/src/device.rs
@@ -8,15 +8,10 @@ use lazy_init::Lazy;
 use rayon::{ThreadPool, ThreadPoolBuilder};
 use sysinfo::{DiskExt, DiskType, System, SystemExt};
 
+use crate::config::Parallelism;
 use crate::files::FileLen;
 use crate::path::Path;
 
-#[derive(Clone, Copy, Debug)]
-pub struct Parallelism {
-    pub random: usize,
-    pub sequential: usize,
-}
-
 impl Parallelism {
     pub fn default_for(disk_type: DiskType) -> Parallelism {
         match disk_type {
@@ -167,13 +162,13 @@ impl DiskDevices {
     fn get_parallelism(
         name: &OsStr,
         disk_type: DiskType,
-        pool_sizes: &mut HashMap<OsString, Parallelism>,
+        pool_sizes: &HashMap<OsString, Parallelism>,
     ) -> Parallelism {
         let mut dev_key = OsString::new();
         dev_key.push("dev:");
         dev_key.push(name);
-        match pool_sizes.remove(&dev_key) {
-            Some(p) => p,
+        match pool_sizes.get(&dev_key) {
+            Some(p) => *p,
             None => {
                 let p = match disk_type {
                     DiskType::SSD => pool_sizes.get(OsStr::new("ssd")),
@@ -183,9 +178,10 @@ impl DiskDevices {
                 };
                 match p {
                     Some(p) => *p,
-                    None => *pool_sizes
+                    None => pool_sizes
                         .get(OsStr::new("default"))
-                        .unwrap_or(&Parallelism::default_for(disk_type)),
+                        .copied()
+                        .unwrap_or_else(|| Parallelism::default_for(disk_type)),
                 }
             }
         }
@@ -197,7 +193,7 @@ impl DiskDevices {
         &mut self,
         name: OsString,
         disk_type: DiskType,
-        pool_sizes: &mut HashMap<OsString, Parallelism>,
+        pool_sizes: &HashMap<OsString, Parallelism>,
     ) -> usize {
         if let Some((index, _)) = self.devices.iter().find_position(|d| d.name == name) {
             index
@@ -227,7 +223,7 @@ impl DiskDevices {
 
     /// Reads the list of partitions and disks from the system and builds the `DiskDevices`
     /// structure from that information.
-    pub fn new(pool_sizes: &mut HashMap<OsString, Parallelism>) -> DiskDevices {
+    pub fn new(pool_sizes: &HashMap<OsString, Parallelism>) -> DiskDevices {
         let mut sys = System::new();
         sys.refresh_disks_list();
         let mut result = DiskDevices {
@@ -249,9 +245,6 @@ impl DiskDevices {
             .mount_points
             .sort_by_key(|(p, _)| cmp::Reverse(p.component_count()));
 
-        pool_sizes.remove(OsStr::new("unknown"));
-        pool_sizes.remove(OsStr::new("ssd"));
-        pool_sizes.remove(OsStr::new("hdd"));
         result
     }
 
@@ -264,6 +257,11 @@ impl DiskDevices {
             .unwrap_or(&self.devices[0])
     }
 
+    /// Returns the disk device by its device name (not mount point)
+    pub fn get_by_name(&self, name: &OsStr) -> Option<&DiskDevice> {
+        self.devices.iter().find(|&d| d.name == name)
+    }
+
     /// Returns the first device on the list
     pub fn get_default(&self) -> &DiskDevice {
         &self.devices[0]
@@ -274,21 +272,21 @@ impl DiskDevices {
         self.devices.len()
     }
 
-    /// Returns true if there are no devices
-    pub fn is_empty(&self) -> bool {
-        self.devices.is_empty()
-    }
-
     /// Returns an iterator over devices
     pub fn iter(&self) -> impl Iterator<Item = &DiskDevice> {
         self.devices.iter()
     }
+
+    /// Returns device_group identifiers recognized by the constructor
+    pub fn device_types() -> Vec<&'static str> {
+        vec!["ssd", "hdd", "removable", "unknown"]
+    }
 }
 
 impl Default for DiskDevices {
     fn default() -> Self {
-        let mut pool_sizes = HashMap::new();
-        Self::new(&mut pool_sizes)
+        let pool_sizes = HashMap::new();
+        Self::new(&pool_sizes)
     }
 }
 
diff --git a/src/files.rs b/src/files.rs
index af64e97..d0009f6 100644
--- a/src/files.rs
+++ b/src/files.rs
@@ -1,3 +1,6 @@
+//! Type-safe wrappers for file position and length and other
+//! file-system related utilities.
+
 use core::fmt;
 use std::cell::RefCell;
 use std::cmp::{max, min};
@@ -15,7 +18,7 @@ use std::os::unix::io::*;
 
 use byte_unit::Byte;
 use bytesize::ByteSize;
-use metrohash::{MetroHash128, MetroHash64};
+use metrohash::MetroHash128;
 use serde::*;
 use smallvec::alloc::fmt::Formatter;
 use smallvec::alloc::str::FromStr;
@@ -90,21 +93,6 @@ impl Sub<FileLen> for FilePos {
 
 /// Represents length of data, in bytes.
 /// Provides more type safety and nicer formatting over using a raw u64.
-///
-/// # Examples
-/// Formatting file length as a human readable string:
-/// ```
-/// use fclones::files::FileLen;
-/// let file_len = FileLen(16000);
-/// let human_readable = format!("{}", file_len);
-/// assert_eq!(human_readable, "16.0 KB");
-/// ```
-///
-/// `FileLen` can be added directly to `FilePos`:
-/// ```
-/// use fclones::files::{FileLen, FilePos};
-/// assert_eq!(FilePos(1000) + FileLen(64), FilePos(1064));
-/// ```
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Serialize, Default)]
 pub struct FileLen(pub u64);
 
@@ -190,7 +178,7 @@ pub trait IntoPath {
 }
 
 #[derive(Clone, Debug)]
-pub struct FileInfo {
+pub(crate) struct FileInfo {
     pub path: Path,
     pub len: FileLen,
     // physical on-disk location of file data for access ordering optimisation
@@ -273,12 +261,16 @@ impl FileInfo {
 
 /// Returns file information for the given path.
 /// On failure, logs an error to stderr and returns `None`.
-pub fn file_info_or_log_err(file: Path, devices: &DiskDevices, log: &Log) -> Option<FileInfo> {
+pub(crate) fn file_info_or_log_err(
+    file: Path,
+    devices: &DiskDevices,
+    log: &Log,
+) -> Option<FileInfo> {
     match FileInfo::new(file, devices) {
         Ok(info) => Some(info),
         Err(e) if e.kind() == ErrorKind::NotFound => None,
         Err(e) => {
-            log.err(e);
+            log.warn(e);
             None
         }
     }
@@ -286,7 +278,7 @@ pub fn file_info_or_log_err(file: Path, devices: &DiskDevices, log: &Log) -> Opt
 
 /// Useful for identifying files in presence of hardlinks
 #[derive(Clone, Copy, PartialEq, Eq, Hash)]
-pub struct FileId {
+pub(crate) struct FileId {
     pub inode: u128,
     pub device: u64,
 }
@@ -335,22 +327,14 @@ impl FileId {
             }
         }
     }
-
-    pub fn get_hash(&self) -> u32 {
-        let mut hasher = MetroHash64::new();
-        self.inode.hash(&mut hasher);
-        self.device.hash(&mut hasher);
-        let hash = hasher.finish();
-        (hash ^ (hash >> 32)) as u32
-    }
 }
 
-pub fn file_id_or_log_err(file: &Path, log: &Log) -> Option<FileId> {
+pub(crate) fn file_id_or_log_err(file: &Path, log: &Log) -> Option<FileId> {
     match FileId::new(&file) {
         Ok(id) => Some(id),
         Err(e) if e.kind() == ErrorKind::NotFound => None,
         Err(e) => {
-            log.err(e);
+            log.warn(e);
             None
         }
     }
@@ -358,7 +342,7 @@ pub fn file_id_or_log_err(file: &Path, log: &Log) -> Option<FileId> {
 
 /// Returns the physical offset of the first data block of the file
 #[cfg(target_os = "linux")]
-pub fn get_physical_file_location(path: &Path) -> io::Result<Option<u64>> {
+pub(crate) fn get_physical_file_location(path: &Path) -> io::Result<Option<u64>> {
     let mut extents = fiemap::fiemap(&path.to_path_buf())?;
     match extents.next() {
         Some(fe) => Ok(Some(fe?.fe_physical)),
@@ -483,7 +467,7 @@ fn evict_page_cache_if_low_mem(file: &mut File, len: FileLen) {
 
 /// Determines how page cache should be used when computing a file hash.
 #[derive(Debug, Copy, Clone)]
-pub enum Caching {
+pub(crate) enum Caching {
     /// Don't send any special cache advice to the OS
     Default,
     /// Prefetch as much data as possible into cache, to speed up sequential access
@@ -564,7 +548,7 @@ fn scan<F: FnMut(&[u8])>(
 
 /// Computes the hash value over at most `len` bytes of the stream.
 /// Returns the number of the bytes read and a 128-bit hash value.
-pub fn stream_hash(
+pub(crate) fn stream_hash(
     stream: &mut impl Read,
     len: FileLen,
     buf_len: usize,
@@ -584,29 +568,7 @@ pub fn stream_hash(
 /// Computes hash of initial `len` bytes of a file.
 /// If the file does not exist or is not readable, print the error to stderr and return `None`.
 /// The returned hash is not cryptograhically secure.
-///
-/// # Example
-/// ```
-/// use fclones::files::{file_hash, FileLen, FilePos, Caching};
-/// use fclones::path::Path;
-/// use std::io::Write;
-/// use std::fs::{File, create_dir_all};
-/// use std::path::PathBuf;
-///
-/// let test_root = PathBuf::from("target/test/file_hash/");
-/// create_dir_all(&test_root).unwrap();
-/// let file1 = test_root.join("file1");
-/// File::create(&file1).unwrap().write_all(b"Test file 1");
-/// let file2 = test_root.join("file2");
-/// File::create(&file2).unwrap().write_all(b"Test file 2");
-///
-/// let hash1 = file_hash(&Path::from(&file1), FilePos(0), FileLen::MAX, 4096, Caching::Default, |_|{}).unwrap();
-/// let hash2 = file_hash(&Path::from(&file2), FilePos(0), FileLen::MAX, 4096, Caching::Default, |_|{}).unwrap();
-/// let hash3 = file_hash(&Path::from(&file2), FilePos(0), FileLen(8), 4096, Caching::Default, |_|{}).unwrap();
-/// assert_ne!(hash1, hash2);
-/// assert_ne!(hash2, hash3);
-/// ```
-pub fn file_hash(
+pub(crate) fn file_hash(
     path: &Path,
     offset: FilePos,
     len: FileLen,
@@ -622,7 +584,7 @@ pub fn file_hash(
 
 /// Computes the file hash or logs an error and returns none if failed.
 /// If file is not found, no error is logged and `None` is returned.
-pub fn file_hash_or_log_err(
+pub(crate) fn file_hash_or_log_err(
     path: &Path,
     offset: FilePos,
     len: FileLen,
@@ -635,7 +597,7 @@ pub fn file_hash_or_log_err(
         Ok(hash) => Some(hash),
         Err(e) if e.kind() == ErrorKind::NotFound => None,
         Err(e) => {
-            log.err(format!(
+            log.warn(format!(
                 "Failed to compute hash of file {}: {}",
                 path.display(),
                 e
@@ -644,3 +606,67 @@ pub fn file_hash_or_log_err(
         }
     }
 }
+
+#[cfg(test)]
+mod test {
+
+    use super::*;
+
+    use crate::path::Path;
+    use std::fs::{create_dir_all, File};
+    use std::io::Write;
+    use std::path::PathBuf;
+
+    #[test]
+    fn test_format_bytes() {
+        let file_len = FileLen(16000);
+        let human_readable = format!("{}", file_len);
+        assert_eq!(human_readable, "16.0 KB");
+    }
+
+    #[test]
+    fn test_file_hash() {
+        let test_root = PathBuf::from("target/test/file_hash/");
+        create_dir_all(&test_root).unwrap();
+        let file1 = test_root.join("file1");
+        File::create(&file1)
+            .unwrap()
+            .write_all(b"Test file 1")
+            .unwrap();
+        let file2 = test_root.join("file2");
+        File::create(&file2)
+            .unwrap()
+            .write_all(b"Test file 2")
+            .unwrap();
+
+        let hash1 = file_hash(
+            &Path::from(&file1),
+            FilePos(0),
+            FileLen::MAX,
+            4096,
+            Caching::Default,
+            |_| {},
+        )
+        .unwrap();
+        let hash2 = file_hash(
+            &Path::from(&file2),
+            FilePos(0),
+            FileLen::MAX,
+            4096,
+            Caching::Default,
+            |_| {},
+        )
+        .unwrap();
+        let hash3 = file_hash(
+            &Path::from(&file2),
+            FilePos(0),
+            FileLen(8),
+            4096,
+            Caching::Default,
+            |_| {},
+        )
+        .unwrap();
+        assert_ne!(hash1, hash2);
+        assert_ne!(hash2, hash3);
+    }
+}
diff --git a/src/group.rs b/src/group.rs
index 78900d9..a92f285 100644
--- a/src/group.rs
+++ b/src/group.rs
@@ -1,19 +1,8 @@
 use std::collections::BTreeMap;
 use std::hash::Hash;
 use std::marker::PhantomData;
-use std::sync::mpsc::{channel, Receiver, Sender};
-use std::sync::Arc;
 
-use crossbeam_utils::thread;
-use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
-use rayon::prelude::ParallelSliceMut;
-use serde::*;
 use smallvec::SmallVec;
-use sysinfo::DiskType;
-
-use crate::device::DiskDevices;
-use crate::files::{FileHash, FileInfo, FileLen};
-use crate::semaphore::Semaphore;
 
 /// Groups items by key.
 /// After all items have been added, this structure can be transformed into
@@ -25,24 +14,7 @@ use crate::semaphore::Semaphore;
 /// The amortized complexity of adding an item is O(1).
 /// The complexity of reading all groups is O(N).
 ///
-/// # Example
-/// ```
-/// use smallvec::SmallVec;
-/// use fclones::group::GroupMap;
-/// let mut map = GroupMap::new(|item: (u32, u32)| (item.0, item.1));
-/// map.add((1, 10));
-/// map.add((2, 20));
-/// map.add((1, 11));
-/// map.add((2, 21));
-///
-/// let mut groups: Vec<_> = map.into_iter().collect();
-///
-/// groups.sort_by_key(|item| item.0);
-/// assert_eq!(groups[0], (1, SmallVec::from_vec(vec![10, 11])));
-/// assert_eq!(groups[1], (2, SmallVec::from_vec(vec![20, 21])));
-/// ```
-///
-pub struct GroupMap<T, K, V, F>
+pub(crate) struct GroupMap<T, K, V, F>
 where
     K: PartialEq + Hash,
     F: Fn(T) -> (K, V),
@@ -75,11 +47,6 @@ where
         let (key, new_item) = (self.split_fn)(item);
         self.groups.entry(key).or_default().push(new_item);
     }
-
-    /// Returns number of groups larger than given min_size
-    pub fn group_count(&self, min_size: usize) -> usize {
-        self.groups.iter().filter(|g| g.1.len() >= min_size).count()
-    }
 }
 
 impl<T, K, V, F> IntoIterator for GroupMap<T, K, V, F>
@@ -95,426 +62,24 @@ where
     }
 }
 
-/// Represents a group of files that have something in common, e.g. same size or same hash
-#[derive(Serialize, Debug)]
-pub struct FileGroup<F> {
-    /// Length of each file
-    pub file_len: FileLen,
-    /// Hash of a part or the whole of the file
-    pub file_hash: FileHash,
-    /// Group of files with the same length and hash
-    pub files: Vec<F>,
-}
-
-/// Computes metrics for reporting summaries of each processing stage.
-pub trait GroupedFileSetMetrics {
-    /// Returns the total count of the files
-    fn total_count(self) -> usize;
-
-    /// Returns the sum of file lengths
-    fn total_size(self) -> FileLen;
-
-    /// Returns the total count of redundant files
-    /// # Arguments
-    /// * `max_rf` - maximum number of replicas allowed (they won't be counted as redundant)
-    fn selected_count(self, rf_over: usize, rf_under: usize) -> usize;
-
-    /// Returns the amount of data in redundant files
-    /// # Arguments
-    /// * `max_rf` - maximum number of replicas allowed (they won't be counted as redundant)
-    fn selected_size(self, rf_over: usize, rf_under: usize) -> FileLen;
-}
-
-impl<'a, I, F> GroupedFileSetMetrics for I
-where
-    I: IntoIterator<Item = &'a FileGroup<F>> + 'a,
-    F: 'a,
-{
-    fn total_count(self) -> usize {
-        self.into_iter().map(|g| g.files.len()).sum()
-    }
-
-    fn total_size(self) -> FileLen {
-        self.into_iter()
-            .map(|g| g.file_len * g.files.len() as u64)
-            .sum()
-    }
-
-    fn selected_count(self, rf_over: usize, rf_under: usize) -> usize {
-        self.into_iter()
-            .filter(|&g| g.files.len() < rf_under)
-            .map(|g| g.files.len().saturating_sub(rf_over))
-            .sum()
-    }
-
-    fn selected_size(self, rf_over: usize, rf_under: usize) -> FileLen {
-        self.into_iter()
-            .filter(|&g| g.files.len() < rf_under)
-            .map(|g| g.file_len * g.files.len().saturating_sub(rf_over) as u64)
-            .sum()
-    }
-}
-
-/// Helper struct to preserve the original file hash and keep it together with file information
-/// Sometimes the old hash must be taken into account, e.g. when combining the prefix hash with
-/// the suffix hash.
-struct HashedFileInfo {
-    file_hash: FileHash,
-    file_info: FileInfo,
-}
-
-/// Partitions files into separate vectors, where each vector holds files persisted
-/// on the same disk device. The vectors are returned in the same order as devices.
-fn partition_by_devices(
-    files: Vec<FileGroup<FileInfo>>,
-    devices: &DiskDevices,
-) -> Vec<Vec<HashedFileInfo>> {
-    let mut result: Vec<Vec<HashedFileInfo>> = Vec::with_capacity(devices.len());
-    for _ in 0..devices.len() {
-        result.push(Vec::new());
-    }
-    for g in files {
-        for f in g.files {
-            let device = &devices[f.get_device_index()];
-            result[device.index].push(HashedFileInfo {
-                file_hash: g.file_hash,
-                file_info: f,
-            });
-        }
-    }
-    result
-}
-
-/// Iterates over grouped files, in parallel
-pub fn flat_iter(files: &[FileGroup<FileInfo>]) -> impl ParallelIterator<Item = &FileInfo> {
-    files.par_iter().flat_map(|g| &g.files)
-}
-
-#[derive(Copy, Clone, Debug)]
-pub enum AccessType {
-    Sequential,
-    Random,
-}
-
-/// Groups files by length and hash computed by given `hash_fn`.
-/// Runs in parallel on dedicated thread pools.
-/// Files on different devices are hashed separately from each other.
-/// File hashes within a single device are computed in the order given by
-/// their `location` field to minimize seek latency.
-///
-/// Caveats: the original grouping is lost. It is possible for two files that
-/// were in the different groups to end up in the same group if they have the same length
-/// and they hash to the same value. If you don't want this, you need to combine the old
-/// hash with the new hash in the provided `hash_fn`.
-pub fn rehash<'a, F1, F2, H>(
-    groups: Vec<FileGroup<FileInfo>>,
-    group_pre_filter: F1,
-    group_post_filter: F2,
-    devices: &DiskDevices,
-    access_type: AccessType,
-    hash_fn: H,
-) -> Vec<FileGroup<FileInfo>>
-where
-    F1: Fn(&FileGroup<FileInfo>) -> bool,
-    F2: Fn(&FileGroup<FileInfo>) -> bool,
-    H: Fn((&mut FileInfo, FileHash)) -> Option<FileHash> + Sync + Send + 'a,
-{
-    // Allow sharing the hash function between threads:
-    type HashFn<'a> = dyn Fn((&mut FileInfo, FileHash)) -> Option<FileHash> + Sync + Send + 'a;
-    let hash_fn: &HashFn<'a> = &hash_fn;
-
-    let (tx, rx): (Sender<HashedFileInfo>, Receiver<HashedFileInfo>) = channel();
-
-    // There is no point in processing groups containing a single file.
-    // Normally when searching for duplicates such groups are filtered out automatically after
-    // each stage, however they are possible when searching for unique files.
-    let (groups_to_process, groups_to_pass): (Vec<_>, Vec<_>) =
-        groups.into_iter().partition(group_pre_filter);
-
-    // This way we can split processing to separate thread-pools, one per device:
-    let files = partition_by_devices(groups_to_process, &devices);
-    let mut hash_map =
-        GroupMap::new(|f: HashedFileInfo| ((f.file_info.len, f.file_hash), f.file_info));
-    let hash_map_ref = &mut hash_map;
-
-    // Scope needed so threads can access shared stuff like groups or shared functions.
-    // The threads we launch are guaranteed to not live longer than this scope.
-    thread::scope(move |s| {
-        // Process all files in background
-        for (mut files, device) in files.into_iter().zip(devices.iter()) {
-            if files.is_empty() {
-                continue;
-            }
-
-            let tx = tx.clone();
-
-            // Launch a separate thread for each device, so we can process
-            // files on each device independently
-            s.spawn(move |_| {
-                // Sort files by their physical location, to reduce disk seek latency
-                if device.disk_type != DiskType::SSD {
-                    files.par_sort_unstable_by_key(|f| f.file_info.location);
-                }
-
-                // Some devices like HDDs may benefit from different amount of parallelism
-                // depending on the access type. Therefore we chose a thread pool appropriate
-                // for the access type
-                let thread_pool = match access_type {
-                    AccessType::Sequential => device.seq_thread_pool(),
-                    AccessType::Random => device.rand_thread_pool(),
-                };
-
-                let thread_count = thread_pool.current_num_threads() as isize;
-
-                // Limit the number of tasks spawned at once into the thread-pool.
-                // Each task creates a heap allocation and reserves memory in the queue.
-                // It is more memory efficient to keep these tasks as long as possible
-                // in our vector. Without this limit we observed over 20% more memory use
-                // when processing 1M of files.
-                let semaphore = Arc::new(Semaphore::new(8 * thread_count));
-
-                // Run hashing on the thread-pool dedicated to the device
-                for mut f in files {
-                    let tx = tx.clone();
-                    let guard = semaphore.clone().access_owned();
-
-                    // Spawning a task into a thread-pool requires a static lifetime,
-                    // because generally the task could outlive caller's stack frame.
-                    // However, this is not the case for rehash function, because
-                    // we don't exit before all tasks are processed.
-                    // In the perfect world we should use scopes for that. Unfortunately
-                    // the current implementation of rayon scopes runs the scope body
-                    // on one of the thread-pool worker threads, so it is not possible
-                    // to safely block inside the scope, because that leads to deadlock
-                    // when the pool has only one thread.
-                    let hash_fn: &HashFn<'static> = unsafe { std::mem::transmute(hash_fn) };
-                    thread_pool.spawn_fifo(move || {
-                        if let Some(hash) = hash_fn((&mut f.file_info, f.file_hash)) {
-                            f.file_hash = hash;
-                            tx.send(f).unwrap();
-                        }
-                        // This forces moving the guard into this task and be released when
-                        // the task is done
-                        drop(guard);
-                    });
-                }
-            });
-        }
-        // Drop the original tx, so all tx are closed when the threads finish and
-        // the next while loop will eventually exit
-        drop(tx);
-
-        // Collect the results from all threads and group them.
-        // Note that this will happen as soon as data are available
-        while let Ok(hashed_file) = rx.recv() {
-            hash_map_ref.add(hashed_file);
-        }
-    })
-    .unwrap();
-
-    // Convert the hashmap into vector, leaving only large-enough groups:
-    hash_map
-        .into_iter()
-        .map(|((len, hash), files)| FileGroup {
-            file_len: len,
-            file_hash: hash,
-            files: files.to_vec(),
-        })
-        .filter(group_post_filter)
-        .chain(groups_to_pass.into_iter())
-        .collect()
-}
-
 #[cfg(test)]
 mod test {
-    use std::sync::atomic::{AtomicBool, Ordering};
-    use std::sync::Mutex;
 
-    use rand::seq::SliceRandom;
-
-    use FileGroup;
-
-    use crate::path::Path;
-
-    use super::*;
-
-    /// Files hashing to different values should be placed into different groups
     #[test]
-    fn test_rehash_puts_files_with_different_hashes_to_different_groups() {
-        let devices = DiskDevices::default();
-        let input = vec![FileGroup {
-            file_len: FileLen(200),
-            file_hash: FileHash(0),
-            files: vec![
-                FileInfo {
-                    len: FileLen(200),
-                    location: 0,
-                    path: Path::from("file1"),
-                },
-                FileInfo {
-                    len: FileLen(200),
-                    location: 35847587,
-                    path: Path::from("file2"),
-                },
-            ],
-        }];
+    fn items_should_be_split_into_groups() {
+        use super::GroupMap;
+        use smallvec::SmallVec;
 
-        let result = rehash(
-            input,
-            |_| true,
-            |_| true,
-            &devices,
-            AccessType::Random,
-            |(fi, _)| Some(FileHash(fi.location as u128)),
-        );
+        let mut map = GroupMap::new(|item: (u32, u32)| (item.0, item.1));
+        map.add((1, 10));
+        map.add((2, 20));
+        map.add((1, 11));
+        map.add((2, 21));
 
-        assert_eq!(result.len(), 2);
-        assert_eq!(result[0].files.len(), 1);
-        assert_eq!(result[1].files.len(), 1);
-        assert_ne!(result[0].files[0].path, result[1].files[0].path);
-    }
+        let mut groups: Vec<_> = map.into_iter().collect();
 
-    /// Files hashing to same values should be placed into the same groups
-    #[test]
-    fn test_rehash_puts_files_with_same_hashes_to_same_groups() {
-        let devices = DiskDevices::default();
-        let input = vec![
-            FileGroup {
-                file_len: FileLen(200),
-                file_hash: FileHash(0),
-                files: vec![FileInfo {
-                    len: FileLen(200),
-                    location: 0,
-                    path: Path::from("file1"),
-                }],
-            },
-            FileGroup {
-                file_len: FileLen(500),
-                file_hash: FileHash(0),
-                files: vec![FileInfo {
-                    len: FileLen(200),
-                    location: 35847587,
-                    path: Path::from("file2"),
-                }],
-            },
-        ];
-
-        let result = rehash(
-            input,
-            |_| true,
-            |_| true,
-            &devices,
-            AccessType::Random,
-            |(_, _)| Some(FileHash(123456)),
-        );
-
-        assert_eq!(result.len(), 1);
-        assert_eq!(result[0].files.len(), 2);
-    }
-
-    #[test]
-    fn test_rehash_can_skip_processing_files() {
-        let devices = DiskDevices::default();
-        let input = vec![FileGroup {
-            file_len: FileLen(200),
-            file_hash: FileHash(0),
-            files: vec![FileInfo {
-                len: FileLen(200),
-                location: 0,
-                path: Path::from("file1"),
-            }],
-        }];
-
-        let called = AtomicBool::new(false);
-        let result = rehash(
-            input,
-            |_| false,
-            |_| true,
-            &devices,
-            AccessType::Random,
-            |(fi, _)| {
-                called.store(true, Ordering::Release);
-                Some(FileHash(fi.location as u128))
-            },
-        );
-
-        assert_eq!(result.len(), 1);
-        assert!(!called.load(Ordering::Acquire));
-    }
-
-    #[test]
-    fn test_rehash_post_filter_removes_groups() {
-        let devices = DiskDevices::default();
-        let input = vec![FileGroup {
-            file_len: FileLen(200),
-            file_hash: FileHash(0),
-            files: vec![
-                FileInfo {
-                    len: FileLen(200),
-                    location: 0,
-                    path: Path::from("file1"),
-                },
-                FileInfo {
-                    len: FileLen(200),
-                    location: 35847587,
-                    path: Path::from("file2"),
-                },
-            ],
-        }];
-
-        let result = rehash(
-            input,
-            |_| true,
-            |g| g.files.len() >= 2,
-            &devices,
-            AccessType::Random,
-            |(fi, _)| Some(FileHash(fi.location as u128)),
-        );
-
-        assert!(result.is_empty())
-    }
-
-    #[test]
-    fn test_rehash_processes_files_in_location_order_on_hdd() {
-        let thread_count = 2;
-        let devices = DiskDevices::single(DiskType::HDD, thread_count);
-        let count = 1000;
-        let mut input = Vec::with_capacity(count);
-        for i in 0..count {
-            input.push(FileGroup {
-                file_len: FileLen(0),
-                file_hash: FileHash(0),
-                files: vec![FileInfo {
-                    len: FileLen(0),
-                    location: i as u64,
-                    path: Path::from(format!("file{}", i)),
-                }],
-            })
-        }
-        input.shuffle(&mut rand::thread_rng());
-
-        let processing_order = Mutex::new(Vec::new());
-        rehash(
-            input,
-            |_| true,
-            |_| true,
-            &devices,
-            AccessType::Random,
-            |(fi, _)| {
-                processing_order.lock().unwrap().push(fi.location as i32);
-                Some(FileHash(fi.location as u128))
-            },
-        );
-        let processing_order = processing_order.into_inner().unwrap();
-
-        // Because we're processing files in parallel, we have no strict guarantee they
-        // will be processed in the exact same order as in the input.
-        // However, we expect some locality so the total distance between subsequent accesses
-        // is low.
-        let mut distance = 0;
-        for i in 0..processing_order.len() - 1 {
-            distance += i32::abs(processing_order[i] - processing_order[i + 1])
-        }
-        assert!(distance < (thread_count * count) as i32)
+        groups.sort_by_key(|item| item.0);
+        assert_eq!(groups[0], (1, SmallVec::from_vec(vec![10, 11])));
+        assert_eq!(groups[1], (2, SmallVec::from_vec(vec![20, 21])));
     }
 }
diff --git a/src/lib.rs b/src/lib.rs
index 3ecae8d..8aebbfb 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,15 +1,1360 @@
+use std::cell::RefCell;
+use std::cmp::Reverse;
+use std::env::current_dir;
+use std::ffi::{OsStr, OsString};
+use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::mpsc::{channel, Receiver, Sender};
+use std::sync::Arc;
+
+use crossbeam_utils::thread;
+use itertools::Itertools;
+use rayon::prelude::*;
+use serde::*;
+use sysinfo::DiskType;
+use thread_local::ThreadLocal;
+
+use crate::config::*;
+use crate::device::{DiskDevice, DiskDevices};
+use crate::files::FileInfo;
+use crate::files::*;
+use crate::group::*;
+use crate::log::Log;
+use crate::path::Path;
+use crate::progress::FastProgressBar;
+use crate::report::Reporter;
+use crate::selector::PathSelector;
+use crate::semaphore::Semaphore;
+use crate::transform::Transform;
+use crate::walk::Walk;
+use console::Term;
+use core::fmt;
+use std::collections::HashMap;
+use std::fmt::{Display, Formatter};
+use std::fs::File;
+use std::io;
+use std::io::{BufWriter, Write};
+
 pub mod config;
-pub mod device;
 pub mod files;
-pub mod group;
 pub mod log;
 pub mod path;
-pub mod pattern;
 pub mod progress;
-pub mod regex;
 pub mod report;
-pub mod selector;
-pub mod semaphore;
-pub mod transform;
-pub mod util;
-pub mod walk;
+
+mod device;
+mod group;
+mod pattern;
+mod regex;
+mod selector;
+mod semaphore;
+mod transform;
+mod util;
+mod walk;
+
+#[derive(Debug)]
+pub struct Error {
+    pub message: String,
+}
+
+impl Error {
+    pub fn new(msg: String) -> Error {
+        Error { message: msg }
+    }
+}
+
+impl std::error::Error for Error {}
+
+impl Display for Error {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.message)
+    }
+}
+
+impl From<String> for Error {
+    fn from(s: String) -> Self {
+        Error { message: s }
+    }
+}
+
+/// Holds stuff needed globally by the whole application
+struct AppCtx<'a> {
+    pub config: &'a Config,
+    pub log: &'a Log,
+    devices: DiskDevices,
+    transform: Option<Transform>,
+    path_selector: PathSelector,
+}
+
+impl<'a> AppCtx<'a> {
+    pub fn new(config: &'a Config, log: &'a Log) -> Result<AppCtx<'a>, Error> {
+        let thread_pool_sizes = config.thread_pool_sizes();
+        let devices = DiskDevices::new(&thread_pool_sizes);
+        let transform = match config.transform() {
+            None => None,
+            Some(Ok(transform)) => Some(transform),
+            Some(Err(e)) => return Err(Error::new(format!("Invalid transform: {}", e))),
+        };
+        let base_dir = Path::from(current_dir().unwrap_or_default());
+        let selector = config
+            .path_selector(&base_dir)
+            .map_err(|e| format!("Invalid pattern: {}", e))?;
+
+        Self::check_pool_config(thread_pool_sizes, &devices)?;
+
+        Ok(AppCtx {
+            config,
+            log,
+            devices,
+            transform,
+            path_selector: selector,
+        })
+    }
+
+    /// Checks if all thread pool names refer to existing pools or devices
+    fn check_pool_config(
+        thread_pool_sizes: HashMap<OsString, Parallelism>,
+        devices: &DiskDevices,
+    ) -> Result<(), Error> {
+        let mut allowed_pool_names = DiskDevices::device_types();
+        allowed_pool_names.push("main");
+        allowed_pool_names.push("default");
+        for (name, _) in thread_pool_sizes.iter() {
+            let name = name.to_string_lossy();
+            match name.strip_prefix("dev:") {
+                Some(name) if devices.get_by_name(OsStr::new(name)).is_none() => {
+                    return Err(Error::new(format!("Unknown device: {}", name)));
+                }
+                None if !allowed_pool_names.contains(&name.as_ref()) => {
+                    return Err(Error::new(format!(
+                        "Unknown thread pool or device type: {}",
+                        name
+                    )));
+                }
+                _ => {}
+            }
+        }
+        Ok(())
+    }
+}
+
+/// A group of files that have something in common, e.g. same size or same hash
+#[derive(Serialize, Debug)]
+pub struct FileGroup<F> {
+    /// Length of each file
+    pub file_len: FileLen,
+    /// Hash of a part or the whole of the file
+    pub file_hash: FileHash,
+    /// Group of files with the same length and hash
+    pub files: Vec<F>,
+}
+
+/// Computes summaries of the results obtained from each grouping stage.
+pub trait StageMetrics {
+    /// Returns the total count of the files
+    fn total_count(self) -> usize;
+
+    /// Returns the sum of file lengths
+    fn total_size(self) -> FileLen;
+
+    /// Returns the total count of redundant files
+    fn selected_count(self, rf_over: usize, rf_under: usize) -> usize;
+
+    /// Returns the amount of data in redundant files
+    fn selected_size(self, rf_over: usize, rf_under: usize) -> FileLen;
+}
+
+impl<'a, I, F> StageMetrics for I
+where
+    I: IntoIterator<Item = &'a FileGroup<F>> + 'a,
+    F: 'a,
+{
+    fn total_count(self) -> usize {
+        self.into_iter().map(|g| g.files.len()).sum()
+    }
+
+    fn total_size(self) -> FileLen {
+        self.into_iter()
+            .map(|g| g.file_len * g.files.len() as u64)
+            .sum()
+    }
+
+    fn selected_count(self, rf_over: usize, rf_under: usize) -> usize {
+        self.into_iter()
+            .filter(|&g| g.files.len() < rf_under)
+            .map(|g| g.files.len().saturating_sub(rf_over))
+            .sum()
+    }
+
+    fn selected_size(self, rf_over: usize, rf_under: usize) -> FileLen {
+        self.into_iter()
+            .filter(|&g| g.files.len() < rf_under)
+            .map(|g| g.file_len * g.files.len().saturating_sub(rf_over) as u64)
+            .sum()
+    }
+}
+
+/// Helper struct to preserve the original file hash and keep it together with file information
+/// Sometimes the old hash must be taken into account, e.g. when combining the prefix hash with
+/// the suffix hash.
+struct HashedFileInfo {
+    file_hash: FileHash,
+    file_info: FileInfo,
+}
+
+/// Partitions files into separate vectors, where each vector holds files persisted
+/// on the same disk device. The vectors are returned in the same order as devices.
+fn partition_by_devices(
+    files: Vec<FileGroup<FileInfo>>,
+    devices: &DiskDevices,
+) -> Vec<Vec<HashedFileInfo>> {
+    let mut result: Vec<Vec<HashedFileInfo>> = Vec::with_capacity(devices.len());
+    for _ in 0..devices.len() {
+        result.push(Vec::new());
+    }
+    for g in files {
+        for f in g.files {
+            let device = &devices[f.get_device_index()];
+            result[device.index].push(HashedFileInfo {
+                file_hash: g.file_hash,
+                file_info: f,
+            });
+        }
+    }
+    result
+}
+
+/// Iterates over grouped files, in parallel
+fn flat_iter(files: &[FileGroup<FileInfo>]) -> impl ParallelIterator<Item = &FileInfo> {
+    files.par_iter().flat_map(|g| &g.files)
+}
+
+#[derive(Copy, Clone, Debug)]
+enum AccessType {
+    Sequential,
+    Random,
+}
+
+/// Groups files by length and hash computed by given `hash_fn`.
+/// Runs in parallel on dedicated thread pools.
+/// Files on different devices are hashed separately from each other.
+/// File hashes within a single device are computed in the order given by
+/// their `location` field to minimize seek latency.
+///
+/// Caveats: the original grouping is lost. It is possible for two files that
+/// were in the different groups to end up in the same group if they have the same length
+/// and they hash to the same value. If you don't want this, you need to combine the old
+/// hash with the new hash in the provided `hash_fn`.
+fn rehash<'a, F1, F2, H>(
+    groups: Vec<FileGroup<FileInfo>>,
+    group_pre_filter: F1,
+    group_post_filter: F2,
+    devices: &DiskDevices,
+    access_type: AccessType,
+    hash_fn: H,
+) -> Vec<FileGroup<FileInfo>>
+where
+    F1: Fn(&FileGroup<FileInfo>) -> bool,
+    F2: Fn(&FileGroup<FileInfo>) -> bool,
+    H: Fn((&mut FileInfo, FileHash)) -> Option<FileHash> + Sync + Send + 'a,
+{
+    // Allow sharing the hash function between threads:
+    type HashFn<'a> = dyn Fn((&mut FileInfo, FileHash)) -> Option<FileHash> + Sync + Send + 'a;
+    let hash_fn: &HashFn<'a> = &hash_fn;
+
+    let (tx, rx): (Sender<HashedFileInfo>, Receiver<HashedFileInfo>) = channel();
+
+    // There is no point in fclonesing groups containing a single file.
+    // Normally when searching for duplicates such groups are filtered out automatically after
+    // each stage, however they are possible when searching for unique files.
+    let (groups_to_fclones, groups_to_pass): (Vec<_>, Vec<_>) =
+        groups.into_iter().partition(group_pre_filter);
+
+    // This way we can split fclonesing to separate thread-pools, one per device:
+    let files = partition_by_devices(groups_to_fclones, &devices);
+    let mut hash_map =
+        GroupMap::new(|f: HashedFileInfo| ((f.file_info.len, f.file_hash), f.file_info));
+    let hash_map_ref = &mut hash_map;
+
+    // Scope needed so threads can access shared stuff like groups or shared functions.
+    // The threads we launch are guaranteed to not live longer than this scope.
+    thread::scope(move |s| {
+        // Process all files in background
+        for (mut files, device) in files.into_iter().zip(devices.iter()) {
+            if files.is_empty() {
+                continue;
+            }
+
+            let tx = tx.clone();
+
+            // Launch a separate thread for each device, so we can fclones
+            // files on each device independently
+            s.spawn(move |_| {
+                // Sort files by their physical location, to reduce disk seek latency
+                if device.disk_type != DiskType::SSD {
+                    files.par_sort_unstable_by_key(|f| f.file_info.location);
+                }
+
+                // Some devices like HDDs may benefit from different amount of parallelism
+                // depending on the access type. Therefore we chose a thread pool appropriate
+                // for the access type
+                let thread_pool = match access_type {
+                    AccessType::Sequential => device.seq_thread_pool(),
+                    AccessType::Random => device.rand_thread_pool(),
+                };
+
+                let thread_count = thread_pool.current_num_threads() as isize;
+
+                // Limit the number of tasks spawned at once into the thread-pool.
+                // Each task creates a heap allocation and reserves memory in the queue.
+                // It is more memory efficient to keep these tasks as long as possible
+                // in our vector. Without this limit we observed over 20% more memory use
+                // when fclonesing 1M of files.
+                let semaphore = Arc::new(Semaphore::new(8 * thread_count));
+
+                // Run hashing on the thread-pool dedicated to the device
+                for mut f in files {
+                    let tx = tx.clone();
+                    let guard = semaphore.clone().access_owned();
+
+                    // Spawning a task into a thread-pool requires a static lifetime,
+                    // because generally the task could outlive caller's stack frame.
+                    // However, this is not the case for rehash function, because
+                    // we don't exit before all tasks are fclonesed.
+                    // In the perfect world we should use scopes for that. Unfortunately
+                    // the current implementation of rayon scopes runs the scope body
+                    // on one of the thread-pool worker threads, so it is not possible
+                    // to safely block inside the scope, because that leads to deadlock
+                    // when the pool has only one thread.
+                    let hash_fn: &HashFn<'static> = unsafe { std::mem::transmute(hash_fn) };
+                    thread_pool.spawn_fifo(move || {
+                        if let Some(hash) = hash_fn((&mut f.file_info, f.file_hash)) {
+                            f.file_hash = hash;
+                            tx.send(f).unwrap();
+                        }
+                        // This forces moving the guard into this task and be released when
+                        // the task is done
+                        drop(guard);
+                    });
+                }
+            });
+        }
+        // Drop the original tx, so all tx are closed when the threads finish and
+        // the next while loop will eventually exit
+        drop(tx);
+
+        // Collect the results from all threads and group them.
+        // Note that this will happen as soon as data are available
+        while let Ok(hashed_file) = rx.recv() {
+            hash_map_ref.add(hashed_file);
+        }
+    })
+    .unwrap();
+
+    // Convert the hashmap into vector, leaving only large-enough groups:
+    hash_map
+        .into_iter()
+        .map(|((len, hash), files)| FileGroup {
+            file_len: len,
+            file_hash: hash,
+            files: files.to_vec(),
+        })
+        .filter(group_post_filter)
+        .chain(groups_to_pass.into_iter())
+        .collect()
+}
+
+/// Walks the directory tree and collects matching files in parallel into a vector
+fn scan_files(ctx: &AppCtx<'_>) -> Vec<Vec<FileInfo>> {
+    let file_collector = ThreadLocal::new();
+    let spinner = ctx.log.spinner("Scanning files");
+    let spinner_tick = &|_: &Path| spinner.tick();
+
+    let config = &ctx.config;
+    let min_size = config.min_size;
+    let max_size = config.max_size.unwrap_or(FileLen::MAX);
+
+    let mut walk = Walk::new();
+    walk.recursive = config.recursive;
+    walk.depth = config.depth.unwrap_or(usize::MAX);
+    walk.skip_hidden = config.skip_hidden;
+    walk.follow_links = config.follow_links;
+    walk.path_selector = ctx.path_selector.clone();
+    walk.log = Some(&ctx.log);
+    walk.on_visit = spinner_tick;
+    walk.run(ctx.config.input_paths(), |path| {
+        file_info_or_log_err(path, &ctx.devices, &ctx.log)
+            .into_iter()
+            .filter(|info| {
+                let l = info.len;
+                l >= min_size && l <= max_size
+            })
+            .for_each(|info| {
+                let vec = file_collector.get_or(|| RefCell::new(Vec::new()));
+                vec.borrow_mut().push(info);
+            });
+    });
+
+    ctx.log
+        .info(format!("Scanned {} file entries", spinner.position()));
+
+    let files: Vec<_> = file_collector.into_iter().map(|r| r.into_inner()).collect();
+
+    let file_count: usize = files.iter().map(|v| v.len()).sum();
+    let total_size: u64 = files.iter().flat_map(|v| v.iter().map(|i| i.len.0)).sum();
+    ctx.log.info(format!(
+        "Found {} ({}) files matching selection criteria",
+        file_count,
+        FileLen(total_size)
+    ));
+    files
+}
+
+fn group_by_size(ctx: &AppCtx<'_>, files: Vec<Vec<FileInfo>>) -> Vec<FileGroup<FileInfo>> {
+    let file_count: usize = files.iter().map(|v| v.len()).sum();
+    let progress = ctx.log.progress_bar("Grouping by size", file_count as u64);
+
+    let mut groups = GroupMap::new(|info: FileInfo| (info.len, info));
+    for files in files.into_iter() {
+        for file in files.into_iter() {
+            progress.tick();
+            groups.add(file);
+        }
+    }
+    let rf_over = ctx.config.rf_over();
+    let rf_under = ctx.config.rf_under();
+
+    let groups: Vec<_> = groups
+        .into_iter()
+        .filter(|(_, files)| files.len() > rf_over)
+        .map(|(l, files)| FileGroup {
+            file_len: l,
+            file_hash: FileHash(0),
+            files: files.into_vec(),
+        })
+        .collect();
+
+    let count: usize = groups.selected_count(rf_over, rf_under);
+    let bytes: FileLen = groups.selected_size(rf_over, rf_under);
+    ctx.log.info(format!(
+        "Found {} ({}) candidates after grouping by size",
+        count, bytes
+    ));
+    groups
+}
+
+/// Removes duplicate files matching by full-path or by inode-id.
+/// Deduplication by inode-id is not performed if the flag to preserve hard-links (-H) is set.
+fn deduplicate<F>(ctx: &AppCtx<'_>, files: &mut Vec<FileInfo>, progress: F)
+where
+    F: Fn(&Path) + Sync + Send,
+{
+    let mut groups = GroupMap::new(|fi: FileInfo| (fi.location, fi));
+    for f in files.drain(..) {
+        groups.add(f)
+    }
+
+    for (_, file_group) in groups.into_iter() {
+        if file_group.len() == 1 {
+            files.extend(file_group.into_iter().inspect(|p| progress(&p.path)));
+        } else if ctx.config.hard_links {
+            files.extend(
+                file_group
+                    .into_iter()
+                    .inspect(|p| progress(&p.path))
+                    .unique_by(|p| p.path.hash128()),
+            )
+        } else {
+            files.extend(
+                file_group
+                    .into_iter()
+                    .inspect(|p| progress(&p.path))
+                    .unique_by(|p| file_id_or_log_err(&p.path, &ctx.log)),
+            )
+        }
+    }
+}
+
+fn remove_same_files(
+    ctx: &AppCtx<'_>,
+    groups: Vec<FileGroup<FileInfo>>,
+) -> Vec<FileGroup<FileInfo>> {
+    let file_count: usize = groups.total_count();
+    let progress = ctx
+        .log
+        .progress_bar("Removing same files", file_count as u64);
+
+    let rf_over = ctx.config.rf_over();
+    let rf_under = ctx.config.rf_under();
+
+    let groups: Vec<_> = groups
+        .into_par_iter()
+        .update(|g| deduplicate(ctx, &mut g.files, |_| progress.tick()))
+        .filter(|g| g.files.len() > rf_over)
+        .collect();
+
+    let count: usize = groups.selected_count(rf_over, rf_under);
+    let bytes: FileLen = groups.selected_size(rf_over, rf_under);
+    ctx.log.info(format!(
+        "Found {} ({}) candidates after grouping by paths {}",
+        count,
+        bytes,
+        if ctx.config.hard_links {
+            ""
+        } else {
+            "and file identifiers"
+        }
+    ));
+    groups
+}
+
+fn atomic_counter_vec(len: usize) -> Vec<AtomicU32> {
+    let mut v = Vec::with_capacity(len);
+    for _ in 0..len {
+        v.push(AtomicU32::new(0));
+    }
+    v
+}
+
+fn update_file_locations(ctx: &AppCtx<'_>, groups: &mut Vec<FileGroup<FileInfo>>) {
+    #[cfg(target_os = "linux")]
+    {
+        let count = groups.total_count();
+        let progress = ctx.log.progress_bar("Fetching extents", count as u64);
+
+        let err_counters = atomic_counter_vec(ctx.devices.len());
+        const MAX_ERR_COUNT_TO_LOG: u32 = 10;
+
+        groups
+            .par_iter_mut()
+            .flat_map(|g| &mut g.files)
+            .update(|fi| {
+                let device: &DiskDevice = &ctx.devices[fi.get_device_index()];
+                if device.disk_type != DiskType::SSD {
+                    if let Err(e) = fi.fetch_physical_location() {
+                        let counter = &err_counters[device.index];
+                        if counter.load(Ordering::Relaxed) < MAX_ERR_COUNT_TO_LOG {
+                            let err_count = counter.fetch_add(1, Ordering::Relaxed);
+                            ctx.log.warn(format!(
+                                "Failed to fetch extents for file {}: {}",
+                                fi.path, e
+                            ));
+                            if err_count == MAX_ERR_COUNT_TO_LOG {
+                                ctx.log.warn(
+                                    "Too many fetch extents errors. More errors will be ignored. \
+                                 Random access performance might be affected on spinning drives.",
+                                )
+                            }
+                        }
+                    }
+                }
+            })
+            .for_each(|_| progress.tick());
+    }
+}
+
+/// Transforms files by piping them to an external program and groups them by their hashes
+fn group_transformed(
+    ctx: &AppCtx<'_>,
+    transform: &Transform,
+    groups: Vec<FileGroup<FileInfo>>,
+) -> Vec<FileGroup<FileInfo>> {
+    let file_count: usize = groups.total_count();
+    let progress = ctx
+        .log
+        .progress_bar("Transforming & grouping", file_count as u64);
+
+    let rf_over = ctx.config.rf_over();
+    let rf_under = ctx.config.rf_under();
+
+    let groups = rehash(
+        groups,
+        |_| true,
+        |g| g.files.len() > rf_over,
+        &ctx.devices,
+        AccessType::Sequential,
+        |(fi, _)| {
+            let result = transform
+                .run_or_log_err(&fi.path, &ctx.log)
+                .map(|(len, hash)| {
+                    fi.len = len;
+                    hash
+                });
+            progress.tick();
+            result
+        },
+    );
+
+    let count = groups.selected_count(rf_over, rf_under);
+    let bytes = groups.selected_size(rf_over, rf_under);
+    ctx.log.info(format!(
+        "Found {} ({}) {} files",
+        count,
+        bytes,
+        ctx.config.search_type()
+    ));
+    groups
+}
+
+/// Returns the maximum value of the given property of the device,
+/// among the devices actually used to store any of the given files
+fn max_device_property<'a>(
+    devices: &DiskDevices,
+    files: impl ParallelIterator<Item = &'a FileInfo>,
+    property_fn: impl Fn(&DiskDevice) -> FileLen + Sync,
+) -> FileLen {
+    files
+        .into_par_iter()
+        .map(|f| property_fn(&devices[f.get_device_index()]))
+        .max()
+        .unwrap_or_else(|| property_fn(devices.get_default()))
+}
+
+/// Returns the desired prefix length for a group of files.
+/// The return value depends on the capabilities of the devices the files are stored on.
+/// Higher values are desired if any of the files resides on an HDD.
+fn prefix_len<'a>(
+    partitions: &DiskDevices,
+    files: impl ParallelIterator<Item = &'a FileInfo>,
+) -> FileLen {
+    max_device_property(partitions, files, |dd| dd.max_prefix_len())
+}
+
+/// Groups files by a hash of their first few thousand bytes.
+fn group_by_prefix(
+    ctx: &AppCtx<'_>,
+    prefix_len: FileLen,
+    groups: Vec<FileGroup<FileInfo>>,
+) -> Vec<FileGroup<FileInfo>> {
+    let pre_filter = |g: &FileGroup<FileInfo>| g.files.len() > 1;
+    let remaining_files = groups.iter().filter(|&g| pre_filter(g)).total_count();
+    let progress = ctx
+        .log
+        .progress_bar("Grouping by prefix", remaining_files as u64);
+
+    let rf_over = ctx.config.rf_over();
+    let rf_under = ctx.config.rf_under();
+
+    let groups = rehash(
+        groups,
+        pre_filter,
+        |g| g.files.len() > rf_over,
+        &ctx.devices,
+        AccessType::Random,
+        |(fi, _)| {
+            progress.tick();
+            let device = &ctx.devices[fi.get_device_index()];
+            let buf_len = device.buf_len();
+            let (caching, prefix_len) = if fi.len <= prefix_len {
+                (Caching::Default, prefix_len)
+            } else {
+                (Caching::Random, device.min_prefix_len())
+            };
+
+            file_hash_or_log_err(
+                &fi.path,
+                FilePos(0),
+                prefix_len,
+                buf_len,
+                caching,
+                |_| {},
+                &ctx.log,
+            )
+        },
+    );
+
+    let count = groups.selected_count(rf_over, rf_under);
+    let bytes = groups.selected_size(rf_over, rf_under);
+    ctx.log.info(format!(
+        "Found {} ({}) candidates after grouping by prefix",
+        count, bytes
+    ));
+    groups
+}
+
+/// Returns the desired suffix length for a group of files.
+/// The return value depends on the capabilities of the devices the files are stored on.
+/// Higher values are desired if any of the files resides on an HDD.
+fn suffix_len<'a>(
+    partitions: &DiskDevices,
+    files: impl ParallelIterator<Item = &'a FileInfo>,
+) -> FileLen {
+    max_device_property(partitions, files, |dd| dd.suffix_len())
+}
+
+fn suffix_threshold<'a>(
+    partitions: &DiskDevices,
+    files: impl ParallelIterator<Item = &'a FileInfo>,
+) -> FileLen {
+    max_device_property(partitions, files, |dd| dd.suffix_threshold())
+}
+
+fn group_by_suffix(ctx: &AppCtx<'_>, groups: Vec<FileGroup<FileInfo>>) -> Vec<FileGroup<FileInfo>> {
+    let suffix_len = suffix_len(&ctx.devices, flat_iter(&groups));
+    let suffix_threshold = suffix_threshold(&ctx.devices, flat_iter(&groups));
+    let pre_filter = |g: &FileGroup<FileInfo>| g.file_len >= suffix_threshold && g.files.len() > 1;
+    let remaining_files = groups.iter().filter(|&g| pre_filter(g)).total_count();
+    let progress = ctx
+        .log
+        .progress_bar("Grouping by suffix", remaining_files as u64);
+
+    let rf_over = ctx.config.rf_over();
+    let rf_under = ctx.config.rf_under();
+
+    let groups = rehash(
+        groups,
+        pre_filter,
+        |g| g.files.len() > rf_over,
+        &ctx.devices,
+        AccessType::Random,
+        |(fi, old_hash)| {
+            progress.tick();
+            let device = &ctx.devices[fi.get_device_index()];
+            let buf_len = device.buf_len();
+            file_hash_or_log_err(
+                &fi.path,
+                fi.len.as_pos() - suffix_len,
+                suffix_len,
+                buf_len,
+                Caching::Default,
+                |_| {},
+                &ctx.log,
+            )
+            .map(|new_hash| old_hash ^ new_hash)
+        },
+    );
+
+    let count = groups.selected_count(rf_over, rf_under);
+    let bytes = groups.selected_size(rf_over, rf_under);
+    ctx.log.info(format!(
+        "Found {} ({}) candidates after grouping by suffix",
+        count, bytes
+    ));
+    groups
+}
+
+fn group_by_contents(
+    ctx: &AppCtx<'_>,
+    min_file_len: FileLen,
+    groups: Vec<FileGroup<FileInfo>>,
+) -> Vec<FileGroup<FileInfo>> {
+    let pre_filter = |g: &FileGroup<FileInfo>| g.files.len() > 1 && g.file_len >= min_file_len;
+    let bytes_to_scan = groups.iter().filter(|&g| pre_filter(g)).total_size();
+    let progress = &ctx
+        .log
+        .bytes_progress_bar("Grouping by contents", bytes_to_scan.0);
+
+    let rf_over = ctx.config.rf_over();
+    let rf_under = ctx.config.rf_under();
+
+    let groups = rehash(
+        groups,
+        pre_filter,
+        |g| g.files.len() > rf_over,
+        &ctx.devices,
+        AccessType::Sequential,
+        |(fi, _)| {
+            let device = &ctx.devices[fi.get_device_index()];
+            let buf_len = device.buf_len();
+            file_hash_or_log_err(
+                &fi.path,
+                FilePos(0),
+                fi.len,
+                buf_len,
+                Caching::Sequential,
+                |delta| progress.inc(delta),
+                &ctx.log,
+            )
+        },
+    );
+
+    let count = groups.selected_count(rf_over, rf_under);
+    let bytes = groups.selected_size(rf_over, rf_under);
+    ctx.log.info(format!(
+        "Found {} ({}) {} files",
+        count,
+        bytes,
+        ctx.config.search_type()
+    ));
+    groups
+}
+
+/// Groups identical files together by 128-bit hash of their contents.
+/// Depending on filtering settings, can find unique, duplicate, over- or under-replicated files.
+///
+/// # Input
+/// The input set of files or paths to scan should be given in the `config.paths` property.
+/// When `config.recursive` is set to true, the search descends into
+/// subdirectories recursively (default is false).
+///
+/// # Output
+/// Returns a vector of groups of absolute paths.
+/// Each group of files has a common hash and length.
+/// Groups are sorted descending by file size.
+///
+/// # Errors
+/// An error is returned immediately if the configuration is invalid.
+/// I/O errors during processing are logged as warnings and unreadable files are skipped.
+/// If panics happen they are likely a result of a bug and should be reported.
+///
+/// # Performance characteristics
+/// The worst-case running time to is roughly proportional to the time required to
+/// open and read all files. Depending on the number of matching files and parameters of the
+/// query, that time can be lower because some files can be skipped from some stages of processing.
+/// The expected memory utilisation is roughly proportional the number of files and
+/// their path lengths.
+///
+/// # Threading
+/// This function blocks caller's thread until all files are processed.
+/// To speed up processing, it spawns multiple threads internally.
+/// Some processing is performed on the default Rayon thread pool, therefore this function
+/// must not be called on Rayon thread pool to avoid a deadlock.
+/// The parallelism level is automatically set based on the type of storage and can be overridden
+/// in the configuration.
+///
+/// # Algorithm
+/// Files are grouped in multiple stages and filtered after each stage.
+/// Files that turn out to be unique at some point are skipped from further stages.
+/// Stages are ordered by increasing I/O cost. On rotational drives,
+/// an attempt is made to sort files by physical data location before each grouping stage
+/// to reduce disk seek times.
+///
+/// 1. Create a list of files to process by walking directory tree if recursive mode selected.
+/// 2. Get length and identifier of each file.
+/// 3. Group files by length.
+/// 4. In each group, remove duplicate files with the same identifier.
+/// 5. Group files by hash of the prefix.
+/// 6. Group files by hash of the suffix.
+/// 7. Group files by hash of their full contents.
+///
+/// # Example
+/// ```
+/// use fclones::log::Log;
+/// use fclones::config::Config;
+/// use fclones::{group_files, write_report};
+/// use std::path::PathBuf;
+///
+/// let log = Log::new();
+/// let mut config = Config::default();
+/// config.paths = vec![PathBuf::from("/path/to/a/dir")];
+/// config.recursive = true;
+///
+/// let groups = group_files(&config, &log).unwrap();
+/// println!("Found {} groups: ", groups.len());
+///
+/// // print standard fclones report to stdout:
+/// write_report(&config, &log, &groups).unwrap();
+/// ```
+pub fn group_files(config: &Config, log: &Log) -> Result<Vec<FileGroup<Path>>, Error> {
+    log.info("Started");
+    let spinner = log.spinner("Initializing");
+    let ctx = AppCtx::new(config, log)?;
+
+    drop(spinner);
+    let matching_files = scan_files(&ctx);
+    let size_groups = group_by_size(&ctx, matching_files);
+    let mut size_groups_pruned = remove_same_files(&ctx, size_groups);
+    update_file_locations(&ctx, &mut size_groups_pruned);
+
+    let groups = match &ctx.transform {
+        Some(transform) => group_transformed(&ctx, transform, size_groups_pruned),
+        _ => {
+            let prefix_len = prefix_len(&ctx.devices, flat_iter(&size_groups_pruned));
+            let prefix_groups = group_by_prefix(&ctx, prefix_len, size_groups_pruned);
+            let suffix_groups = group_by_suffix(&ctx, prefix_groups);
+            group_by_contents(&ctx, prefix_len, suffix_groups)
+        }
+    };
+    let mut groups: Vec<_> = groups
+        .into_par_iter()
+        .map(|g| FileGroup {
+            file_len: g.file_len,
+            file_hash: g.file_hash,
+            files: g.files.into_iter().map(|fi| fi.path).collect(),
+        })
+        .collect();
+    groups.retain(|g| g.files.len() < ctx.config.rf_under());
+    groups.par_sort_by_key(|g| Reverse((g.file_len, g.file_hash)));
+    groups.par_iter_mut().for_each(|g| g.files.sort());
+    Ok(groups)
+}
+
+/// Creates a reporter that writes to the standard output
+fn stdout_reporter(progress: Arc<FastProgressBar>) -> Reporter<Box<dyn Write>> {
+    let stdout = Term::stdout();
+    let is_term = stdout.is_term();
+    if is_term {
+        progress.finish_and_clear()
+    }
+    let out: Box<dyn Write> = Box::new(BufWriter::new(stdout));
+    Reporter::new(out, is_term, progress)
+}
+
+/// Creates a reporter that writes to the given file.
+fn file_reporter(
+    path: &std::path::Path,
+    progress: Arc<FastProgressBar>,
+) -> io::Result<Reporter<Box<dyn Write>>> {
+    let file = File::create(path)?;
+    let out: Box<dyn Write> = Box::new(BufWriter::new(file));
+    Ok(Reporter::new(out, false, progress))
+}
+
+/// Writes the list of groups to a file or the standard output.
+///
+/// # Parameters
+/// - `config.output`: a path to the output file, `None` for standard output
+/// - `config.format`: selects the format of the output, see [`config::OutputFormat`]
+/// - `log`: used for drawing a progress bar to standard error
+/// - `groups`: list of groups of files to print, e.g. obtained from [`group_files`]
+///
+/// # Errors
+/// Returns [`io::Error`] on I/O write error or if the output file cannot be created.
+pub fn write_report(config: &Config, log: &Log, groups: &[FileGroup<Path>]) -> io::Result<()> {
+    let remaining_files = groups.total_count();
+    let progress = log.progress_bar("Writing report", remaining_files as u64);
+
+    // No progress bar when we write to terminal
+    let mut reporter = match &config.output {
+        Some(path) => file_reporter(path, progress)?,
+        None => stdout_reporter(progress),
+    };
+
+    match &config.format {
+        OutputFormat::Text => reporter.write_as_text(groups),
+        OutputFormat::Fdupes => reporter.write_as_fdupes(groups),
+        OutputFormat::Csv => reporter.write_as_csv(groups),
+        OutputFormat::Json => reporter.write_as_json(groups),
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::fs::{hard_link, File, OpenOptions};
+    use std::io::{Read, Write};
+    use std::path::PathBuf;
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::Mutex;
+
+    use rand::seq::SliceRandom;
+
+    use crate::path::Path;
+    use crate::util::test::*;
+
+    use super::*;
+
+    const MAX_PREFIX_LEN: usize = 256 * 1024;
+    const MAX_SUFFIX_LEN: usize = 256 * 1024;
+
+    /// Files hashing to different values should be placed into different groups
+    #[test]
+    fn test_rehash_puts_files_with_different_hashes_to_different_groups() {
+        let devices = DiskDevices::default();
+        let input = vec![FileGroup {
+            file_len: FileLen(200),
+            file_hash: FileHash(0),
+            files: vec![
+                FileInfo {
+                    len: FileLen(200),
+                    location: 0,
+                    path: Path::from("file1"),
+                },
+                FileInfo {
+                    len: FileLen(200),
+                    location: 35847587,
+                    path: Path::from("file2"),
+                },
+            ],
+        }];
+
+        let result = rehash(
+            input,
+            |_| true,
+            |_| true,
+            &devices,
+            AccessType::Random,
+            |(fi, _)| Some(FileHash(fi.location as u128)),
+        );
+
+        assert_eq!(result.len(), 2);
+        assert_eq!(result[0].files.len(), 1);
+        assert_eq!(result[1].files.len(), 1);
+        assert_ne!(result[0].files[0].path, result[1].files[0].path);
+    }
+
+    /// Files hashing to same values should be placed into the same groups
+    #[test]
+    fn test_rehash_puts_files_with_same_hashes_to_same_groups() {
+        let devices = DiskDevices::default();
+        let input = vec![
+            FileGroup {
+                file_len: FileLen(200),
+                file_hash: FileHash(0),
+                files: vec![FileInfo {
+                    len: FileLen(200),
+                    location: 0,
+                    path: Path::from("file1"),
+                }],
+            },
+            FileGroup {
+                file_len: FileLen(500),
+                file_hash: FileHash(0),
+                files: vec![FileInfo {
+                    len: FileLen(200),
+                    location: 35847587,
+                    path: Path::from("file2"),
+                }],
+            },
+        ];
+
+        let result = rehash(
+            input,
+            |_| true,
+            |_| true,
+            &devices,
+            AccessType::Random,
+            |(_, _)| Some(FileHash(123456)),
+        );
+
+        assert_eq!(result.len(), 1);
+        assert_eq!(result[0].files.len(), 2);
+    }
+
+    #[test]
+    fn test_rehash_can_skip_processing_files() {
+        let devices = DiskDevices::default();
+        let input = vec![FileGroup {
+            file_len: FileLen(200),
+            file_hash: FileHash(0),
+            files: vec![FileInfo {
+                len: FileLen(200),
+                location: 0,
+                path: Path::from("file1"),
+            }],
+        }];
+
+        let called = AtomicBool::new(false);
+        let result = rehash(
+            input,
+            |_| false,
+            |_| true,
+            &devices,
+            AccessType::Random,
+            |(fi, _)| {
+                called.store(true, Ordering::Release);
+                Some(FileHash(fi.location as u128))
+            },
+        );
+
+        assert_eq!(result.len(), 1);
+        assert!(!called.load(Ordering::Acquire));
+    }
+
+    #[test]
+    fn test_rehash_post_filter_removes_groups() {
+        let devices = DiskDevices::default();
+        let input = vec![FileGroup {
+            file_len: FileLen(200),
+            file_hash: FileHash(0),
+            files: vec![
+                FileInfo {
+                    len: FileLen(200),
+                    location: 0,
+                    path: Path::from("file1"),
+                },
+                FileInfo {
+                    len: FileLen(200),
+                    location: 35847587,
+                    path: Path::from("file2"),
+                },
+            ],
+        }];
+
+        let result = rehash(
+            input,
+            |_| true,
+            |g| g.files.len() >= 2,
+            &devices,
+            AccessType::Random,
+            |(fi, _)| Some(FileHash(fi.location as u128)),
+        );
+
+        assert!(result.is_empty())
+    }
+
+    #[test]
+    fn test_rehash_processes_files_in_location_order_on_hdd() {
+        let thread_count = 2;
+        let devices = DiskDevices::single(DiskType::HDD, thread_count);
+        let count = 1000;
+        let mut input = Vec::with_capacity(count);
+        for i in 0..count {
+            input.push(FileGroup {
+                file_len: FileLen(0),
+                file_hash: FileHash(0),
+                files: vec![FileInfo {
+                    len: FileLen(0),
+                    location: i as u64,
+                    path: Path::from(format!("file{}", i)),
+                }],
+            })
+        }
+        input.shuffle(&mut rand::thread_rng());
+
+        let processing_order = Mutex::new(Vec::new());
+        rehash(
+            input,
+            |_| true,
+            |_| true,
+            &devices,
+            AccessType::Random,
+            |(fi, _)| {
+                processing_order.lock().unwrap().push(fi.location as i32);
+                Some(FileHash(fi.location as u128))
+            },
+        );
+        let processing_order = processing_order.into_inner().unwrap();
+
+        // Because we're processing files in parallel, we have no strict guarantee they
+        // will be processed in the exact same order as in the input.
+        // However, we expect some locality so the total distance between subsequent accesses
+        // is low.
+        let mut distance = 0;
+        for i in 0..processing_order.len() - 1 {
+            distance += i32::abs(processing_order[i] - processing_order[i + 1])
+        }
+        assert!(distance < (thread_count * count) as i32)
+    }
+
+    #[test]
+    fn identical_small_files() {
+        with_dir("target/test/main/identical_small_files", |root| {
+            let file1 = root.join("file1");
+            let file2 = root.join("file2");
+            write_test_file(&file1, b"aaa", b"", b"");
+            write_test_file(&file2, b"aaa", b"", b"");
+
+            let log = test_log();
+            let mut config = Config::default();
+            config.paths = vec![file1, file2];
+            let results = group_files(&config, &log).unwrap();
+            assert_eq!(results.len(), 1);
+            assert_eq!(results[0].file_len, FileLen(3));
+            assert_eq!(results[0].files.len(), 2);
+        });
+    }
+
+    #[test]
+    fn identical_large_files() {
+        with_dir("target/test/main/identical_large_files", |root| {
+            let file1 = root.join("file1");
+            let file2 = root.join("file2");
+            write_test_file(&file1, &[0; MAX_PREFIX_LEN], &[1; 4096], &[2; 4096]);
+            write_test_file(&file2, &[0; MAX_PREFIX_LEN], &[1; 4096], &[2; 4096]);
+
+            let log = test_log();
+            let mut config = Config::default();
+            config.paths = vec![file1, file2];
+
+            let results = group_files(&config, &log).unwrap();
+            assert_eq!(results.len(), 1);
+            assert_eq!(results[0].files.len(), 2);
+        });
+    }
+
+    #[test]
+    fn files_differing_by_size() {
+        with_dir("target/test/main/files_differing_by_size", |root| {
+            let file1 = root.join("file1");
+            let file2 = root.join("file2");
+            write_test_file(&file1, b"aaaa", b"", b"");
+            write_test_file(&file2, b"aaa", b"", b"");
+
+            let log = test_log();
+            let mut config = Config::default();
+            config.paths = vec![file1.clone(), file2.clone()];
+            config.rf_over = Some(0);
+
+            let results = group_files(&config, &log).unwrap();
+            assert_eq!(results.len(), 2);
+            assert_eq!(
+                results[0].files,
+                vec![Path::from(file1.canonicalize().unwrap())]
+            );
+            assert_eq!(
+                results[1].files,
+                vec![Path::from(file2.canonicalize().unwrap())]
+            );
+        });
+    }
+
+    #[test]
+    fn files_differing_by_prefix() {
+        with_dir("target/test/main/files_differing_by_prefix", |root| {
+            let file1 = root.join("file1");
+            let file2 = root.join("file2");
+            write_test_file(&file1, b"aaa", b"", b"");
+            write_test_file(&file2, b"bbb", b"", b"");
+
+            let log = test_log();
+            let mut config = Config::default();
+            config.paths = vec![file1.clone(), file2.clone()];
+            config.unique = true;
+
+            let results = group_files(&config, &log).unwrap();
+            assert_eq!(results.len(), 2);
+            assert_eq!(results[0].files.len(), 1);
+            assert_eq!(results[1].files.len(), 1);
+        });
+    }
+
+    #[test]
+    fn files_differing_by_suffix() {
+        with_dir("target/test/main/files_differing_by_suffix", |root| {
+            let file1 = root.join("file1");
+            let file2 = root.join("file2");
+            let prefix = [0; MAX_PREFIX_LEN];
+            let mid = [1; MAX_PREFIX_LEN + MAX_SUFFIX_LEN];
+            write_test_file(&file1, &prefix, &mid, b"suffix1");
+            write_test_file(&file2, &prefix, &mid, b"suffix2");
+
+            let log = test_log();
+            let mut config = Config::default();
+            config.paths = vec![file1.clone(), file2.clone()];
+            config.unique = true;
+
+            let results = group_files(&config, &log).unwrap();
+            assert_eq!(results.len(), 2);
+            assert_eq!(results[0].files.len(), 1);
+            assert_eq!(results[1].files.len(), 1);
+        });
+    }
+
+    #[test]
+    fn files_differing_by_middle() {
+        with_dir("target/test/main/files_differing_by_middle", |root| {
+            let file1 = root.join("file1");
+            let file2 = root.join("file2");
+            let prefix = [0; MAX_PREFIX_LEN];
+            let suffix = [1; MAX_SUFFIX_LEN];
+            write_test_file(&file1, &prefix, b"middle1", &suffix);
+            write_test_file(&file2, &prefix, b"middle2", &suffix);
+
+            let log = test_log();
+            let mut config = Config::default();
+            config.paths = vec![file1.clone(), file2.clone()];
+            config.unique = true;
+
+            let results = group_files(&config, &log).unwrap();
+            assert_eq!(results.len(), 2);
+            assert_eq!(results[0].files.len(), 1);
+            assert_eq!(results[1].files.len(), 1);
+        });
+    }
+
+    #[test]
+    fn hard_links() {
+        with_dir("target/test/main/hard_links", |root| {
+            let file1 = root.join("file1");
+            let file2 = root.join("file2");
+            write_test_file(&file1, b"aaa", b"", b"");
+            hard_link(&file1, &file2).unwrap();
+
+            let log = test_log();
+            let mut config = Config::default();
+            config.paths = vec![file1.clone(), file2.clone()];
+            config.unique = true;
+
+            let results = group_files(&config, &log).unwrap();
+            assert_eq!(results.len(), 1);
+            assert_eq!(results[0].files.len(), 1);
+        });
+    }
+
+    #[test]
+    fn duplicate_input_files() {
+        with_dir("target/test/main/duplicate_input_files", |root| {
+            let file1 = root.join("file1");
+            write_test_file(&file1, b"foo", b"", b"");
+            let log = test_log();
+            let mut config = Config::default();
+            config.paths = vec![file1.clone(), file1.clone(), file1.clone()];
+            config.unique = true;
+            config.hard_links = true;
+
+            let results = group_files(&config, &log).unwrap();
+            assert_eq!(results.len(), 1);
+            assert_eq!(results[0].files.len(), 1);
+        });
+    }
+
+    #[test]
+    #[cfg(unix)]
+    fn duplicate_input_files_non_canonical() {
+        use std::os::unix::fs::symlink;
+
+        with_dir(
+            "target/test/main/duplicate_input_files_non_canonical",
+            |root| {
+                let dir = root.join("dir");
+                symlink(&root, &dir).unwrap();
+
+                let file1 = root.join("file1");
+                let file2 = root.join("dir/file1");
+                write_test_file(&file1, b"foo", b"", b"");
+
+                let log = test_log();
+                let mut config = Config::default();
+                config.paths = vec![file1.clone(), file2.clone()];
+                config.unique = true;
+                config.hard_links = true;
+
+                let results = group_files(&config, &log).unwrap();
+                assert_eq!(results.len(), 1);
+                assert_eq!(results[0].files.len(), 1);
+            },
+        );
+    }
+
+    #[test]
+    fn report() {
+        with_dir("target/test/main/report", |root| {
+            let file = root.join("file1");
+            write_test_file(&file, b"foo", b"", b"");
+
+            let report_file = root.join("report.txt");
+            let log = test_log();
+            let mut config = Config::default();
+            config.paths = vec![file.clone()];
+            config.unique = true;
+            config.output = Some(report_file.clone());
+
+            let results = group_files(&config, &log).unwrap();
+            write_report(&config, &log, &results).unwrap();
+
+            assert!(report_file.exists());
+            let mut report = String::new();
+            File::open(report_file)
+                .unwrap()
+                .read_to_string(&mut report)
+                .unwrap();
+            assert!(report.contains("file1"))
+        });
+    }
+
+    fn write_test_file(path: &PathBuf, prefix: &[u8], mid: &[u8], suffix: &[u8]) {
+        let mut file = OpenOptions::new()
+            .write(true)
+            .create(true)
+            .open(&path)
+            .unwrap();
+        file.write(prefix).unwrap();
+        file.write(mid).unwrap();
+        file.write(suffix).unwrap();
+    }
+
+    fn test_log() -> Log {
+        let mut log = Log::new();
+        log.no_progress = true;
+        log
+    }
+}
diff --git a/src/log.rs b/src/log.rs
index a9ec4d6..ba800a7 100644
--- a/src/log.rs
+++ b/src/log.rs
@@ -1,3 +1,5 @@
+//! Logging and progress reporting.
+
 use std::sync::{Arc, Mutex, Weak};
 
 use console::style;
@@ -102,7 +104,8 @@ impl Log {
             "{} {}: {} {}",
             style(timestamp.format(Self::TIMESTAMP_FMT))
                 .for_stderr()
-                .green(),
+                .dim()
+                .white(),
             style(&self.program_name).for_stderr().yellow(),
             style(" info:").for_stderr().green(),
             msg
@@ -116,7 +119,8 @@ impl Log {
             "{} {}: {} {}",
             style(timestamp.format(Self::TIMESTAMP_FMT))
                 .for_stderr()
-                .green(),
+                .dim()
+                .white(),
             style(&self.program_name).for_stderr().yellow(),
             style(" warn:").for_stderr().yellow(),
             msg
@@ -130,7 +134,8 @@ impl Log {
             "{} {}: {} {}",
             style(timestamp.format(Self::TIMESTAMP_FMT))
                 .for_stderr()
-                .green(),
+                .dim()
+                .white(),
             style(&self.program_name).for_stderr().yellow(),
             style("error:").for_stderr().red(),
             msg
diff --git a/src/main.rs b/src/main.rs
index 43f3253..ad4d05e 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,45 +1,35 @@
-use std::cell::RefCell;
-use std::cmp::Reverse;
-use std::collections::HashMap;
-use std::env::current_dir;
-use std::ffi::{OsStr, OsString};
 use std::fs::File;
-use std::io::{BufWriter, Write};
 use std::process::exit;
-#[cfg(target_os = "linux")]
-use std::sync::atomic::{AtomicU32, Ordering};
-use std::sync::Arc;
 
-use console::{style, Term};
+use console::style;
 use indoc::indoc;
-use itertools::Itertools;
-use rayon::prelude::*;
 use regex::Regex;
 use structopt::StructOpt;
-use thread_local::ThreadLocal;
 
-use fclones::config::*;
-use fclones::device::{DiskDevice, DiskDevices, Parallelism};
-use fclones::files::*;
-use fclones::group::*;
+use fclones::config::{Config, Parallelism};
 use fclones::log::Log;
-use fclones::path::Path;
-use fclones::pattern::ESCAPE_CHAR;
-use fclones::progress::FastProgressBar;
-use fclones::report::Reporter;
-use fclones::transform::Transform;
-use fclones::walk::Walk;
+use fclones::path::PATH_ESCAPE_CHAR;
+
+use fclones::{group_files, write_report};
+
+use std::collections::HashMap;
+use std::ffi::{OsStr, OsString};
 
-struct AppCtx {
-    config: Config,
-    log: Log,
-    devices: DiskDevices,
+fn paint_help(s: &str) -> String {
+    let escape_regex = "{escape}";
+    let code_regex = Regex::new(r"`([^`]+)`").unwrap();
+    let title_regex = Regex::new(r"(?m)^# *(.*?)$").unwrap();
+    let s = s.replace(escape_regex, PATH_ESCAPE_CHAR);
+    let s = code_regex.replace_all(s.as_ref(), style("$1").green().to_string().as_str());
+    title_regex
+        .replace_all(s.as_ref(), style("$1").yellow().to_string().as_str())
+        .to_string()
 }
 
 /// Configures global thread pool to use desired number of threads
-fn configure_main_thread_pool(pool_sizes: &mut HashMap<OsString, Parallelism>) {
-    let parallelism = pool_sizes.remove(OsStr::new("main")).unwrap_or_else(|| {
-        *pool_sizes
+fn configure_main_thread_pool(pool_sizes: &HashMap<OsString, Parallelism>) {
+    let parallelism = pool_sizes.get(OsStr::new("main")).unwrap_or_else(|| {
+        pool_sizes
             .get(OsStr::new("default"))
             .unwrap_or(&Parallelism {
                 sequential: 0,
@@ -53,488 +43,6 @@ fn configure_main_thread_pool(pool_sizes: &mut HashMap<OsString, Parallelism>) {
         .unwrap();
 }
 
-/// Walks the directory tree and collects matching files in parallel into a vector
-fn scan_files(ctx: &AppCtx) -> Vec<Vec<FileInfo>> {
-    let base_dir = Path::from(current_dir().unwrap_or_default());
-    let path_selector = ctx.config.path_selector(&ctx.log, &base_dir);
-    let file_collector = ThreadLocal::new();
-    let spinner = ctx.log.spinner("Scanning files");
-    let spinner_tick = &|_: &Path| spinner.tick();
-
-    let config = &ctx.config;
-    let min_size = config.min_size;
-    let max_size = config.max_size.unwrap_or(FileLen::MAX);
-
-    let mut walk = Walk::new();
-    walk.recursive = config.recursive;
-    walk.depth = config.depth.unwrap_or(usize::MAX);
-    walk.skip_hidden = config.skip_hidden;
-    walk.follow_links = config.follow_links;
-    walk.path_selector = path_selector;
-    walk.log = Some(&ctx.log);
-    walk.on_visit = spinner_tick;
-    walk.run(ctx.config.input_paths(), |path| {
-        file_info_or_log_err(path, &ctx.devices, &ctx.log)
-            .into_iter()
-            .filter(|info| {
-                let l = info.len;
-                l >= min_size && l <= max_size
-            })
-            .for_each(|info| {
-                let vec = file_collector.get_or(|| RefCell::new(Vec::new()));
-                vec.borrow_mut().push(info);
-            });
-    });
-
-    ctx.log
-        .info(format!("Scanned {} file entries", spinner.position()));
-
-    let files: Vec<_> = file_collector.into_iter().map(|r| r.into_inner()).collect();
-
-    let file_count: usize = files.iter().map(|v| v.len()).sum();
-    let total_size: u64 = files.iter().flat_map(|v| v.iter().map(|i| i.len.0)).sum();
-    ctx.log.info(format!(
-        "Found {} ({}) files matching selection criteria",
-        file_count,
-        FileLen(total_size)
-    ));
-    files
-}
-
-fn group_by_size(ctx: &AppCtx, files: Vec<Vec<FileInfo>>) -> Vec<FileGroup<FileInfo>> {
-    let file_count: usize = files.iter().map(|v| v.len()).sum();
-    let progress = ctx.log.progress_bar("Grouping by size", file_count as u64);
-
-    let mut groups = GroupMap::new(|info: FileInfo| (info.len, info));
-    for files in files.into_iter() {
-        for file in files.into_iter() {
-            progress.tick();
-            groups.add(file);
-        }
-    }
-    let rf_over = ctx.config.rf_over();
-    let rf_under = ctx.config.rf_under();
-
-    let groups: Vec<_> = groups
-        .into_iter()
-        .filter(|(_, files)| files.len() > rf_over)
-        .map(|(l, files)| FileGroup {
-            file_len: l,
-            file_hash: FileHash(0),
-            files: files.into_vec(),
-        })
-        .collect();
-
-    let count: usize = groups.selected_count(rf_over, rf_under);
-    let bytes: FileLen = groups.selected_size(rf_over, rf_under);
-    ctx.log.info(format!(
-        "Found {} ({}) candidates after grouping by size",
-        count, bytes
-    ));
-    groups
-}
-
-/// Removes duplicate files matching by full-path or by inode-id.
-/// Deduplication by inode-id is not performed if the flag to preserve hard-links (-H) is set.
-fn deduplicate<F>(ctx: &AppCtx, files: &mut Vec<FileInfo>, progress: F)
-where
-    F: Fn(&Path) + Sync + Send,
-{
-    let mut groups = GroupMap::new(|fi: FileInfo| (fi.location, fi));
-    for f in files.drain(..) {
-        groups.add(f)
-    }
-
-    for (_, file_group) in groups.into_iter() {
-        if file_group.len() == 1 {
-            files.extend(file_group.into_iter().inspect(|p| progress(&p.path)));
-        } else if ctx.config.hard_links {
-            files.extend(
-                file_group
-                    .into_iter()
-                    .inspect(|p| progress(&p.path))
-                    .unique_by(|p| p.path.hash128()),
-            )
-        } else {
-            files.extend(
-                file_group
-                    .into_iter()
-                    .inspect(|p| progress(&p.path))
-                    .unique_by(|p| file_id_or_log_err(&p.path, &ctx.log)),
-            )
-        }
-    }
-}
-
-fn remove_same_files(ctx: &AppCtx, groups: Vec<FileGroup<FileInfo>>) -> Vec<FileGroup<FileInfo>> {
-    let file_count: usize = groups.total_count();
-    let progress = ctx
-        .log
-        .progress_bar("Removing same files", file_count as u64);
-
-    let rf_over = ctx.config.rf_over();
-    let rf_under = ctx.config.rf_under();
-
-    let groups: Vec<_> = groups
-        .into_par_iter()
-        .update(|g| deduplicate(ctx, &mut g.files, |_| progress.tick()))
-        .filter(|g| g.files.len() > rf_over)
-        .collect();
-
-    let count: usize = groups.selected_count(rf_over, rf_under);
-    let bytes: FileLen = groups.selected_size(rf_over, rf_under);
-    ctx.log.info(format!(
-        "Found {} ({}) candidates after grouping by paths {}",
-        count,
-        bytes,
-        if ctx.config.hard_links {
-            ""
-        } else {
-            "and file identifiers"
-        }
-    ));
-    groups
-}
-
-#[cfg(target_os = "linux")]
-fn atomic_counter_vec(len: usize) -> Vec<AtomicU32> {
-    let mut v = Vec::with_capacity(len);
-    for _ in 0..len {
-        v.push(AtomicU32::new(0));
-    }
-    v
-}
-
-#[cfg(target_os = "linux")]
-fn update_file_locations(ctx: &AppCtx, groups: &mut Vec<FileGroup<FileInfo>>) {
-    let count = groups.total_count();
-    let progress = ctx.log.progress_bar("Fetching extents", count as u64);
-
-    let err_counters = atomic_counter_vec(ctx.devices.len());
-    const MAX_ERR_COUNT_TO_LOG: u32 = 10;
-
-    groups
-        .par_iter_mut()
-        .flat_map(|g| &mut g.files)
-        .update(|fi| {
-            let device: &DiskDevice = &ctx.devices[fi.get_device_index()];
-            if device.disk_type != sysinfo::DiskType::SSD {
-                if let Err(e) = fi.fetch_physical_location() {
-                    let counter = &err_counters[device.index];
-                    if counter.load(Ordering::Relaxed) < MAX_ERR_COUNT_TO_LOG {
-                        let err_count = counter.fetch_add(1, Ordering::SeqCst);
-                        if err_count < MAX_ERR_COUNT_TO_LOG {
-                            ctx.log.warn(format!(
-                                "Failed to fetch extents for file {}: {}",
-                                fi.path, e
-                            ));
-                            if err_count + 1 == MAX_ERR_COUNT_TO_LOG {
-                                ctx.log.warn(
-                                    "Too many fetch extents errors. \
-                                    More errors will be ignored. \
-                                    Random access performance might be affected on spinning drives.",
-                                )
-                            }
-                        }
-                    }
-                }
-            }
-        })
-        .for_each(|_| progress.tick());
-}
-
-/// Transforms files by piping them to an external program and groups them by their hashes
-fn group_transformed(
-    ctx: &AppCtx,
-    transform: &Transform,
-    groups: Vec<FileGroup<FileInfo>>,
-) -> Vec<FileGroup<FileInfo>> {
-    let file_count: usize = groups.total_count();
-    let progress = ctx
-        .log
-        .progress_bar("Transforming & grouping", file_count as u64);
-
-    let rf_over = ctx.config.rf_over();
-    let rf_under = ctx.config.rf_under();
-
-    let groups = rehash(
-        groups,
-        |_| true,
-        |g| g.files.len() > rf_over,
-        &ctx.devices,
-        AccessType::Sequential,
-        |(fi, _)| {
-            let result = transform
-                .run_or_log_err(&fi.path, &ctx.log)
-                .map(|(len, hash)| {
-                    fi.len = len;
-                    hash
-                });
-            progress.tick();
-            result
-        },
-    );
-
-    let count = groups.selected_count(rf_over, rf_under);
-    let bytes = groups.selected_size(rf_over, rf_under);
-    ctx.log.info(format!(
-        "Found {} ({}) {} files",
-        count,
-        bytes,
-        ctx.config.search_type()
-    ));
-    groups
-}
-
-/// Returns the maximum value of the given property of the device,
-/// among the devices actually used to store any of the given files
-fn max_device_property<'a>(
-    devices: &DiskDevices,
-    files: impl ParallelIterator<Item = &'a FileInfo>,
-    property_fn: impl Fn(&DiskDevice) -> FileLen + Sync,
-) -> FileLen {
-    files
-        .into_par_iter()
-        .map(|f| property_fn(&devices[f.get_device_index()]))
-        .max()
-        .unwrap_or_else(|| property_fn(devices.get_default()))
-}
-
-/// Returns the desired prefix length for a group of files.
-/// The return value depends on the capabilities of the devices the files are stored on.
-/// Higher values are desired if any of the files resides on an HDD.
-fn prefix_len<'a>(
-    partitions: &DiskDevices,
-    files: impl ParallelIterator<Item = &'a FileInfo>,
-) -> FileLen {
-    max_device_property(partitions, files, |dd| dd.max_prefix_len())
-}
-
-/// Groups files by a hash of their first few thousand bytes.
-fn group_by_prefix(
-    ctx: &AppCtx,
-    prefix_len: FileLen,
-    groups: Vec<FileGroup<FileInfo>>,
-) -> Vec<FileGroup<FileInfo>> {
-    let pre_filter = |g: &FileGroup<FileInfo>| g.files.len() > 1;
-    let remaining_files = groups.iter().filter(|&g| pre_filter(g)).total_count();
-    let progress = ctx
-        .log
-        .progress_bar("Grouping by prefix", remaining_files as u64);
-
-    let rf_over = ctx.config.rf_over();
-    let rf_under = ctx.config.rf_under();
-
-    let groups = rehash(
-        groups,
-        pre_filter,
-        |g| g.files.len() > rf_over,
-        &ctx.devices,
-        AccessType::Random,
-        |(fi, _)| {
-            progress.tick();
-            let device = &ctx.devices[fi.get_device_index()];
-            let buf_len = device.buf_len();
-            let (caching, prefix_len) = if fi.len <= prefix_len {
-                (Caching::Default, prefix_len)
-            } else {
-                (Caching::Random, device.min_prefix_len())
-            };
-
-            file_hash_or_log_err(
-                &fi.path,
-                FilePos(0),
-                prefix_len,
-                buf_len,
-                caching,
-                |_| {},
-                &ctx.log,
-            )
-        },
-    );
-
-    let count = groups.selected_count(rf_over, rf_under);
-    let bytes = groups.selected_size(rf_over, rf_under);
-    ctx.log.info(format!(
-        "Found {} ({}) candidates after grouping by prefix",
-        count, bytes
-    ));
-    groups
-}
-
-/// Returns the desired suffix length for a group of files.
-/// The return value depends on the capabilities of the devices the files are stored on.
-/// Higher values are desired if any of the files resides on an HDD.
-fn suffix_len<'a>(
-    partitions: &DiskDevices,
-    files: impl ParallelIterator<Item = &'a FileInfo>,
-) -> FileLen {
-    max_device_property(partitions, files, |dd| dd.suffix_len())
-}
-
-fn suffix_threshold<'a>(
-    partitions: &DiskDevices,
-    files: impl ParallelIterator<Item = &'a FileInfo>,
-) -> FileLen {
-    max_device_property(partitions, files, |dd| dd.suffix_threshold())
-}
-
-fn group_by_suffix(ctx: &AppCtx, groups: Vec<FileGroup<FileInfo>>) -> Vec<FileGroup<FileInfo>> {
-    let suffix_len = suffix_len(&ctx.devices, flat_iter(&groups));
-    let suffix_threshold = suffix_threshold(&ctx.devices, flat_iter(&groups));
-    let pre_filter = |g: &FileGroup<FileInfo>| g.file_len >= suffix_threshold && g.files.len() > 1;
-    let remaining_files = groups.iter().filter(|&g| pre_filter(g)).total_count();
-    let progress = ctx
-        .log
-        .progress_bar("Grouping by suffix", remaining_files as u64);
-
-    let rf_over = ctx.config.rf_over();
-    let rf_under = ctx.config.rf_under();
-
-    let groups = rehash(
-        groups,
-        pre_filter,
-        |g| g.files.len() > rf_over,
-        &ctx.devices,
-        AccessType::Random,
-        |(fi, old_hash)| {
-            progress.tick();
-            let device = &ctx.devices[fi.get_device_index()];
-            let buf_len = device.buf_len();
-            file_hash_or_log_err(
-                &fi.path,
-                fi.len.as_pos() - suffix_len,
-                suffix_len,
-                buf_len,
-                Caching::Default,
-                |_| {},
-                &ctx.log,
-            )
-            .map(|new_hash| old_hash ^ new_hash)
-        },
-    );
-
-    let count = groups.selected_count(rf_over, rf_under);
-    let bytes = groups.selected_size(rf_over, rf_under);
-    ctx.log.info(format!(
-        "Found {} ({}) candidates after grouping by suffix",
-        count, bytes
-    ));
-    groups
-}
-
-fn group_by_contents(
-    ctx: &AppCtx,
-    min_file_len: FileLen,
-    groups: Vec<FileGroup<FileInfo>>,
-) -> Vec<FileGroup<FileInfo>> {
-    let pre_filter = |g: &FileGroup<FileInfo>| g.files.len() > 1 && g.file_len >= min_file_len;
-    let bytes_to_scan = groups.iter().filter(|&g| pre_filter(g)).total_size();
-    let progress = &ctx
-        .log
-        .bytes_progress_bar("Grouping by contents", bytes_to_scan.0);
-
-    let rf_over = ctx.config.rf_over();
-    let rf_under = ctx.config.rf_under();
-
-    let groups = rehash(
-        groups,
-        pre_filter,
-        |g| g.files.len() > rf_over,
-        &ctx.devices,
-        AccessType::Sequential,
-        |(fi, _)| {
-            let device = &ctx.devices[fi.get_device_index()];
-            let buf_len = device.buf_len();
-            file_hash_or_log_err(
-                &fi.path,
-                FilePos(0),
-                fi.len,
-                buf_len,
-                Caching::Sequential,
-                |delta| progress.inc(delta),
-                &ctx.log,
-            )
-        },
-    );
-
-    let count = groups.selected_count(rf_over, rf_under);
-    let bytes = groups.selected_size(rf_over, rf_under);
-    ctx.log.info(format!(
-        "Found {} ({}) {} files",
-        count,
-        bytes,
-        ctx.config.search_type()
-    ));
-    groups
-}
-
-fn write_report(ctx: &AppCtx, groups: &[FileGroup<Path>]) {
-    let remaining_files = groups.total_count();
-    let progress = ctx
-        .log
-        .progress_bar("Writing report", remaining_files as u64);
-
-    // No progress bar when we write to terminal
-    let mut reporter = match &ctx.config.output {
-        Some(path) => file_reporter(ctx, path, progress),
-        None => stdout_reporter(progress),
-    };
-
-    let result = match &ctx.config.format {
-        OutputFormat::Text => reporter.write_as_text(groups),
-        OutputFormat::Fdupes => reporter.write_as_fdupes(groups),
-        OutputFormat::Csv => reporter.write_as_csv(groups),
-        OutputFormat::Json => reporter.write_as_json(groups),
-    };
-    match result {
-        Ok(()) => (),
-        Err(e) => ctx.log.err(format!("Failed to write report: {}", e)),
-    }
-}
-
-/// Creates a reporter that writes to the standard output
-fn stdout_reporter(progress: Arc<FastProgressBar>) -> Reporter<Box<dyn Write>> {
-    let stdout = Term::stdout();
-    let is_term = stdout.is_term();
-    if is_term {
-        progress.finish_and_clear()
-    }
-    let out: Box<dyn Write> = Box::new(BufWriter::new(stdout));
-    Reporter::new(out, is_term, progress)
-}
-
-/// Creates a reporter that writes to the given file.
-/// Writes and error message and exists the application if the file cannot be open.
-fn file_reporter(
-    ctx: &AppCtx,
-    path: &std::path::Path,
-    progress: Arc<FastProgressBar>,
-) -> Reporter<Box<dyn Write>> {
-    match File::create(path) {
-        Ok(file) => {
-            let out: Box<dyn Write> = Box::new(BufWriter::new(file));
-            Reporter::new(out, false, progress)
-        }
-        Err(e) => {
-            ctx.log
-                .err(format!("Could not create {}: {}", path.display(), e));
-            exit(1)
-        }
-    }
-}
-
-fn paint_help(s: &str) -> String {
-    let escape_regex = "{escape}";
-    let code_regex = Regex::new(r"`([^`]+)`").unwrap();
-    let title_regex = Regex::new(r"(?m)^# *(.*?)$").unwrap();
-    let s = s.replace(escape_regex, ESCAPE_CHAR);
-    let s = code_regex.replace_all(s.as_ref(), style("$1").green().to_string().as_str());
-    title_regex
-        .replace_all(s.as_ref(), style("$1").yellow().to_string().as_str())
-        .to_string()
-}
-
 fn main() {
     let mut log = Log::new();
     let after_help = &paint_help(indoc!(
@@ -570,33 +78,18 @@ fn main() {
         }
         exit(1);
     });
+
     let config: Config = Config::from_clap(&matches);
     if config.quiet {
         log.no_progress = true;
     }
 
-    log.info("Started");
-    let mut thread_pool_sizes = config.thread_pool_sizes();
-    configure_main_thread_pool(&mut thread_pool_sizes);
-
-    let device_scanning_spinner = log.spinner("Fetching disk device list");
-    let devices = DiskDevices::new(&mut thread_pool_sizes);
-    drop(device_scanning_spinner);
-
-    let ctx = AppCtx {
-        log,
-        config,
-        devices,
-    };
-    ctx.config.check_transform(&ctx.log);
-    ctx.config
-        .check_thread_pools(&ctx.log, &mut thread_pool_sizes);
-
-    if let Some(output) = &ctx.config.output {
+    configure_main_thread_pool(&config.thread_pool_sizes());
+    if let Some(output) = &config.output {
         // Try to create the output file now and fail early so that
         // the user doesn't waste time to only find that the report cannot be written at the end:
         if let Err(e) = File::create(output) {
-            ctx.log.err(format!(
+            log.err(format!(
                 "Cannot create output file {}: {}",
                 output.display(),
                 e
@@ -605,282 +98,15 @@ fn main() {
         }
     }
 
-    let results = process(&ctx);
-    write_report(&ctx, &results)
-}
-
-// Extracted for testing
-fn process(ctx: &AppCtx) -> Vec<FileGroup<Path>> {
-    let matching_files = scan_files(&ctx);
-    let size_groups = group_by_size(&ctx, matching_files);
-
-    #[allow(unused_mut)]
-    let mut size_groups_pruned = remove_same_files(&ctx, size_groups);
-
-    #[cfg(target_os = "linux")]
-    update_file_locations(ctx, &mut size_groups_pruned);
-
-    let groups = match ctx.config.transform(&ctx.log) {
-        Some(transform) => group_transformed(ctx, &transform, size_groups_pruned),
-        None => {
-            let prefix_len = prefix_len(&ctx.devices, flat_iter(&size_groups_pruned));
-            let prefix_groups = group_by_prefix(&ctx, prefix_len, size_groups_pruned);
-            let suffix_groups = group_by_suffix(&ctx, prefix_groups);
-            group_by_contents(&ctx, prefix_len, suffix_groups)
+    let results = match group_files(&config, &log) {
+        Ok(groups) => groups,
+        Err(e) => {
+            log.err(e);
+            exit(1)
         }
     };
-    let mut groups: Vec<_> = groups
-        .into_par_iter()
-        .map(|g| FileGroup {
-            file_len: g.file_len,
-            file_hash: g.file_hash,
-            files: g.files.into_iter().map(|fi| fi.path).collect(),
-        })
-        .collect();
-    groups.retain(|g| g.files.len() < ctx.config.rf_under());
-    groups.par_sort_by_key(|g| Reverse((g.file_len, g.file_hash)));
-    groups.par_iter_mut().for_each(|g| g.files.sort());
-    groups
-}
-
-#[cfg(test)]
-mod test {
-    use std::fs::{hard_link, OpenOptions};
-    use std::io::{Read, Write};
-    use std::path::PathBuf;
-
-    use fclones::path::Path;
-    use fclones::util::test::*;
-
-    use super::*;
-
-    const MAX_PREFIX_LEN: usize = 256 * 1024;
-    const MAX_SUFFIX_LEN: usize = 256 * 1024;
-
-    #[test]
-    fn identical_small_files() {
-        with_dir("target/test/main/identical_small_files", |root| {
-            let file1 = root.join("file1");
-            let file2 = root.join("file2");
-            write_test_file(&file1, b"aaa", b"", b"");
-            write_test_file(&file2, b"aaa", b"", b"");
 
-            let mut ctx = test_ctx();
-            ctx.config.paths = vec![file1, file2];
-            let results = process(&ctx);
-            assert_eq!(results.len(), 1);
-            assert_eq!(results[0].file_len, FileLen(3));
-            assert_eq!(results[0].files.len(), 2);
-        });
-    }
-
-    #[test]
-    fn identical_large_files() {
-        with_dir("target/test/main/identical_large_files", |root| {
-            let file1 = root.join("file1");
-            let file2 = root.join("file2");
-            write_test_file(&file1, &[0; MAX_PREFIX_LEN], &[1; 4096], &[2; 4096]);
-            write_test_file(&file2, &[0; MAX_PREFIX_LEN], &[1; 4096], &[2; 4096]);
-
-            let mut ctx = test_ctx();
-            ctx.config.paths = vec![file1, file2];
-
-            let results = process(&ctx);
-            assert_eq!(results.len(), 1);
-            assert_eq!(results[0].files.len(), 2);
-        });
-    }
-
-    #[test]
-    fn files_differing_by_size() {
-        with_dir("target/test/main/files_differing_by_size", |root| {
-            let file1 = root.join("file1");
-            let file2 = root.join("file2");
-            write_test_file(&file1, b"aaaa", b"", b"");
-            write_test_file(&file2, b"aaa", b"", b"");
-
-            let mut ctx = test_ctx();
-            ctx.config.paths = vec![file1.clone(), file2.clone()];
-            ctx.config.rf_over = Some(0);
-
-            let results = process(&ctx);
-            assert_eq!(results.len(), 2);
-            assert_eq!(
-                results[0].files,
-                vec![Path::from(file1.canonicalize().unwrap())]
-            );
-            assert_eq!(
-                results[1].files,
-                vec![Path::from(file2.canonicalize().unwrap())]
-            );
-        });
-    }
-
-    #[test]
-    fn files_differing_by_prefix() {
-        with_dir("target/test/main/files_differing_by_prefix", |root| {
-            let file1 = root.join("file1");
-            let file2 = root.join("file2");
-            write_test_file(&file1, b"aaa", b"", b"");
-            write_test_file(&file2, b"bbb", b"", b"");
-
-            let mut ctx = test_ctx();
-            ctx.config.paths = vec![file1.clone(), file2.clone()];
-            ctx.config.unique = true;
-
-            let results = process(&ctx);
-            assert_eq!(results.len(), 2);
-            assert_eq!(results[0].files.len(), 1);
-            assert_eq!(results[1].files.len(), 1);
-        });
-    }
-
-    #[test]
-    fn files_differing_by_suffix() {
-        with_dir("target/test/main/files_differing_by_suffix", |root| {
-            let file1 = root.join("file1");
-            let file2 = root.join("file2");
-            let prefix = [0; MAX_PREFIX_LEN];
-            let mid = [1; MAX_PREFIX_LEN + MAX_SUFFIX_LEN];
-            write_test_file(&file1, &prefix, &mid, b"suffix1");
-            write_test_file(&file2, &prefix, &mid, b"suffix2");
-
-            let mut ctx = test_ctx();
-            ctx.config.paths = vec![file1.clone(), file2.clone()];
-            ctx.config.unique = true;
-
-            let results = process(&ctx);
-            assert_eq!(results.len(), 2);
-            assert_eq!(results[0].files.len(), 1);
-            assert_eq!(results[1].files.len(), 1);
-        });
-    }
-
-    #[test]
-    fn files_differing_by_middle() {
-        with_dir("target/test/main/files_differing_by_middle", |root| {
-            let file1 = root.join("file1");
-            let file2 = root.join("file2");
-            let prefix = [0; MAX_PREFIX_LEN];
-            let suffix = [1; MAX_SUFFIX_LEN];
-            write_test_file(&file1, &prefix, b"middle1", &suffix);
-            write_test_file(&file2, &prefix, b"middle2", &suffix);
-
-            let mut ctx = test_ctx();
-            ctx.config.paths = vec![file1.clone(), file2.clone()];
-            ctx.config.unique = true;
-
-            let results = process(&ctx);
-            assert_eq!(results.len(), 2);
-            assert_eq!(results[0].files.len(), 1);
-            assert_eq!(results[1].files.len(), 1);
-        });
-    }
-
-    #[test]
-    fn hard_links() {
-        with_dir("target/test/main/hard_links", |root| {
-            let file1 = root.join("file1");
-            let file2 = root.join("file2");
-            write_test_file(&file1, b"aaa", b"", b"");
-            hard_link(&file1, &file2).unwrap();
-
-            let mut ctx = test_ctx();
-            ctx.config.paths = vec![file1.clone(), file2.clone()];
-            ctx.config.unique = true;
-
-            let results = process(&ctx);
-            assert_eq!(results.len(), 1);
-            assert_eq!(results[0].files.len(), 1);
-        });
-    }
-
-    #[test]
-    fn duplicate_input_files() {
-        with_dir("target/test/main/duplicate_input_files", |root| {
-            let file1 = root.join("file1");
-            write_test_file(&file1, b"foo", b"", b"");
-            let mut ctx = test_ctx();
-            ctx.config.paths = vec![file1.clone(), file1.clone(), file1.clone()];
-            ctx.config.unique = true;
-            ctx.config.hard_links = true;
-
-            let results = process(&ctx);
-            assert_eq!(results.len(), 1);
-            assert_eq!(results[0].files.len(), 1);
-        });
-    }
-
-    #[test]
-    #[cfg(unix)]
-    fn duplicate_input_files_non_canonical() {
-        use std::os::unix::fs::symlink;
-
-        with_dir(
-            "target/test/main/duplicate_input_files_non_canonical",
-            |root| {
-                let dir = root.join("dir");
-                symlink(&root, &dir).unwrap();
-
-                let file1 = root.join("file1");
-                let file2 = root.join("dir/file1");
-                write_test_file(&file1, b"foo", b"", b"");
-
-                let mut ctx = test_ctx();
-                ctx.config.paths = vec![file1.clone(), file2.clone()];
-                ctx.config.unique = true;
-                ctx.config.hard_links = true;
-
-                let results = process(&ctx);
-                assert_eq!(results.len(), 1);
-                assert_eq!(results[0].files.len(), 1);
-            },
-        );
-    }
-
-    #[test]
-    fn report() {
-        with_dir("target/test/main/report", |root| {
-            let file = root.join("file1");
-            write_test_file(&file, b"foo", b"", b"");
-
-            let report_file = root.join("report.txt");
-            let mut ctx = test_ctx();
-            ctx.config.paths = vec![file.clone()];
-            ctx.config.unique = true;
-            ctx.config.output = Some(report_file.clone());
-
-            let results = process(&ctx);
-            write_report(&ctx, &results);
-
-            assert!(report_file.exists());
-            let mut report = String::new();
-            File::open(report_file)
-                .unwrap()
-                .read_to_string(&mut report)
-                .unwrap();
-            assert!(report.contains("file1"))
-        });
-    }
-
-    fn write_test_file(path: &PathBuf, prefix: &[u8], mid: &[u8], suffix: &[u8]) {
-        let mut file = OpenOptions::new()
-            .write(true)
-            .create(true)
-            .open(&path)
-            .unwrap();
-        file.write(prefix).unwrap();
-        file.write(mid).unwrap();
-        file.write(suffix).unwrap();
-    }
-
-    fn test_ctx() -> AppCtx {
-        let mut log = Log::new();
-        log.no_progress = true;
-        AppCtx {
-            log,
-            config: Default::default(),
-            devices: Default::default(),
-        }
+    if let Err(e) = write_report(&config, &log, &results) {
+        log.err(format!("Failed to write report: {}", e))
     }
 }
diff --git a/src/path.rs b/src/path.rs
index c90a878..1c748ea 100644
--- a/src/path.rs
+++ b/src/path.rs
@@ -1,3 +1,5 @@
+//! Memory-efficient file path representation.
+
 use std::ffi::{CStr, CString, OsString};
 use std::fmt;
 use std::fmt::Display;
@@ -12,11 +14,18 @@ use smallvec::SmallVec;
 
 use crate::path::string::{c_to_os_str, os_to_c_str};
 
-/// Memory-efficient path representation.
-/// When storing multiple paths with common parent, the standard PathBuf would keep
-/// the parent duplicated in memory, wasting a lot of memory.
-/// This shares the common parent between many paths.
-/// The price is a tiny cost of managing Arc references.
+#[cfg(unix)]
+pub const PATH_ESCAPE_CHAR: &str = "\\";
+
+#[cfg(windows)]
+pub const PATH_ESCAPE_CHAR: &str = "^";
+
+/// Memory-efficient file path representation.
+///
+/// When storing multiple paths with common parent, the standard [`std::path::PathBuf`]
+/// would keep the parent path text duplicated in memory, wasting a lot of memory.
+/// This structure here shares the common parent between many paths by reference-counted
+/// references.
 #[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
 pub struct Path {
     parent: Option<Arc<Path>>,
diff --git a/src/pattern.rs b/src/pattern.rs
index 718bc84..10bf555 100644
--- a/src/pattern.rs
+++ b/src/pattern.rs
@@ -11,6 +11,7 @@ use nom::sequence::tuple;
 use nom::IResult;
 use regex::escape;
 
+use crate::path::PATH_ESCAPE_CHAR;
 use crate::regex::Regex;
 
 #[derive(Debug)]
@@ -65,12 +66,6 @@ enum Scope {
     RoundBrackets,
 }
 
-#[cfg(unix)]
-pub const ESCAPE_CHAR: &str = "\\";
-
-#[cfg(windows)]
-pub const ESCAPE_CHAR: &str = "^";
-
 impl Pattern {
     /// Creates `Pattern` instance from raw regular expression. Supports PCRE syntax.
     pub fn regex(pattern: &str) -> Result<Pattern, PatternError> {
@@ -176,7 +171,7 @@ impl Pattern {
     /// Parses a UNIX glob and converts it to a regular expression
     fn glob_to_regex(scope: Scope, glob: &str) -> IResult<&str, String> {
         // pass escaped characters as-is:
-        let p_escaped = map(tuple((tag(ESCAPE_CHAR), anychar)), |(_, c)| {
+        let p_escaped = map(tuple((tag(PATH_ESCAPE_CHAR), anychar)), |(_, c)| {
             escape(c.to_string().as_str())
         });
 
diff --git a/src/progress.rs b/src/progress.rs
index 4f58b33..1a401d5 100644
--- a/src/progress.rs
+++ b/src/progress.rs
@@ -1,3 +1,5 @@
+//! Fast, concurrent, lockless progress bars.
+
 use atomic_counter::{AtomicCounter, RelaxedCounter};
 use console::style;
 use indicatif::{ProgressBar, ProgressDrawTarget, ProgressStyle};
@@ -11,24 +13,6 @@ use std::time::Duration;
 /// This wrapper uses `atomic_counter::RelaxedCounter` to keep shared state without ever blocking
 /// writers. That state is copied repeatedly by a background thread to an underlying
 /// `ProgressBar` at a low rate.
-///
-/// # Example
-/// ```
-/// use indicatif::{ProgressBar, ProgressDrawTarget};
-/// use rayon::prelude::*;
-/// use fclones::progress::FastProgressBar;
-/// use std::time::Duration;
-///
-/// let collection = vec![0; 100000];
-/// let pb = ProgressBar::new(collection.len() as u64);
-/// let pb = FastProgressBar::wrap(pb);
-/// collection.par_iter()
-///     .inspect(|x| pb.tick())
-///     .for_each(|x| ());
-/// pb.abandon();
-/// assert_eq!(pb.position(), 100000);
-/// assert_eq!(pb.last_displayed_position(), 100000);
-/// ```
 pub struct FastProgressBar {
     counter: Arc<RelaxedCounter>,
     progress_bar: Arc<ProgressBar>,
@@ -205,3 +189,25 @@ impl Drop for FastProgressBar {
         }
     }
 }
+
+#[cfg(test)]
+mod test {
+
+    use super::*;
+    use rayon::prelude::*;
+
+    #[test]
+    fn all_ticks_should_be_counted() {
+        let collection = vec![0; 100000];
+        let pb = ProgressBar::new(collection.len() as u64);
+        pb.set_draw_target(ProgressDrawTarget::hidden());
+        let pb = FastProgressBar::wrap(pb);
+        collection
+            .par_iter()
+            .inspect(|_| pb.tick())
+            .for_each(|_| ());
+        pb.abandon();
+        assert_eq!(pb.position(), 100000);
+        assert_eq!(pb.last_displayed_position(), 100000);
+    }
+}
diff --git a/src/report.rs b/src/report.rs
index 0681c67..372c219 100644
--- a/src/report.rs
+++ b/src/report.rs
@@ -1,3 +1,5 @@
+//! Output formatting.
+
 use std::io;
 use std::io::Write;
 use std::sync::Arc;
@@ -5,9 +7,9 @@ use std::sync::Arc;
 use console::style;
 use serde::{Serialize, Serializer};
 
-use crate::group::FileGroup;
 use crate::path::Path;
 use crate::progress::FastProgressBar;
+use crate::FileGroup;
 
 pub struct Reporter<W: Write> {
     out: W,
diff --git a/src/selector.rs b/src/selector.rs
index 2ad8f72..7875d96 100644
--- a/src/selector.rs
+++ b/src/selector.rs
@@ -9,7 +9,7 @@ use crate::pattern::Pattern;
 /// A path is selected only if it matches at least one include pattern
 /// and doesn't match any exclude patterns.
 /// An empty include pattern vector matches all paths.
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct PathSelector {
     base_dir: Arc<Path>,
     included_names: Vec<Pattern>,
diff --git a/src/semaphore.rs b/src/semaphore.rs
index e2e21be..399360a 100644
--- a/src/semaphore.rs
+++ b/src/semaphore.rs
@@ -38,7 +38,7 @@ pub struct OwnedSemaphoreGuard {
     sem: Arc<Semaphore>,
 }
 
-#[allow(clippy::mutex_atomic)]
+#[allow(clippy::mutex_atomic, unused)]
 impl Semaphore {
     /// Creates a new semaphore with the initial count specified.
     ///
diff --git a/src/util.rs b/src/util.rs
index d918995..89ca4a1 100644
--- a/src/util.rs
+++ b/src/util.rs
@@ -1,3 +1,4 @@
+#[cfg(test)]
 pub mod test {
     use std::fs::{create_dir_all, remove_dir_all};
     use std::path::PathBuf;
diff --git a/src/walk.rs b/src/walk.rs
index 9aefce3..75a4b01 100644
--- a/src/walk.rs
+++ b/src/walk.rs
@@ -102,48 +102,6 @@ impl<'a> Walk<'a> {
     /// The `consumer` must be able to receive items from many threads.
     /// Inaccessible files are skipped, but errors are printed to stderr.
     /// The order of files is not specified and may be different every time.
-    ///
-    /// # Example
-    /// ```
-    /// use fclones::walk::*;
-    /// use fclones::path::Path;
-    /// use std::fs::{File, create_dir_all, remove_dir_all, create_dir};
-    /// use std::path::PathBuf;
-    /// use std::sync::Mutex;
-    /// use std::env::current_dir;
-    ///
-    /// let test_root = current_dir().unwrap().join(PathBuf::from("target/test/walk/"));
-    /// let dir1 = test_root.join("dir1");
-    /// let dir2 = test_root.join("dir2");
-    /// let dir3 = dir2.join("dir3");
-    ///
-    /// remove_dir_all(&test_root).ok();
-    /// create_dir_all(&test_root).unwrap();
-    /// create_dir(&dir1).unwrap();
-    /// create_dir(&dir2).unwrap();
-    /// create_dir(&dir3).unwrap();
-    ///
-    /// let file11 = dir1.join("file11.txt");
-    /// let file12 = dir1.join("file12.txt");
-    /// let file2 = dir2.join("file2.txt");
-    /// let file3 = dir3.join("file3.txt");
-    /// File::create(&file11).unwrap();
-    /// File::create(&file12).unwrap();
-    /// File::create(&file2).unwrap();
-    /// File::create(&file3).unwrap();
-    ///
-    /// let receiver = Mutex::new(Vec::<PathBuf>::new());
-    /// let mut walk = Walk::new();
-    /// walk.run(vec![Path::from(&test_root)], |path| {
-    ///     receiver.lock().unwrap().push(path.to_path_buf())
-    /// });
-    ///
-    /// let mut results = receiver.lock().unwrap();
-    /// results.sort();
-    /// assert_eq!(*results, vec![file11, file12, file3, file2]);
-    ///
-    /// remove_dir_all(&test_root).ok();
-    /// ```
     pub fn run<I, F>(&self, roots: I, consumer: F)
     where
         I: IntoIterator<Item = Path> + Send,
@@ -174,7 +132,7 @@ impl<'a> Walk<'a> {
     {
         if self.path_selector.matches_dir(&path) {
             Entry::from_path(path.clone())
-                .map_err(|e| self.log_err(format!("Failed to stat {}: {}", path.display(), e)))
+                .map_err(|e| self.log_warn(format!("Failed to stat {}: {}", path.display(), e)))
                 .into_iter()
                 .for_each(|entry| self.visit_entry(entry, scope, level, state))
         }
@@ -243,7 +201,7 @@ impl<'a> Walk<'a> {
         if self.follow_links {
             match self.resolve_link(&path) {
                 Ok(target) => self.visit_path(target, scope, level, state),
-                Err(e) => self.log_err(format!("Failed to read link {}: {}", path.display(), e)),
+                Err(e) => self.log_warn(format!("Failed to read link {}: {}", path.display(), e)),
             }
         }
     }
@@ -267,7 +225,7 @@ impl<'a> Walk<'a> {
                         scope.spawn(move |s| self.visit_entry(entry, s, level + 1, state))
                     }
                 }
-                Err(e) => self.log_err(format!("Failed to read dir {}: {}", path.display(), e)),
+                Err(e) => self.log_warn(format!("Failed to read dir {}: {}", path.display(), e)),
             }
         }
     }
@@ -312,9 +270,9 @@ impl<'a> Walk<'a> {
         }
     }
 
-    /// Logs an error
-    fn log_err(&self, msg: String) {
-        self.log.iter().for_each(|l| l.err(&msg))
+    /// Logs a warning
+    fn log_warn(&self, msg: String) {
+        self.log.iter().for_each(|l| l.warn(&msg))
     }
 }
 


Test patch: