Repository: pkolaczk/fclones (language: rust)

Issue:
Support JSON input for remove/link
```
$ fclones remove ... < dupes.json
fclones: error: Input error: Not a default fclones report. Formats other than the default one are not supported yet.
```

I found it very useful to process the `fclones group` JSON output with `jq` and would like continue the workflow with `fclones remove`.


Discussion / hints:

Fix patch:
diff --git a/Cargo.lock b/Cargo.lock
index 03ee36d..78ab947 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -60,9 +60,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
 
 [[package]]
 name = "bitflags"
-version = "1.2.1"
+version = "1.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
+checksum = "2da1976d75adbe5fbc88130ecd119529cf1cc6a93ae1546d8696ee66f0d21af1"
 
 [[package]]
 name = "block-utils"
@@ -109,15 +109,15 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
 
 [[package]]
 name = "bytesize"
-version = "1.0.1"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81a18687293a1546b67c246452202bbbf143d239cb43494cc163da14979082da"
+checksum = "6c58ec36aac5066d5ca17df51b3e70279f5670a72102f5752cb7e7c856adfc70"
 
 [[package]]
 name = "cc"
-version = "1.0.68"
+version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787"
+checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2"
 
 [[package]]
 name = "cfg-if"
@@ -242,9 +242,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-deque"
-version = "0.8.0"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
+checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
 dependencies = [
  "cfg-if 1.0.0",
  "crossbeam-epoch",
@@ -334,9 +334,9 @@ dependencies = [
 
 [[package]]
 name = "dunce"
-version = "1.0.1"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2641c4a7c0c4101df53ea572bffdc561c146f6c2eb09e4df02bc4811e3feeb4"
+checksum = "453440c271cf5577fd2a40e4942540cb7d0d2f85e27c8d07dd0023c925a67541"
 
 [[package]]
 name = "either"
@@ -358,7 +358,7 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
 
 [[package]]
 name = "fclones"
-version = "0.12.3"
+version = "0.13.0"
 dependencies = [
  "atomic-counter",
  "block-utils",
@@ -413,9 +413,9 @@ dependencies = [
 
 [[package]]
 name = "filetime"
-version = "0.2.14"
+version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d34cfa13a63ae058bfa601fe9e313bbdb3746427c1459185464ce0fcf62e1e8"
+checksum = "975ccf83d8d9d0d84682850a38c8169027be83368805971cc4f238c2b245bc98"
 dependencies = [
  "cfg-if 1.0.0",
  "libc",
@@ -454,9 +454,9 @@ dependencies = [
 
 [[package]]
 name = "hermit-abi"
-version = "0.1.18"
+version = "0.1.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
 dependencies = [
  "libc",
 ]
@@ -539,9 +539,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.95"
+version = "0.2.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "789da6d93f1b866ffe175afc5322a4d76c038605a1c3319bb57b06967ca98a36"
+checksum = "a7f823d141fe0a24df1e23b4af4e3c7ba9e5966ec514ea068c93024aa7deb765"
 
 [[package]]
 name = "libudev-sys"
@@ -726,9 +726,9 @@ checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
 
 [[package]]
 name = "once_cell"
-version = "1.7.2"
+version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3"
+checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56"
 
 [[package]]
 name = "parse-zoneinfo"
@@ -783,9 +783,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.27"
+version = "1.0.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038"
+checksum = "5c7ed8b8c7b886ea3ed7dde405212185f423ab44682667c8c6dd14aa1d9f6612"
 dependencies = [
  "unicode-xid",
 ]
@@ -801,9 +801,9 @@ dependencies = [
 
 [[package]]
 name = "rand"
-version = "0.8.3"
+version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e"
+checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
 dependencies = [
  "libc",
  "rand_chacha",
@@ -813,9 +813,9 @@ dependencies = [
 
 [[package]]
 name = "rand_chacha"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
 dependencies = [
  "ppv-lite86",
  "rand_core",
@@ -823,18 +823,18 @@ dependencies = [
 
 [[package]]
 name = "rand_core"
-version = "0.6.2"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7"
+checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
 dependencies = [
  "getrandom",
 ]
 
 [[package]]
 name = "rand_hc"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73"
+checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7"
 dependencies = [
  "rand_core",
 ]
@@ -866,9 +866,9 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.2.8"
+version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc"
+checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff"
 dependencies = [
  "bitflags",
 ]
@@ -931,18 +931,18 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
 [[package]]
 name = "serde"
-version = "1.0.126"
+version = "1.0.127"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03"
+checksum = "f03b9878abf6d14e6779d3f24f07b2cfa90352cfec4acc5aab8f1ac7f146fae8"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.126"
+version = "1.0.127"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43"
+checksum = "a024926d3432516606328597e0f224a51355a493b49fdd67e9209187cbe55ecc"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -951,9 +951,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.64"
+version = "1.0.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79"
+checksum = "336b10da19a12ad094b59d870ebde26a45402e5b470add4b5fd03c5048a32127"
 dependencies = [
  "itoa",
  "ryu",
@@ -992,9 +992,9 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
 
 [[package]]
 name = "structopt"
-version = "0.3.21"
+version = "0.3.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5277acd7ee46e63e5168a80734c9f6ee81b1367a7d8772a2d765df2a3705d28c"
+checksum = "69b041cdcb67226aca307e6e7be44c8806423d83e018bd662360a93dabce4d71"
 dependencies = [
  "clap",
  "lazy_static",
@@ -1003,9 +1003,9 @@ dependencies = [
 
 [[package]]
 name = "structopt-derive"
-version = "0.4.14"
+version = "0.4.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90"
+checksum = "7813934aecf5f51a54775e00068c237de98489463968231a51746bbbc03f9c10"
 dependencies = [
  "heck",
  "proc-macro-error",
@@ -1034,9 +1034,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "1.0.72"
+version = "1.0.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82"
+checksum = "1873d832550d4588c3dbc20f01361ab00bfe741048f71e3fecf145a7cc18b29c"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1104,18 +1104,18 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.25"
+version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa6f76457f59514c7eeb4e59d891395fab0b2fd1d40723ae737d64153392e9c6"
+checksum = "93119e4feac1cbe6c798c34d3a53ea0026b0b1de6a120deef895137c0529bfe2"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.25"
+version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a36768c0fbf1bb15eca10defa29526bda730a2376c2ab4393ccfa16fb1a318d"
+checksum = "060d69a0afe7796bf42e9e2ff91f5ee691fb15c53d38b4b62a9a53eb23164745"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1163,9 +1163,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-segmentation"
-version = "1.7.1"
+version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796"
+checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"
 
 [[package]]
 name = "unicode-width"
diff --git a/Cargo.toml b/Cargo.toml
index d1cb037..ebddb07 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "fclones"
-version = "0.12.3"
+version = "0.13.0"
 description = "Finds duplicate, unique, under- or over-replicated files"
 authors = ["Piotr Kołaczkowski <pkolaczk@gmail.com>"]
 repository = "https://github.com/pkolaczk/fclones"
diff --git a/README.md b/README.md
index bc71c3b..f85fd14 100644
--- a/README.md
+++ b/README.md
@@ -184,8 +184,10 @@ Exclude a part of the directory tree from the scan:
     fclones group / --exclude '/dev/**' '/proc/**'    
 
 ### Removing Files
-To remove files or replace them by links, you need to send the default report produced by
+To remove files or replace them by links, you need to send the report produced by
 `fclones group` to the standard input of `fclones remove` or `fclones link` command.
+The report format is detected automatically. Currently, `default` and `json` report 
+formats are supported. 
 
 Assuming the list of duplicates has been saved in file `dupes.txt`, the following commands would remove
 the redundant files: 
diff --git a/src/config.rs b/src/config.rs
index af5edc6..c2004d6 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -2,6 +2,7 @@
 
 use std::collections::HashMap;
 use std::ffi::OsString;
+use std::fmt::{Display, Formatter};
 use std::io;
 use std::io::{stdin, BufRead, BufReader};
 use std::path::PathBuf;
@@ -31,6 +32,17 @@ impl OutputFormat {
     }
 }
 
+impl Display for OutputFormat {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        match self {
+            OutputFormat::Default => f.pad("default"),
+            OutputFormat::Fdupes => f.pad("fdupes"),
+            OutputFormat::Csv => f.pad("csv"),
+            OutputFormat::Json => f.pad("json"),
+        }
+    }
+}
+
 impl FromStr for OutputFormat {
     type Err = String;
 
diff --git a/src/dedupe.rs b/src/dedupe.rs
index 3ee4171..9cffcc7 100644
--- a/src/dedupe.rs
+++ b/src/dedupe.rs
@@ -89,7 +89,7 @@ pub enum FsCommand {
 
 impl FsCommand {
     fn remove(path: &Path) -> io::Result<()> {
-        let _ = FileLock::new(&path)?;
+        let _ = FileLock::new(path)?;
         fs::remove_file(path.to_path_buf())
             .map_err(|e| io::Error::new(e.kind(), format!("Failed to remove file {}: {}", path, e)))
     }
@@ -174,14 +174,14 @@ impl FsCommand {
     /// If the function fails, moves the file back to the original location.
     /// If the function succeeds, removes the file permanently.
     fn safe_remove<R>(path: &Path, f: impl Fn(&Path) -> io::Result<R>, log: &Log) -> io::Result<R> {
-        let _ = FileLock::new(&path)?; // don't remove a locked file
+        let _ = FileLock::new(path)?; // don't remove a locked file
         let tmp = Self::temp_file(path);
-        Self::rename(&path, &tmp)?;
-        let result = match f(&path) {
+        Self::rename(path, &tmp)?;
+        let result = match f(path) {
             Ok(result) => result,
             Err(e) => {
                 // Try to undo the move if possible
-                if let Err(remove_err) = Self::rename(&tmp, &path) {
+                if let Err(remove_err) = Self::rename(&tmp, path) {
                     log.warn(format!(
                         "Failed to undo move from {} to {}: {}",
                         &path, &tmp, remove_err
@@ -616,7 +616,7 @@ where
 {
     groups
         .into_par_iter()
-        .flat_map(move |group| match partition(group, &config, log) {
+        .flat_map(move |group| match partition(group, config, log) {
             Ok(group) => group.dedupe_script(op),
             Err(e) => {
                 log.warn(e);
diff --git a/src/files.rs b/src/files.rs
index 9b35352..9a32925 100644
--- a/src/files.rs
+++ b/src/files.rs
@@ -93,7 +93,9 @@ impl Sub<FileLen> for FilePos {
 
 /// Represents length of data, in bytes.
 /// Provides more type safety and nicer formatting over using a raw u64.
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Serialize, Default)]
+#[derive(
+    Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Deserialize, Serialize, Default,
+)]
 pub struct FileLen(pub u64);
 
 impl FileLen {
@@ -257,7 +259,7 @@ impl FileInfo {
 
     #[cfg(target_os = "linux")]
     pub fn fetch_physical_location(&mut self) -> io::Result<u64> {
-        let new_location = get_physical_file_location(&self.path())?;
+        let new_location = get_physical_file_location(self.path())?;
         if let Some(new_location) = new_location {
             self.location = self.location & DEVICE_MASK | (new_location >> 8) & OFFSET_MASK;
         }
@@ -349,7 +351,7 @@ impl FileId {
 }
 
 pub(crate) fn file_id_or_log_err(file: &Path, log: &Log) -> Option<FileId> {
-    match FileId::new(&file) {
+    match FileId::new(file) {
         Ok(id) => Some(id),
         Err(e) if e.kind() == ErrorKind::NotFound => None,
         Err(e) => {
@@ -411,6 +413,17 @@ impl Serialize for FileHash {
     }
 }
 
+impl<'de> Deserialize<'de> for FileHash {
+    fn deserialize<D>(deserializer: D) -> Result<FileHash, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let s = String::deserialize(deserializer)?;
+        let hash_value = u128::from_str_radix(s.as_str(), 16).map_err(serde::de::Error::custom)?;
+        Ok(FileHash(hash_value))
+    }
+}
+
 #[cfg(unix)]
 fn to_off_t(offset: u64) -> libc::off_t {
     min(libc::off_t::MAX as u64, offset) as libc::off_t
@@ -475,7 +488,7 @@ fn evict_page_cache_if_low_mem(file: &mut File, len: FileLen) {
             let free_ratio = free_mem as f32 / total_mem as f32;
             if free_ratio < 0.05 {
                 evict_page_cache(
-                    &file,
+                    file,
                     FilePos::zero() + skipped_prefix_len,
                     len - skipped_prefix_len,
                 );
@@ -499,7 +512,7 @@ pub(crate) enum Caching {
 /// Opens a file and positions it at the given offset.
 /// Additionally, sends the advice to the operating system about how many bytes will be read.
 fn open(path: &Path, offset: FilePos, len: FileLen, cache_policy: Caching) -> io::Result<File> {
-    let mut file = open_noatime(&path)?;
+    let mut file = open_noatime(path)?;
     configure_readahead(&file, offset, len, cache_policy);
     if offset > FilePos::zero() {
         file.seek(offset.into())?;
@@ -628,14 +641,14 @@ pub(crate) fn file_hash_or_log_err(
 
 #[cfg(test)]
 mod test {
-
-    use super::*;
-
-    use crate::path::Path;
     use std::fs::{create_dir_all, File};
     use std::io::Write;
     use std::path::PathBuf;
 
+    use crate::path::Path;
+
+    use super::*;
+
     #[test]
     fn test_format_bytes() {
         let file_len = FileLen(16000);
diff --git a/src/lib.rs b/src/lib.rs
index 934bf3f..51462dc 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -151,7 +151,7 @@ impl<'a> AppCtx<'a> {
 }
 
 /// A group of files that have something in common, e.g. same size or same hash
-#[derive(Serialize, Debug, PartialEq, Eq, Clone)]
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
 pub struct FileGroup<F> {
     /// Length of each file
     pub file_len: FileLen,
@@ -283,7 +283,7 @@ where
         groups.into_iter().partition(group_pre_filter);
 
     // This way we can split fclonesing to separate thread-pools, one per device:
-    let files = partition_by_devices(groups_to_fclones, &devices);
+    let files = partition_by_devices(groups_to_fclones, devices);
     let mut hash_map =
         GroupMap::new(|f: HashedFileInfo| ((f.file_info.len, f.file_hash), f.file_info));
     let hash_map_ref = &mut hash_map;
@@ -391,10 +391,10 @@ fn scan_files(ctx: &AppCtx<'_>) -> Vec<Vec<FileInfo>> {
     walk.skip_hidden = config.skip_hidden;
     walk.follow_links = config.follow_links;
     walk.path_selector = ctx.path_selector.clone();
-    walk.log = Some(&ctx.log);
+    walk.log = Some(ctx.log);
     walk.on_visit = spinner_tick;
     walk.run(ctx.config.input_paths(), |path| {
-        file_info_or_log_err(path, &ctx.devices, &ctx.log)
+        file_info_or_log_err(path, &ctx.devices, ctx.log)
             .into_iter()
             .filter(|info| {
                 let l = info.len;
@@ -480,7 +480,7 @@ where
                 file_group
                     .into_iter()
                     .inspect(|p| progress(&p.path))
-                    .unique_by(|p| file_id_or_log_err(&p.path, &ctx.log)),
+                    .unique_by(|p| file_id_or_log_err(&p.path, ctx.log)),
             )
         }
     }
@@ -586,7 +586,7 @@ fn group_transformed(
         AccessType::Sequential,
         |(fi, _)| {
             let result = transform
-                .run_or_log_err(&fi.path, &ctx.log)
+                .run_or_log_err(&fi.path, ctx.log)
                 .map(|(len, hash)| {
                     fi.len = len;
                     hash
@@ -669,7 +669,7 @@ fn group_by_prefix(
                 buf_len,
                 caching,
                 |_| {},
-                &ctx.log,
+                ctx.log,
             )
         },
     );
@@ -729,7 +729,7 @@ fn group_by_suffix(ctx: &AppCtx<'_>, groups: Vec<FileGroup<FileInfo>>) -> Vec<Fi
                 buf_len,
                 Caching::Default,
                 |_| {},
-                &ctx.log,
+                ctx.log,
             )
             .map(|new_hash| old_hash ^ new_hash)
         },
@@ -774,7 +774,7 @@ fn group_by_contents(
                 buf_len,
                 Caching::Sequential,
                 |delta| progress.inc(delta),
-                &ctx.log,
+                ctx.log,
             )
         },
     );
@@ -918,13 +918,13 @@ pub fn write_report(config: &GroupConfig, log: &Log, groups: &[FileGroup<Path>])
             let iter = groups.iter().inspect(|_g| progress.tick());
             let file = BufWriter::new(File::create(path)?);
             let mut reporter = ReportWriter::new(file, false);
-            reporter.write(&config.format, &header, iter)
+            reporter.write(config.format, &header, iter)
         }
         None => {
             let term = Term::stdout();
             let color = term.is_term();
             let mut reporter = ReportWriter::new(BufWriter::new(term), color);
-            reporter.write(&config.format, &header, groups.iter())
+            reporter.write(config.format, &header, groups.iter())
         }
     }
 }
diff --git a/src/main.rs b/src/main.rs
index afef538..97bf1d0 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -13,7 +13,7 @@ use structopt::StructOpt;
 
 use fclones::config::{Command, Config, DedupeConfig, GroupConfig, Parallelism};
 use fclones::log::Log;
-use fclones::report::TextReportReader;
+use fclones::report::open_report;
 use fclones::{dedupe, log_script, run_script, DedupeOp};
 use fclones::{group_files, write_report, Error};
 
@@ -100,9 +100,9 @@ fn run_group(mut config: GroupConfig, log: &mut Log) -> Result<(), Error> {
     }
 
     log.info("Started grouping");
-    let results = group_files(&config, &log).map_err(|e| Error::new(e.message))?;
+    let results = group_files(&config, log).map_err(|e| Error::new(e.message))?;
 
-    write_report(&config, &log, &results)
+    write_report(&config, log, &results)
         .map_err(|e| Error::new(format!("Failed to write report: {}", e)))
 }
 
@@ -121,10 +121,11 @@ fn get_output_writer(config: &DedupeConfig) -> Result<Box<dyn Write + Send>, Err
 }
 
 pub fn run_dedupe(op: DedupeOp, config: DedupeConfig, log: &mut Log) -> Result<(), Error> {
+    let input_error = |e: io::Error| format!("Input error: {}", e);
     let mut dedupe_config = config;
-    let reader = TextReportReader::new(stdin()).map_err(|e| format!("Input error: {}", e))?;
-
-    let find_config: Config = Config::from_iter_safe(&reader.header.command).map_err(|e| {
+    let mut reader = open_report(stdin()).map_err(input_error)?;
+    let header = reader.read_header().map_err(input_error)?;
+    let find_config: Config = Config::from_iter_safe(&header.command).map_err(|e| {
         let message: String = extract_error_cause(&e.message);
         format!("Unrecognized earlier fclones configuration: {}", message)
     })?;
@@ -141,7 +142,7 @@ pub fn run_dedupe(op: DedupeOp, config: DedupeConfig, log: &mut Log) -> Result<(
 
     dedupe_config.rf_over = Some(rf_over);
     if dedupe_config.modified_before.is_none() {
-        dedupe_config.modified_before = Some(reader.header.timestamp);
+        dedupe_config.modified_before = Some(header.timestamp);
     }
 
     if dedupe_config.dry_run {
@@ -151,15 +152,17 @@ pub fn run_dedupe(op: DedupeOp, config: DedupeConfig, log: &mut Log) -> Result<(
     }
 
     let mut result: Result<(), io::Error> = Ok(());
-    let group_count = reader.header.stats.map(|s| s.group_count as u64);
+    let group_count = header.stats.map(|s| s.group_count as u64);
     let progress = match group_count {
         _ if dedupe_config.dry_run && dedupe_config.output.is_none() => log.hidden(),
         Some(group_count) => log.progress_bar("Deduplicating", group_count),
         None => log.spinner("Deduplicating"),
     };
 
-    let groups = reader
-        .groups
+    let groups = reader.read_groups();
+
+    let groups = groups
+        .map_err(input_error)?
         .iterator()
         .map(|g| match g {
             Ok(g) => Some(g),
diff --git a/src/path.rs b/src/path.rs
index b3fa657..d599994 100644
--- a/src/path.rs
+++ b/src/path.rs
@@ -159,7 +159,7 @@ impl Path {
     }
 
     pub fn display(&self) -> &Self {
-        &self
+        self
     }
 
     /// Returns a hash of the full path. Useful for deduplicating paths without making path clones.
@@ -270,7 +270,7 @@ impl Path {
 
 impl AsRef<Path> for Path {
     fn as_ref(&self) -> &Path {
-        &self
+        self
     }
 }
 
diff --git a/src/report.rs b/src/report.rs
index 96d687a..365f9e9 100644
--- a/src/report.rs
+++ b/src/report.rs
@@ -1,31 +1,29 @@
 //! Output formatting.
 
+use std::borrow::Borrow;
+use std::cell::Cell;
+use std::cmp::min;
+use std::fmt::Display;
 use std::io;
 use std::io::{BufRead, BufReader, Error, ErrorKind, Read, Write};
 
+use chrono::{DateTime, FixedOffset};
 use console::style;
+use fallible_iterator::FallibleIterator;
+use itertools::Itertools;
 use lazy_static::lazy_static;
 use regex::Regex;
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 
 use crate::config::OutputFormat;
 use crate::files::{FileHash, FileLen};
-
+use crate::path::Path;
 use crate::util::IteratorWrapper;
 use crate::{FileGroup, TIMESTAMP_FMT};
-use chrono::{DateTime, FixedOffset};
-use fallible_iterator::FallibleIterator;
-
-use std::borrow::Borrow;
-use std::cell::Cell;
-
-use crate::path::Path;
-use std::cmp::min;
-use std::fmt::Display;
 
 /// Describes how many redundant files were found, in how many groups,
 /// how much space can be reclaimed, etc.
-#[derive(Debug, Eq, PartialEq, Serialize)]
+#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
 pub struct FileStats {
     pub group_count: usize,
     pub redundant_file_count: usize,
@@ -33,7 +31,7 @@ pub struct FileStats {
 }
 
 /// Data in the header of the whole report.
-#[derive(Debug, Eq, PartialEq, Serialize)]
+#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
 pub struct ReportHeader {
     /// The program version that produced the report
     pub version: String,
@@ -46,6 +44,25 @@ pub struct ReportHeader {
     pub stats: Option<FileStats>,
 }
 
+/// A helper struct that allows to serialize the report with serde.
+/// Together with `IteratorWrapper` used as `groups` it allows to serialize
+/// a report in a streaming way, without the need to keep all groups in memory at once.
+#[derive(Serialize)]
+struct SerializableReport<'a, G: Serialize> {
+    header: &'a ReportHeader,
+    groups: G,
+}
+
+/// A structure for holding contents of the report after fully deserializing the report.
+/// Used only by report readers that deserialize the whole report at once.
+/// Paths are represented as strings, because strings are more memory efficient than Path here,
+/// because we can't do prefix compression that `Path` was designed for.
+#[derive(Deserialize)]
+struct DeserializedReport {
+    header: ReportHeader,
+    groups: Vec<FileGroup<String>>,
+}
+
 /// Formats and writes duplicate files report to a stream.
 /// Supports many formats: text, csv, json, etc.
 pub struct ReportWriter<W: Write> {
@@ -242,13 +259,7 @@ impl<W: Write> ReportWriter<W> {
         I: IntoIterator<Item = P>,
         P: Serialize,
     {
-        #[derive(Serialize)]
-        struct Report<'a, G: Serialize> {
-            header: &'a ReportHeader,
-            groups: G,
-        }
-
-        let report = Report {
+        let report = SerializableReport {
             header,
             groups: IteratorWrapper(Cell::new(Some(groups))),
         };
@@ -260,7 +271,7 @@ impl<W: Write> ReportWriter<W> {
     /// Writes the report in the format given by `format` parameter.
     pub fn write<I, G, P>(
         &mut self,
-        format: &OutputFormat,
+        format: OutputFormat,
         header: &ReportHeader,
         groups: I,
     ) -> io::Result<()>
@@ -278,10 +289,24 @@ impl<W: Write> ReportWriter<W> {
     }
 }
 
+/// Iterator over groups of files, read form the report
+pub type GroupIterator = dyn FallibleIterator<Item = FileGroup<Path>, Error = io::Error> + Send;
+
+/// Reads a report from a stream.
+pub trait ReportReader {
+    /// Reads the header. Must be called exactly once before reading the groups.
+    /// Reports an io::Error with ErrorKind::InvalidData
+    /// if the report header is malformed.
+    fn read_header(&mut self) -> io::Result<ReportHeader>;
+
+    /// Opens an iterator over groups.
+    fn read_groups(self: Box<Self>) -> io::Result<Box<GroupIterator>>;
+}
+
 /// Iterates the contents of the report.
 /// Each emitted item is a group of duplicate files.
-pub struct TextReportIterator<R: Read> {
-    stream: BufReader<R>,
+pub struct TextReportIterator<R: BufRead> {
+    stream: R,
     line_buf: String,
     stopped_on_error: bool,
 }
@@ -296,9 +321,9 @@ struct GroupHeader {
 
 impl<R> TextReportIterator<R>
 where
-    R: Read,
+    R: BufRead,
 {
-    fn new(input: BufReader<R>) -> TextReportIterator<R> {
+    fn new(input: R) -> TextReportIterator<R> {
         TextReportIterator {
             stream: input,
             line_buf: String::new(),
@@ -332,7 +357,7 @@ where
                 Regex::new(r"^([a-f0-9]{32}), ([0-9]+) B [^*]* \* ([0-9]+):").unwrap();
         }
 
-        let captures = GROUP_HEADER_RE.captures(&header_str).ok_or_else(|| {
+        let captures = GROUP_HEADER_RE.captures(header_str).ok_or_else(|| {
             Error::new(
                 ErrorKind::InvalidData,
                 format!("Malformed group header: {}", header_str),
@@ -372,7 +397,7 @@ where
     }
 }
 
-impl<R: Read> FallibleIterator for TextReportIterator<R> {
+impl<R: BufRead + 'static> FallibleIterator for TextReportIterator<R> {
     type Item = FileGroup<Path>;
     type Error = std::io::Error;
 
@@ -403,24 +428,24 @@ impl<R: Read> FallibleIterator for TextReportIterator<R> {
 /// Currently supports only the default text report format.
 /// Does not load the whole report into memory.
 /// Allows iterating over groups of files.
-pub struct TextReportReader<R: Read> {
-    pub header: ReportHeader,
-    pub groups: TextReportIterator<R>,
+pub struct TextReportReader<R: BufRead> {
+    pub stream: R,
 }
 
-impl<R: Read> TextReportReader<R> {
-    fn read_line(stream: &mut impl BufRead) -> io::Result<String> {
+impl<R: BufRead> TextReportReader<R> {
+    /// Creates a new reader for reading from the given stream
+    pub fn new(stream: R) -> TextReportReader<R> {
+        TextReportReader { stream }
+    }
+
+    fn read_line(&mut self) -> io::Result<String> {
         let mut line_buf = String::new();
-        stream.read_line(&mut line_buf)?;
+        self.stream.read_line(&mut line_buf)?;
         Ok(line_buf)
     }
 
-    fn read_extract(
-        stream: &mut impl BufRead,
-        regex: &Regex,
-        msg: &str,
-    ) -> io::Result<Vec<String>> {
-        let line = Self::read_line(stream)?;
+    fn read_extract(&mut self, regex: &Regex, msg: &str) -> io::Result<Vec<String>> {
+        let line = self.read_line()?;
         Ok(regex
             .captures(line.trim())
             .ok_or_else(|| Error::new(ErrorKind::InvalidData, msg.to_owned()))?
@@ -429,14 +454,10 @@ impl<R: Read> TextReportReader<R> {
             .map(|c| c.unwrap().as_str().to_owned())
             .collect())
     }
+}
 
-    /// Creates a new reader for reading from the given stream and
-    /// decodes the report header.
-    /// Reports an io::Error with ErrorKind::InvalidData
-    /// if the report header is malformed.
-    pub fn new(input_stream: R) -> io::Result<TextReportReader<R>> {
-        let mut stream = BufReader::new(input_stream);
-
+impl<R: BufRead + Send + 'static> ReportReader for TextReportReader<R> {
+    fn read_header(&mut self) -> io::Result<ReportHeader> {
         lazy_static! {
             static ref VERSION_RE: Regex =
                 Regex::new(r"^# Report by fclones ([0-9]+\.[0-9]+\.[0-9]+)").unwrap();
@@ -449,54 +470,38 @@ impl<R: Read> TextReportReader<R> {
                     .unwrap();
         }
 
-        let version = Self::read_extract(
-            &mut stream,
-            &VERSION_RE,
-            "Not a default fclones report. \
-            Formats other than the default one are not supported yet.",
-        )?
-        .swap_remove(0);
-        let timestamp = Self::read_extract(
-            &mut stream,
-            &TIMESTAMP_RE,
-            "Malformed header: Missing timestamp",
-        )?
-        .swap_remove(0);
+        let version = self
+            .read_extract(&VERSION_RE, "Malformed header: Missing fclones version")?
+            .swap_remove(0);
+        let timestamp = self
+            .read_extract(&TIMESTAMP_RE, "Malformed header: Missing timestamp")?
+            .swap_remove(0);
         let timestamp = DateTime::parse_from_str(&timestamp, TIMESTAMP_FMT).map_err(|e| {
             Error::new(
                 ErrorKind::InvalidData,
                 format!("Malformed header: Failed to parse timestamp: {}", e),
             )
         })?;
-        let command = Self::read_extract(
-            &mut stream,
-            &COMMAND_RE,
-            "Malformed header: Missing command",
-        )?
-        .swap_remove(0);
+        let command = self
+            .read_extract(&COMMAND_RE, "Malformed header: Missing command")?
+            .swap_remove(0);
         let command = shell_words::split(&command).map_err(|e| {
             Error::new(
                 ErrorKind::InvalidData,
                 format!("Malformed header: Failed to parse command arguments: {}", e),
             )
         })?;
-        let group_count = Self::read_extract(
-            &mut stream,
-            &GROUP_COUNT_RE,
-            "Malformed header: Missing group count",
-        )?
-        .swap_remove(0);
+        let group_count = self
+            .read_extract(&GROUP_COUNT_RE, "Malformed header: Missing group count")?
+            .swap_remove(0);
         let group_count: usize = group_count.parse().map_err(|e| {
             Error::new(
                 ErrorKind::InvalidData,
                 format!("Malformed header: Failed to parse group count: {}", e),
             )
         })?;
-        let stats_line = Self::read_extract(
-            &mut stream,
-            &STATS_RE,
-            "Malformed header: Missing file statistics line",
-        )?;
+        let stats_line =
+            self.read_extract(&STATS_RE, "Malformed header: Missing file statistics line")?;
         let redundant_file_size = FileLen(stats_line[0].parse().map_err(|e| {
             Error::new(
                 ErrorKind::InvalidData,
@@ -516,7 +521,7 @@ impl<R: Read> TextReportReader<R> {
             )
         })?;
 
-        let header = ReportHeader {
+        Ok(ReportHeader {
             version,
             timestamp,
             command,
@@ -525,28 +530,90 @@ impl<R: Read> TextReportReader<R> {
                 redundant_file_count,
                 redundant_file_size,
             }),
-        };
-        Ok(TextReportReader {
-            header,
-            groups: TextReportIterator::new(stream),
         })
     }
+
+    fn read_groups(
+        self: Box<Self>,
+    ) -> io::Result<Box<dyn FallibleIterator<Item = FileGroup<Path>, Error = Error> + Send>> {
+        Ok(Box::new(TextReportIterator::new(self.stream)))
+    }
+}
+
+/// Reads a report from a JSON file.
+/// Currently it is not very memory efficient, because limited to reading the whole file and
+/// deserializing all data into memory.
+pub struct JsonReportReader {
+    report: DeserializedReport,
+}
+
+impl JsonReportReader {
+    pub fn new<R: Read>(stream: R) -> io::Result<JsonReportReader> {
+        let report: DeserializedReport = serde_json::from_reader(stream).map_err(|e| {
+            Error::new(
+                ErrorKind::InvalidData,
+                format!("Failed to deserialize JSON report: {}", e),
+            )
+        })?;
+        Ok(JsonReportReader { report })
+    }
+}
+
+impl ReportReader for JsonReportReader {
+    fn read_header(&mut self) -> io::Result<ReportHeader> {
+        Ok(self.report.header.clone())
+    }
+
+    fn read_groups(self: Box<Self>) -> io::Result<Box<GroupIterator>> {
+        let iter = self.report.groups.into_iter().map(|g| {
+            Ok(FileGroup {
+                file_len: g.file_len,
+                file_hash: g.file_hash,
+                files: g.files.iter().map(|s| Path::from(s.as_str())).collect_vec(),
+            })
+        });
+        let iter = fallible_iterator::convert(iter);
+        Ok(Box::new(iter))
+    }
+}
+
+/// Returns a `ReportReader` that can read and decode the report from the given stream.
+/// Automatically detects the type of the report.
+pub fn open_report(r: impl Read + Send + 'static) -> io::Result<Box<dyn ReportReader>> {
+    let mut buf_reader = BufReader::with_capacity(16 * 1024, r);
+    let preview = buf_reader.fill_buf()?;
+    let preview = String::from_utf8_lossy(preview);
+    if preview.starts_with('{') {
+        Ok(Box::new(JsonReportReader::new(buf_reader)?))
+    } else if preview.starts_with('#') {
+        Ok(Box::new(TextReportReader::new(buf_reader)))
+    } else {
+        Err(io::Error::new(
+            ErrorKind::InvalidData,
+            format!(
+                "Unknown report format. Supported formats are: {}, {}",
+                OutputFormat::Default,
+                OutputFormat::Json
+            ),
+        ))
+    }
 }
 
 #[cfg(test)]
 mod test {
-    use super::*;
+    use tempfile::NamedTempFile;
+
     use crate::files::{FileHash, FileLen};
     use crate::path::Path;
 
-    use chrono::Utc;
-    use tempfile::NamedTempFile;
+    use super::*;
 
     fn dummy_report_header() -> ReportHeader {
         ReportHeader {
             command: vec!["fclones".to_owned(), "find".to_owned(), ".".to_owned()],
             version: env!("CARGO_PKG_VERSION").to_owned(),
-            timestamp: DateTime::from(Utc::now()),
+            timestamp: DateTime::parse_from_str("2021-08-27 12:11:23.456 +0000", TIMESTAMP_FMT)
+                .unwrap(),
             stats: Some(FileStats {
                 group_count: 4,
                 redundant_file_count: 234,
@@ -557,23 +624,21 @@ mod test {
 
     #[test]
     fn test_text_report_reader_reads_header() {
-        let header = dummy_report_header();
+        let header1 = dummy_report_header();
         let groups: Vec<FileGroup<Path>> = vec![];
 
         let output = NamedTempFile::new().unwrap();
         let input = output.reopen().unwrap();
 
         let mut writer = ReportWriter::new(output, false);
-        writer.write_as_text(&header, groups.into_iter()).unwrap();
-
-        let reader = TextReportReader::new(input).unwrap();
-        assert_eq!(reader.header.version, header.version);
-        assert_eq!(reader.header.command, header.command);
-        assert_eq!(
-            reader.header.timestamp.timestamp(),
-            header.timestamp.timestamp()
-        );
-        assert_eq!(reader.header.stats, header.stats);
+        writer.write_as_text(&header1, groups.into_iter()).unwrap();
+
+        let mut reader = TextReportReader::new(BufReader::new(input));
+        let header2 = reader.read_header().unwrap();
+        assert_eq!(header2.version, header1.version);
+        assert_eq!(header2.command, header1.command);
+        assert_eq!(header2.timestamp.timestamp(), header1.timestamp.timestamp());
+        assert_eq!(header2.stats, header1.stats);
     }
 
     #[test]
@@ -597,9 +662,10 @@ mod test {
 
         let mut writer = ReportWriter::new(output, false);
         writer.write_as_text(&header, groups.iter()).unwrap();
-        let reader = TextReportReader::new(input).unwrap();
+        let mut reader = Box::new(TextReportReader::new(BufReader::new(input)));
+        reader.read_header().unwrap();
 
-        let groups2: Vec<_> = reader.groups.collect().unwrap();
+        let groups2: Vec<_> = reader.read_groups().unwrap().collect().unwrap();
         assert_eq!(groups, groups2);
     }
 
@@ -649,4 +715,70 @@ mod test {
         assert!(g.files.contains(&Path::from("/file3")));
         assert!(g.files.contains(&Path::from("/file4")));
     }
+
+    #[test]
+    fn test_json_report_header() {
+        let header1 = dummy_report_header();
+        let groups: Vec<FileGroup<Path>> = vec![];
+
+        let output = NamedTempFile::new().unwrap();
+        let input = output.reopen().unwrap();
+
+        let mut writer = ReportWriter::new(output, false);
+        writer.write_as_json(&header1, groups.into_iter()).unwrap();
+
+        let mut reader = JsonReportReader::new(input).unwrap();
+        let header2 = reader.read_header().unwrap();
+        assert_eq!(header2.version, header1.version);
+        assert_eq!(header2.command, header1.command);
+        assert_eq!(header2.timestamp.timestamp(), header1.timestamp.timestamp());
+        assert_eq!(header2.stats, header1.stats);
+    }
+
+    #[test]
+    fn test_json_report_reader_reads_files() {
+        let header = dummy_report_header();
+        let groups = vec![
+            FileGroup {
+                file_len: FileLen(100),
+                file_hash: FileHash(0x00112233445566778899aabbccddeeff),
+                files: vec![Path::from("a"), Path::from("b")],
+            },
+            FileGroup {
+                file_len: FileLen(40),
+                file_hash: FileHash(0x0000000000000555555555ffffffffff),
+                files: vec![Path::from("c"), Path::from("d")],
+            },
+        ];
+
+        let output = NamedTempFile::new().unwrap();
+        let input = output.reopen().unwrap();
+
+        let mut writer = ReportWriter::new(output, false);
+        writer.write_as_json(&header, groups.iter()).unwrap();
+        let mut reader = Box::new(JsonReportReader::new(input).unwrap());
+        reader.read_header().unwrap();
+
+        let groups2: Vec<_> = reader.read_groups().unwrap().collect().unwrap();
+        assert_eq!(groups, groups2);
+    }
+
+    fn write_read_header(header: &ReportHeader, format: OutputFormat) -> ReportHeader {
+        let groups: Vec<FileGroup<Path>> = vec![];
+        let output = NamedTempFile::new().unwrap();
+        let input = output.reopen().unwrap();
+        let mut writer = ReportWriter::new(output, false);
+        writer.write(format, header, groups.iter()).unwrap();
+        let mut reader = open_report(input).unwrap();
+        reader.read_header().unwrap()
+    }
+
+    #[test]
+    fn test_format_autodetection() {
+        let header = dummy_report_header();
+        let reread_header_1 = write_read_header(&header, OutputFormat::Default);
+        let reread_header_2 = write_read_header(&header, OutputFormat::Json);
+        assert_eq!(header, reread_header_1);
+        assert_eq!(header, reread_header_2);
+    }
 }
diff --git a/src/transform.rs b/src/transform.rs
index 26c2998..117bad6 100644
--- a/src/transform.rs
+++ b/src/transform.rs
@@ -240,7 +240,7 @@ impl Transform {
 
     /// Processes the input file, and computes the length and hash of the returned output stream
     pub fn run(&self, input: &Path) -> io::Result<Output> {
-        let (args, input_conf, output_conf) = self.make_args(&input);
+        let (args, input_conf, output_conf) = self.make_args(input);
         let mut command = Self::build_command(&args, &input_conf, &output_conf)?;
         let result = Self::execute(&mut command, &output_conf)?;
         input_conf.cleanup()?;
@@ -302,7 +302,7 @@ impl Transform {
 
         if let OutputConf::Named(output) = output_conf {
             command.stdout(Stdio::null());
-            Self::create_named_pipe(&output)?;
+            Self::create_named_pipe(output)?;
         } else {
             command.stdout(Stdio::piped());
         }
diff --git a/src/walk.rs b/src/walk.rs
index 636cb12..f91a326 100644
--- a/src/walk.rs
+++ b/src/walk.rs
@@ -204,7 +204,7 @@ impl<'a> Walk<'a> {
         's: 'w,
     {
         if self.follow_links {
-            match self.resolve_link(&path) {
+            match self.resolve_link(path) {
                 Ok(target) => self.visit_path(target, scope, level, state),
                 Err(e) => self.log_warn(format!("Failed to read link {}: {}", path.display(), e)),
             }


Test patch: