Repository: pkolaczk/fclones (language: rust)

Issue:
fclones group --transform flag output not useable by fclones move 
Running the following command, (which compares RAW files using their embedded thumbnail):
```
fclones group --cache --transform 'dcraw -c -e $IN' --format json -o output.json /mnt/photos
```
results in `fclones move` throwing the following:
```
[2022-05-11 15:25:33.626] fclones:  warn: Skipping file '/mnt/h/Photo Archive/2020/2020-08-19/f51562240.raf' with length 56.1 MB different than the group length 2657671
```
As `dcraw -c` outputs to stdout, my thought is that fclones is saving the temp file's filesize, instead of the original file's filesize to the output json.

This causes a subsequent `fclones move` to throw the warning about the filesize mismatch, preventing deduplication.


Discussion / hints:
Thanks!
I can see two things wrong here: 

1. as you noted, the size check should not be done if --transform was used, or it should be done against the original file size (but it is not saved or kept anywhere, so unlikely to happen soon, I'd have to change the output format)

2. it should not allow to use `--cached` with `--transform`, because caching is not supported in that case (it has no effect)



Fix patch:
diff --git a/src/config.rs b/src/config.rs
index 1261ebe..ecaafb5 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -591,9 +591,19 @@ pub struct DedupeConfig {
     #[structopt(long = "isolate", value_name = "path", parse(from_os_str))]
     pub isolated_roots: Vec<Path>,
 
-    /// Skips locking files before performing an action on them.
+    /// Doesn't lock files before performing an action on them.
     #[structopt(long)]
     pub no_lock: bool,
+
+    /// Allows the size of a file to be different than the size recorded during grouping.
+    ///
+    /// By default, files are checked for size to prevent accidentally removing a file
+    /// that was modified since grouping.
+    /// However, if `--transform` was used when grouping, the data sizes recorded in the `fclones group`
+    /// report likely don't match the on-disk sizes of the files. Therefore,
+    /// this flag is set automatically if `--transform` was used.
+    #[structopt(long)]
+    pub no_check_size: bool,
 }
 
 #[derive(Debug, StructOpt)]
diff --git a/src/dedupe.rs b/src/dedupe.rs
index 0c1d66e..bd5359f 100644
--- a/src/dedupe.rs
+++ b/src/dedupe.rs
@@ -776,18 +776,20 @@ fn partition(
 
     // If file has a different length, then we really know it has been modified.
     // Therefore, it does not belong to the group and we can safely skip it.
-    files.retain(|m| {
-        let len_ok = m.metadata.len() == file_len;
-        if !len_ok {
-            log.warn(format!(
-                "Skipping file {} with length {} different than the group length {}",
-                m.path.display(),
-                m.metadata.len(),
-                file_len.0,
-            ));
-        }
-        len_ok
-    });
+    if !config.no_check_size {
+        files.retain(|m| {
+            let len_ok = m.metadata.len() == file_len;
+            if !len_ok {
+                log.warn(format!(
+                    "Skipping file {} with length {} different than the group length {}",
+                    m.path.display(),
+                    m.metadata.len(),
+                    file_len.0,
+                ));
+            }
+            len_ok
+        });
+    }
 
     // Bail out as well if any file has been modified after `config.modified_before`.
     // We need to skip the whole group, because we don't know if these files are really different.
diff --git a/src/main.rs b/src/main.rs
index f4baa31..ea1205a 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -159,25 +159,27 @@ pub fn run_dedupe(op: DedupeOp, config: DedupeConfig, log: &mut Log) -> Result<(
     let header = reader.read_header().map_err(input_error)?;
     let prev_command_config = get_command_config(&header)?;
 
-    if dedupe_config.rf_over.is_none() {
-        match &prev_command_config.command {
-            Command::Group(c) => dedupe_config.rf_over = Some(c.rf_over()),
-            _ => {
-                return Err(Error::from(
-                    "Could not extract --rf-over setting from the earlier fclones configuration.",
-                ))
-            }
+    if let Command::Group(c) = &prev_command_config.command {
+        // we cannot check size if a transformation was applied, because the transformation
+        // may change the size of the data and the recorded data size
+        // would not match the physical size of the file
+        dedupe_config.no_check_size = c.transform.is_some();
+
+        if dedupe_config.rf_over.is_none() {
+            dedupe_config.rf_over = Some(c.rf_over())
         }
-    };
-
-    if dedupe_config.isolated_roots.is_empty() {
-        if let Command::Group(c) = &prev_command_config.command {
-            if c.isolate {
-                dedupe_config.isolated_roots = c.input_paths().collect();
-            }
+        if dedupe_config.isolated_roots.is_empty() && c.isolate {
+            dedupe_config.isolated_roots = c.input_paths().collect();
         }
     }
 
+    if dedupe_config.rf_over.is_none() {
+        return Err(Error::from(
+            "Could not extract --rf-over setting from the earlier fclones configuration. \
+            Please set --rf-over explicitly.",
+        ));
+    };
+
     if dedupe_config.modified_before.is_none() {
         dedupe_config.modified_before = Some(header.timestamp);
     }


Test patch: