Repository: pkolaczk/fclones (language: rust)

Issue:
test_file_hash_* tests sometimes fail
If you run the `test_file_hash_*` tests repeatedly on a Linux machine with 16 or more threads (fewer untested), you should quickly see test failures:

```
while true; do nice cargo test test_file_hash || break; done
```

```
     Running unittests src/lib.rs (target/debug/deps/fclones-0a0ce4b66c0e4c3b)

running 7 tests
test hasher::test::test_file_hash_metro_128 ... ok
test hasher::test::test_file_hash_sha3_512 ... FAILED
test hasher::test::test_file_hash_blake3 ... FAILED
test hasher::test::test_file_hash_sha3_256 ... FAILED
test hasher::test::test_file_hash_sha256 ... FAILED
test hasher::test::test_file_hash_xxh3 ... FAILED
test hasher::test::test_file_hash_sha512 ... ok

failures:

---- hasher::test::test_file_hash_sha3_512 stdout ----
thread 'hasher::test::test_file_hash_sha3_512' panicked at 'assertion failed: `(left != right)`
  left: `FileHash([166, 159, 115, 204, 162, 58, 154, 197, 200, 181, 103, 220, 24, 90, 117, 110, 151, 201, 130, 22, 79, 226, 88, 89, 224, 209, 220, 193, 71, 92, 128, 166, 21, 178, 18, 58, 241, 245, 249, 76, 17, 227, 233, 64, 44, 58, 197, 88, 245, 0, 25, 157, 149, 182, 211, 227, 1, 117, 133, 134, 40, 29, 205, 38])`,
 right: `FileHash([166, 159, 115, 204, 162, 58, 154, 197, 200, 181, 103, 220, 24, 90, 117, 110, 151, 201, 130, 22, 79, 226, 88, 89, 224, 209, 220, 193, 71, 92, 128, 166, 21, 178, 18, 58, 241, 245, 249, 76, 17, 227, 233, 64, 44, 58, 197, 88, 245, 0, 25, 157, 149, 182, 211, 227, 1, 117, 133, 134, 40, 29, 205, 38])`', src/hasher.rs:670:9
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace

---- hasher::test::test_file_hash_blake3 stdout ----
thread 'hasher::test::test_file_hash_blake3' panicked at 'assertion failed: `(left != right)`
  left: `FileHash([175, 19, 73, 185, 245, 249, 161, 166, 160, 64, 77, 234, 54, 220, 201, 73, 155, 203, 37, 201, 173, 193, 18, 183, 204, 154, 147, 202, 228, 31, 50, 98])`,
 right: `FileHash([175, 19, 73, 185, 245, 249, 161, 166, 160, 64, 77, 234, 54, 220, 201, 73, 155, 203, 37, 201, 173, 193, 18, 183, 204, 154, 147, 202, 228, 31, 50, 98])`', src/hasher.rs:670:9

---- hasher::test::test_file_hash_sha3_256 stdout ----
thread 'hasher::test::test_file_hash_sha3_256' panicked at 'assertion failed: `(left != right)`
  left: `FileHash([167, 255, 198, 248, 191, 30, 215, 102, 81, 193, 71, 86, 160, 97, 214, 98, 245, 128, 255, 77, 228, 59, 73, 250, 130, 216, 10, 75, 128, 248, 67, 74])`,
 right: `FileHash([167, 255, 198, 248, 191, 30, 215, 102, 81, 193, 71, 86, 160, 97, 214, 98, 245, 128, 255, 77, 228, 59, 73, 250, 130, 216, 10, 75, 128, 248, 67, 74])`', src/hasher.rs:671:9

---- hasher::test::test_file_hash_sha256 stdout ----
thread 'hasher::test::test_file_hash_sha256' panicked at 'assertion failed: `(left != right)`
  left: `FileHash([227, 176, 196, 66, 152, 252, 28, 20, 154, 251, 244, 200, 153, 111, 185, 36, 39, 174, 65, 228, 100, 155, 147, 76, 164, 149, 153, 27, 120, 82, 184, 85])`,
 right: `FileHash([227, 176, 196, 66, 152, 252, 28, 20, 154, 251, 244, 200, 153, 111, 185, 36, 39, 174, 65, 228, 100, 155, 147, 76, 164, 149, 153, 27, 120, 82, 184, 85])`', src/hasher.rs:671:9

---- hasher::test::test_file_hash_xxh3 stdout ----
thread 'hasher::test::test_file_hash_xxh3' panicked at 'assertion failed: `(left != right)`
  left: `FileHash([127, 73, 141, 70, 36, 195, 1, 96, 216, 152, 71, 1, 211, 6, 170, 153])`,
 right: `FileHash([127, 73, 141, 70, 36, 195, 1, 96, 216, 152, 71, 1, 211, 6, 170, 153])`', src/hasher.rs:671:9


failures:
    hasher::test::test_file_hash_blake3
    hasher::test::test_file_hash_sha256
    hasher::test::test_file_hash_sha3_256
    hasher::test::test_file_hash_sha3_512
    hasher::test::test_file_hash_xxh3

test result: FAILED. 2 passed; 5 failed; 0 ignored; 0 measured; 152 filtered out; finished in 0.00s

error: test failed, to rerun pass `--lib`
```

I believe this happens because in [test_file_hash](https://github.com/pkolaczk/fclones/blob/f9e3b37800c12e8aade0f97cd864f6485c956d66/src/hasher.rs#L644-L658), several hasher tests run concurrently but use the same, overlapping temporary files. The files are briefly 0 bytes long, and so when multiple tests run concurrently, sometimes both files are 0 bytes and the hashes are unexpectedly equal.

e.g. the `[227, 176, 196, 66, 152, 252, 28, 20, 154, 251, 244, 200, 153, 111, 185, 36, 39, 174, 65, 228, 100, 155, 147, 76, 164, 149, 153, 27, 120, 82, 184, 85]` above equals `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855` which is the SHA256 hash for empty input.



Discussion / hints:

Fix patch:
diff --git a/src/hasher.rs b/src/hasher.rs
index 84d2609..1484ea2 100644
--- a/src/hasher.rs
+++ b/src/hasher.rs
@@ -633,29 +633,19 @@ fn file_hash<H: StreamHasher>(
 #[cfg(test)]
 mod test {
     use metrohash::MetroHash128;
-    use std::fs::{create_dir_all, File};
     use std::io::Write;
-    use std::path::PathBuf;
+    use tempfile::NamedTempFile;
 
     use crate::file::{FileChunk, FileLen, FilePos};
     use crate::hasher::{file_hash, StreamHasher};
     use crate::path::Path;
 
     fn test_file_hash<H: StreamHasher>() {
-        let test_root = PathBuf::from("target/test/file_hash/");
-        create_dir_all(&test_root).unwrap();
-
-        let file1 = test_root.join("file1");
-        File::create(&file1)
-            .unwrap()
-            .write_all(b"Test file 1")
-            .unwrap();
-
-        let file2 = test_root.join("file2");
-        File::create(&file2)
-            .unwrap()
-            .write_all(b"Test file 2")
-            .unwrap();
+        let mut file1 = NamedTempFile::new().unwrap();
+        file1.write_all(b"Test file 1").unwrap();
+
+        let mut file2 = NamedTempFile::new().unwrap();
+        file2.write_all(b"Test file 2").unwrap();
 
         let file1 = Path::from(&file1);
         let file2 = Path::from(&file2);


Test patch: