Repository: pkolaczk/fclones (language: rust)

Issue:
Compiling to linux ARM
Great work - already had a play with the utility on macos and it works great.

Just feedback, I also tried to compile to manjaro on the rpi4 and the compile failed on the hashing crate. I might ask the author o the fashhash-sys crate what would be required to allow it to compile on the arm architecture.

```
fclones git:(master) cargo build --release
   Compiling fasthash-sys v0.3.2
   Compiling getrandom v0.1.14
   Compiling num_cpus v1.13.0
   Compiling atty v0.2.14
error: failed to run custom build command for `fasthash-sys v0.3.2`

Caused by:
  process didn't exit successfully: `/home/stuart/rust_projects/fclones/target/release/build/fasthash-sys-3bfb9e86593b1584/build-script-build` (exit code: 101)
--- stdout
TARGET = Some("aarch64-unknown-linux-gnu")
OPT_LEVEL = Some("3")
TARGET = Some("aarch64-unknown-linux-gnu")
HOST = Some("aarch64-unknown-linux-gnu")
TARGET = Some("aarch64-unknown-linux-gnu")
TARGET = Some("aarch64-unknown-linux-gnu")
HOST = Some("aarch64-unknown-linux-gnu")
CC_aarch64-unknown-linux-gnu = None
CC_aarch64_unknown_linux_gnu = None
HOST_CC = None
CC = None
HOST = Some("aarch64-unknown-linux-gnu")
TARGET = Some("aarch64-unknown-linux-gnu")
HOST = Some("aarch64-unknown-linux-gnu")
CFLAGS_aarch64-unknown-linux-gnu = None
CFLAGS_aarch64_unknown_linux_gnu = None
HOST_CFLAGS = None
CFLAGS = None
DEBUG = Some("false")
running: "cc" "-O3" "-ffunction-sections" "-fdata-sections" "-fPIC" "-Wno-implicit-fallthrough" "-Wno-unknown-attributes" "-msse4.2" "-maes" "-mavx" "-mavx2" "-DT1HA0_RUNTIME_SELECT=1" "-DT1HA0_AESNI_AVAILABLE=1" "-Wall" "-Wextra" "-o" "/home/stuart/rust_projects/fclones/target/release/build/fasthash-sys-fc57bf495c3381b2/out/src/fasthash.o" "-c" "src/fasthash.cpp"
cargo:warning=cc: error: unrecognized command line option \u2018-msse4.2\u2019
cargo:warning=cc: error: unrecognized command line option \u2018-maes\u2019
cargo:warning=cc: error: unrecognized command line option \u2018-mavx\u2019
cargo:warning=cc: error: unrecognized command line option \u2018-mavx2\u2019
exit code: 1

--- stderr
thread 'main' panicked at '

Internal error occurred: Command "cc" "-O3" "-ffunction-sections" "-fdata-sections" "-fPIC" "-Wno-implicit-fallthrough" "-Wno-unknown-attributes" "-msse4.2" "-maes" "-mavx" "-mavx2" "-DT1HA0_RUNTIME_SELECT=1" "-DT1HA0_AESNI_AVAILABLE=1" "-Wall" "-Wextra" "-o" "/home/stuart/rust_projects/fclones/target/release/build/fasthash-sys-fc57bf495c3381b2/out/src/fasthash.o" "-c" "src/fasthash.cpp" with args "cc" did not execute successfully (status code exit code: 1).

', /home/stuart/.cargo/registry/src/github.com-1ecc6299db9ec823/gcc-0.3.55/src/lib.rs:1672:5
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace

warning: build failed, waiting for other jobs to finish...
error: build failed
```
-- feel free to close this issue - I just wanted to give feedback.
 


Discussion / hints:
Thank you. Before closing, let me try a different hash crate. Maybe metrohash 128 would work? 

Fix patch:
diff --git a/Cargo.toml b/Cargo.toml
index 96c923d..675ad0a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -16,10 +16,10 @@ clap = "2.33.0"
 console = "0.11.2"
 csv = "1.1.3"
 dashmap = "3.11.1"
-fasthash = "0.4.0"
 indicatif = { version = "0.14.0", features = ["with_rayon"] }
 indoc = "0.3.5"
 itertools = "0.9.0"
+metrohash = "1.0.6"
 nom = "5.1.1"
 pcre2 = { git = "https://github.com/pkolaczk/rust-pcre2.git" }
 rayon = "1.3.0"
diff --git a/README.md b/README.md
index 2a8cf75..53aa59b 100644
--- a/README.md
+++ b/README.md
@@ -93,8 +93,9 @@ the previous stage must complete fully before the next one is started.
    omitted. Same as in steps 4 and 5, split groups and remove the ones that are too small.
 7. Write report to the stdout.          
     
-Note that there is no byte-by-byte comparison of files anywhere. A fast but good 128-bit T1HA hash function
-is used and you don't need to worry about hash collisions. At 10^15 files, the probability of collision is
+Note that there is no byte-by-byte comparison of files anywhere. A fast and good 128-bit 
+[MetroHash](http://www.jandrewrogers.com/2015/05/27/metrohash/) hash function
+is used and you don't need to worry about hash collisions. At 10<sup>15</sup> files, the probability of collision is
 0.000000001, without taking into account the requirement for the files to also match by size.
     
 ## Benchmarks
diff --git a/src/files.rs b/src/files.rs
index e519196..dfbb9a3 100644
--- a/src/files.rs
+++ b/src/files.rs
@@ -15,7 +15,7 @@ use std::os::unix::fs::OpenOptionsExt;
 use std::os::unix::io::*;
 
 use bytesize::ByteSize;
-use fasthash::{t1ha2::Hasher128, FastHasher, HasherExt, Murmur3Hasher};
+use metrohash::{MetroHash128, MetroHash64};
 use serde::*;
 use smallvec::alloc::fmt::Formatter;
 use smallvec::alloc::str::FromStr;
@@ -248,7 +248,7 @@ pub struct FileId {
 
 impl FileId {
     pub fn hash(&self) -> u32 {
-        let mut hasher: Murmur3Hasher = Default::default();
+        let mut hasher = MetroHash64::new();
         self.inode.hash(&mut hasher);
         self.device.hash(&mut hasher);
         hasher.finish() as u32
@@ -320,8 +320,9 @@ impl Serialize for FileHash {
 /// There is one such buffer per thread.
 pub const BUF_LEN: usize = 16 * 1024;
 
-fn to_off_t(offset: u64) -> i64 {
-    min(i64::MAX as u64, offset) as i64
+#[cfg(unix)]
+fn to_off_t(offset: u64) -> libc::off_t {
+    min(libc::off_t::MAX as u64, offset) as libc::off_t
 }
 
 /// Wrapper for `posix_fadvise`. Ignores errors.
@@ -484,16 +485,15 @@ pub fn file_hash(
     cache_policy: Caching,
     progress: impl Fn(usize),
 ) -> io::Result<FileHash> {
-    let mut hasher = Hasher128::new();
+    let mut hasher = MetroHash128::new();
     let mut file = open(path, offset, len, cache_policy)?;
     scan(&mut file, len, |buf| {
-        let mut block_hasher = Hasher128::new();
-        block_hasher.write(buf);
-        hasher.write_u128(block_hasher.finish_ext());
+        hasher.write(buf);
         (progress)(buf.len());
     })?;
     evict_page_cache_if_low_mem(&mut file, len);
-    Ok(FileHash(hasher.finish_ext()))
+    let (a, b) = hasher.finish128();
+    Ok(FileHash(((a as u128) << 64) | b as u128))
 }
 
 /// Computes the file hash or logs an error and returns none if failed.
diff --git a/src/group.rs b/src/group.rs
index 1eeef5c..bd492c8 100644
--- a/src/group.rs
+++ b/src/group.rs
@@ -155,11 +155,11 @@ pub trait SplitGroups<G, H> {
     ///
     /// # Example
     /// ```
+    /// use metrohash::MetroHash128;
     /// use rayon::iter::ParallelIterator;
-    /// use fclones::files::{FileLen, FileHash};
-    /// use fasthash::spooky::Hasher128;
-    /// use fasthash::{FastHasher, HasherExt};
     /// use std::hash::Hasher;
+    ///
+    /// use fclones::files::{FileLen, FileHash};
     /// use fclones::group::{FileGroup, SplitGroups};
     /// use fclones::path::Path;
     ///
@@ -169,9 +169,9 @@ pub trait SplitGroups<G, H> {
     /// ]};
     ///
     /// fn hash(len: FileLen, hash: Option<FileHash>, path: &Path) -> Option<FileHash> {
-    ///     let mut hasher = Hasher128::new();
+    ///     let mut hasher = MetroHash128::new();
     ///     hasher.write(path.to_string().as_bytes());
-    ///     Some(FileHash(hasher.finish_ext()))
+    ///     Some(FileHash(hasher.finish() as u128))
     /// }
     ///
     /// let mut groups = vec![g].split(1, hash);
diff --git a/src/path.rs b/src/path.rs
index 1cc2c63..8f8ad0d 100644
--- a/src/path.rs
+++ b/src/path.rs
@@ -1,16 +1,17 @@
+#[cfg(unix)]
+use nix::NixPath;
+use nom::lib::std::fmt::Formatter;
+use serde::{Serialize, Serializer};
+use smallvec::SmallVec;
 use std::ffi::{CStr, CString, OsStr, OsString};
 use std::fmt::Display;
 use std::hash::Hash;
+#[cfg(unix)]
 use std::os::unix::ffi::OsStrExt;
 use std::path::{Component, PathBuf};
 use std::sync::Arc;
 use std::{fmt, io};
 
-use nix::NixPath;
-use nom::lib::std::fmt::Formatter;
-use serde::{Serialize, Serializer};
-use smallvec::SmallVec;
-
 /// Memory-efficient path representation.
 /// When storing multiple paths with common parent, the standard PathBuf would keep
 /// the parent duplicated in memory, wasting a lot of memory.
diff --git a/src/walk.rs b/src/walk.rs
index ec7405d..62586e5 100644
--- a/src/walk.rs
+++ b/src/walk.rs
@@ -5,7 +5,7 @@ use std::io;
 use std::sync::Arc;
 
 use dashmap::DashSet;
-use fasthash::{t1ha2::Hasher128, FastHasher, HasherExt};
+use metrohash::MetroHash128;
 use rayon::Scope;
 
 use crate::log::Log;
@@ -186,9 +186,10 @@ impl<'a> Walk<'a> {
     /// We need 128-bits so that collisions are not a problem.
     /// Thanks to using a long hash we can be sure collisions won't be a problem.
     fn path_hash(path: &Path) -> u128 {
-        let mut h = Hasher128::new();
+        let mut h = MetroHash128::new();
         path.hash(&mut h);
-        h.finish_ext()
+        let (a, b) = h.finish128();
+        (a as u128) << 64 | b as u128
     }
 
     /// Visits a path that was already converted to an `Entry` so the entry type is known.


Test patch: