Repository: wcampbell0x2a/backhand (language: rust)

Issue:
Test out libz-sys
They recently released a version that doesn't require cmake https://github.com/rust-lang/libz-sys/releases/tag/1.1.17


Discussion / hints:

Fix patch:
diff --git a/.github/workflows/binaries.yml b/.github/workflows/binaries.yml
index 098239a3..5384bb20 100644
--- a/.github/workflows/binaries.yml
+++ b/.github/workflows/binaries.yml
@@ -45,8 +45,7 @@ jobs:
           target: ${{ matrix.job.target }}
 
       - run: $BUILD_CMD build -p backhand-cli --bin add-backhand --bin replace-backhand --features xz-static --locked --target ${{ matrix.job.target }} --profile=dist
-      # default features, but replace gzip with gzip-zune-inflate
-      - run: $BUILD_CMD build -p backhand-cli --bin unsquashfs-backhand --locked --target ${{ matrix.job.target }} --profile=dist --no-default-features --features zstd,xz-static,gzip-zune-inflate
+      - run: $BUILD_CMD build -p backhand-cli --bin unsquashfs-backhand --locked --target ${{ matrix.job.target }} --profile=dist --no-default-features --features zstd,xz-static,gzip
       - name: archive
         run: |
           tar -czvf backhand-${{ matrix.job.target }}.tar.gz \
diff --git a/BENCHMARK.md b/BENCHMARK.md
index f3ea16ed..2e793169 100644
--- a/BENCHMARK.md
+++ b/BENCHMARK.md
@@ -59,68 +59,75 @@ NUMA:
 $ ./bench.bash
 ```
 
-## Wall time: `backhand/unsquashfs-3e25c7d` vs `squashfs-tools/unsquashfs-4.6.1`
+## Wall time: `backhand/unsquashfs` vs `squashfs-tools/unsquashfs-4.6.1`
 ### `openwrt-22.03.2-ath79-generic-tplink_archer-a7-v5-squashfs-factory.bin`
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `backhand-dist-v0.17.0` | 50.2 ± 3.1 | 43.7 | 58.3 | 1.02 ± 0.09 |
-| `backhand-dist` | 49.5 ± 3.2 | 44.2 | 57.8 | 1.00 |
-| `backhand-dist-musl` | 78.7 ± 5.1 | 66.0 | 90.2 | 1.59 ± 0.15 |
-| `squashfs-tools` | 111.8 ± 11.9 | 84.2 | 133.1 | 2.26 ± 0.28 |
+| `backhand-dist-v0.18.0` | 50.0 ± 2.7 | 46.4 | 56.3 | 1.02 ± 0.09 |
+| `backhand-dist` | 49.0 ± 3.4 | 42.4 | 56.6 | 1.00 |
+| `backhand-dist-musl` | 77.7 ± 4.7 | 66.7 | 87.9 | 1.59 ± 0.15 |
+| `squashfs-tools` | 108.8 ± 13.0 | 84.7 | 142.8 | 2.22 ± 0.31 |
 ### `openwrt-22.03.2-ipq40xx-generic-netgear_ex6100v2-squashfs-factory.img`
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `backhand-dist-v0.17.0` | 51.0 ± 3.3 | 44.1 | 58.8 | 1.01 ± 0.08 |
-| `backhand-dist` | 50.3 ± 2.6 | 45.1 | 58.9 | 1.00 |
-| `backhand-dist-musl` | 78.0 ± 4.3 | 71.3 | 89.8 | 1.55 ± 0.12 |
-| `squashfs-tools` | 112.9 ± 16.7 | 90.1 | 159.0 | 2.24 ± 0.35 |
+| `backhand-dist-v0.18.0` | 51.0 ± 2.9 | 45.1 | 57.3 | 1.02 ± 0.09 |
+| `backhand-dist` | 50.2 ± 3.1 | 45.9 | 60.7 | 1.00 |
+| `backhand-dist-musl` | 77.0 ± 4.4 | 69.6 | 85.8 | 1.54 ± 0.13 |
+| `squashfs-tools` | 112.2 ± 14.2 | 81.0 | 137.9 | 2.24 ± 0.31 |
 ### `870D97.squashfs`
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `backhand-dist-v0.17.0` | 258.9 ± 12.5 | 228.4 | 275.5 | 1.35 ± 0.11 |
-| `backhand-dist` | 252.8 ± 14.9 | 220.2 | 273.7 | 1.32 ± 0.12 |
-| `backhand-dist-musl` | 429.0 ± 27.9 | 367.4 | 462.5 | 2.24 ± 0.20 |
-| `squashfs-tools` | 191.7 ± 12.4 | 168.9 | 215.0 | 1.00 |
+| `backhand-dist-v0.18.0` | 252.7 ± 15.0 | 217.5 | 271.6 | 1.32 ± 0.10 |
+| `backhand-dist` | 253.3 ± 15.0 | 219.3 | 271.4 | 1.32 ± 0.10 |
+| `backhand-dist-musl` | 423.6 ± 25.7 | 370.5 | 456.1 | 2.21 ± 0.17 |
+| `squashfs-tools` | 191.6 ± 9.5 | 166.1 | 214.2 | 1.00 |
 ### `img-1571203182_vol-ubi_rootfs.ubifs`
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `backhand-dist-v0.17.0` | 226.1 ± 7.9 | 214.7 | 240.6 | 1.00 |
-| `backhand-dist` | 226.4 ± 6.4 | 216.6 | 240.0 | 1.00 ± 0.04 |
-| `backhand-dist-musl` | 360.7 ± 10.8 | 342.2 | 384.5 | 1.60 ± 0.07 |
-| `squashfs-tools` | 288.9 ± 11.9 | 269.6 | 320.4 | 1.28 ± 0.07 |
+| `backhand-dist-v0.18.0` | 225.4 ± 7.0 | 214.0 | 242.2 | 1.00 ± 0.04 |
+| `backhand-dist` | 224.3 ± 6.3 | 214.5 | 246.7 | 1.00 |
+| `backhand-dist-musl` | 360.1 ± 12.9 | 343.9 | 395.1 | 1.61 ± 0.07 |
+| `squashfs-tools` | 285.1 ± 10.3 | 259.5 | 305.5 | 1.27 ± 0.06 |
 ### `2611E3.squashfs`
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `backhand-dist-v0.17.0` | 106.4 ± 4.3 | 97.9 | 117.6 | 1.00 ± 0.06 |
-| `backhand-dist` | 106.0 ± 4.7 | 97.5 | 119.3 | 1.00 |
-| `backhand-dist-musl` | 171.5 ± 6.3 | 151.2 | 185.2 | 1.62 ± 0.09 |
-| `squashfs-tools` | 187.3 ± 15.6 | 160.7 | 223.4 | 1.77 ± 0.17 |
+| `backhand-dist-v0.18.0` | 104.9 ± 4.6 | 96.4 | 120.9 | 1.00 |
+| `backhand-dist` | 104.9 ± 4.0 | 98.1 | 119.2 | 1.00 ± 0.06 |
+| `backhand-dist-musl` | 172.2 ± 7.8 | 152.3 | 191.2 | 1.64 ± 0.10 |
+| `squashfs-tools` | 188.6 ± 14.1 | 159.1 | 217.2 | 1.80 ± 0.16 |
 ### `Plexamp-4.6.1.AppImage`
 | Command | Mean [ms] | Min [ms] | Max [ms] | Relative |
 |:---|---:|---:|---:|---:|
-| `backhand-dist-v0.17.0` | 490.1 ± 6.8 | 477.9 | 506.7 | 2.43 ± 0.10 |
-| `backhand-dist` | 511.3 ± 6.8 | 498.6 | 527.0 | 2.54 ± 0.10 |
-| `backhand-dist-musl` | 870.0 ± 8.2 | 856.6 | 890.6 | 4.32 ± 0.17 |
-| `squashfs-tools` | 201.4 ± 7.7 | 188.2 | 222.9 | 1.00 |
+| `backhand-dist-v0.18.0` | 689.8 ± 7.9 | 675.7 | 705.9 | 3.41 ± 0.15 |
+| `backhand-dist` | 438.0 ± 4.2 | 430.1 | 448.3 | 2.16 ± 0.10 |
+| `backhand-dist-musl` | 519.5 ± 3.2 | 511.8 | 526.1 | 2.57 ± 0.11 |
+| `squashfs-tools` | 202.4 ± 8.9 | 187.0 | 223.5 | 1.00 |
 ### `crates-io.squashfs`
 | Command | Mean [s] | Min [s] | Max [s] | Relative |
 |:---|---:|---:|---:|---:|
-| `backhand-dist-v0.17.0` | 1.423 ± 0.009 | 1.406 | 1.451 | 1.20 ± 0.01 |
-| `backhand-dist` | 1.392 ± 0.009 | 1.375 | 1.411 | 1.18 ± 0.01 |
-| `backhand-dist-musl` | 1.181 ± 0.004 | 1.174 | 1.190 | 1.00 |
-| `squashfs-tools` | 1.819 ± 0.042 | 1.706 | 1.908 | 1.54 ± 0.04 |
+| `backhand-dist-v0.18.0` | 1.395 ± 0.011 | 1.372 | 1.422 | 1.20 ± 0.01 |
+| `backhand-dist` | 1.372 ± 0.008 | 1.358 | 1.387 | 1.18 ± 0.01 |
+| `backhand-dist-musl` | 1.160 ± 0.005 | 1.151 | 1.176 | 1.00 |
+| `squashfs-tools` | 1.827 ± 0.046 | 1.745 | 1.990 | 1.57 ± 0.04 |
+### `airootfs.sfs`
+| Command | Mean [µs] | Min [µs] | Max [µs] | Relative |
+|:---|---:|---:|---:|---:|
+| `backhand-dist-v0.18.0` | 813.8 ± 62.2 | 720.6 | 948.4 | 1.27 ± 0.13 |
+| `backhand-dist` | 793.0 ± 92.2 | 708.4 | 1340.4 | 1.23 ± 0.17 |
+| `backhand-dist-musl` | 1110.4 ± 88.0 | 893.1 | 1381.3 | 1.73 ± 0.18 |
+| `squashfs-tools` | 643.0 ± 43.3 | 571.8 | 798.9 | 1.00 |
 
-## Heap Usage: `backhand/unsquashfs-master` vs `squashfs-tools/unsquashfs-4.6.1`
+## Heap Usage: `backhand/unsquashfs` vs `squashfs-tools/unsquashfs-4.6.1`
 ```
-$ cargo +stable build -p backhand-cli --bins --locked --profile=dist --no-default-features --features xz --features gzip-zune-inflate
+$ cargo +stable build -p backhand-cli --bins --locked --profile=dist
 ```
 
 | Command | Peak Heap Memory Consumption |
 | :------ | ---------------------------: |
-| `heaptrack ./target/dist/unsquashfs-backhand --quiet -f -d $(mktemp -d) backhand-test/test-assets/test_re815_xev160/870D97.squashfs` | 36.5MB |
-| `heaptrack unsquashfs -quiet -no-progress -d $(mktemp -d) backhand-test/test-assets/test_re815_xev160/870D97.squashfs` | 75.7MB |
+| `heaptrack ./target/dist/unsquashfs-backhand --quiet -f -d $(mktemp -d) backhand-test/test-assets/test_re815_xev160/870D97.squashfs` | 34.4MB |
+| `heaptrack unsquashfs -quiet -no-progress -d $(mktemp -d) backhand-test/test-assets/test_re815_xev160/870D97.squashfs` | 76.8MB |
 
 | Command | Peak Heap Memory Consumption |
 | :------ | ---------------------------: |
-| `heaptrack ./target/dist/unsquashfs-backhand --quiet -f -d $(mktemp -d) backhand-test/test-assets/test_tplink_ax1800/img-1571203182_vol-ubi_rootfs.ubifs` | 56.6MB |
-| `heaptrack unsquashfs -d $(mktemp -d) backhand-test/test-assets/test_tplink_ax1800/img-1571203182_vol-ubi_rootfs.ubifs` | 114.4MB |
+| `heaptrack ./target/dist/unsquashfs-backhand --quiet -f -d $(mktemp -d) backhand-test/test-assets/test_tplink_ax1800/img-1571203182_vol-ubi_rootfs.ubifs` | 52.3MB |
+| `heaptrack unsquashfs -d $(mktemp -d) backhand-test/test-assets/test_tplink_ax1800/img-1571203182_vol-ubi_rootfs.ubifs` | 103.4MB |
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c6f69a26..f07c562f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### `backhand`
+- Use feature `zlib-ng` for `flate2`, which is enabled when compression option `Gzip` is used. This enables the backend to use [zlib-ng](https://github.com/zlib-ng/zlib-ng), which is faster by default! ([#562](https://github.com/wcampbell0x2a/backhand/pull/562))
 - Remove duplicated data when addding new files to a `FilesystemWriter`. This also applies this behavior to the `add` and `replace` binaries. This is controllable with `FilesystemWriter::set_no_duplicate_files`. ([#603](https://github.com/wcampbell0x2a/backhand/pull/603)), ([#594](https://github.com/wcampbell0x2a/backhand/pull/594))
 - Increase speed of internal `HashMap`s, by switching to `xxhash` and just using the `inode` as the key in other places.
 - Changed `SuperBlock::Flags` to be public.
diff --git a/Cargo.lock b/Cargo.lock
index a6e9a599..bca50afa 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -105,6 +105,7 @@ dependencies = [
  "document-features",
  "flate2",
  "libdeflater",
+ "libz-sys",
  "rust-lzo",
  "solana-nohash-hasher",
  "tempfile",
@@ -195,12 +196,13 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
 
 [[package]]
 name = "cc"
-version = "1.0.90"
+version = "1.0.98"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5"
+checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f"
 dependencies = [
  "jobserver",
  "libc",
+ "once_cell",
 ]
 
 [[package]]
@@ -292,6 +294,15 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce"
 
+[[package]]
+name = "cmake"
+version = "0.1.50"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "color-print"
 version = "0.3.6"
@@ -590,6 +601,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253"
 dependencies = [
  "crc32fast",
+ "libz-ng-sys",
  "miniz_oxide",
 ]
 
@@ -765,9 +777,9 @@ dependencies = [
 
 [[package]]
 name = "jobserver"
-version = "0.1.28"
+version = "0.1.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6"
+checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e"
 dependencies = [
  "libc",
 ]
@@ -811,6 +823,28 @@ dependencies = [
  "libdeflate-sys",
 ]
 
+[[package]]
+name = "libz-ng-sys"
+version = "1.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4436751a01da56f1277f323c80d584ffad94a3d14aecd959dd0dff75aa73a438"
+dependencies = [
+ "cmake",
+ "libc",
+]
+
+[[package]]
+name = "libz-sys"
+version = "1.1.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.4.13"
@@ -1608,6 +1642,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
 
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
 [[package]]
 name = "version_check"
 version = "0.9.4"
diff --git a/backhand/Cargo.toml b/backhand/Cargo.toml
index fba6006d..d7c4fcff 100644
--- a/backhand/Cargo.toml
+++ b/backhand/Cargo.toml
@@ -19,7 +19,7 @@ rustdoc-args = ["--cfg", "docsrs"]
 deku = "0.17.0"
 tracing = { version = "0.1.40" }
 thiserror = "1.0.63"
-flate2 = { version = "1.0.33", optional = true }
+flate2 = { version = "1.0.33", optional = true, features = ["zlib-ng"] }
 zune-inflate = { version = "0.2.54", optional = true, default-features = false, features = ["zlib"] }
 xz2 = { version = "0.1.7", optional = true }
 rust-lzo = { version = "0.6.2", optional = true }
@@ -28,6 +28,8 @@ zstd-safe = { version = "7.2.1", optional = true }
 document-features = { version = "0.2.10", optional = true }
 xxhash-rust = { version = "0.8.12", features = ["xxh64"] }
 solana-nohash-hasher = "0.2.1"
+# Use the fastest implementation (libz-ng) for flate2 but remove dependence on CMake
+libz-sys = { version = "1.1.20", features = ["zlib-ng-no-cmake-experimental-community-maintained"], default-features = false }
 
 [features]
 default = ["xz", "gzip", "zstd"]
diff --git a/bench.bash b/bench.bash
index 1800c4f2..47e36f01 100755
--- a/bench.bash
+++ b/bench.bash
@@ -1,7 +1,7 @@
 #!/bin/bash
 set -ex
 
-LAST_RELEASE="v0.17.0"
+LAST_RELEASE="v0.18.0"
 
 BACKHAND_LAST_RELEASE="./last-release/bin/unsquashfs-backhand"
 BACKHAND="./target/dist/unsquashfs-backhand"
@@ -9,7 +9,7 @@ BACKHAND_MUSL="./target/x86_64-unknown-linux-musl/dist/unsquashfs-backhand"
 UNSQUASHFS="/usr/bin/unsquashfs"
 
 # Using dynamic linked xz for perf reasons and matching unsquashfs in this testing
-FLAGS="--bins --locked --profile=dist --no-default-features --features xz --features gzip-zune-inflate --features zstd"
+FLAGS="--bins --locked --profile=dist --no-default-features --features xz --features zstd --features gzip"
 
 bench () {
     echo ""
@@ -42,7 +42,7 @@ bench "backhand-test/test-assets/test_re815_xev160/870D97.squashfs" 0x0 2_re815
 # xz
 bench "backhand-test/test-assets/test_tplink_ax1800/img-1571203182_vol-ubi_rootfs.ubifs" 0x0 3_ax18000
 # xz
-#bench "test-assets/test_archlinux_iso_rootfs/airootfs.sfs" 0x0
+bench "test-assets/test_archlinux_iso_rootfs/airootfs.sfs" 0x0
 # xz
 bench "backhand-test/test-assets/test_er605_v2_2/2611E3.squashfs" 0x0 4_er605
 # gzip


Test patch: