Repository: static-web-server/static-web-server (language: rust)

Issue:
Add support for zstd encoder
**Describe the solution you'd like**
Currently SWS only supports brotli, deflate, and gzip. The async_compression library has support for zstd which would be beneficial when SWS is used in combination with other Rust tools, Ex. Vector.dev

https://docs.rs/async-compression/latest/async_compression/#compression-algorithm



Discussion / hints:
Just out of curiosity. Can you please elaborate a bit more about what's your use case for zstd as a new content encoding?
@joseluisq Adding zstd as a content encoder can lead to faster load times, improved user experience, reduced bandwidth costs, and increased energy efficiency. These benefits make zstd an excellent choice for modern web services looking to optimize their performance and deliver the best possible experience to users.

There's several study cases explaining how zstd is used on service to service communications to reduce network i/o usage compared to gzip/brotli

Sources:
- https://aws.amazon.com/blogs/containers/reducing-aws-fargate-startup-times-with-zstd-compressed-container-images/
- https://docs.aws.amazon.com/athena/latest/ug/compression-support-zstd-levels.html
- https://engineering.fb.com/2018/12/19/core-data/zstandard/
- http://www.bioinformaticszen.com/post/use-zstd-for-raw-fastq/
- https://daniel.haxx.se/blog/2020/08/19/curl-7-72-0-more-compression/

I'm currently using Caddy 2 to serve static content with zstd encoding. For some use cases using something like SWS would be ideal, even more if I could serve the content using zstd. 
Ok, I got it.
Then, I think it can be used for backend-to-backend communication and backend-to-native-mobile/desktop apps.
I also saw that the QUIC protocol is including support for zstd (maybe browsers someday too).

Since Zstd looks can provide a good balance between compression ratio and decompression speed so why not.

Also noting that we could add on-the-fly zstd compression and pre-compressed file serving in a similar way to how is done for gzip/brotli actually.


Fix patch:
diff --git a/Cargo.lock b/Cargo.lock
index b8fdfc54..ea11cd5f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -59,6 +59,8 @@ dependencies = [
  "memchr",
  "pin-project-lite",
  "tokio",
+ "zstd",
+ "zstd-safe",
 ]
 
 [[package]]
@@ -171,6 +173,9 @@ name = "cc"
 version = "1.0.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
+dependencies = [
+ "jobserver",
+]
 
 [[package]]
 name = "cfg-if"
@@ -442,9 +447,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 
 [[package]]
 name = "headers-accept-encoding"
-version = "0.3.8"
+version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "450f8482d4d4ec4db6d82d0e8ce6c3cd5421dd469bc0104e43c635c11a645314"
+checksum = "f31dd148a6d59ca552b4b30a018188701f588c0d398da9e280a455775e450d69"
 dependencies = [
  "base64 0.13.1",
  "bitflags",
@@ -619,6 +624,15 @@ version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6"
 
+[[package]]
+name = "jobserver"
+version = "0.1.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "js-sys"
 version = "0.3.61"
@@ -840,6 +854,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
+[[package]]
+name = "pkg-config"
+version = "0.3.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
+
 [[package]]
 name = "proc-macro-error"
 version = "1.0.4"
@@ -1700,3 +1720,33 @@ name = "zeroize"
 version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9"
+
+[[package]]
+name = "zstd"
+version = "0.11.2+zstd.1.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4"
+dependencies = [
+ "zstd-safe",
+]
+
+[[package]]
+name = "zstd-safe"
+version = "5.0.2+zstd.1.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db"
+dependencies = [
+ "libc",
+ "zstd-sys",
+]
+
+[[package]]
+name = "zstd-sys"
+version = "2.0.8+zstd.1.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+]
diff --git a/Cargo.toml b/Cargo.toml
index 138b25b5..d8499e25 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -43,13 +43,13 @@ http2 = ["tls"]
 
 [dependencies]
 anyhow = "1.0"
-async-compression = { version = "0.3", default-features = false, features = ["brotli", "deflate", "gzip", "tokio"] }
+async-compression = { version = "0.3", default-features = false, features = ["brotli", "deflate", "gzip", "zstd", "tokio"] }
 bcrypt = "0.14"
 bytes = "1.4"
 form_urlencoded = "1.1"
 futures-util = { version = "0.3", default-features = false, features = ["sink"] }
 globset = { version = "0.4", features = ["serde1"] }
-headers = { package = "headers-accept-encoding", version = "0.3" }
+headers = { package = "headers-accept-encoding", version = "1.0" }
 http = "0.2"
 http-serde = "1.1"
 humansize = { version = "2.1", features = ["impl_style"] }
diff --git a/src/compression.rs b/src/compression.rs
index a39b1c18..110723b6 100644
--- a/src/compression.rs
+++ b/src/compression.rs
@@ -3,7 +3,7 @@
 
 // Part of the file is borrowed from <https://github.com/seanmonstar/warp/pull/513>*
 
-use async_compression::tokio::bufread::{BrotliEncoder, DeflateEncoder, GzipEncoder};
+use async_compression::tokio::bufread::{BrotliEncoder, DeflateEncoder, GzipEncoder, ZstdEncoder};
 use bytes::Bytes;
 use futures_util::Stream;
 use headers::{AcceptEncoding, ContentCoding, ContentType, HeaderMap, HeaderMapExt};
@@ -91,6 +91,10 @@ pub fn auto(
             let (head, body) = resp.into_parts();
             return Ok(brotli(head, body.into()));
         }
+        if encoding == ContentCoding::ZSTD {
+            let (head, body) = resp.into_parts();
+            return Ok(zstd(head, body.into()));
+        }
     }
 
     Ok(resp)
@@ -149,6 +153,21 @@ pub fn brotli(
     Response::from_parts(head, body)
 }
 
+/// Create a wrapping handler that compresses the Body of a [`Response`](hyper::Response)
+/// using zstd, adding `content-encoding: zstd` to the Response's [`HeaderMap`](hyper::HeaderMap)
+pub fn zstd(
+    mut head: http::response::Parts,
+    body: CompressableBody<Body, hyper::Error>,
+) -> Response<Body> {
+    tracing::trace!("compressing response body on the fly using zstd");
+
+    let body = Body::wrap_stream(ReaderStream::new(ZstdEncoder::new(StreamReader::new(body))));
+    let header = create_encoding_header(head.headers.remove(CONTENT_ENCODING), ContentCoding::ZSTD);
+    head.headers.remove(CONTENT_LENGTH);
+    head.headers.append(CONTENT_ENCODING, header);
+    Response::from_parts(head, body)
+}
+
 /// Given an optional existing encoding header, appends to the existing or creates a new one.
 pub fn create_encoding_header(existing: Option<HeaderValue>, coding: ContentCoding) -> HeaderValue {
     if let Some(val) = existing {
diff --git a/src/compression_static.rs b/src/compression_static.rs
index 85578b2c..8948cf64 100644
--- a/src/compression_static.rs
+++ b/src/compression_static.rs
@@ -36,6 +36,8 @@ pub async fn precompressed_variant<'a>(
         Some(ContentCoding::GZIP | ContentCoding::DEFLATE) => "gz",
         // https://peazip.github.io/brotli-compressed-file-format.html
         Some(ContentCoding::BROTLI) => "br",
+        // https://datatracker.ietf.org/doc/html/rfc8878
+        Some(ContentCoding::ZSTD) => "zst",
         _ => {
             tracing::trace!(
                 "preferred encoding based on the file extension was not determined, skipping"


Test patch: