Repository: seanmonstar/reqwest
Pull request #1250 (2021-04-20 21:47:52)

Support Deflate decoding
Also adds `deflate()` and `no_deflate()` methods.


One thing I should mention is that I was able to confirm that this works with `http://example.org` with the header `accept-encoding: deflate`. The same cannot be said about `httpbin.org/deflate` which fails with `deflate decompression error`.


Patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3d463f7..30f83a6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -66,6 +66,7 @@ jobs:
           - "feat.: blocking"
           - "feat.: gzip"
           - "feat.: brotli"
+          - "feat.: deflate"
           - "feat.: json"
           - "feat.: multipart"
           - "feat.: stream"
@@ -89,21 +90,21 @@ jobs:
           - name: windows / stable-x86_64-msvc
             os: windows-latest
             target: x86_64-pc-windows-msvc
-            features: "--features blocking,gzip,brotli,json,multipart"
+            features: "--features blocking,gzip,brotli,deflate,json,multipart"
           - name: windows / stable-i686-msvc
             os: windows-latest
             target: i686-pc-windows-msvc
-            features: "--features blocking,gzip,brotli,json,multipart"
+            features: "--features blocking,gzip,brotli,deflate,json,multipart"
           - name: windows / stable-x86_64-gnu
             os: windows-latest
             rust: stable-x86_64-pc-windows-gnu
             target: x86_64-pc-windows-gnu
-            features: "--features blocking,gzip,brotli,json,multipart"
+            features: "--features blocking,gzip,brotli,deflate,json,multipart"
           - name: windows / stable-i686-gnu
             os: windows-latest
             rust: stable-i686-pc-windows-gnu
             target: i686-pc-windows-gnu
-            features: "--features blocking,gzip,brotli,json,multipart"
+            features: "--features blocking,gzip,brotli,deflate,json,multipart"
 
           - name: "feat.: default-tls disabled"
             features: "--no-default-features"
@@ -123,6 +124,8 @@ jobs:
             features: "--features gzip"
           - name: "feat.: brotli"
             features: "--features brotli"
+          - name: "feat.: deflate"
+            features: "--features deflate"
           - name: "feat.: json"
             features: "--features json"
           - name: "feat.: multipart"
diff --git a/Cargo.toml b/Cargo.toml
index 50c4f38..c50df9f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -49,6 +49,8 @@ gzip = ["async-compression", "async-compression/gzip", "tokio-util"]
 
 brotli = ["async-compression", "async-compression/brotli", "tokio-util"]
 
+deflate = ["async-compression", "async-compression/deflate", "tokio-util"]
+
 json = ["serde_json"]
 
 multipart = ["mime_guess"]
@@ -216,6 +218,11 @@ name = "brotli"
 path = "tests/brotli.rs"
 required-features = ["brotli"]
 
+[[test]]
+name = "deflate"
+path = "tests/deflate.rs"
+required-features = ["deflate"]
+
 [[test]]
 name = "multipart"
 path = "tests/multipart.rs"
diff --git a/src/async_impl/client.rs b/src/async_impl/client.rs
index 4bdad5d..04830a6 100644
--- a/src/async_impl/client.rs
+++ b/src/async_impl/client.rs
@@ -536,6 +536,29 @@ impl ClientBuilder {
         self
     }
 
+    /// Enable auto deflate decompression by checking the `Content-Encoding` response header.
+    ///
+    /// If auto deflate decompression is turned on:
+    ///
+    /// - When sending a request and if the request's headers do not already contain
+    ///   an `Accept-Encoding` **and** `Range` values, the `Accept-Encoding` header is set to `deflate`.
+    ///   The request body is **not** automatically compressed.
+    /// - When receiving a response, if it's headers contain a `Content-Encoding` value that
+    ///   equals to `deflate`, both values `Content-Encoding` and `Content-Length` are removed from the
+    ///   headers' set. The response body is automatically decompressed.
+    ///
+    /// If the `deflate` feature is turned on, the default option is enabled.
+    ///
+    /// # Optional
+    ///
+    /// This requires the optional `deflate` feature to be enabled
+    #[cfg(feature = "deflate")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "deflate")))]
+    pub fn deflate(mut self, enable: bool) -> ClientBuilder {
+        self.config.accepts.deflate = enable;
+        self
+    }
+
     /// Disable auto response body gzip decompression.
     ///
     /// This method exists even if the optional `gzip` feature is not enabled.
@@ -570,6 +593,23 @@ impl ClientBuilder {
         }
     }
 
+    /// Disable auto response body deflate decompression.
+    ///
+    /// This method exists even if the optional `deflate` feature is not enabled.
+    /// This can be used to ensure a `Client` doesn't use deflate decompression
+    /// even if another dependency were to enable the optional `deflate` feature.
+    pub fn no_deflate(self) -> ClientBuilder {
+        #[cfg(feature = "deflate")]
+        {
+            self.deflate(false)
+        }
+
+        #[cfg(not(feature = "deflate"))]
+        {
+            self
+        }
+    }
+
     // Redirect options
 
     /// Set a `RedirectPolicy` for this client.
diff --git a/src/async_impl/decoder.rs b/src/async_impl/decoder.rs
index 50f762f..38adeeb 100644
--- a/src/async_impl/decoder.rs
+++ b/src/async_impl/decoder.rs
@@ -9,15 +9,18 @@ use async_compression::tokio::bufread::GzipDecoder;
 #[cfg(feature = "brotli")]
 use async_compression::tokio::bufread::BrotliDecoder;
 
+#[cfg(feature = "deflate")]
+use async_compression::tokio::bufread::DeflateDecoder;
+
 use bytes::Bytes;
 use futures_core::Stream;
 use futures_util::stream::Peekable;
 use http::HeaderMap;
 use hyper::body::HttpBody;
 
-#[cfg(any(feature = "gzip", feature = "brotli"))]
+#[cfg(any(feature = "gzip", feature = "brotli", feature = "deflate"))]
 use tokio_util::codec::{BytesCodec, FramedRead};
-#[cfg(any(feature = "gzip", feature = "brotli"))]
+#[cfg(any(feature = "gzip", feature = "brotli", feature = "deflate"))]
 use tokio_util::io::StreamReader;
 
 use super::super::Body;
@@ -29,6 +32,8 @@ pub(super) struct Accepts {
     pub(super) gzip: bool,
     #[cfg(feature = "brotli")]
     pub(super) brotli: bool,
+    #[cfg(feature = "deflate")]
+    pub(super) deflate: bool,
 }
 
 /// A response decompressor over a non-blocking stream of chunks.
@@ -50,8 +55,12 @@ enum Inner {
     #[cfg(feature = "brotli")]
     Brotli(FramedRead<BrotliDecoder<StreamReader<Peekable<IoStream>, Bytes>>, BytesCodec>),
 
+    /// A `Deflate` decoder will uncompress the deflated response content before returning it.
+    #[cfg(feature = "deflate")]
+    Deflate(FramedRead<DeflateDecoder<StreamReader<Peekable<IoStream>, Bytes>>, BytesCodec>),
+
     /// A decoder that doesn't have a value yet.
-    #[cfg(any(feature = "brotli", feature = "gzip"))]
+    #[cfg(any(feature = "brotli", feature = "gzip", feature = "deflate"))]
     Pending(Pending),
 }
 
@@ -65,6 +74,8 @@ enum DecoderType {
     Gzip,
     #[cfg(feature = "brotli")]
     Brotli,
+    #[cfg(feature = "deflate")]
+    Deflate,
 }
 
 impl fmt::Debug for Decoder {
@@ -120,68 +131,49 @@ impl Decoder {
         }
     }
 
-    #[cfg(feature = "gzip")]
-    fn detect_gzip(headers: &mut HeaderMap) -> bool {
-        use http::header::{CONTENT_ENCODING, CONTENT_LENGTH, TRANSFER_ENCODING};
-        use log::warn;
+    /// A deflate decoder.
+    ///
+    /// This decoder will buffer and decompress chunks that are deflated.
+    #[cfg(feature = "deflate")]
+    fn deflate(body: Body) -> Decoder {
+        use futures_util::StreamExt;
 
-        let content_encoding_gzip: bool;
-        let mut is_gzip = {
-            content_encoding_gzip = headers
-                .get_all(CONTENT_ENCODING)
-                .iter()
-                .any(|enc| enc == "gzip");
-            content_encoding_gzip
-                || headers
-                    .get_all(TRANSFER_ENCODING)
-                    .iter()
-                    .any(|enc| enc == "gzip")
-        };
-        if is_gzip {
-            if let Some(content_length) = headers.get(CONTENT_LENGTH) {
-                if content_length == "0" {
-                    warn!("gzip response with content-length of 0");
-                    is_gzip = false;
-                }
-            }
-        }
-        if is_gzip {
-            headers.remove(CONTENT_ENCODING);
-            headers.remove(CONTENT_LENGTH);
+        Decoder {
+            inner: Inner::Pending(Pending(
+                IoStream(body.into_stream()).peekable(),
+                DecoderType::Deflate,
+            )),
         }
-        is_gzip
     }
 
-    #[cfg(feature = "brotli")]
-    fn detect_brotli(headers: &mut HeaderMap) -> bool {
+    #[cfg(any(feature = "brotli", feature = "gzip", feature = "deflate"))]
+    fn detect_encoding(headers: &mut HeaderMap, encoding_str: &str) -> bool {
         use http::header::{CONTENT_ENCODING, CONTENT_LENGTH, TRANSFER_ENCODING};
         use log::warn;
 
-        let content_encoding_gzip: bool;
-        let mut is_brotli = {
-            content_encoding_gzip = headers
+        let mut is_content_encoded = {
+            headers
                 .get_all(CONTENT_ENCODING)
                 .iter()
-                .any(|enc| enc == "br");
-            content_encoding_gzip
+                .any(|enc| enc == encoding_str)
                 || headers
                     .get_all(TRANSFER_ENCODING)
                     .iter()
-                    .any(|enc| enc == "br")
+                    .any(|enc| enc == encoding_str)
         };
-        if is_brotli {
+        if is_content_encoded {
             if let Some(content_length) = headers.get(CONTENT_LENGTH) {
                 if content_length == "0" {
-                    warn!("brotli response with content-length of 0");
-                    is_brotli = false;
+                    warn!("{} response with content-length of 0", encoding_str);
+                    is_content_encoded = false;
                 }
             }
         }
-        if is_brotli {
+        if is_content_encoded {
             headers.remove(CONTENT_ENCODING);
             headers.remove(CONTENT_LENGTH);
         }
-        is_brotli
+        is_content_encoded
     }
 
     /// Constructs a Decoder from a hyper request.
@@ -193,18 +185,25 @@ impl Decoder {
     pub(super) fn detect(_headers: &mut HeaderMap, body: Body, _accepts: Accepts) -> Decoder {
         #[cfg(feature = "gzip")]
         {
-            if _accepts.gzip && Decoder::detect_gzip(_headers) {
+            if _accepts.gzip && Decoder::detect_encoding(_headers, "gzip") {
                 return Decoder::gzip(body);
             }
         }
 
         #[cfg(feature = "brotli")]
         {
-            if _accepts.brotli && Decoder::detect_brotli(_headers) {
+            if _accepts.brotli && Decoder::detect_encoding(_headers, "br") {
                 return Decoder::brotli(body);
             }
         }
 
+        #[cfg(feature = "deflate")]
+        {
+            if _accepts.deflate && Decoder::detect_encoding(_headers, "deflate") {
+                return Decoder::deflate(body);
+            }
+        }
+
         Decoder::plain_text(body)
     }
 }
@@ -215,7 +214,7 @@ impl Stream for Decoder {
     fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Option<Self::Item>> {
         // Do a read or poll for a pending decoder value.
         match self.inner {
-            #[cfg(any(feature = "brotli", feature = "gzip"))]
+            #[cfg(any(feature = "brotli", feature = "gzip", feature = "deflate"))]
             Inner::Pending(ref mut future) => match Pin::new(future).poll(cx) {
                 Poll::Ready(Ok(inner)) => {
                     self.inner = inner;
@@ -243,6 +242,14 @@ impl Stream for Decoder {
                     None => Poll::Ready(None),
                 };
             }
+            #[cfg(feature = "deflate")]
+            Inner::Deflate(ref mut decoder) => {
+                return match futures_core::ready!(Pin::new(decoder).poll_next(cx)) {
+                    Some(Ok(bytes)) => Poll::Ready(Some(Ok(bytes.freeze()))),
+                    Some(Err(err)) => Poll::Ready(Some(Err(crate::error::decode_io(err)))),
+                    None => Poll::Ready(None),
+                };
+            }
         }
     }
 }
@@ -269,7 +276,7 @@ impl HttpBody for Decoder {
         match self.inner {
             Inner::PlainText(ref body) => HttpBody::size_hint(body),
             // the rest are "unknown", so default
-            #[cfg(any(feature = "brotli", feature = "gzip"))]
+            #[cfg(any(feature = "brotli", feature = "gzip", feature = "deflate"))]
             _ => http_body::SizeHint::default(),
         }
     }
@@ -312,6 +319,11 @@ impl Future for Pending {
                 GzipDecoder::new(StreamReader::new(_body)),
                 BytesCodec::new(),
             )))),
+            #[cfg(feature = "deflate")]
+            DecoderType::Deflate => Poll::Ready(Ok(Inner::Deflate(FramedRead::new(
+                DeflateDecoder::new(StreamReader::new(_body)),
+                BytesCodec::new(),
+            )))),
         }
     }
 }
@@ -337,15 +349,21 @@ impl Accepts {
             gzip: false,
             #[cfg(feature = "brotli")]
             brotli: false,
+            #[cfg(feature = "deflate")]
+            deflate: false,
         }
     }
 
     pub(super) fn as_str(&self) -> Option<&'static str> {
-        match (self.is_gzip(), self.is_brotli()) {
-            (true, true) => Some("gzip, br"),
-            (true, false) => Some("gzip"),
-            (false, true) => Some("br"),
-            _ => None,
+        match (self.is_gzip(), self.is_brotli(), self.is_deflate()) {
+            (true, true, true) => Some("gzip, br, deflate"),
+            (true, true, false) => Some("gzip, br"),
+            (true, false, true) => Some("gzip, deflate"),
+            (false, true, true) => Some("br, deflate"),
+            (true, false, false) => Some("gzip"),
+            (false, true, false) => Some("br"),
+            (false, false, true) => Some("deflate"),
+            (false, false, false) => None,
         }
     }
 
@@ -372,6 +390,18 @@ impl Accepts {
             false
         }
     }
+
+    fn is_deflate(&self) -> bool {
+        #[cfg(feature = "deflate")]
+        {
+            self.deflate
+        }
+
+        #[cfg(not(feature = "deflate"))]
+        {
+            false
+        }
+    }
 }
 
 impl Default for Accepts {
@@ -381,6 +411,8 @@ impl Default for Accepts {
             gzip: true,
             #[cfg(feature = "brotli")]
             brotli: true,
+            #[cfg(feature = "deflate")]
+            deflate: true,
         }
     }
 }
diff --git a/src/blocking/client.rs b/src/blocking/client.rs
index 568a741..0dce1d2 100644
--- a/src/blocking/client.rs
+++ b/src/blocking/client.rs
@@ -257,6 +257,28 @@ impl ClientBuilder {
         self.with_inner(|inner| inner.brotli(enable))
     }
 
+    /// Enable auto deflate decompression by checking the `Content-Encoding` response header.
+    ///
+    /// If auto deflate decompresson is turned on:
+    ///
+    /// - When sending a request and if the request's headers do not already contain
+    ///   an `Accept-Encoding` **and** `Range` values, the `Accept-Encoding` header is set to `deflate`.
+    ///   The request body is **not** automatically compressed.
+    /// - When receiving a response, if it's headers contain a `Content-Encoding` value that
+    ///   equals to `deflate`, both values `Content-Encoding` and `Content-Length` are removed from the
+    ///   headers' set. The response body is automatically decompressed.
+    ///
+    /// If the `deflate` feature is turned on, the default option is enabled.
+    ///
+    /// # Optional
+    ///
+    /// This requires the optional `deflate` feature to be enabled
+    #[cfg(feature = "deflate")]
+    #[cfg_attr(docsrs, doc(cfg(feature = "deflate")))]
+    pub fn deflate(self, enable: bool) -> ClientBuilder {
+        self.with_inner(|inner| inner.deflate(enable))
+    }
+
     /// Disable auto response body gzip decompression.
     ///
     /// This method exists even if the optional `gzip` feature is not enabled.
@@ -275,6 +297,15 @@ impl ClientBuilder {
         self.with_inner(|inner| inner.no_brotli())
     }
 
+    /// Disable auto response body deflate decompression.
+    ///
+    /// This method exists even if the optional `deflate` feature is not enabled.
+    /// This can be used to ensure a `Client` doesn't use deflate decompression
+    /// even if another dependency were to enable the optional `deflate` feature.
+    pub fn no_deflate(self) -> ClientBuilder {
+        self.with_inner(|inner| inner.no_deflate())
+    }
+
     // Redirect options
 
     /// Set a `redirect::Policy` for this client.
diff --git a/src/lib.rs b/src/lib.rs
index f158f78..f8b4c74 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -182,6 +182,7 @@
 //! - **cookies**: Provides cookie session support.
 //! - **gzip**: Provides response body gzip decompression.
 //! - **brotli**: Provides response body brotli decompression.
+//! - **deflate**: Provides response body deflate decompression.
 //! - **json**: Provides serialization and deserialization for JSON bodies.
 //! - **multipart**: Provides functionality for multipart forms.
 //! - **stream**: Adds support for `futures::Stream`.


Test patch:
diff --git a/tests/client.rs b/tests/client.rs
index dc53a3a..2fbddc2 100644
--- a/tests/client.rs
+++ b/tests/client.rs
@@ -24,6 +24,12 @@ async fn auto_headers() {
                 .unwrap()
                 .contains("br"));
         }
+        if cfg!(feature = "deflate") {
+            assert!(req.headers()["accept-encoding"]
+                .to_str()
+                .unwrap()
+                .contains("deflate"));
+        }
 
         http::Response::default()
     });
diff --git a/tests/deflate.rs b/tests/deflate.rs
new file mode 100644
index 0000000..8b16530
--- /dev/null
+++ b/tests/deflate.rs
@@ -0,0 +1,151 @@
+mod support;
+use std::io::Write;
+use support::*;
+
+#[tokio::test]
+async fn deflate_response() {
+    deflate_case(10_000, 4096).await;
+}
+
+#[tokio::test]
+async fn deflate_single_byte_chunks() {
+    deflate_case(10, 1).await;
+}
+
+#[tokio::test]
+async fn test_deflate_empty_body() {
+    let server = server::http(move |req| async move {
+        assert_eq!(req.method(), "HEAD");
+
+        http::Response::builder()
+            .header("content-encoding", "deflate")
+            .header("content-length", 100)
+            .body(Default::default())
+            .unwrap()
+    });
+
+    let client = reqwest::Client::new();
+    let res = client
+        .head(&format!("http://{}/deflate", server.addr()))
+        .send()
+        .await
+        .unwrap();
+
+    let body = res.text().await.unwrap();
+
+    assert_eq!(body, "");
+}
+
+#[tokio::test]
+async fn test_accept_header_is_not_changed_if_set() {
+    let server = server::http(move |req| async move {
+        assert_eq!(req.headers()["accept"], "application/json");
+        assert!(req.headers()["accept-encoding"]
+            .to_str()
+            .unwrap()
+            .contains("deflate"));
+        http::Response::default()
+    });
+
+    let client = reqwest::Client::new();
+
+    let res = client
+        .get(&format!("http://{}/accept", server.addr()))
+        .header(
+            reqwest::header::ACCEPT,
+            reqwest::header::HeaderValue::from_static("application/json"),
+        )
+        .send()
+        .await
+        .unwrap();
+
+    assert_eq!(res.status(), reqwest::StatusCode::OK);
+}
+
+#[tokio::test]
+async fn test_accept_encoding_header_is_not_changed_if_set() {
+    let server = server::http(move |req| async move {
+        assert_eq!(req.headers()["accept"], "*/*");
+        assert_eq!(req.headers()["accept-encoding"], "identity");
+        http::Response::default()
+    });
+
+    let client = reqwest::Client::new();
+
+    let res = client
+        .get(&format!("http://{}/accept-encoding", server.addr()))
+        .header(
+            reqwest::header::ACCEPT_ENCODING,
+            reqwest::header::HeaderValue::from_static("identity"),
+        )
+        .send()
+        .await
+        .unwrap();
+
+    assert_eq!(res.status(), reqwest::StatusCode::OK);
+}
+
+async fn deflate_case(response_size: usize, chunk_size: usize) {
+    use futures_util::stream::StreamExt;
+
+    let content: String = (0..response_size)
+        .into_iter()
+        .map(|i| format!("test {}", i))
+        .collect();
+    let mut encoder = libflate::deflate::Encoder::new(Vec::new());
+    match encoder.write(content.as_bytes()) {
+        Ok(n) => assert!(n > 0, "Failed to write to encoder."),
+        _ => panic!("Failed to deflate encode string."),
+    };
+
+    let deflated_content = encoder.finish().into_result().unwrap();
+
+    let mut response = format!(
+        "\
+         HTTP/1.1 200 OK\r\n\
+         Server: test-accept\r\n\
+         Content-Encoding: deflate\r\n\
+         Content-Length: {}\r\n\
+         \r\n",
+        &deflated_content.len()
+    )
+    .into_bytes();
+    response.extend(&deflated_content);
+
+    let server = server::http(move |req| {
+        assert!(req.headers()["accept-encoding"]
+            .to_str()
+            .unwrap()
+            .contains("deflate"));
+
+        let deflated = deflated_content.clone();
+        async move {
+            let len = deflated.len();
+            let stream =
+                futures_util::stream::unfold((deflated, 0), move |(deflated, pos)| async move {
+                    let chunk = deflated.chunks(chunk_size).nth(pos)?.to_vec();
+
+                    Some((chunk, (deflated, pos + 1)))
+                });
+
+            let body = hyper::Body::wrap_stream(stream.map(Ok::<_, std::convert::Infallible>));
+
+            http::Response::builder()
+                .header("content-encoding", "deflate")
+                .header("content-length", len)
+                .body(body)
+                .unwrap()
+        }
+    });
+
+    let client = reqwest::Client::new();
+
+    let res = client
+        .get(&format!("http://{}/deflate", server.addr()))
+        .send()
+        .await
+        .expect("response");
+
+    let body = res.text().await.expect("text");
+    assert_eq!(body, content);
+}