Repository: ducaale/xh (rust)

Issue:
bug (?): xh removes the content-encoding header
Hi, and thanks for xh! I've been using it to develop a server framework, and noticed that although xh does a great job of automatically processing gzip and brotli encoded response bodies, it seems to _remove_ the Content-Encoding header from verbose display. If this is in fact a bug, would you like a patch PR?

Pull request:
Replace reqwest's auto-decompression with a custom one
Doing so will allow us to preserve `Content-Encoding` and `Content-Length` for compressed responses. This also opens the door for handling responses from servers that [misinterpret](https://stackoverflow.com/a/3932260/5915221) the `deflate` encoding as `deflate` instead of `deflate + zlib headers` e.g `xh example.org accept-encoding:deflate`.

~~One side effect from this PR is that the `--download` flag no longer decompresses responses but since we are explicitly sending `accept-identity: identity` and HTTPie doesn't handle it perfectly either, maybe we can ignore it?~~

```
$ http httpbin.org/gzip -d

HTTP/1.1 200 OK
Access-Control-Allow-Credentials: true
Access-Control-Allow-Origin: *
Content-Encoding: gzip
Content-Length: 216
Content-Type: application/json
Date: Sun, 13 Mar 2022 14:58:59 GMT
Server: gunicorn/19.9.0

Downloading 216.00 B to "gzip.json"
 |
http: error: Incomplete download: size=216; downloaded=279

Done. 279.00 B in 0.00061s (448.50 kB/s)
```

Fixes #210 

Patch:
diff --git a/Cargo.lock b/Cargo.lock
index 2bbe77e..18b9418 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -52,20 +52,6 @@ dependencies = [
  "wait-timeout",
 ]
 
-[[package]]
-name = "async-compression"
-version = "0.3.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5443ccbb270374a2b1055fc72da40e1f237809cd6bb0e97e66d264cd138473a6"
-dependencies = [
- "brotli",
- "flate2",
- "futures-core",
- "memchr",
- "pin-project-lite",
- "tokio",
-]
-
 [[package]]
 name = "atty"
 version = "0.2.14"
@@ -121,9 +107,9 @@ dependencies = [
 
 [[package]]
 name = "brotli"
-version = "3.3.2"
+version = "3.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "71cb90ade945043d3d53597b2fc359bb063db8ade2bcffe7997351d0756e9d50"
+checksum = "f838e47a451d5a8fa552371f80024dd6ace9b7acdf25c4c3d0f9bc6816fb1c39"
 dependencies = [
  "alloc-no-stdlib",
  "alloc-stdlib",
@@ -420,9 +406,9 @@ dependencies = [
 
 [[package]]
 name = "flate2"
-version = "1.0.21"
+version = "1.0.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80edafed416a46fb378521624fab1cfa2eb514784fd8921adbe8a8d8321da811"
+checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f"
 dependencies = [
  "cfg-if",
  "crc32fast",
@@ -1315,7 +1301,6 @@ version = "0.11.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "66d2927ca2f685faf0fc620ac4834690d29e7abb153add10f5812eef20b5e280"
 dependencies = [
- "async-compression",
  "base64",
  "bytes",
  "cookie",
@@ -1347,7 +1332,6 @@ dependencies = [
  "tokio-native-tls",
  "tokio-rustls",
  "tokio-socks",
- "tokio-util",
  "url",
  "wasm-bindgen",
  "wasm-bindgen-futures",
@@ -2148,6 +2132,7 @@ dependencies = [
  "anyhow",
  "assert_cmd",
  "atty",
+ "brotli",
  "chardetng",
  "clap",
  "clap_complete",
@@ -2157,6 +2142,7 @@ dependencies = [
  "dirs",
  "encoding_rs",
  "encoding_rs_io",
+ "flate2",
  "form_urlencoded",
  "hyper",
  "indicatif",
diff --git a/Cargo.toml b/Cargo.toml
index b90beae..73fe21e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,13 +9,12 @@ documentation = "https://github.com/ducaale/xh"
 homepage = "https://github.com/ducaale/xh"
 repository = "https://github.com/ducaale/xh"
 readme = "README.md"
-keywords = ["http"]
-categories = ["command-line-utilities"]
 exclude = ["assets/xhs", "assets/xhs.1.gz"]
 
 [dependencies]
 anyhow = "1.0.38"
 atty = "0.2"
+brotli = { version = "3.3.0", default-features = false, features = ["std"] }
 chardetng = "0.1.15"
 clap = { version = "3.1.0", features = ["derive", "wrap_help"] }
 clap_complete = "3.1.0"
@@ -25,6 +24,7 @@ digest_auth = "0.3.0"
 dirs = "3.0.1"
 encoding_rs = "0.8.28"
 encoding_rs_io = "0.1.7"
+flate2 = "1.0.22"
 indicatif = "0.16.2"
 jsonxf = "1.1.0"
 memchr = "2.4.1"
@@ -47,7 +47,7 @@ url = "2.2.2"
 [dependencies.reqwest]
 version = "0.11.6"
 default-features = false
-features = ["rustls-tls", "rustls-tls-webpki-roots", "rustls-tls-native-roots", "json", "gzip", "brotli", "deflate", "multipart", "blocking", "socks", "cookies"]
+features = ["rustls-tls", "rustls-tls-webpki-roots", "rustls-tls-native-roots", "json", "multipart", "blocking", "socks", "cookies"]
 
 [dependencies.syntect]
 version = "4.4"
diff --git a/src/auth.rs b/src/auth.rs
index 1b7684f..1bb20e2 100644
--- a/src/auth.rs
+++ b/src/auth.rs
@@ -83,7 +83,7 @@ impl<'a> DigestAuthMiddleware<'a> {
 
 impl<'a> Middleware for DigestAuthMiddleware<'a> {
     fn handle(&mut self, mut ctx: Context, mut request: Request) -> Result<Response> {
-        let response = self.next(&mut ctx, clone_request(&mut request)?)?;
+        let mut response = self.next(&mut ctx, clone_request(&mut request)?)?;
         match response.headers().get(WWW_AUTHENTICATE) {
             Some(wwwauth) if response.status() == StatusCode::UNAUTHORIZED => {
                 let mut context = digest_auth::AuthContext::new(
@@ -99,7 +99,7 @@ impl<'a> Middleware for DigestAuthMiddleware<'a> {
                 request
                     .headers_mut()
                     .insert(AUTHORIZATION, HeaderValue::from_str(&answer)?);
-                self.print(&mut ctx, response, &mut request)?;
+                self.print(&mut ctx, &mut response, &mut request)?;
                 Ok(self.next(&mut ctx, request)?)
             }
             _ => Ok(response),
diff --git a/src/decoder.rs b/src/decoder.rs
new file mode 100644
index 0000000..e257ef1
--- /dev/null
+++ b/src/decoder.rs
@@ -0,0 +1,175 @@
+use std::io::{self, Read};
+use std::str::FromStr;
+
+use brotli::Decompressor as BrotliDecoder;
+use flate2::read::{GzDecoder, ZlibDecoder};
+use reqwest::header::{HeaderMap, CONTENT_ENCODING, CONTENT_LENGTH, TRANSFER_ENCODING};
+
+#[derive(Debug)]
+pub enum CompressionType {
+    Gzip,
+    Deflate,
+    Brotli,
+}
+
+impl FromStr for CompressionType {
+    type Err = anyhow::Error;
+    fn from_str(value: &str) -> anyhow::Result<CompressionType> {
+        match value {
+            "gzip" => Ok(CompressionType::Gzip),
+            "deflate" => Ok(CompressionType::Deflate),
+            "br" => Ok(CompressionType::Brotli),
+            _ => Err(anyhow::anyhow!("unknown compression type")),
+        }
+    }
+}
+
+// See https://github.com/seanmonstar/reqwest/blob/9bd4e90ec3401c2c5bc435c58954f3d52ab53e99/src/async_impl/decoder.rs#L150
+pub fn get_compression_type(headers: &HeaderMap) -> Option<CompressionType> {
+    let mut compression_type = headers
+        .get_all(CONTENT_ENCODING)
+        .iter()
+        .find_map(|value| value.to_str().ok().and_then(|value| value.parse().ok()));
+
+    if compression_type.is_none() {
+        compression_type = headers
+            .get_all(TRANSFER_ENCODING)
+            .iter()
+            .find_map(|value| value.to_str().ok().and_then(|value| value.parse().ok()));
+    }
+
+    if compression_type.is_some() {
+        if let Some(content_length) = headers.get(CONTENT_LENGTH) {
+            if content_length == "0" {
+                return None;
+            }
+        }
+    }
+
+    compression_type
+}
+
+struct InnerReader<R: Read> {
+    reader: R,
+    has_errored: bool,
+}
+
+impl<R: Read> InnerReader<R> {
+    fn new(reader: R) -> Self {
+        InnerReader {
+            reader,
+            has_errored: false,
+        }
+    }
+}
+
+impl<R: Read> Read for InnerReader<R> {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        match self.reader.read(buf) {
+            Ok(len) => Ok(len),
+            Err(e) => {
+                self.has_errored = true;
+                Err(e)
+            }
+        }
+    }
+}
+
+enum Decoder<R: Read> {
+    PlainText(InnerReader<R>),
+    Gzip(GzDecoder<InnerReader<R>>),
+    Deflate(ZlibDecoder<InnerReader<R>>),
+    Brotli(BrotliDecoder<InnerReader<R>>),
+}
+
+impl<R: Read> Read for Decoder<R> {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        match self {
+            Decoder::PlainText(decoder) => decoder.read(buf),
+            Decoder::Gzip(decoder) => decoder.read(buf).map_err(|e| {
+                if decoder.get_ref().has_errored {
+                    e
+                } else {
+                    io::Error::new(
+                        e.kind(),
+                        format!("error decoding gzip response body: {}", e),
+                    )
+                }
+            }),
+            Decoder::Deflate(decoder) => decoder.read(buf).map_err(|e| {
+                if decoder.get_ref().has_errored {
+                    e
+                } else {
+                    io::Error::new(
+                        e.kind(),
+                        format!("error decoding deflate response body: {}", e),
+                    )
+                }
+            }),
+            Decoder::Brotli(decoder) => decoder.read(buf).map_err(|e| {
+                if decoder.get_ref().has_errored {
+                    e
+                } else {
+                    io::Error::new(
+                        e.kind(),
+                        format!("error decoding brotli response body: {}", e),
+                    )
+                }
+            }),
+        }
+    }
+}
+
+pub fn decompress(
+    reader: &mut impl Read,
+    compression_type: Option<CompressionType>,
+) -> impl Read + '_ {
+    let reader = InnerReader::new(reader);
+    match compression_type {
+        Some(CompressionType::Gzip) => Decoder::Gzip(GzDecoder::new(reader)),
+        Some(CompressionType::Deflate) => Decoder::Deflate(ZlibDecoder::new(reader)),
+        Some(CompressionType::Brotli) => Decoder::Brotli(BrotliDecoder::new(reader, 4096)),
+        None => Decoder::PlainText(reader),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn decode_errors_are_prepended_with_custom_message() {
+        let uncompressed_data = String::from("Hello world");
+        let mut uncompressed_data = uncompressed_data.as_bytes();
+        let mut reader = decompress(&mut uncompressed_data, Some(CompressionType::Gzip));
+        let mut buffer = Vec::new();
+        match reader.read_to_end(&mut buffer) {
+            Ok(_) => unreachable!("gzip should fail to decompress an uncompressed data"),
+            Err(e) => {
+                assert!(e
+                    .to_string()
+                    .starts_with("error decoding gzip response body:"))
+            }
+        }
+    }
+
+    #[test]
+    fn underlying_read_errors_are_not_modified() {
+        struct SadReader;
+        impl Read for SadReader {
+            fn read(&mut self, _buf: &mut [u8]) -> io::Result<usize> {
+                Err(io::Error::new(io::ErrorKind::Other, "oh no!"))
+            }
+        }
+
+        let mut sad_reader = SadReader;
+        let mut reader = decompress(&mut sad_reader, Some(CompressionType::Gzip));
+        let mut buffer = Vec::new();
+        match reader.read_to_end(&mut buffer) {
+            Ok(_) => unreachable!("SadReader should never be read"),
+            Err(e) => {
+                assert!(e.to_string().starts_with("oh no!"))
+            }
+        }
+    }
+}
diff --git a/src/download.rs b/src/download.rs
index 9314e1c..c07fb6c 100644
--- a/src/download.rs
+++ b/src/download.rs
@@ -13,6 +13,7 @@ use reqwest::{
     StatusCode,
 };
 
+use crate::decoder::{decompress, get_compression_type};
 use crate::regex;
 use crate::utils::{copy_largebuf, test_pretend_term};
 
@@ -245,7 +246,12 @@ pub fn download_file(
 
     match pb {
         Some(ref pb) => {
-            copy_largebuf(&mut pb.wrap_read(response), &mut buffer, false)?;
+            let compression_type = get_compression_type(response.headers());
+            copy_largebuf(
+                &mut decompress(&mut pb.wrap_read(response), compression_type),
+                &mut buffer,
+                false,
+            )?;
             let downloaded_length = pb.position() - starting_length;
             pb.finish_and_clear();
             let time_taken = starting_time.elapsed();
@@ -261,7 +267,12 @@ pub fn download_file(
             }
         }
         None => {
-            copy_largebuf(&mut response, &mut buffer, false)?;
+            let compression_type = get_compression_type(response.headers());
+            copy_largebuf(
+                &mut decompress(&mut response, compression_type),
+                &mut buffer,
+                false,
+            )?;
         }
     }
 
diff --git a/src/main.rs b/src/main.rs
index 3db4ed6..7875cec 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -2,6 +2,7 @@
 mod auth;
 mod buffer;
 mod cli;
+mod decoder;
 mod download;
 mod formatting;
 mod middleware;
@@ -152,7 +153,10 @@ fn run(args: Cli) -> Result<i32> {
         .use_rustls_tls()
         .http2_adaptive_window(true)
         .redirect(reqwest::redirect::Policy::none())
-        .timeout(timeout);
+        .timeout(timeout)
+        .no_gzip()
+        .no_deflate()
+        .no_brotli();
 
     if let Some(Some(tls_version)) = args.ssl {
         client = client
@@ -466,13 +470,13 @@ fn run(args: Cli) -> Result<i32> {
     }
 
     if !args.offline {
-        let response = {
+        let mut response = {
             let history_print = args.history_print.unwrap_or(print);
             let mut client = ClientWithMiddleware::new(&client);
             if args.all {
                 client = client.with_printer(|prev_response, next_request| {
                     if history_print.response_headers {
-                        printer.print_response_headers(&prev_response)?;
+                        printer.print_response_headers(prev_response)?;
                     }
                     if history_print.response_body {
                         printer.print_response_body(
@@ -528,7 +532,7 @@ fn run(args: Cli) -> Result<i32> {
                 )?;
             }
         } else if print.response_body {
-            printer.print_response_body(response, response_charset, response_mime)?;
+            printer.print_response_body(&mut response, response_charset, response_mime)?;
         }
     }
 
diff --git a/src/middleware.rs b/src/middleware.rs
index 5d8b5d7..8e1f6fc 100644
--- a/src/middleware.rs
+++ b/src/middleware.rs
@@ -3,14 +3,14 @@ use reqwest::blocking::{Client, Request, Response};
 
 pub struct Context<'a, 'b> {
     client: &'a Client,
-    printer: Option<&'a mut (dyn FnMut(Response, &mut Request) -> Result<()> + 'b)>,
+    printer: Option<&'a mut (dyn FnMut(&mut Response, &mut Request) -> Result<()> + 'b)>,
     middlewares: &'a mut [Box<dyn Middleware + 'b>],
 }
 
 impl<'a, 'b> Context<'a, 'b> {
     fn new(
         client: &'a Client,
-        printer: Option<&'a mut (dyn FnMut(Response, &mut Request) -> Result<()> + 'b)>,
+        printer: Option<&'a mut (dyn FnMut(&mut Response, &mut Request) -> Result<()> + 'b)>,
         middlewares: &'a mut [Box<dyn Middleware + 'b>],
     ) -> Self {
         Context {
@@ -39,7 +39,12 @@ pub trait Middleware {
         ctx.execute(request)
     }
 
-    fn print(&self, ctx: &mut Context, response: Response, request: &mut Request) -> Result<()> {
+    fn print(
+        &self,
+        ctx: &mut Context,
+        response: &mut Response,
+        request: &mut Request,
+    ) -> Result<()> {
         if let Some(ref mut printer) = ctx.printer {
             printer(response, request)?;
         }
@@ -50,7 +55,7 @@ pub trait Middleware {
 
 pub struct ClientWithMiddleware<'a, T>
 where
-    T: FnMut(Response, &mut Request) -> Result<()>,
+    T: FnMut(&mut Response, &mut Request) -> Result<()>,
 {
     client: &'a Client,
     printer: Option<T>,
@@ -59,7 +64,7 @@ where
 
 impl<'a, T> ClientWithMiddleware<'a, T>
 where
-    T: FnMut(Response, &mut Request) -> Result<()> + 'a,
+    T: FnMut(&mut Response, &mut Request) -> Result<()> + 'a,
 {
     pub fn new(client: &'a Client) -> Self {
         ClientWithMiddleware {
diff --git a/src/printer.rs b/src/printer.rs
index 7e144b3..756db82 100644
--- a/src/printer.rs
+++ b/src/printer.rs
@@ -12,6 +12,7 @@ use reqwest::header::{
 use reqwest::Version;
 use url::Url;
 
+use crate::decoder::{decompress, get_compression_type};
 use crate::{
     buffer::Buffer,
     cli::{Pretty, Theme},
@@ -423,14 +424,16 @@ impl Printer {
 
     pub fn print_response_body(
         &mut self,
-        mut response: Response,
+        response: &mut Response,
         encoding: Option<&'static Encoding>,
         mime: Option<&str>,
     ) -> anyhow::Result<()> {
         let url = response.url().clone();
         let content_type =
             mime.map_or_else(|| get_content_type(response.headers()), ContentType::from);
-        let encoding = encoding.or_else(|| get_charset(&response));
+        let encoding = encoding.or_else(|| get_charset(response));
+        let compression_type = get_compression_type(response.headers());
+        let mut body = decompress(response, compression_type);
 
         if !self.buffer.is_terminal() {
             if (self.color || self.indent_json) && content_type.is_text() {
@@ -449,24 +452,25 @@ impl Printer {
                 if self.stream {
                     self.print_body_stream(
                         content_type,
-                        &mut decode_stream(&mut response, encoding, &url)?,
+                        &mut decode_stream(&mut body, encoding, &url)?,
                     )?;
                 } else {
-                    let bytes = response.bytes()?;
-                    let text = decode_blob_unconditional(&bytes, encoding, &url);
+                    let mut buf = Vec::new();
+                    body.read_to_end(&mut buf)?;
+                    let text = decode_blob_unconditional(&buf, encoding, &url);
                     self.print_body_text(content_type, &text)?;
                 }
             } else if self.stream {
-                copy_largebuf(&mut response, &mut self.buffer, true)?;
+                copy_largebuf(&mut body, &mut self.buffer, true)?;
             } else {
-                let body = response.bytes()?;
-                self.buffer.print(&body)?;
+                let mut buf = Vec::new();
+                body.read_to_end(&mut buf)?;
+                self.buffer.print(&buf)?;
             }
         } else if self.stream {
-            match self.print_body_stream(
-                content_type,
-                &mut decode_stream(&mut response, encoding, &url)?,
-            ) {
+            match self
+                .print_body_stream(content_type, &mut decode_stream(&mut body, encoding, &url)?)
+            {
                 Ok(_) => {
                     self.buffer.print("\n")?;
                 }
@@ -476,24 +480,25 @@ impl Printer {
                 Err(err) => return Err(err.into()),
             }
         } else {
-            // Note that .decode() behaves like String::from_utf8_lossy()
-            let bytes = response.bytes()?;
-            let text = match decode_blob(&bytes, encoding, &url) {
+            let mut buf = Vec::new();
+            body.read_to_end(&mut buf)?;
+            match decode_blob(&buf, encoding, &url) {
                 None => {
                     self.buffer.print(BINARY_SUPPRESSOR)?;
                     return Ok(());
                 }
-                Some(text) => text,
+                Some(text) => {
+                    self.print_body_text(content_type, &text)?;
+                    self.buffer.print("\n")?;
+                }
             };
-            self.print_body_text(content_type, &text)?;
-            self.buffer.print("\n")?;
         }
         self.buffer.flush()?;
         Ok(())
     }
 }
 
-pub enum ContentType {
+enum ContentType {
     Json,
     Html,
     Xml,
@@ -506,7 +511,7 @@ pub enum ContentType {
 }
 
 impl ContentType {
-    pub fn is_text(&self) -> bool {
+    fn is_text(&self) -> bool {
         !matches!(
             self,
             ContentType::Unknown | ContentType::UrlencodedForm | ContentType::Multipart
@@ -542,14 +547,14 @@ impl From<&str> for ContentType {
     }
 }
 
-pub fn get_content_type(headers: &HeaderMap) -> ContentType {
+fn get_content_type(headers: &HeaderMap) -> ContentType {
     headers
         .get(CONTENT_TYPE)
         .and_then(|value| value.to_str().ok())
         .map_or(ContentType::Unknown, ContentType::from)
 }
 
-pub fn valid_json(text: &str) -> bool {
+fn valid_json(text: &str) -> bool {
     serde_json::from_str::<serde::de::IgnoredAny>(text).is_ok()
 }
 
@@ -598,7 +603,7 @@ fn decode_blob_unconditional<'a>(
 /// output is valid UTF-8, but a differently configured DecodeReaderBytes can
 /// produce invalid UTF-8.
 fn decode_stream<'a>(
-    response: &'a mut Response,
+    stream: &'a mut impl Read,
     encoding: Option<&'static Encoding>,
     url: &Url,
 ) -> io::Result<impl Read + 'a> {
@@ -608,7 +613,7 @@ fn decode_stream<'a>(
     // large enough for a best-effort attempt.
     // (16 is otherwise used because 0 seems dangerous, but it shouldn't matter.)
     let capacity = if encoding.is_some() { 16 } else { 16 * 1024 };
-    let mut reader = BufReader::with_capacity(capacity, response);
+    let mut reader = BufReader::with_capacity(capacity, stream);
     let encoding = match encoding {
         Some(encoding) => encoding,
         None => {
diff --git a/src/redirect.rs b/src/redirect.rs
index 70c8dc9..202fef3 100644
--- a/src/redirect.rs
+++ b/src/redirect.rs
@@ -36,7 +36,7 @@ impl Middleware for RedirectFollower {
                     self.max_redirects
                 ));
             }
-            self.print(&mut ctx, response, &mut next_request)?;
+            self.print(&mut ctx, &mut response, &mut next_request)?;
             request = clone_request(&mut next_request)?;
             response = self.next(&mut ctx, next_request)?;
         }
diff --git a/src/request_items.rs b/src/request_items.rs
index aaeaf01..219a2e7 100644
--- a/src/request_items.rs
+++ b/src/request_items.rs
@@ -269,7 +269,7 @@ impl RequestItems {
                     let key = HeaderName::from_bytes(key.as_bytes())?;
                     let value = HeaderValue::from_str(value)?;
                     headers_to_unset.remove(&key);
-                    headers.append(key, value);
+                    headers.insert(key, value);
                 }
                 RequestItem::HttpHeaderToUnset(key) => {
                     let key = HeaderName::from_bytes(key.as_bytes())?;


Test patch:
diff --git a/tests/cli.rs b/tests/cli.rs
index 821fc6e..4657447 100644
--- a/tests/cli.rs
+++ b/tests/cli.rs
@@ -2,7 +2,7 @@
 mod server;
 
 use std::collections::{HashMap, HashSet};
-use std::fs::{create_dir_all, read_to_string, File, OpenOptions};
+use std::fs::{self, File, OpenOptions};
 use std::future::Future;
 use std::io::Write;
 use std::iter::FromIterator;
@@ -214,20 +214,6 @@ fn header() {
         .success();
 }
 
-#[test]
-fn multiple_headers_with_same_key() {
-    let server = server::http(|req| async move {
-        let mut hello_header = req.headers().get_all("hello").iter();
-        assert_eq!(hello_header.next().unwrap(), &"world");
-        assert_eq!(hello_header.next().unwrap(), &"people");
-        hyper::Response::default()
-    });
-    get_command()
-        .args(&[&server.base_url(), "hello:world", "hello:people"])
-        .assert()
-        .success();
-}
-
 #[test]
 fn query_param() {
     let server = server::http(|req| async move {
@@ -311,7 +297,7 @@ fn download() {
         .arg(server.base_url())
         .assert()
         .success();
-    assert_eq!(read_to_string(&outfile).unwrap(), "file contents\n");
+    assert_eq!(fs::read_to_string(&outfile).unwrap(), "file contents\n");
 }
 
 #[test]
@@ -353,9 +339,12 @@ fn download_generated_filename() {
         .assert()
         .success();
 
-    assert_eq!(read_to_string(dir.path().join("bar.json")).unwrap(), "file");
     assert_eq!(
-        read_to_string(dir.path().join("bar.json-1")).unwrap(),
+        fs::read_to_string(dir.path().join("bar.json")).unwrap(),
+        "file"
+    );
+    assert_eq!(
+        fs::read_to_string(dir.path().join("bar.json-1")).unwrap(),
         "file"
     );
 }
@@ -375,7 +364,10 @@ fn download_supplied_filename() {
         .current_dir(&dir)
         .assert()
         .success();
-    assert_eq!(read_to_string(dir.path().join("foo.bar")).unwrap(), "file");
+    assert_eq!(
+        fs::read_to_string(dir.path().join("foo.bar")).unwrap(),
+        "file"
+    );
 }
 
 #[test]
@@ -394,7 +386,7 @@ fn download_supplied_unquoted_filename() {
         .assert()
         .success();
     assert_eq!(
-        read_to_string(dir.path().join("foo bar baz")).unwrap(),
+        fs::read_to_string(dir.path().join("foo bar baz")).unwrap(),
         "file"
     );
 }
@@ -1812,7 +1804,7 @@ fn named_sessions() {
         .collect(),
     );
 
-    let session_content = read_to_string(path_to_session).unwrap();
+    let session_content = fs::read_to_string(path_to_session).unwrap();
 
     assert_eq!(
         serde_json::from_str::<serde_json::Value>(&session_content).unwrap(),
@@ -1854,7 +1846,7 @@ fn anonymous_sessions() {
 
     server.assert_hits(1);
 
-    let session_content = read_to_string(path_to_session).unwrap();
+    let session_content = fs::read_to_string(path_to_session).unwrap();
 
     assert_eq!(
         serde_json::from_str::<serde_json::Value>(&session_content).unwrap(),
@@ -1900,8 +1892,10 @@ fn anonymous_read_only_session() {
         .success();
 
     assert_eq!(
-        serde_json::from_str::<serde_json::Value>(&read_to_string(session_file.path()).unwrap())
-            .unwrap(),
+        serde_json::from_str::<serde_json::Value>(
+            &fs::read_to_string(session_file.path()).unwrap()
+        )
+        .unwrap(),
         old_session_content
     );
 }
@@ -1930,7 +1924,7 @@ fn session_files_are_created_in_read_only_mode() {
         .assert()
         .success();
 
-    let session_content = read_to_string(path_to_session).unwrap();
+    let session_content = fs::read_to_string(path_to_session).unwrap();
     assert_eq!(
         serde_json::from_str::<serde_json::Value>(&session_content).unwrap(),
         serde_json::json!({
@@ -1980,7 +1974,7 @@ fn named_read_only_session() {
             "hello": "world"
         }
     });
-    create_dir_all(path_to_session.parent().unwrap()).unwrap();
+    fs::create_dir_all(path_to_session.parent().unwrap()).unwrap();
     File::create(&path_to_session).unwrap();
     std::fs::write(&path_to_session, old_session_content.to_string()).unwrap();
 
@@ -1993,7 +1987,7 @@ fn named_read_only_session() {
         .success();
 
     assert_eq!(
-        serde_json::from_str::<serde_json::Value>(&read_to_string(path_to_session).unwrap())
+        serde_json::from_str::<serde_json::Value>(&fs::read_to_string(path_to_session).unwrap())
             .unwrap(),
         old_session_content
     );
@@ -2045,7 +2039,7 @@ fn expired_cookies_are_removed_from_session() {
         .assert()
         .success();
 
-    let session_content = read_to_string(session_file.path()).unwrap();
+    let session_content = fs::read_to_string(session_file.path()).unwrap();
     assert_eq!(
         serde_json::from_str::<serde_json::Value>(&session_content).unwrap(),
         serde_json::json!({
@@ -2116,7 +2110,7 @@ fn cookies_override_each_other_in_the_correct_order() {
 
     server.assert_hits(1);
 
-    let session_content = read_to_string(session_file.path()).unwrap();
+    let session_content = fs::read_to_string(session_file.path()).unwrap();
     assert_eq!(
         serde_json::from_str::<serde_json::Value>(&session_content).unwrap(),
         serde_json::json!({
@@ -2226,7 +2220,7 @@ fn auth_netrc_is_not_persisted_in_session() {
 
     server.assert_hits(1);
 
-    let session_content = read_to_string(path_to_session).unwrap();
+    let session_content = fs::read_to_string(path_to_session).unwrap();
     assert_eq!(
         serde_json::from_str::<serde_json::Value>(&session_content).unwrap(),
         serde_json::json!({
@@ -2763,3 +2757,101 @@ fn tilde_expanded_in_request_items() {
         .stdout(contains("random data"))
         .success();
 }
+
+#[test]
+fn gzip() {
+    let server = server::http(|_req| async move {
+        let compressed_bytes = fs::read("./tests/fixtures/responses/hello_world.gz").unwrap();
+        hyper::Response::builder()
+            .header("date", "N/A")
+            .header("content-encoding", "gzip")
+            .body(compressed_bytes.into())
+            .unwrap()
+    });
+
+    get_command()
+        .arg(server.base_url())
+        .assert()
+        .stdout(indoc! {r#"
+            HTTP/1.1 200 OK
+            Content-Encoding: gzip
+            Content-Length: 48
+            Date: N/A
+
+            Hello world
+
+        "#});
+}
+
+#[test]
+fn deflate() {
+    let server = server::http(|_req| async move {
+        let compressed_bytes = fs::read("./tests/fixtures/responses/hello_world.zz").unwrap();
+        hyper::Response::builder()
+            .header("date", "N/A")
+            .header("content-encoding", "deflate")
+            .body(compressed_bytes.into())
+            .unwrap()
+    });
+
+    get_command()
+        .arg(server.base_url())
+        .assert()
+        .stdout(indoc! {r#"
+            HTTP/1.1 200 OK
+            Content-Encoding: deflate
+            Content-Length: 20
+            Date: N/A
+
+            Hello world
+
+        "#});
+}
+
+#[test]
+fn brotli() {
+    let server = server::http(|_req| async move {
+        let compressed_bytes = fs::read("./tests/fixtures/responses/hello_world.br").unwrap();
+        hyper::Response::builder()
+            .header("date", "N/A")
+            .header("content-encoding", "br")
+            .body(compressed_bytes.into())
+            .unwrap()
+    });
+
+    get_command()
+        .arg(server.base_url())
+        .assert()
+        .stdout(indoc! {r#"
+            HTTP/1.1 200 OK
+            Content-Encoding: br
+            Content-Length: 17
+            Date: N/A
+
+            Hello world
+
+        "#});
+}
+
+#[test]
+fn empty_response_with_content_encoding() {
+    let server = server::http(|_req| async move {
+        hyper::Response::builder()
+            .header("date", "N/A")
+            .header("content-encoding", "gzip")
+            .body("".into())
+            .unwrap()
+    });
+
+    get_command()
+        .arg(server.base_url())
+        .assert()
+        .stdout(indoc! {r#"
+            HTTP/1.1 200 OK
+            Content-Encoding: gzip
+            Content-Length: 0
+            Date: N/A
+
+
+        "#});
+}
diff --git a/tests/fixtures/responses/README.md b/tests/fixtures/responses/README.md
new file mode 100644
index 0000000..dadf983
--- /dev/null
+++ b/tests/fixtures/responses/README.md
@@ -0,0 +1,12 @@
+# Test Fixtures: Compressed Responses
+
+```sh
+$ echo "Hello world" > hello_world
+$ pigz hello_world # hello_world.gz
+
+$ echo "Hello world" > hello_world
+$ pigz -z hello_world # hello_world.zz
+
+$ echo "Hello world" > hello_world
+$ brotli hello_world # hello_world.br
+```