Repository: mitmproxy/mitmproxy

Issue:
Crash of har_dump.py
I sometimes get either `b'' is not JSON serializable` or  `Object of type 'bytes' is not JSON serializable`

I used mitmproxy to extract such a flow and uploaded it [here](https://gist.github.com/Junkern/495d4c7c1b572d7749f0291adfb0dd15)

##### Steps to reproduce the problem:

1.
2.
3.


##### Any other comments? What have you tried so far?

It seems, that the error mostly occurs for `POST` and `HEAD` requests. Sometimes with an empty response body, sometimes not. Seldomly a `GET` requests triggers this error

##### System information
mitmproxy 1.0.2
macOs Sierra 10.12.3

Edit: Updated the Gist link to point to a working binary gist


Discussion:
Thanks. Could you please upload the flow somewhere else? Gists don't work with binary data, the file is corrupted.
I now used <https://github.com/mbostock/gistup> to upload binary data as a gist.
[Link](https://gist.github.com/Junkern/495d4c7c1b572d7749f0291adfb0dd15)
That works, thanks!
A short update from my side: I used the fix from https://github.com/mitmproxy/mitmproxy/pull/1650/files to address this issue. Seems to work as well.
@Junkern Does that work in case of an empty response?
I implemented the fix in the har_dump and tested it on the flow with an empty response (the one  I posted above). No error was thrown

Resolving patch:
diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py
--- a/examples/complex/har_dump.py
+++ b/examples/complex/har_dump.py
@@ -7,6 +7,7 @@
 import sys
 import base64
 import zlib
+import os
 
 from datetime import datetime
 import pytz
@@ -166,7 +167,7 @@ def done():
         if dump_file.endswith('.zhar'):
             raw = zlib.compress(raw, 9)
 
-        with open(dump_file, "wb") as f:
+        with open(os.path.expanduser(dump_file), "wb") as f:
             f.write(raw)
 
         mitmproxy.ctx.log("HAR dump finished (wrote %s bytes to file)" % len(json_dump))
diff --git a/mitmproxy/net/http/encoding.py b/mitmproxy/net/http/encoding.py
--- a/mitmproxy/net/http/encoding.py
+++ b/mitmproxy/net/http/encoding.py
@@ -31,8 +31,8 @@ def decode(encoded: Union[str, bytes], encoding: str, errors: str='strict') -> U
     Raises:
         ValueError, if decoding fails.
     """
-    if len(encoded) == 0:
-        return encoded
+    if encoded is None:
+        return None
 
     global _cache
     cached = (
@@ -72,8 +72,8 @@ def encode(decoded: Union[str, bytes], encoding: str, errors: str='strict') -> U
     Raises:
         ValueError, if encoding fails.
     """
-    if len(decoded) == 0:
-        return decoded
+    if decoded is None:
+        return None
 
     global _cache
     cached = (
@@ -86,10 +86,7 @@ def encode(decoded: Union[str, bytes], encoding: str, errors: str='strict') -> U
         return _cache.encoded
     try:
         try:
-            value = decoded
-            if isinstance(value, str):
-                value = decoded.encode()
-            encoded = custom_encode[encoding](value)
+            encoded = custom_encode[encoding](decoded)
         except KeyError:
             encoded = codecs.encode(decoded, encoding, errors)
         if encoding in ("gzip", "deflate", "br"):
@@ -114,12 +111,14 @@ def identity(content):
     return content
 
 
-def decode_gzip(content):
+def decode_gzip(content: bytes) -> bytes:
+    if not content:
+        return b""
     gfile = gzip.GzipFile(fileobj=BytesIO(content))
     return gfile.read()
 
 
-def encode_gzip(content):
+def encode_gzip(content: bytes) -> bytes:
     s = BytesIO()
     gf = gzip.GzipFile(fileobj=s, mode='wb')
     gf.write(content)
@@ -127,15 +126,17 @@ def encode_gzip(content):
     return s.getvalue()
 
 
-def decode_brotli(content):
+def decode_brotli(content: bytes) -> bytes:
+    if not content:
+        return b""
     return brotli.decompress(content)
 
 
-def encode_brotli(content):
+def encode_brotli(content: bytes) -> bytes:
     return brotli.compress(content)
 
 
-def decode_deflate(content):
+def decode_deflate(content: bytes) -> bytes:
     """
         Returns decompressed data for DEFLATE. Some servers may respond with
         compressed data without a zlib header or checksum. An undocumented
@@ -144,13 +145,15 @@ def decode_deflate(content):
 
         http://bugs.python.org/issue5784
     """
+    if not content:
+        return b""
     try:
         return zlib.decompress(content)
     except zlib.error:
         return zlib.decompress(content, -15)
 
 
-def encode_deflate(content):
+def encode_deflate(content: bytes) -> bytes:
     """
         Returns compressed content, always including zlib header and checksum.
     """


Test patch:
diff --git a/test/mitmproxy/net/http/test_encoding.py b/test/mitmproxy/net/http/test_encoding.py
--- a/test/mitmproxy/net/http/test_encoding.py
+++ b/test/mitmproxy/net/http/test_encoding.py
@@ -21,26 +21,55 @@ def test_identity(encoder):
     'deflate',
 ])
 def test_encoders(encoder):
-    assert "" == encoding.decode("", encoder)
+    """
+        This test is for testing byte->byte encoding/decoding
+    """
+    assert encoding.decode(None, encoder) is None
+    assert encoding.encode(None, encoder) is None
+
     assert b"" == encoding.decode(b"", encoder)
 
-    assert "string" == encoding.decode(
+    assert b"string" == encoding.decode(
         encoding.encode(
-            "string",
+            b"string",
             encoder
         ),
         encoder
     )
-    assert b"string" == encoding.decode(
+
+    with pytest.raises(TypeError):
+        encoding.encode("string", encoder)
+
+    with pytest.raises(TypeError):
+        encoding.decode("string", encoder)
+    with pytest.raises(ValueError):
+        encoding.decode(b"foobar", encoder)
+
+
+@pytest.mark.parametrize("encoder", [
+    'utf8',
+    'latin-1'
+])
+def test_encoders_strings(encoder):
+    """
+        This test is for testing byte->str decoding
+        and str->byte encoding
+    """
+    assert "" == encoding.decode(b"", encoder)
+
+    assert "string" == encoding.decode(
         encoding.encode(
-            b"string",
+            "string",
             encoder
         ),
         encoder
     )
 
-    with pytest.raises(ValueError):
-        encoding.decode(b"foobar", encoder)
+    with pytest.raises(TypeError):
+        encoding.encode(b"string", encoder)
+
+    with pytest.raises(TypeError):
+        encoding.decode("foobar", encoder)
 
 
 def test_cache():