Repository: mitmproxy/mitmproxy

Issue:
fix crash at cut addon
fixes #3965
App crashed when trying to copy the response content using "cut.clip @focus response.raw_content"
Crash log:

```
Traceback (most recent call last):
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/mitmproxy/master.py", line 86, in run_loop
    loop()
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/urwid/main_loop.py", line 286, in run
    self._run()
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/urwid/main_loop.py", line 384, in _run
    self.event_loop.run()
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/urwid/main_loop.py", line 1484, in run
    reraise(*exc_info)
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/urwid/compat.py", line 58, in reraise
    raise value
  File "/usr/local/Cellar/python@3.8/3.8.1/Frameworks/Python.framework/Versions/3.8/lib/python3.8/asyncio/events.py", line 81, in _run
    self._context.run(self._callback, *self._args)
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/urwid/raw_display.py", line 403, in <lambda>
    wrapper = lambda: self.parse_input(
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/urwid/raw_display.py", line 502, in parse_input
    callback(processed, processed_codes)
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/urwid/main_loop.py", line 411, in _update
    self.process_input(keys)
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/urwid/main_loop.py", line 511, in process_input
    k = self._topmost_widget.keypress(self.screen_size, k)
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/mitmproxy/tools/console/window.py", line 310, in keypress
    k = super().keypress(size, k)
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/urwid/container.py", line 1119, in keypress
    return self.footer.keypress((maxcol,),key)
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/mitmproxy/tools/console/statusbar.py", line 201, in keypress
    return self.ab.keypress(*args, **kwargs)
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/mitmproxy/tools/console/statusbar.py", line 149, in keypress
    self.prompt_execute(text)
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/mitmproxy/tools/console/statusbar.py", line 169, in prompt_execute
    msg = p(txt)
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/mitmproxy/tools/console/statusbar.py", line 115, in execute_command
    execute(txt)
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/mitmproxy/tools/console/commandexecutor.py", line 17, in __call__
    ret = self.master.commands.execute(cmd)
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/mitmproxy/command.py", line 265, in execute
    return self._call_strings(command_name, args)
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/mitmproxy/command.py", line 251, in _call_strings
    return self.commands[command_name].call(args)
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/mitmproxy/command.py", line 122, in call
    ret = self.func(*bound_args.args, **bound_args.kwargs)
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/mitmproxy/command.py", line 295, in wrapper
    return function(*args, **kwargs)
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/mitmproxy/addons/cut.py", line 133, in clip
    fp.write(strutils.always_str(v))  # type: ignore
  File "/usr/local/Cellar/mitmproxy/5.0.1_1/libexec/lib/python3.8/site-packages/mitmproxy/utils/strutils.py", line 26, in always_str
    return str_or_bytes.decode(*decode_args)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8b in position 1: invalid start byte

```


Discussion:
@mhils we can use this or with codecs, we can use try and catch with different encoding but I think this approach is better. what are your thoughts about this?
Any hint about the build-wheel error?
Shouldn't we just switch to `io.BinaryIO` in the case where exactly one column and row is selected? 

https://github.com/mitmproxy/mitmproxy/blob/dac0bfe786a8d1dfdd97f29d6bb262ed258153fa/mitmproxy/addons/cut.py#L130-L134

> If there is exactly one row and one column, the data is written to file as-is, with raw bytes preserved.

The user is trying to copy bytes to the clipboard, no need to force it into a string. Or is there? I don't know if you can even put raw bytes to the clipboard?


---

**Welp** https://github.com/asweigart/pyperclip/issues/63 :smile: 

Is this even a good idea? What's the use case? Maybe we should just tell the user that they cannot put binary data into the clipboard and that they can export it to a file instead?
>The user is trying to copy bytes to the clipboard, no need to force it into a string. Or is there? I don't know if you can even put raw bytes to the clipboard?
>
I think pyperclip does not support binary data for the clipboard. 
>Is this even a good idea? What's the use case? Maybe we should just tell the user that they cannot put binary data into the clipboard and that they can export it to a file instead?
>
May be we can notify that if one row and one column and binary data then it's decoded, right now it shows 
`# Send cuts to the clipboard. If there are multiple flows or cuts, the`
 `# format is UTF-8 encoded CSV. If there is exactly one row and one`
 `# column, the data is written to file as-is, with raw bytes preserved.`
`cut.clip flows cuts`

I don't think it's right for single cut?
But let's hear maintainers thought about this.
I wasn't even aware of the `cut.clip` feature so I'm not sure what the original intended use-cases are. I thought about this and if I would use it to quickly process the data somewhere else (e.g. paste it into CyberChef). I would like binary stuff to be Base64 encoded. I think that's the most common way to represent binary as strings and well supported. I can't think of a use-case where you would actually want raw bytes in your clipboard, I personally don't know where I could actually paste it. I mean maybe I could paste an image right into Gimp?
So maybe we can unify this and also include Base64 in the CSV (if it cannot be represented as text)?

But like I said I didn't even know about this feature and I'm not invested in it at all. I'm just trying to kill some time rn :smile: 
> Just guessing the encoding is can of works we really don't want to open.

I mean :see_no_evil:

https://github.com/mitmproxy/mitmproxy/blob/7fdcbb09e6034ab1f76724965cfdf45f3d775129/mitmproxy/net/http/message.py#L172-L187

This is not about guessing encoding for arbitrary binary data. The original error report / crash was about `raw_content` which can be gzip or whatever. And mitmproxy also crashes when `content` is an image or other binary. So this definitely needs to be fixed. The question is how much responsibility do we give the user? If they're using `content` instead of `text` they might already made a decision that they want binary? So the only question is how we represent these binary cases, without doing any magic. In the best case we could add them raw to the clipboard, which pyperclip doesn't support. And also I haven't checked what happens with binary if it's a CSV, maybe the csv lib already does something useful we can steal?
>And also I haven't checked what happens with binary if it's a CSV
>
This also does not store bytes directly that's why when writing to file in more row or column the data is converted into str.
As saving to file also failed(in case of either more flow or more cuts) along with copy to clipboard as we try to decode them. if we try to preserve the raw data how about we store data as python byte object in the string(example "b'\x80\x40something'") in both cases.
@mhils, @Prinzhorn 
It's definitely a bug that we crash here, but for most purposes, you really want to use `.text` instead of `.raw_content` with `cut.clip`. Binary content and the clipboard doesn't mix well.
Thanks, @mhils for the help. cheers

Resolving patch:
diff --git a/mitmproxy/addons/cut.py b/mitmproxy/addons/cut.py
--- a/mitmproxy/addons/cut.py
+++ b/mitmproxy/addons/cut.py
@@ -8,7 +8,6 @@
 from mitmproxy import flow
 from mitmproxy import ctx
 from mitmproxy import certs
-from mitmproxy.utils import strutils
 import mitmproxy.types
 
 import pyperclip
@@ -54,6 +53,14 @@ def extract(cut: str, f: flow.Flow) -> typing.Union[str, bytes]:
     return str(current or "")
 
 
+def extract_str(cut: str, f: flow.Flow) -> str:
+    ret = extract(cut, f)
+    if isinstance(ret, bytes):
+        return repr(ret)
+    else:
+        return ret
+
+
 class Cut:
     @command.command("cut")
     def cut(
@@ -110,10 +117,8 @@ def save(
                 with open(path, "a" if append else "w", newline='', encoding="utf8") as tfp:
                     writer = csv.writer(tfp)
                     for f in flows:
-                        vals = [extract(c, f) for c in cuts]
-                        writer.writerow(
-                            [strutils.always_str(x) or "" for x in vals]  # type: ignore
-                        )
+                        vals = [extract_str(c, f) for c in cuts]
+                        writer.writerow(vals)
                 ctx.log.alert("Saved %s cuts over %d flows as CSV." % (len(cuts), len(flows)))
         except OSError as e:
             ctx.log.error(str(e))
@@ -132,16 +137,14 @@ def clip(
         v: typing.Union[str, bytes]
         fp = io.StringIO(newline="")
         if len(cuts) == 1 and len(flows) == 1:
-            v = extract(cuts[0], flows[0])
-            fp.write(strutils.always_str(v))  # type: ignore
+            v = extract_str(cuts[0], flows[0])
+            fp.write(v)
             ctx.log.alert("Clipped single cut.")
         else:
             writer = csv.writer(fp)
             for f in flows:
-                vals = [extract(c, f) for c in cuts]
-                writer.writerow(
-                    [strutils.always_str(v) for v in vals]
-                )
+                vals = [extract_str(c, f) for c in cuts]
+                writer.writerow(vals)
             ctx.log.alert("Clipped %s cuts as CSV." % len(cuts))
         try:
             pyperclip.copy(fp.getvalue())


Test patch:
diff --git a/test/mitmproxy/addons/test_cut.py b/test/mitmproxy/addons/test_cut.py
--- a/test/mitmproxy/addons/test_cut.py
+++ b/test/mitmproxy/addons/test_cut.py
@@ -59,6 +59,12 @@ def test_extract(tdata):
     assert "CERTIFICATE" in cut.extract("server_conn.certificate_list", tf)
 
 
+def test_extract_str():
+    tf = tflow.tflow()
+    tf.request.raw_content = b"\xFF"
+    assert cut.extract_str("request.raw_content", tf) == r"b'\xff'"
+
+
 def test_headername():
     with pytest.raises(exceptions.CommandError):
         cut.headername("header[foo.")
@@ -115,7 +121,7 @@ def test_cut_save(tmpdir):
         tctx.command(c.save, "@all", "request.method", f)
         assert qr(f).splitlines() == [b"GET", b"GET"]
         tctx.command(c.save, "@all", "request.method,request.content", f)
-        assert qr(f).splitlines() == [b"GET,content", b"GET,content"]
+        assert qr(f).splitlines() == [b"GET,b'content'", b"GET,b'content'"]
 
 
 @pytest.mark.parametrize("exception, log_message", [