Repository: mitmproxy/mitmproxy

Issue:
Mitmproxy with received random data crashes, when trying to edit a form 
##### Steps to reproduce the problem:

1. Run mitmproxy.
2. `pathoc -c example.com:80 localhost:8080 'post:/:b@200'`
3. Choose the corresponding flow.
4. Press `e`.
5. Form.
**Traceback:**
```
Traceback (most recent call last):
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/master.py", line 216, in run
    self.loop.run()
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/main_loop.py", line 278, in run
    self._run()
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/main_loop.py", line 376, in _run
    self.event_loop.run()
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/main_loop.py", line 682, in run
    self._loop()
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/main_loop.py", line 719, in _loop
    self._watch_files[fd]()
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/raw_display.py", line 393, in <lambda>
    event_loop, callback, self.get_available_raw_input())
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/raw_display.py", line 493, in parse_input
    callback(processed, processed_codes)
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/main_loop.py", line 403, in _update
    self.process_input(keys)
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/main_loop.py", line 503, in process_input
    k = self._topmost_widget.keypress(self.screen_size, k)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/window.py", line 293, in keypress
    k = super().keypress(size, k)
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/container.py", line 1128, in keypress
    return self.body.keypress( (maxcol, remaining), key )
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/window.py", line 36, in keypress
    ret = super().keypress(size, key)
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/container.py", line 1128, in keypress
    return self.body.keypress( (maxcol, remaining), key )
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/container.py", line 592, in keypress
    *self.calculate_padding_filler(size, True)), key)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/overlay.py", line 130, in keypress
    self.master.keymap.handle("global", key)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/keymap.py", line 130, in handle
    return self.executor(b.command)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/commandexecutor.py", line 17, in __call__
    ret = self.master.commands.call(cmd)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/command.py", line 221, in call
    return self.call_args(parts[0], parts[1:])
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/command.py", line 212, in call_args
    return self.commands[path].call(args)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/command.py", line 101, in call
    ret = self.func(*pargs)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/command.py", line 251, in wrapper
    return function(*args, **kwargs)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/consoleaddons.py", line 174, in nav_select
    self.master.inject_key("m_select")
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/master.py", line 181, in inject_key
    self.loop.process_input([key])
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/main_loop.py", line 503, in process_input
    k = self._topmost_widget.keypress(self.screen_size, k)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/window.py", line 293, in keypress
    k = super().keypress(size, k)
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/container.py", line 1128, in keypress
    return self.body.keypress( (maxcol, remaining), key )
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/window.py", line 36, in keypress
    ret = super().keypress(size, key)
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/container.py", line 1128, in keypress
    return self.body.keypress( (maxcol, remaining), key )
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/container.py", line 592, in keypress
    *self.calculate_padding_filler(size, True)), key)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/overlay.py", line 121, in keypress
    signals.pop_view_state.send(self)
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/blinker/base.py", line 267, in send
    for receiver in self.receivers_for(sender)]
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/blinker/base.py", line 267, in <listcomp>
    for receiver in self.receivers_for(sender)]
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/window.py", line 230, in pop
    if self.focus_stack().pop():
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/window.py", line 98, in pop
    self.call("layout_popping")
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/window.py", line 110, in call
    getattr(self.top_window(), name)(*args, **kwargs)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/grideditor/base.py", line 463, in layout_popping
    self.call(self._w, "layout_popping")
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/grideditor/base.py", line 441, in call
    f(*args, **kwargs)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/grideditor/base.py", line 313, in layout_popping
    self.callback(self.data_out(res), *self.cb_args, **self.cb_kwargs)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/grideditor/base.py", line 456, in set_data_update
    self.set_data(vals, flow)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/grideditor/editors.py", line 65, in set_data
    flow.request.urlencoded_form = vals
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/net/http/request.py", line 462, in urlencoded_form
    self._set_urlencoded_form(value)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/net/http/request.py", line 444, in _set_urlencoded_form
    self.content = mitmproxy.net.http.url.encode(form_data, self.content.decode()).encode()
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xa8 in position 0: invalid start byte
```

##### System information

Mitmproxy: 3.0.0.dev0004 (commit 68c32d8) 
Python:    3.5.2
OpenSSL:   OpenSSL 1.1.0g  2 Nov 2017
Platform:  Linux-4.4.0-109-generic-x86_64-with-Ubuntu-16.04-xenial

Mitmproxy with received random data crashes, when trying to edit a form 
##### Steps to reproduce the problem:

1. Run mitmproxy.
2. `pathoc -c example.com:80 localhost:8080 'post:/:b@200'`
3. Choose the corresponding flow.
4. Press `e`.
5. Form.
**Traceback:**
```
Traceback (most recent call last):
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/master.py", line 216, in run
    self.loop.run()
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/main_loop.py", line 278, in run
    self._run()
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/main_loop.py", line 376, in _run
    self.event_loop.run()
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/main_loop.py", line 682, in run
    self._loop()
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/main_loop.py", line 719, in _loop
    self._watch_files[fd]()
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/raw_display.py", line 393, in <lambda>
    event_loop, callback, self.get_available_raw_input())
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/raw_display.py", line 493, in parse_input
    callback(processed, processed_codes)
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/main_loop.py", line 403, in _update
    self.process_input(keys)
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/main_loop.py", line 503, in process_input
    k = self._topmost_widget.keypress(self.screen_size, k)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/window.py", line 293, in keypress
    k = super().keypress(size, k)
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/container.py", line 1128, in keypress
    return self.body.keypress( (maxcol, remaining), key )
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/window.py", line 36, in keypress
    ret = super().keypress(size, key)
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/container.py", line 1128, in keypress
    return self.body.keypress( (maxcol, remaining), key )
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/container.py", line 592, in keypress
    *self.calculate_padding_filler(size, True)), key)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/overlay.py", line 130, in keypress
    self.master.keymap.handle("global", key)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/keymap.py", line 130, in handle
    return self.executor(b.command)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/commandexecutor.py", line 17, in __call__
    ret = self.master.commands.call(cmd)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/command.py", line 221, in call
    return self.call_args(parts[0], parts[1:])
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/command.py", line 212, in call_args
    return self.commands[path].call(args)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/command.py", line 101, in call
    ret = self.func(*pargs)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/command.py", line 251, in wrapper
    return function(*args, **kwargs)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/consoleaddons.py", line 174, in nav_select
    self.master.inject_key("m_select")
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/master.py", line 181, in inject_key
    self.loop.process_input([key])
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/main_loop.py", line 503, in process_input
    k = self._topmost_widget.keypress(self.screen_size, k)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/window.py", line 293, in keypress
    k = super().keypress(size, k)
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/container.py", line 1128, in keypress
    return self.body.keypress( (maxcol, remaining), key )
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/window.py", line 36, in keypress
    ret = super().keypress(size, key)
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/container.py", line 1128, in keypress
    return self.body.keypress( (maxcol, remaining), key )
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/urwid/container.py", line 592, in keypress
    *self.calculate_padding_filler(size, True)), key)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/overlay.py", line 121, in keypress
    signals.pop_view_state.send(self)
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/blinker/base.py", line 267, in send
    for receiver in self.receivers_for(sender)]
  File "/home/kajoj/Mitmproxy/mitmproxy/venv/lib/python3.5/site-packages/blinker/base.py", line 267, in <listcomp>
    for receiver in self.receivers_for(sender)]
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/window.py", line 230, in pop
    if self.focus_stack().pop():
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/window.py", line 98, in pop
    self.call("layout_popping")
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/window.py", line 110, in call
    getattr(self.top_window(), name)(*args, **kwargs)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/grideditor/base.py", line 463, in layout_popping
    self.call(self._w, "layout_popping")
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/grideditor/base.py", line 441, in call
    f(*args, **kwargs)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/grideditor/base.py", line 313, in layout_popping
    self.callback(self.data_out(res), *self.cb_args, **self.cb_kwargs)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/grideditor/base.py", line 456, in set_data_update
    self.set_data(vals, flow)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/tools/console/grideditor/editors.py", line 65, in set_data
    flow.request.urlencoded_form = vals
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/net/http/request.py", line 462, in urlencoded_form
    self._set_urlencoded_form(value)
  File "/home/kajoj/Mitmproxy/mitmproxy/mitmproxy/net/http/request.py", line 444, in _set_urlencoded_form
    self.content = mitmproxy.net.http.url.encode(form_data, self.content.decode()).encode()
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xa8 in position 0: invalid start byte
```

##### System information

Mitmproxy: 3.0.0.dev0004 (commit 68c32d8) 
Python:    3.5.2
OpenSSL:   OpenSSL 1.1.0g  2 Nov 2017
Platform:  Linux-4.4.0-109-generic-x86_64-with-Ubuntu-16.04-xenial



Discussion:
Well the problem is that we are assuming the content to be of `utf-8` type here right? as mentioned before by @mhils let's use `get_text`
Well the problem is that we are assuming the content to be of `utf-8` type here right? as mentioned before by @mhils let's use `get_text`

Resolving patch:
diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py
--- a/mitmproxy/net/http/request.py
+++ b/mitmproxy/net/http/request.py
@@ -429,10 +429,7 @@ def constrain_encoding(self):
     def _get_urlencoded_form(self):
         is_valid_content_type = "application/x-www-form-urlencoded" in self.headers.get("content-type", "").lower()
         if is_valid_content_type:
-            try:
-                return tuple(mitmproxy.net.http.url.decode(self.content.decode()))
-            except ValueError:
-                pass
+            return tuple(mitmproxy.net.http.url.decode(self.get_text(strict=False)))
         return ()
 
     def _set_urlencoded_form(self, form_data):
@@ -441,7 +438,7 @@ def _set_urlencoded_form(self, form_data):
         This will overwrite the existing content if there is one.
         """
         self.headers["content-type"] = "application/x-www-form-urlencoded"
-        self.content = mitmproxy.net.http.url.encode(form_data, self.content.decode()).encode()
+        self.content = mitmproxy.net.http.url.encode(form_data, self.get_text(strict=False)).encode()
 
     @property
     def urlencoded_form(self):


Test patch:
diff --git a/test/mitmproxy/net/http/test_request.py b/test/mitmproxy/net/http/test_request.py
--- a/test/mitmproxy/net/http/test_request.py
+++ b/test/mitmproxy/net/http/test_request.py
@@ -351,10 +351,10 @@ def test_get_urlencoded_form(self):
         request.headers["Content-Type"] = "application/x-www-form-urlencoded"
         assert list(request.urlencoded_form.items()) == [("foobar", "baz")]
         request.raw_content = b"\xFF"
-        assert len(request.urlencoded_form) == 0
+        assert len(request.urlencoded_form) == 1
 
     def test_set_urlencoded_form(self):
-        request = treq()
+        request = treq(content=b"\xec\xed")
         request.urlencoded_form = [('foo', 'bar'), ('rab', 'oof')]
         assert request.headers["Content-Type"] == "application/x-www-form-urlencoded"
         assert request.content