Repository: mitmproxy/mitmproxy

Issue:
mitmdump crashes when delaying recorded responses with the kill flag
##### Steps to reproduce the problem:
1. Record the server responses of some website: `mitmdump -w rec.out --anticache`
2. Replay the server responses with the `--kill` flag and a script that delays one or more responses without blocking: `mitmdump -s script.py -S rec.out --kill --no-pop`
   
   script.py:
   
   ```
   import re
   import time
   
   from mitmproxy.script import concurrent
   
   delayUrl=".*/some_url.js"
   
   @concurrent
   def request(context, flow):
     if re.match(delayUrl, flow.request.url):
       print('delay request: %s%s' % (flow.request.host, flow.request.path))
       time.sleep(5)
     print('start request: %s%s' % (flow.request.host, flow.request.path))
   
   def response(context, flow):
     if flow.error:
       print('Error: ' + str(flow.error))
   ```
##### What is the expected behavior?

The server responses are replayed, the responses matched by the `delayUrl` pattern are delayed by 5s, and responses not found in the recording are killed.
##### What went wrong?

The script crashes:

```
Error: Connection killed
Traceback (most recent call last):
  File "/home/christofferqa/mitmproxy/venv/bin/mitmdump", line 9, in <module>
    load_entry_point('mitmproxy', 'console_scripts', 'mitmdump')()
  File "/home/christofferqa/mitmproxy/mitmproxy/main.py", line 102, in mitmdump
    master.run()
  File "/home/christofferqa/mitmproxy/mitmproxy/dump.py", line 354, in run
    return flow.FlowMaster.run(self)
  File "/home/christofferqa/mitmproxy/mitmproxy/controller.py", line 128, in run
    self.tick(self.masterq, 0.1)
  File "/home/christofferqa/mitmproxy/mitmproxy/flow.py", line 828, in tick
    return super(FlowMaster, self).tick(q, timeout)
  File "/home/christofferqa/mitmproxy/mitmproxy/controller.py", line 113, in tick
    self.handle(*msg)
  File "/home/christofferqa/mitmproxy/mitmproxy/controller.py", line 135, in handle
    m(obj)
  File "/home/christofferqa/mitmproxy/mitmproxy/dump.py", line 337, in handle_response
    self._process_flow(f)
  File "/home/christofferqa/mitmproxy/mitmproxy/dump.py", line 321, in _process_flow
    self.state.delete_flow(f)
  File "/home/christofferqa/mitmproxy/mitmproxy/flow.py", line 584, in delete_flow
    self.flows._remove(f)
  File "/home/christofferqa/mitmproxy/mitmproxy/flow.py", line 496, in _remove
    self._list.remove(f)
ValueError: list.remove(x): x not in list
Traceback (most recent call last):
  File "/home/christofferqa/mitmproxy/mitmproxy/proxy/server.py", line 121, in handle
    root_layer()
TypeError: 'NoneType' object is not callable

mitmproxy has crashed!
Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy
----------------------------------------
Error in processing of request from 127.0.0.1:41084
Traceback (most recent call last):
  File "/home/christofferqa/mitmproxy/netlib/tcp.py", line 854, in connection_thread
    self.handle_client_connection(connection, client_address)
  File "/home/christofferqa/mitmproxy/mitmproxy/proxy/server.py", line 64, in handle_client_connection
    h.handle()
  File "/home/christofferqa/mitmproxy/mitmproxy/proxy/server.py", line 152, in handle
    self.channel.tell("clientdisconnect", root_layer)
  File "/home/christofferqa/mitmproxy/mitmproxy/controller.py", line 68, in tell
    m.reply = DummyReply()
AttributeError: 'NoneType' object has no attribute 'reply'
```

---

Mitmproxy Version: [master](https://github.com/mitmproxy/mitmproxy/tree/dfcfa6263cd8d2208155a45d35ef4434dc967be2) (0.17)
Operating System: Ubuntu 14.04 LTS

mitmdump crashes when delaying recorded responses with the kill flag
##### Steps to reproduce the problem:
1. Record the server responses of some website: `mitmdump -w rec.out --anticache`
2. Replay the server responses with the `--kill` flag and a script that delays one or more responses without blocking: `mitmdump -s script.py -S rec.out --kill --no-pop`
   
   script.py:
   
   ```
   import re
   import time
   
   from mitmproxy.script import concurrent
   
   delayUrl=".*/some_url.js"
   
   @concurrent
   def request(context, flow):
     if re.match(delayUrl, flow.request.url):
       print('delay request: %s%s' % (flow.request.host, flow.request.path))
       time.sleep(5)
     print('start request: %s%s' % (flow.request.host, flow.request.path))
   
   def response(context, flow):
     if flow.error:
       print('Error: ' + str(flow.error))
   ```
##### What is the expected behavior?

The server responses are replayed, the responses matched by the `delayUrl` pattern are delayed by 5s, and responses not found in the recording are killed.
##### What went wrong?

The script crashes:

```
Error: Connection killed
Traceback (most recent call last):
  File "/home/christofferqa/mitmproxy/venv/bin/mitmdump", line 9, in <module>
    load_entry_point('mitmproxy', 'console_scripts', 'mitmdump')()
  File "/home/christofferqa/mitmproxy/mitmproxy/main.py", line 102, in mitmdump
    master.run()
  File "/home/christofferqa/mitmproxy/mitmproxy/dump.py", line 354, in run
    return flow.FlowMaster.run(self)
  File "/home/christofferqa/mitmproxy/mitmproxy/controller.py", line 128, in run
    self.tick(self.masterq, 0.1)
  File "/home/christofferqa/mitmproxy/mitmproxy/flow.py", line 828, in tick
    return super(FlowMaster, self).tick(q, timeout)
  File "/home/christofferqa/mitmproxy/mitmproxy/controller.py", line 113, in tick
    self.handle(*msg)
  File "/home/christofferqa/mitmproxy/mitmproxy/controller.py", line 135, in handle
    m(obj)
  File "/home/christofferqa/mitmproxy/mitmproxy/dump.py", line 337, in handle_response
    self._process_flow(f)
  File "/home/christofferqa/mitmproxy/mitmproxy/dump.py", line 321, in _process_flow
    self.state.delete_flow(f)
  File "/home/christofferqa/mitmproxy/mitmproxy/flow.py", line 584, in delete_flow
    self.flows._remove(f)
  File "/home/christofferqa/mitmproxy/mitmproxy/flow.py", line 496, in _remove
    self._list.remove(f)
ValueError: list.remove(x): x not in list
Traceback (most recent call last):
  File "/home/christofferqa/mitmproxy/mitmproxy/proxy/server.py", line 121, in handle
    root_layer()
TypeError: 'NoneType' object is not callable

mitmproxy has crashed!
Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy
----------------------------------------
Error in processing of request from 127.0.0.1:41084
Traceback (most recent call last):
  File "/home/christofferqa/mitmproxy/netlib/tcp.py", line 854, in connection_thread
    self.handle_client_connection(connection, client_address)
  File "/home/christofferqa/mitmproxy/mitmproxy/proxy/server.py", line 64, in handle_client_connection
    h.handle()
  File "/home/christofferqa/mitmproxy/mitmproxy/proxy/server.py", line 152, in handle
    self.channel.tell("clientdisconnect", root_layer)
  File "/home/christofferqa/mitmproxy/mitmproxy/controller.py", line 68, in tell
    m.reply = DummyReply()
AttributeError: 'NoneType' object has no attribute 'reply'
```

---

Mitmproxy Version: [master](https://github.com/mitmproxy/mitmproxy/tree/dfcfa6263cd8d2208155a45d35ef4434dc967be2) (0.17)
Operating System: Ubuntu 14.04 LTS

mitmdump crashes when delaying recorded responses with the kill flag
##### Steps to reproduce the problem:
1. Record the server responses of some website: `mitmdump -w rec.out --anticache`
2. Replay the server responses with the `--kill` flag and a script that delays one or more responses without blocking: `mitmdump -s script.py -S rec.out --kill --no-pop`
   
   script.py:
   
   ```
   import re
   import time
   
   from mitmproxy.script import concurrent
   
   delayUrl=".*/some_url.js"
   
   @concurrent
   def request(context, flow):
     if re.match(delayUrl, flow.request.url):
       print('delay request: %s%s' % (flow.request.host, flow.request.path))
       time.sleep(5)
     print('start request: %s%s' % (flow.request.host, flow.request.path))
   
   def response(context, flow):
     if flow.error:
       print('Error: ' + str(flow.error))
   ```
##### What is the expected behavior?

The server responses are replayed, the responses matched by the `delayUrl` pattern are delayed by 5s, and responses not found in the recording are killed.
##### What went wrong?

The script crashes:

```
Error: Connection killed
Traceback (most recent call last):
  File "/home/christofferqa/mitmproxy/venv/bin/mitmdump", line 9, in <module>
    load_entry_point('mitmproxy', 'console_scripts', 'mitmdump')()
  File "/home/christofferqa/mitmproxy/mitmproxy/main.py", line 102, in mitmdump
    master.run()
  File "/home/christofferqa/mitmproxy/mitmproxy/dump.py", line 354, in run
    return flow.FlowMaster.run(self)
  File "/home/christofferqa/mitmproxy/mitmproxy/controller.py", line 128, in run
    self.tick(self.masterq, 0.1)
  File "/home/christofferqa/mitmproxy/mitmproxy/flow.py", line 828, in tick
    return super(FlowMaster, self).tick(q, timeout)
  File "/home/christofferqa/mitmproxy/mitmproxy/controller.py", line 113, in tick
    self.handle(*msg)
  File "/home/christofferqa/mitmproxy/mitmproxy/controller.py", line 135, in handle
    m(obj)
  File "/home/christofferqa/mitmproxy/mitmproxy/dump.py", line 337, in handle_response
    self._process_flow(f)
  File "/home/christofferqa/mitmproxy/mitmproxy/dump.py", line 321, in _process_flow
    self.state.delete_flow(f)
  File "/home/christofferqa/mitmproxy/mitmproxy/flow.py", line 584, in delete_flow
    self.flows._remove(f)
  File "/home/christofferqa/mitmproxy/mitmproxy/flow.py", line 496, in _remove
    self._list.remove(f)
ValueError: list.remove(x): x not in list
Traceback (most recent call last):
  File "/home/christofferqa/mitmproxy/mitmproxy/proxy/server.py", line 121, in handle
    root_layer()
TypeError: 'NoneType' object is not callable

mitmproxy has crashed!
Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy
----------------------------------------
Error in processing of request from 127.0.0.1:41084
Traceback (most recent call last):
  File "/home/christofferqa/mitmproxy/netlib/tcp.py", line 854, in connection_thread
    self.handle_client_connection(connection, client_address)
  File "/home/christofferqa/mitmproxy/mitmproxy/proxy/server.py", line 64, in handle_client_connection
    h.handle()
  File "/home/christofferqa/mitmproxy/mitmproxy/proxy/server.py", line 152, in handle
    self.channel.tell("clientdisconnect", root_layer)
  File "/home/christofferqa/mitmproxy/mitmproxy/controller.py", line 68, in tell
    m.reply = DummyReply()
AttributeError: 'NoneType' object has no attribute 'reply'
```

---

Mitmproxy Version: [master](https://github.com/mitmproxy/mitmproxy/tree/dfcfa6263cd8d2208155a45d35ef4434dc967be2) (0.17)
Operating System: Ubuntu 14.04 LTS



Discussion:
Thanks for the detailed report! :smiley:

Hi! I' looking at this bug. I guess this is caused by the incompatibility of `--kill` and the `@concurrent` decorator.

When the `request` method in the script using `@concurrent` decorator, it will create a new thread and run the following code in it, including kill a flow object. Almost at the same time, the main thread will deal with the response while don't know the flow has been killed by another thread.

If we remove the `@concurrent` in the script, no errors appear any more. The bugs involving are always tricky... Any ideas? @mhils @Kriechi 

Thanks for the detailed report! :smiley:

Hi! I' looking at this bug. I guess this is caused by the incompatibility of `--kill` and the `@concurrent` decorator.

When the `request` method in the script using `@concurrent` decorator, it will create a new thread and run the following code in it, including kill a flow object. Almost at the same time, the main thread will deal with the response while don't know the flow has been killed by another thread.

If we remove the `@concurrent` in the script, no errors appear any more. The bugs involving are always tricky... Any ideas? @mhils @Kriechi 

Thanks for the detailed report! :smiley:

Hi! I' looking at this bug. I guess this is caused by the incompatibility of `--kill` and the `@concurrent` decorator.

When the `request` method in the script using `@concurrent` decorator, it will create a new thread and run the following code in it, including kill a flow object. Almost at the same time, the main thread will deal with the response while don't know the flow has been killed by another thread.

If we remove the `@concurrent` in the script, no errors appear any more. The bugs involving are always tricky... Any ideas? @mhils @Kriechi 


Resolving patch:
diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py
--- a/mitmproxy/flow.py
+++ b/mitmproxy/flow.py
@@ -806,7 +806,7 @@ def do_server_playback(self, flow):
             response.is_replay = True
             if self.refresh_server_playback:
                 response.refresh()
-            flow.reply(response)
+            flow.response = response
             if self.server_playback.count() == 0:
                 self.stop_server_playback()
             return True
@@ -905,11 +905,11 @@ def load_flows(self, fr):
     def load_flows_file(self, path):
         path = os.path.expanduser(path)
         try:
-            f = file(path, "rb")
-            freader = FlowReader(f)
+            with open(path, "rb") as f:
+                freader = FlowReader(f)
+                return self.load_flows(freader)
         except IOError as v:
             raise FlowReadError(v.strerror)
-        return self.load_flows(freader)
 
     def process_new_request(self, f):
         if self.stickycookie_state:
@@ -924,11 +924,8 @@ def process_new_request(self, f):
 
         if self.server_playback:
             pb = self.do_server_playback(f)
-            if not pb:
-                if self.kill_nonreplay:
+            if not pb and self.kill_nonreplay:
                     f.kill(self)
-                else:
-                    f.reply()
 
     def process_new_response(self, f):
         if self.stickycookie_state:
diff --git a/mitmproxy/script/concurrent.py b/mitmproxy/script/concurrent.py
--- a/mitmproxy/script/concurrent.py
+++ b/mitmproxy/script/concurrent.py
@@ -8,22 +8,23 @@
 
 class ReplyProxy(object):
 
-    def __init__(self, original_reply, script_thread):
-        self.original_reply = original_reply
+    def __init__(self, reply_func, script_thread):
+        self.reply_func = reply_func
         self.script_thread = script_thread
-        self._ignore_call = True
-        self.lock = threading.Lock()
+        self.master_reply = None
 
-    def __call__(self, *args, **kwargs):
-        with self.lock:
-            if self._ignore_call:
-                self.script_thread.start()
-                self._ignore_call = False
-                return
-        self.original_reply(*args, **kwargs)
+    def __call__(self, *args):
+        if self.master_reply is None:
+            self.master_reply = args
+            self.script_thread.start()
+            return
+        self.reply_func(*args)
+
+    def done(self):
+        self.reply_func(*self.master_reply)
 
     def __getattr__(self, k):
-        return getattr(self.original_reply, k)
+        return getattr(self.reply_func, k)
 
 
 def _handle_concurrent_reply(fn, o, *args, **kwargs):
@@ -34,7 +35,7 @@ def _handle_concurrent_reply(fn, o, *args, **kwargs):
     def run():
         fn(*args, **kwargs)
         # If the script did not call .reply(), we have to do it now.
-        reply_proxy()
+        reply_proxy.done()
 
     script_thread = ScriptThread(target=run)
 


Test patch: