Repository: dask/dask

Issue:
Bug: threaded scheduler is not interruptible
```
import dask, time
dask.__version__
> u'0.14.1'

# can be interrupted
fun = dask.delayed(lambda x: time.sleep(5))
dask.compute([fun(x) for x in range(100)], get = dask.threaded.get)

# can't be interrupted
fun2 = dask.delayed(lambda x: sum([1 for x in range(10000000)]))
dask.compute([fun2(x) for x in range(100)], get = dask.threaded.get)
```

Using the threaded scheduler, I noticed that some tasks cannot be interrupted by pressing Ctrl+C in a python console. Directly after pressing Ctrl+C the CPU usage of that process jumps up: It uses all available minus one cores. Judging by the output of top, the process seems to spend the computing time in the kernel space. 




Discussion:
Thanks for the bug report. This is due to https://bugs.python.org/issue1360, and is fixed in python 3. A workaround is to always specify a timeout to `get` (can be extremely long). Fix coming soon. Minimal example:

```python
from time import sleep
from multiprocessing.pool import ThreadPool
from dask.compatibility import Queue

p = ThreadPool()
queue = Queue()
p.apply_async(sleep, args=(10,), callback=queue.put)
queue.get()   # This will block KeyboardInterrupt in python 2.7
# queue.get(1000000)  # This won't
```

Resolving patch:
diff --git a/dask/async.py b/dask/async.py
--- a/dask/async.py
+++ b/dask/async.py
@@ -114,10 +114,11 @@
 """
 from __future__ import absolute_import, division, print_function
 
+import os
 import sys
 import traceback
 
-from .compatibility import Queue
+from .compatibility import Queue, Empty
 from .core import (istask, flatten, reverse_dict, get_dependencies, ishashable,
                    has_tasks)
 from .context import _globals
@@ -127,6 +128,30 @@
 from .utils_test import add, inc  # noqa: F401
 
 
+if sys.version_info.major < 3:
+    # Due to a bug in python 2.7 Queue.get, if a timeout isn't specified then
+    # `Queue.get` can't be interrupted. A workaround is to specify an extremely
+    # long timeout, which then allows it to be interrupted.
+    # For more information see: https://bugs.python.org/issue1360
+    def queue_get(q):
+        return q.get(block=True, timeout=(365 * 24 * 60 * 60))
+
+elif os.name == 'nt':
+    # Python 3 windows Queue.get also doesn't handle interrupts properly. To
+    # workaround this we poll at a sufficiently large interval that it
+    # shouldn't affect performance, but small enough that users trying to kill
+    # an application shouldn't care.
+    def queue_get(q):
+        while True:
+            try:
+                return q.get(block=True, timeout=0.1)
+            except Empty:
+                pass
+else:
+    def queue_get(q):
+        return q.get()
+
+
 DEBUG = False
 
 
@@ -488,7 +513,7 @@ def fire_task():
 
         # Main loop, wait on tasks to finish, insert new ones
         while state['waiting'] or state['ready'] or state['running']:
-            key, res_info = queue.get()
+            key, res_info = queue_get(queue)
             res, tb, worker_id = loads(res_info)
             if isinstance(res, BaseException):
                 if rerun_exceptions_locally:


Test patch:
diff --git a/dask/tests/test_threaded.py b/dask/tests/test_threaded.py
--- a/dask/tests/test_threaded.py
+++ b/dask/tests/test_threaded.py
@@ -1,6 +1,9 @@
-from multiprocessing.pool import ThreadPool
+import os
 import threading
-from threading import Thread
+import signal
+import subprocess
+import sys
+from multiprocessing.pool import ThreadPool
 from time import time, sleep
 
 import pytest
@@ -52,7 +55,7 @@ def f(i):
     before = threading.active_count()
 
     for i in range(20):
-        t = Thread(target=f, args=(1,))
+        t = threading.Thread(target=f, args=(1,))
         t.daemon = True
         t.start()
         t.join()
@@ -91,7 +94,7 @@ def test_f():
 
     threads = []
     for i in range(20):
-        t = Thread(target=test_f)
+        t = threading.Thread(target=test_f)
         t.daemon = True
         t.start()
         threads.append(t)
@@ -100,3 +103,63 @@ def test_f():
         thread.join()
 
     assert L == [1] * 20
+
+
+code = """
+import sys
+from dask.threaded import get
+
+def signal_started():
+    sys.stdout.write('started\\n')
+    sys.stdout.flush()
+
+def long_task(x):
+    out = 0
+    N = 100000
+    for i in range(N):
+        for j in range(N):
+            out += 1
+
+dsk = {('x', i): (long_task, 'started') for i in range(100)}
+dsk['started'] = (signal_started,)
+dsk['x'] = (sum, list(dsk.keys()))
+get(dsk, 'x')
+"""
+
+
+# TODO: this test passes locally on windows, but fails on appveyor
+# because the ctrl-c event also tears down their infrastructure.
+# There's probably a better way to test this, but for now we'll mark
+# it slow (slow tests are skipped on appveyor).
+@pytest.mark.slow
+def test_interrupt():
+    try:
+        proc = subprocess.Popen([sys.executable, '-c', code],
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE)
+        # Wait for scheduler to start
+        msg = proc.stdout.readline()
+        if msg != 'started\n' and proc.poll() is not None:
+            assert False, "subprocess failed"
+        # Scheduler has started, send an interrupt
+        sigint = signal.CTRL_C_EVENT if os.name == 'nt' else signal.SIGINT
+        try:
+            proc.send_signal(sigint)
+            # Wait a bit for it to die
+            start = time()
+            while time() - start < 0.5:
+                if proc.poll() is not None:
+                    break
+                sleep(0.05)
+            else:
+                assert False, "KeyboardInterrupt Failed"
+        except KeyboardInterrupt:
+            # On windows the interrupt is also raised in this process.
+            # That's silly, ignore it.
+            pass
+        # Ensure KeyboardInterrupt in traceback
+        stderr = proc.stderr.read()
+        assert "KeyboardInterrupt" in stderr.decode()
+    except:
+        proc.terminate()
+        raise