Repository: blaze/blaze

Issue:
importing blaze starts `cpu_count() + 3` threads
Not exactly a bug, but maybe an improvement request -

In debugging some context switching in our app, I was surprised to see a number of extra python Threads running.  It turned out they are started by the import of blaze, because [`blaze/__init__.py`](https://github.com/blaze/blaze/blob/737953bfd8c1bfd819eb41f1d9ad17d969081ba9/blaze/__init__.py#L18) imports `blaze/utils.py`, which [instantiates a `multiprocessing.ThreadPool`](https://github.com/blaze/blaze/blob/737953bfd8c1bfd819eb41f1d9ad17d969081ba9/blaze/utils.py#L27) at module scope.  The pool starts 3 helper threads plus 1 thread per cpu.

I _think_ this pool is only used by [h5py](https://github.com/blaze/blaze/blob/e0474f3131ea50c24113b6133178cee98fa4b727/blaze/compute/h5py.py#L127). Is its creation something that could be made lazy, such that apps don't need to run the extra threads just to import blaze, especially if they don't compute with h5py?



Discussion:

Resolving patch:
diff --git a/blaze/compute/h5py.py b/blaze/compute/h5py.py
--- a/blaze/compute/h5py.py
+++ b/blaze/compute/h5py.py
@@ -6,6 +6,10 @@
 from datashape import DataShape, to_numpy
 from toolz import curry
 
+import psutil
+
+from multiprocessing.pool import ThreadPool
+
 from ..partition import partitions
 from ..expr import Reduction, Field, symbol
 from ..expr import Expr, Slice, ElemWise
@@ -15,7 +19,8 @@
 
 from .core import compute
 from ..dispatch import dispatch
-from ..utils import available_memory, thread_pool
+from ..utils import available_memory
+
 
 __all__ = []
 
@@ -25,6 +30,7 @@ def pre_compute(expr, data, scope=None, **kwargs):
     """ Don't push slices into memory, they're about to come in anyway """
     return data
 
+
 @dispatch(Expr, h5py.Dataset)
 def pre_compute(expr, data, scope=None, **kwargs):
     """ Bring dataset into memory if it's small relative to memory """
@@ -36,6 +42,7 @@ def pre_compute(expr, data, scope=None, **kwargs):
     else:
         return data
 
+
 @dispatch(Expr, h5py.Dataset)
 def post_compute(expr, data, scope=None):
     """ Bring dataset into memory if it's small relative to memory """
@@ -47,6 +54,7 @@ def post_compute(expr, data, scope=None):
     else:
         return data
 
+
 @dispatch(Symbol, h5py.Dataset)
 def optimize(expr, data):
     return expr
@@ -80,6 +88,7 @@ def optimize(expr, data):
     else:
         return expr
 
+
 @dispatch(Symbol, (h5py.File, h5py.Group, h5py.Dataset))
 def compute_up(expr, data, **kwargs):
     return data
@@ -123,8 +132,21 @@ def compute_chunk(source, target, chunk, chunk_expr, parts):
     target[target_part] = result
 
 
+thread_pool = None
+
+def _get_map(map):
+    global thread_pool
+
+    if map is None:
+        if thread_pool is None:
+            thread_pool = ThreadPool(psutil.cpu_count())
+        return thread_pool.map
+    else:
+        return map
+
+
 @dispatch(Expr, h5py.Dataset)
-def compute_down(expr, data, map=thread_pool.map, **kwargs):
+def compute_down(expr, data, map=None, **kwargs):
     """ Compute expressions on H5Py datasets by operating on chunks
 
     This uses blaze.expr.split to break a full-array-computation into a
@@ -139,6 +161,8 @@ def compute_down(expr, data, map=thread_pool.map, **kwargs):
     The expression must contain some sort of Reduction.  Both the intermediate
     result and the final result are assumed to fit into memory
     """
+    map = _get_map(map)
+
     leaf = expr._leaves()[0]
     if not any(isinstance(node, Reduction) for node in path(expr, leaf)):
         raise MDNotImplementedError()
@@ -160,9 +184,10 @@ def compute_down(expr, data, map=thread_pool.map, **kwargs):
     target_parts = list(partitions(intermediate, chunksize=chunk_expr.shape,
                                    keepdims=True))
 
-
-    parts = list(map(curry(compute_chunk, data, intermediate, chunk, chunk_expr),
-                           zip(source_parts, target_parts)))
+    list(map(
+        curry(compute_chunk, data, intermediate, chunk, chunk_expr),
+        zip(source_parts, target_parts)
+    ))
 
     # Compute on the aggregate
     return compute(agg_expr, {agg: intermediate})
diff --git a/blaze/utils.py b/blaze/utils.py
--- a/blaze/utils.py
+++ b/blaze/utils.py
@@ -12,9 +12,8 @@
 
 from toolz.curried.operator import setitem
 
-from itertools import islice, chain
+from itertools import islice
 from collections import Iterator
-from multiprocessing.pool import ThreadPool
 
 # these are used throughout blaze, don't remove them
 from odo.utils import tmpfile, filetext, filetexts, raises, keywords, ignoring
@@ -28,8 +27,6 @@
 
 from .dispatch import dispatch
 
-thread_pool = ThreadPool(psutil.cpu_count())
-
 
 def nth_list(n, seq):
     """


Test patch: