Repository: dask/dask

Issue:
"CI Additional – mindeps-bag-delayed" step failing
This step is apparently skipped in this repo, but was the most useful CI step in [celsiustx/dask](https://github.com/celsiustx/dask/) for a few months. It broke at some point recently though:
```
py.test dask --runslow 
============================= test session starts ==============================
platform linux -- Python 3.7.9, pytest-6.2.2, py-1.10.0, pluggy-0.13.1
rootdir: /home/runner/work/dask/dask, configfile: setup.cfg
plugins: forked-1.3.0, xdist-2.2.1
collected 757 items / 1 error / 8 skipped / 748 selected

==================================== ERRORS ====================================
____________ ERROR collecting dask/tests/test_graph_manipulation.py ____________
ImportError while importing test module '/home/runner/work/dask/dask/dask/tests/test_graph_manipulation.py'.
Hint: make sure your test modules/packages have valid Python names.
Traceback:
dask/array/__init__.py:2: in <module>
    from .blockwise import blockwise, atop
dask/array/blockwise.py:9: in <module>
    from ..blockwise import blockwise as core_blockwise
dask/blockwise.py:4: in <module>
    import numpy as np
E   ModuleNotFoundError: No module named 'numpy'

The above exception was the direct cause of the following exception:
/usr/share/miniconda3/envs/test-environment/lib/python3.7/importlib/__init__.py:127: in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
dask/tests/test_graph_manipulation.py:10: in <module>
    from dask.graph_manipulation import bind, checkpoint, chunks, clone, wait_on
dask/graph_manipulation.py:8: in <module>
    from .array import Array
dask/array/__init__.py:253: in <module>
    raise ImportError(str(e) + "\n\n" + msg) from e
E   ImportError: No module named 'numpy'
E   
E   Dask array requirements are not installed.
E   
E   Please either conda or pip install as follows:
E   
E     conda install dask                 # either conda install
E     python -m pip install "dask[array]" --upgrade  # or python -m pip install
```
([example](https://github.com/celsiustx/dask/runs/1922876494?check_suite_focus=true#step:5:7))

I'm bisecting now to see if I can narrow it to a specific commit: https://github.com/celsiustx/dask/compare/5694253ba...02b466d2b


Discussion:
Thanks for reporting @ryan-williams. I suspect this started happening when https://github.com/dask/dask/pull/7109 was merged. cc @crusaderky for visibility 
Yea, my bisect points at https://github.com/dask/dask/commit/02b466d2bf82c9cdd420b492adf765d14db8e8ea, and I'm noticing now that the error occurs in `dask/tests/test_graph_manipulation.py` which was added in that commit / #7109

Resolving patch:
diff --git a/dask/blockwise.py b/dask/blockwise.py
--- a/dask/blockwise.py
+++ b/dask/blockwise.py
@@ -1,10 +1,9 @@
 import itertools
 import warnings
 
-import numpy as np
-
 import tlz as toolz
 
+from .compatibility import prod
 from .core import reverse_dict, flatten, keys_in_tasks
 from .delayed import unpack_collections
 from .highlevelgraph import HighLevelGraph, Layer
@@ -272,7 +271,7 @@ def __iter__(self):
         return iter(self._dict)
 
     def __len__(self):
-        return int(np.prod(list(self._out_numblocks().values())))
+        return int(prod(self._out_numblocks().values()))
 
     def _out_numblocks(self):
         d = {}
@@ -434,8 +433,8 @@ def cull(self, keys, all_hlg_keys):
             if key[0] == self.output:
                 output_blocks.add(key[1:])
         culled_deps = self._cull_dependencies(all_hlg_keys, output_blocks)
-        out_size = tuple([self.dims[i] for i in self.output_indices])
-        if np.prod(out_size) != len(culled_deps):
+        out_size_iter = (self.dims[i] for i in self.output_indices)
+        if prod(out_size_iter) != len(culled_deps):
             culled_layer = self._cull(output_blocks)
             return culled_layer, culled_deps
         else:
diff --git a/dask/compatibility.py b/dask/compatibility.py
--- a/dask/compatibility.py
+++ b/dask/compatibility.py
@@ -9,9 +9,8 @@
 
 try:
     from dataclasses import is_dataclass, fields as dataclass_fields
-
 except ImportError:
-
+    # Python < 3.7
     def is_dataclass(x):
         return False
 
@@ -19,4 +18,15 @@ def dataclass_fields(x):
         return []
 
 
+try:
+    from math import prod
+except ImportError:
+    # Python < 3.8
+    def prod(iterable, *, start=1):
+        acc = start
+        for el in iterable:
+            acc *= el
+        return acc
+
+
 PY_VERSION = LooseVersion(".".join(map(str, sys.version_info[:3])))
diff --git a/dask/graph_manipulation.py b/dask/graph_manipulation.py
--- a/dask/graph_manipulation.py
+++ b/dask/graph_manipulation.py
@@ -5,8 +5,6 @@
 import uuid
 from typing import Callable, Hashable, Optional, Set, Tuple, TypeVar
 
-from .array import Array
-from .bag import Bag
 from .base import (
     clone_key,
     get_collection_name,
@@ -16,7 +14,6 @@
 )
 from .blockwise import blockwise
 from .core import flatten
-from .dataframe import DataFrame, Series
 from .delayed import Delayed, delayed
 from .highlevelgraph import BasicLayer, HighLevelGraph, Layer
 
@@ -73,6 +70,35 @@ def checkpoint(*collections) -> Delayed:
     return Delayed(name, dsk)
 
 
+def _can_apply_blockwise(collection):
+    """Return True if _map_blocks can be sped up via blockwise operations; False
+    otherwise.
+
+    FIXME this returns False for collections that wrap around around da.Array, such as
+          pint.Quantity, xarray DataArray, Dataset, and Variable.
+    """
+    try:
+        from .bag import Bag
+
+        if isinstance(collection, Bag):
+            return True
+    except ImportError:
+        pass
+    try:
+        from .array import Array
+
+        if isinstance(collection, Array):
+            return True
+    except ImportError:
+        pass
+    try:
+        from .dataframe import DataFrame, Series
+
+        return isinstance(collection, (DataFrame, Series))
+    except ImportError:
+        return False
+
+
 def _build_map_layer(
     func: Callable, name: str, collection, dependencies: Tuple[Delayed, ...] = ()
 ) -> Layer:
@@ -91,11 +117,11 @@ def _build_map_layer(
         Zero or more Delayed objects, which will be passed as arbitrary variadic args to
         func after the collection's chunk
     """
-    if isinstance(collection, (Array, DataFrame, Series, Bag)):
+    if _can_apply_blockwise(collection):
         # Use a Blockwise layer
-        if isinstance(collection, Array):
+        try:
             numblocks = collection.numblocks
-        else:
+        except AttributeError:
             numblocks = (collection.npartitions,)
         indices = tuple(i for i, _ in enumerate(numblocks))
         kwargs = {"_deps": dependencies} if dependencies else {}


Test patch:
diff --git a/continuous_integration/scripts/test_imports.sh b/continuous_integration/scripts/test_imports.sh
--- a/continuous_integration/scripts/test_imports.sh
+++ b/continuous_integration/scripts/test_imports.sh
@@ -17,9 +17,9 @@ test_import () {
 # Note: in setup.py, bag and delayed require cloudpickle, but it's omitted here as it is
 # only a dependency for real-life usage and unit tests
 test_import ""                                "import dask, dask.multiprocessing, dask.threaded, dask.optimization"
-test_import "toolz"                           "import dask.delayed"
+test_import "toolz"                           "import dask.base, dask.delayed, dask.graph_manipulation"
 test_import "fsspec toolz partd"              "import dask.bag"
-test_import "toolz numpy toolz"               "import dask.array"
+test_import "numpy toolz"                     "import dask.array"
 test_import "fsspec numpy pandas toolz partd" "import dask.dataframe"
 test_import "bokeh"                           "import dask.diagnostics"
 test_import "distributed"                     "import dask.distributed"