Repository: numpy/numpy

Issue:
BUG? hstack/vstack/column_stack no longer accept generators
As reported by @ilayn over in https://github.com/numpy/numpy/issues/12262:

> There are other backwards incompatible changes for hstack and column_stack which were explicitly accepting generator expressions but now they don't.
>
> Xref : scipy/scipy#9405


Discussion:
This is definitely due to my `__array_function__` refactor to add dispatchers to these functions (specifically https://github.com/numpy/numpy/pull/12115). The new dispatcher functions now exhaust the generators before they reach functions.

This is certainly not supported behavior -- the docstrings for these functions describes these parameters as accepting a "sequence of ndarrays" and generators are not sequences.

That said, if scipy relied on this behavior it would be better to handle this change more gracefully. My proposal:
- For now, raise a FutureWarning when passed a generator, as determined by `isinstance(arg, types.GeneratorType)`. Skip dispatch to `__array_function__` implementations in this case.
- In the future, raise TypeError.

This will result in a small performance penalty, but I don't see any good alternatives. In particular, I don't see any way we could correctly implement these functions with dispatching with support for generator arguments.
Scikit-learn relied on this also in many places. Even if it wasn't officially supported I would think a deprecation cycle / FutureWarning would be good. If scipy and scikit-learn rely on it, chances are lots of people were.
@shoyer That sounds like a good solution to me.
Yeah, sounds sensible given the strong indication of downstream dependency from big projects.
I think we will need this fix for `stack`, `column_stack`, `dstack`, `vstack` and `hstack`. Anything else?

`concatenate` uses `PySequence_Check` internally, so it doesn't need any changes. But note that that this is a more relaxed check than `isinstance(..., collections.abc.Sequence)` (https://stackoverflow.com/questions/43566044/what-is-pythons-sequence-protocol). At the very least, NumPy arrays should be fine to use as "sequences" here.
To be honest I don't see the benefit of generators in this context since at some point all elements have to be ... generated and be present in the memory all at once. Thus I would be in favor of deprecating and eventually removing them if they make `__array_function__` perform better which is way more important than this detail.
For scikit-learn, I think they used this just because it looked a little cleaner with two fewer characters, e.g.,
```python
self.rows_ = np.vstack(self.row_labels_ == c
                       for c in range(self.n_clusters))
```
vs
```python
self.rows_ = np.vstack([self.row_labels_ == c
                        for c in range(self.n_clusters)])
```
@shoyer that seems a likely explanation and I don't think we actually care about adding two characters. Obviously we do care about scikit-learn breaking with the next numpy release, but I think we're all on the same page there, i.e. do a FutureWarning and eventually remove.

Resolving patch:
diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py
--- a/numpy/core/shape_base.py
+++ b/numpy/core/shape_base.py
@@ -5,6 +5,8 @@
 
 import functools
 import operator
+import types
+import warnings
 
 from . import numeric as _nx
 from . import overrides
@@ -204,11 +206,22 @@ def atleast_3d(*arys):
         return res
 
 
-def _vstack_dispatcher(tup):
-    return tup
+def _arrays_for_stack_dispatcher(arrays, stacklevel=4):
+    if not hasattr(arrays, '__getitem__') and hasattr(arrays, '__iter__'):
+        warnings.warn('arrays to stack must be passed as a "sequence" type '
+                      'such as list or tuple. Support for non-sequence '
+                      'iterables such as generators is deprecated as of '
+                      'NumPy 1.16 and will raise an error in the future.',
+                      FutureWarning, stacklevel=stacklevel)
+        return ()
+    return arrays
 
 
-@array_function_dispatch(_vstack_dispatcher)
+def _vhstack_dispatcher(tup):
+    return _arrays_for_stack_dispatcher(tup)
+
+
+@array_function_dispatch(_vhstack_dispatcher)
 def vstack(tup):
     """
     Stack arrays in sequence vertically (row wise).
@@ -264,11 +277,7 @@ def vstack(tup):
     return _nx.concatenate([atleast_2d(_m) for _m in tup], 0)
 
 
-def _hstack_dispatcher(tup):
-    return tup
-
-
-@array_function_dispatch(_hstack_dispatcher)
+@array_function_dispatch(_vhstack_dispatcher)
 def hstack(tup):
     """
     Stack arrays in sequence horizontally (column wise).
@@ -325,6 +334,7 @@ def hstack(tup):
 
 
 def _stack_dispatcher(arrays, axis=None, out=None):
+    arrays = _arrays_for_stack_dispatcher(arrays, stacklevel=6)
     for a in arrays:
         yield a
     if out is not None:
diff --git a/numpy/lib/shape_base.py b/numpy/lib/shape_base.py
--- a/numpy/lib/shape_base.py
+++ b/numpy/lib/shape_base.py
@@ -11,6 +11,7 @@
 from numpy.core.multiarray import normalize_axis_index
 from numpy.core import overrides
 from numpy.core import vstack, atleast_3d
+from numpy.core.shape_base import _arrays_for_stack_dispatcher
 from numpy.lib.index_tricks import ndindex
 from numpy.matrixlib.defmatrix import matrix  # this raises all the right alarm bells
 
@@ -591,7 +592,7 @@ def expand_dims(a, axis):
 
 
 def _column_stack_dispatcher(tup):
-    return tup
+    return _arrays_for_stack_dispatcher(tup)
 
 
 @array_function_dispatch(_column_stack_dispatcher)
@@ -638,7 +639,7 @@ def column_stack(tup):
 
 
 def _dstack_dispatcher(tup):
-    return tup
+    return _arrays_for_stack_dispatcher(tup)
 
 
 @array_function_dispatch(_dstack_dispatcher)


Test patch:
diff --git a/numpy/core/tests/test_shape_base.py b/numpy/core/tests/test_shape_base.py
--- a/numpy/core/tests/test_shape_base.py
+++ b/numpy/core/tests/test_shape_base.py
@@ -1,6 +1,7 @@
 from __future__ import division, absolute_import, print_function
 
 import warnings
+import sys
 import numpy as np
 from numpy.core import (
     array, arange, atleast_1d, atleast_2d, atleast_3d, block, vstack, hstack,
@@ -10,7 +11,7 @@
                                    _block_concatenate, _block_slicing)
 from numpy.testing import (
     assert_, assert_raises, assert_array_equal, assert_equal,
-    assert_raises_regex, assert_almost_equal
+    assert_raises_regex, assert_warns, assert_almost_equal
     )
 
 from numpy.compat import long
@@ -155,6 +156,14 @@ def test_2D_array(self):
         desired = array([[1, 1], [2, 2]])
         assert_array_equal(res, desired)
 
+    def test_generator(self):
+        with assert_warns(FutureWarning):
+            hstack((np.arange(3) for _ in range(2)))
+        if sys.version_info.major > 2:
+            # map returns a list on Python 2
+            with assert_warns(FutureWarning):
+                hstack(map(lambda x: x, np.ones((3, 2))))
+
 
 class TestVstack(object):
     def test_non_iterable(self):
@@ -191,6 +200,10 @@ def test_2D_array2(self):
         desired = array([[1, 2], [1, 2]])
         assert_array_equal(res, desired)
 
+    def test_generator(self):
+        with assert_warns(FutureWarning):
+            vstack((np.arange(3) for _ in range(2)))
+
 
 class TestConcatenate(object):
     def test_returns_copy(self):
@@ -354,7 +367,7 @@ def test_stack():
     arrays = [np.random.randn(3, 4) for _ in range(10)]
     axes = [0, 1, 2, -1, -2, -3]
     expected_shapes = [(10, 3, 4), (3, 10, 4), (3, 4, 10),
-                        (3, 4, 10), (3, 10, 4), (10, 3, 4)]
+                       (3, 4, 10), (3, 10, 4), (10, 3, 4)]
     for axis, expected_shape in zip(axes, expected_shapes):
         assert_equal(np.stack(arrays, axis).shape, expected_shape)
     # empty arrays
@@ -372,6 +385,10 @@ def test_stack():
                         stack, [np.zeros((3, 3)), np.zeros(3)], axis=1)
     assert_raises_regex(ValueError, 'must have the same shape',
                         stack, [np.arange(2), np.arange(3)])
+    # generator is deprecated
+    with assert_warns(FutureWarning):
+        result = stack((x for x in range(3)))
+    assert_array_equal(result, np.array([0, 1, 2]))
 
 
 # See for more information on how to parametrize a whole class
diff --git a/numpy/lib/tests/test_shape_base.py b/numpy/lib/tests/test_shape_base.py
--- a/numpy/lib/tests/test_shape_base.py
+++ b/numpy/lib/tests/test_shape_base.py
@@ -457,6 +457,7 @@ def test_unequal_split(self):
         a = np.arange(10)
         assert_raises(ValueError, split, a, 3)
 
+
 class TestColumnStack(object):
     def test_non_iterable(self):
         assert_raises(TypeError, column_stack, 1)
@@ -481,6 +482,10 @@ def test_2D_arrays(self):
         actual = np.column_stack((a, b))
         assert_equal(actual, expected)
 
+    def test_generator(self):
+        with assert_warns(FutureWarning):
+            column_stack((np.arange(3) for _ in range(2)))
+
 
 class TestDstack(object):
     def test_non_iterable(self):
@@ -514,6 +519,10 @@ def test_2D_array2(self):
         desired = np.array([[[1, 1], [2, 2]]])
         assert_array_equal(res, desired)
 
+    def test_generator(self):
+        with assert_warns(FutureWarning):
+            dstack((np.arange(3) for _ in range(2)))
+
 
 # array_split has more comprehensive test of splitting.
 # only do simple test on hsplit, vsplit, and dsplit