Repository: dask/dask

Issue:
Add meta to the repr for dask arrays?
It seems like `meta` would be nice to know for debugging. It's already in the HTML repr -- what about putting it in the text repr, too?

For example, instead of:
```
dask.array<array, shape=(3, 3), dtype=float64, chunksize=(3, 3)>
```
we could have:
```
dask.array<array, shape=(3, 3), dtype=float64, chunksize=(3, 3), meta=sparse.COO>
```

Or maybe drop the existing `name` field in favor of meta, since that isn't especially informative.

This came up in https://github.com/pydata/xarray/pull/3211, cc @crusaderky 

Discussion:

Patch:
diff --git a/dask/array/core.py b/dask/array/core.py
index 286dbcbc6..c6f79f4b5 100644
--- a/dask/array/core.py
+++ b/dask/array/core.py
@@ -492,7 +492,7 @@ def map_blocks(
     ...     return np.array([a.max(), b.max()])
 
     >>> da.map_blocks(func, x, y, chunks=(2,), dtype='i8')
-    dask.array<func, shape=(20,), dtype=int64, chunksize=(2,)>
+    dask.array<func, shape=(20,), dtype=int64, chunksize=(2,), chunktype=numpy.ndarray>
 
     >>> _.compute()
     array([ 99,   9, 199,  19, 299,  29, 399,  39, 499,  49, 599,  59, 699,
@@ -534,7 +534,7 @@ def map_blocks(
     ...     return np.arange(loc[0], loc[1])
 
     >>> da.map_blocks(func, chunks=((4, 4),), dtype=np.float_)
-    dask.array<func, shape=(8,), dtype=float64, chunksize=(4,)>
+    dask.array<func, shape=(8,), dtype=float64, chunksize=(4,), chunktype=numpy.ndarray>
 
     >>> _.compute()
     array([0, 1, 2, 3, 4, 5, 6, 7])
@@ -543,7 +543,7 @@ def map_blocks(
     the optional ``token`` keyword argument.
 
     >>> x.map_blocks(lambda x: x + 1, name='increment')  # doctest: +SKIP
-    dask.array<increment, shape=(100,), dtype=int64, chunksize=(10,)>
+    dask.array<increment, shape=(100,), dtype=int64, chunksize=(10,), chunktype=numpy.ndarray>
     """
     if not callable(func):
         msg = (
@@ -1153,15 +1153,17 @@ class Array(DaskMethodsMixin):
 
         >>> import dask.array as da
         >>> da.ones((10, 10), chunks=(5, 5), dtype='i4')
-        dask.array<..., shape=(10, 10), dtype=int32, chunksize=(5, 5)>
+        dask.array<..., shape=(10, 10), dtype=int32, chunksize=(5, 5), chunktype=numpy.ndarray>
         """
         chunksize = str(self.chunksize)
         name = self.name.rsplit("-", 1)[0]
-        return "dask.array<%s, shape=%s, dtype=%s, chunksize=%s>" % (
+        return "dask.array<%s, shape=%s, dtype=%s, chunksize=%s, chunktype=%s.%s>" % (
             name,
             self.shape,
             self.dtype,
             chunksize,
+            type(self._meta).__module__.split(".")[0],
+            type(self._meta).__name__,
         )
 
     def _repr_html_(self):
@@ -2890,7 +2892,7 @@ def from_delayed(value, shape, dtype=None, meta=None, name=None):
     >>> value = dask.delayed(np.ones)(5)
     >>> array = da.from_delayed(value, (5,), dtype=float)
     >>> array
-    dask.array<from-value, shape=(5,), dtype=float64, chunksize=(5,)>
+    dask.array<from-value, shape=(5,), dtype=float64, chunksize=(5,), chunktype=numpy.ndarray>
     >>> array.compute()
     array([1., 1., 1., 1., 1.])
     """
@@ -3569,11 +3571,11 @@ def asarray(a, **kwargs):
     >>> import numpy as np
     >>> x = np.arange(3)
     >>> da.asarray(x)
-    dask.array<array, shape=(3,), dtype=int64, chunksize=(3,)>
+    dask.array<array, shape=(3,), dtype=int64, chunksize=(3,), chunktype=numpy.ndarray>
 
     >>> y = [[1, 2, 3], [4, 5, 6]]
     >>> da.asarray(y)
-    dask.array<array, shape=(2, 3), dtype=int64, chunksize=(2, 3)>
+    dask.array<array, shape=(2, 3), dtype=int64, chunksize=(2, 3), chunktype=numpy.ndarray>
     """
     if isinstance(a, Array):
         return a
@@ -3607,11 +3609,11 @@ def asanyarray(a):
     >>> import numpy as np
     >>> x = np.arange(3)
     >>> da.asanyarray(x)
-    dask.array<array, shape=(3,), dtype=int64, chunksize=(3,)>
+    dask.array<array, shape=(3,), dtype=int64, chunksize=(3,), chunktype=numpy.ndarray>
 
     >>> y = [[1, 2, 3], [4, 5, 6]]
     >>> da.asanyarray(y)
-    dask.array<array, shape=(2, 3), dtype=int64, chunksize=(2, 3)>
+    dask.array<array, shape=(2, 3), dtype=int64, chunksize=(2, 3), chunktype=numpy.ndarray>
     """
     if isinstance(a, Array):
         return a
diff --git a/dask/array/random.py b/dask/array/random.py
index 6846a2cf1..97a814d3a 100644
--- a/dask/array/random.py
+++ b/dask/array/random.py
@@ -222,7 +222,8 @@ class RandomState(object):
                 if a < 0:
                     raise ValueError("a must be greater than 0")
             else:
-                a = asarray(a).rechunk(a.shape)
+                a = asarray(a)
+                a = a.rechunk(a.shape)
                 dtype = a.dtype
                 if a.ndim != 1:
                     raise ValueError("a must be one dimensional")
diff --git a/dask/dataframe/io/io.py b/dask/dataframe/io/io.py
index 336c2a610..0e0e0c1e0 100644
--- a/dask/dataframe/io/io.py
+++ b/dask/dataframe/io/io.py
@@ -516,7 +516,7 @@ def to_records(df):
     Examples
     --------
     >>> df.to_records()  # doctest: +SKIP
-    dask.array<shape=(nan,), dtype=(numpy.record, [('ind', '<f8'), ('x', 'O'), ('y', '<i8')]), chunksize=(nan,)>
+    dask.array<to_records, shape=(nan,), dtype=(numpy.record, [('ind', '<f8'), ('x', 'O'), ('y', '<i8')]), chunksize=(nan,), chunktype=numpy.ndarray>  # noqa: E501
 
     See Also
     --------
diff --git a/docs/source/array-chunks.rst b/docs/source/array-chunks.rst
index af5c6f47d..32e6b9d50 100644
--- a/docs/source/array-chunks.rst
+++ b/docs/source/array-chunks.rst
@@ -83,7 +83,7 @@ This can also happen when creating a Dask array from a Dask DataFrame:
 
    >>> ddf = dask.dataframe.from_pandas(...)
    >>> ddf.to_dask_array()
-   ... dask.array<values, shape=(nan, 2), dtype=float64, chunksize=(nan, 2)>
+   ... dask.array<values, shape=(nan, 2), dtype=float64, chunksize=(nan, 2), chunktype=numpy.ndarray>
 
 For details on how to avoid unknown chunk sizes, look at how to create a Dask
 array from a Dask DataFrame in the :doc:`documentation on Dask array creation
@@ -268,4 +268,4 @@ These values can also be used when creating arrays with operations like
 .. code-block:: python
 
    >>> dask.array.ones((10000, 10000), chunks=(-1, 'auto'))
-   dask.array<wrapped, shape=(10000, 10000), dtype=float64, chunksize=(10000, 1250)>
+   dask.array<wrapped, shape=(10000, 10000), dtype=float64, chunksize=(10000, 1250), chunktype=numpy.ndarray>
diff --git a/docs/source/array-creation.rst b/docs/source/array-creation.rst
index 23bbd7b0a..d849935d3 100644
--- a/docs/source/array-creation.rst
+++ b/docs/source/array-creation.rst
@@ -141,7 +141,7 @@ There are several ways to create a Dask array from a Dask DataFrame. Dask DataFr
 
    >>> df = dask.dataframes.from_pandas(...)
    >>> df.to_dask_array()
-   dask.array<values, shape=(nan, 3), dtype=float64, chunksize=(nan, 3)>
+   dask.array<values, shape=(nan, 3), dtype=float64, chunksize=(nan, 3), chunktype=numpy.ndarray>
 
 This mirrors the `to_numpy
 <https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_numpy.html>`_
@@ -150,7 +150,7 @@ function in Pandas. The ``values`` attribute is also supported:
 .. code-block:: python
 
    >>> df.values
-   dask.array<values, shape=(nan, 3), dtype=float64, chunksize=(nan, 3)>
+   dask.array<values, shape=(nan, 3), dtype=float64, chunksize=(nan, 3), chunktype=numpy.ndarray>
 
 However, these arrays do not have known chunk sizes because dask.dataframe does
 not track the number of rows in each partition. This means that some operations
@@ -161,7 +161,7 @@ The chunk sizes can be computed:
 .. code-block:: python
 
    >>> df.to_dask_array(lengths=True)
-   dask.array<array, shape=(100, 3), dtype=float64, chunksize=(50, 3)>
+   dask.array<array, shape=(100, 3), dtype=float64, chunksize=(50, 3), chunktype=numpy.ndarray>
 
 Specifying ``lengths=True`` triggers immediate computation of the chunk sizes.
 This enables downstream computations that rely on having known chunk sizes
@@ -171,7 +171,7 @@ The Dask DataFrame ``to_records`` method also returns a Dask Array, but does not
 information:
 
    >>> df.to_records()
-   dask.array<to_records, shape=(nan,), dtype=(numpy.record, [('index', '<i8'), ('x', '<f8'), ('y', '<f8'), ('z', '<f8')]), chunksize=(nan,)>
+   dask.array<to_records, shape=(nan,), dtype=(numpy.record, [('index', '<i8'), ('x', '<f8'), ('y', '<f8'), ('z', '<f8')]), chunksize=(nan,), chunktype=numpy.ndarray>
 
 If you have a function that converts a Pandas DataFrame into a NumPy array,
 then calling ``map_partitions`` with that function on a Dask DataFrame will
@@ -180,7 +180,7 @@ produce a Dask array:
 .. code-block:: python
 
    >>> df.map_partitions(np.asarray)
-   dask.array<asarray, shape=(nan, 3), dtype=float64, chunksize=(nan, 3)>
+   dask.array<asarray, shape=(nan, 3), dtype=float64, chunksize=(nan, 3), chunktype=numpy.ndarray>
 
 
 Interactions with NumPy arrays
@@ -205,7 +205,7 @@ chunk shape:
    >>> y = np.ones(10)
    >>> z = x + y
    >>> z
-   dask.array<add, shape=(10,), dtype=float64, chunksize=(5,)>
+   dask.array<add, shape=(10,), dtype=float64, chunksize=(5,), chunktype=numpy.ndarray>
 
 These interactions work not just for NumPy arrays but for any object that has
 shape and dtype attributes and implements NumPy slicing syntax.
diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 13fd4a373..55ed07e2d 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,76 @@
 Changelog
 =========
 
+2.3.0 / 2019-08-16
+------------------
+
+Array
++++++
+
+- Raise exception when ``from_array`` is given a dask array (:pr:`5280`) `David Hoese`_
+- Avoid adjusting gufunc's meta dtype twice (:pr:`5274`) `Peter Andreas Entschev`_
+- Add ``meta=`` keyword to map_blocks and add test with sparse (:pr:`5269`) `Matthew Rocklin`_
+- Add rollaxis and moveaxis (:pr:`4822`) `Tobias de Jong`_
+- Always increment old chunk index (:pr:`5256`) `James Bourbeau`_
+- Shuffle dask array (:pr:`3901`) `Tom Augspurger`_
+- Fix ordering when indexing a dask array with a bool dask array (:pr:`5151`) `James Bourbeau`_
+
+Bag
++++
+
+- Add workaround for memory leaks in bag generators (:pr:`5208`) `Marco Neumann`_
+
+Core
+++++
+
+- Set strict xfail option (:pr:`5220`) `James Bourbeau`_
+- test-upstream (:pr:`5267`) `Tom Augspurger`_
+- Fixed HDFS CI failure (:pr:`5234`) `Tom Augspurger`_
+- Error nicely if no file size inferred (:pr:`5231`) `Jim Crist`_
+- A few changes to ``config.set`` (:pr:`5226`) `Jim Crist`_
+- Fixup black string normalization (:pr:`5227`) `Jim Crist`_
+- Pin NumPy in windows tests (:pr:`5228`) `Jim Crist`_
+- Ensure parquet tests are skipped if fastparquet and pyarrow not installed (:pr:`5217`) `James Bourbeau`_
+- Add fsspec to readthedocs (:pr:`5207`) `Matthew Rocklin`_
+- Bump NumPy and Pandas to 1.17 and 0.25 in CI test (:pr:`5179`) `John A Kirkham`_
+
+DataFrame
++++++++++
+
+- Fix ``DataFrame.query`` docstring (incorrect numexpr API) (:pr:`5271`) `Doug Davis`_
+- Parquet metadata-handling improvements (:pr:`5218`) `Richard J Zamora`_
+- Improve messaging around sorted parquet columns for index (:pr:`5265`) `Martin Durant`_
+- Add ``rearrange_by_divisions`` and ``set_index`` support for cudf (:pr:`5205`) `Richard J Zamora`_
+- Fix ``groupby.std()`` with integer colum names (:pr:`5096`) `Nicolas Hug`_
+- Add ``Series.__iter__`` (:pr:`5071`) `Blane`_
+- Generalize ``hash_pandas_object`` to work for non-pandas backends (:pr:`5184`) `GALI PREM SAGAR`_
+- Add rolling cov (:pr:`5154`) `Ivars Geidans`_
+- Add columns argument in drop function (:pr:`5223`) `Henrique Ribeiro`_
+
+Documentation
++++++++++++++
+
+- Update institutional FAQ doc (:pr:`5277`) `Matthew Rocklin`_
+- Add draft of institutional FAQ (:pr:`5214`) `Matthew Rocklin`_
+- Make boxes for dask-spark page (:pr:`5249`) `Martin Durant`_
+- Add motivation for shuffle docs (:pr:`5213`) `Matthew Rocklin`_
+- Fix links and API entries for best-practices (:pr:`5246`) `Martin Durant`_
+- Remove "bytes" (internal data ingestion) doc page (:pr:`5242`) `Martin Durant`_
+- Redirect from our local distributed page to distributed.dask.org (:pr:`5248`) `Matthew Rocklin`_
+- Cleanup API page (:pr:`5247`) `Matthew Rocklin`_
+- Remove excess endlines from install docs (:pr:`5243`) `Matthew Rocklin`_
+- Remove item list in phases of computation doc (:pr:`5245`) `Martin Durant`_
+- Remove custom graphs from the TOC sidebar (:pr:`5241`) `Matthew Rocklin`_
+- Remove experimental status of custom collections (:pr:`5236`) `James Bourbeau`_
+- Adds table of contents to Why Dask? (:pr:`5244`) `James Bourbeau`_
+- Moves bag overview to top-level bag page (:pr:`5240`) `James Bourbeau`_
+- Remove use-cases in favor of stories.dask.org (:pr:`5238`) `Matthew Rocklin`_
+- Removes redundant TOC information in index.rst (:pr:`5235`) `James Bourbeau`_
+- Elevate dashboard in distributed diagnostics documentation (:pr:`5239`) `Martin Durant`_
+- Updates "add" layer in HLG docs example (:pr:`5237`) `James Bourbeau`_
+- Update GUFunc documentation (:pr:`5232`) `Matthew Rocklin`_
+
+
 2.2.0 / 2019-08-01
 ------------------
 
@@ -2362,3 +2432,7 @@ Other
 .. _`Loïc Estève`: https://github.com/lesteve
 .. _`Xavier Holt`: https://github.com/xavi-ai
 .. _`Sarah Bird`: https://github.com/birdsarah
+.. _`Doug Davis`: https://github.com/douglasdavis
+.. _`Nicolas Hug`: https://github.com/NicolasHug
+.. _`Blane`: https://github.com/BlaneG
+.. _`Ivars Geidans`: https://github.com/ivarsfg


Test patch:
diff --git a/dask/array/tests/test_array_core.py b/dask/array/tests/test_array_core.py
index 82372437e..b72d5facd 100644
--- a/dask/array/tests/test_array_core.py
+++ b/dask/array/tests/test_array_core.py
@@ -1419,6 +1419,16 @@ def test_repr():
     assert len(str(d)) < 1000
 
 
+def test_repr_meta():
+    d = da.ones((4, 4), chunks=(2, 2))
+    assert "chunktype=numpy.ndarray" in repr(d)
+
+    # Test non-numpy meta
+    sparse = pytest.importorskip("sparse")
+    s = d.map_blocks(sparse.COO)
+    assert "chunktype=sparse.COO" in repr(s)
+
+
 def test_slicing_with_ellipsis():
     x = np.arange(256).reshape((4, 4, 4, 4))
     d = da.from_array(x, chunks=((2, 2, 2, 2)))
diff --git a/dask/array/tests/test_random.py b/dask/array/tests/test_random.py
index e0de8a5b9..11164522f 100644
--- a/dask/array/tests/test_random.py
+++ b/dask/array/tests/test_random.py
@@ -237,14 +237,16 @@ def test_choice():
     assert res.dtype == np_dtype
     assert res.shape == size
 
-    np_a = np.array([1, 3, 5, 7, 9], dtype="f8")
+    py_a = [1, 3, 5, 7, 9]
+    np_a = np.array(py_a, dtype="f8")
     da_a = da.from_array(np_a, chunks=2)
 
-    for a in [np_a, da_a]:
+    for a in [py_a, np_a, da_a]:
         x = da.random.choice(a, size=size, chunks=chunks)
         res = x.compute()
-        assert x.dtype == np_a.dtype
-        assert res.dtype == np_a.dtype
+        expected_dtype = np.asarray(a).dtype
+        assert x.dtype == expected_dtype
+        assert res.dtype == expected_dtype
         assert set(np.unique(res)).issubset(np_a)
 
     np_p = np.array([0, 0.2, 0.2, 0.3, 0.3])