internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: dask/dask

Issue:
Add ptp
Would be nice to have [`ptp`]( https://docs.scipy.org/doc/numpy/reference/generated/numpy.ptp.html ) for Dask Arrays. Should be relatively straightforward as it can be composed from the difference between the `min` and `max` of the array.


Discussion:
Seems reasonable

On Mon, Sep 18, 2017 at 12:23 PM, jakirkham <notifications@github.com>
wrote:

> Would be nice to have ptp
> <https://docs.scipy.org/doc/numpy/reference/generated/numpy.ptp.html> for
> Dask Arrays. Should be relatively straightforward as it can be composed
> from the difference between the min and max of the array.
>
> —
> You are receiving this because you are subscribed to this thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/2687>, or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AASszLzAy_dKoRdmlkwRy5dW6zQxC3bEks5sjpkVgaJpZM4PbLAu>
> .
>


Resolving patch:
diff --git a/dask/array/__init__.py b/dask/array/__init__.py
--- a/dask/array/__init__.py
+++ b/dask/array/__init__.py
@@ -5,7 +5,7 @@
                    atop, to_hdf5, to_npy_stack, from_npy_stack, from_delayed,
                    asarray, asanyarray, broadcast_to)
 from .routines import (take, choose, argwhere, where, coarsen, insert,
-                       ravel, roll, unique, squeeze, topk, diff, ediff1d,
+                       ravel, roll, unique, squeeze, topk, ptp, diff, ediff1d,
                        bincount, digitize, histogram, cov, array, dstack,
                        vstack, hstack, compress, extract, round, count_nonzero,
                        flatnonzero, nonzero, around, isnull, notnull, isclose,
diff --git a/dask/array/routines.py b/dask/array/routines.py
--- a/dask/array/routines.py
+++ b/dask/array/routines.py
@@ -160,6 +160,11 @@ def dot(a, b):
     return tensordot(a, b, axes=((a.ndim - 1,), (b.ndim - 2,)))
 
 
+@wraps(np.ptp)
+def ptp(a, axis=None):
+    return a.max(axis=axis) - a.min(axis=axis)
+
+
 @wraps(np.diff)
 def diff(a, n=1, axis=-1):
     a = asarray(a)


Test patch:
diff --git a/dask/array/tests/test_routines.py b/dask/array/tests/test_routines.py
--- a/dask/array/tests/test_routines.py
+++ b/dask/array/tests/test_routines.py
@@ -104,6 +104,20 @@ def test_dot_method():
     assert_eq(a.dot(b), x.dot(y))
 
 
+@pytest.mark.parametrize('shape, axis', [
+    [(10, 15, 20), None],
+    [(10, 15, 20), 0],
+    [(10, 15, 20), 1],
+    [(10, 15, 20), 2],
+    [(10, 15, 20), -1],
+])
+def test_ptp(shape, axis):
+    a = np.random.randint(0, 10, shape)
+    d = da.from_array(a, chunks=(len(shape) * (5,)))
+
+    assert_eq(da.ptp(d, axis), np.ptp(a, axis))
+
+
 @pytest.mark.parametrize('shape, axis', [
     [(10, 15, 20), 0],
     [(10, 15, 20), 1],
Repository: dask/dask

Issue:
Optionally require pytest-runner
There is a suggestion that `pytest-runner` only be added to `setup_requires` when `test` or `pytest` is in the `setup.py` invocation. Should be easy enough to do. Am raising it here for consideration.

ref: https://github.com/pytest-dev/pytest-runner#considerations
xref: https://github.com/conda-forge/dask-core-feedstock/issues/1


Discussion:
cc-ing @Horta who added `pytest-runner`.

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+import sys
 from os.path import exists
 from setuptools import setup
 import versioneer
@@ -22,6 +23,11 @@
 
 tests = [p + '.tests' for p in packages]
 
+# Only include pytest-runner in setup_requires if we're invoking tests
+if {'pytest', 'test', 'ptr'}.intersection(sys.argv):
+    setup_requires = ['pytest-runner']
+else:
+    setup_requires = []
 
 setup(name='dask',
       version=versioneer.get_version(),
@@ -35,7 +41,7 @@
       packages=packages + tests,
       long_description=(open('README.rst').read() if exists('README.rst')
                         else ''),
-      setup_requires=['pytest-runner'],
+      setup_requires=setup_requires,
       tests_require=['pytest'],
       extras_require=extras_require,
       zip_safe=False)


Test patch:
Repository: dask/dask

Issue:
BUG: dask.array no longer supports indexing with np.int64
To reproduce:
```
In [20]: x = da.ones((2,), chunks=1)

In [21]: x[0]
Out[21]: dask.array<getitem, shape=(), dtype=float64, chunksize=()>

In [22]: x[np.int64(0)]
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-22-8268a5ebce06> in <module>()
----> 1 x[np.int64(0)]

~/dev/dask/dask/array/core.py in __getitem__(self, index)
   1220
   1221         from .slicing import normalize_index, slice_with_dask_array
-> 1222         index2 = normalize_index(index, self.shape)
   1223
   1224         if any(isinstance(i, Array) for i in index2):

~/dev/dask/dask/array/slicing.py in normalize_index(idx, shape)
    760     idx = idx + (slice(None),) * (len(shape) - n_sliced_dims)
    761     if len([i for i in idx if i is not None]) > len(shape):
--> 762         raise IndexError("Too many indices for array")
    763
    764     none_shape = []

IndexError: Too many indices for array
```

This is a recent regression affecting xarray users, see https://github.com/pydata/xarray/issues/1591. (Apparently we normalize all integer indexing to `np.int64`.)


Discussion:
This seems to be caused by that `np.int64` (and also `np.int`) has `ndim` attribute.

Will send a PR.

Resolving patch:
diff --git a/dask/array/slicing.py b/dask/array/slicing.py
--- a/dask/array/slicing.py
+++ b/dask/array/slicing.py
@@ -751,7 +751,7 @@ def normalize_index(idx, shape):
     idx = replace_ellipsis(len(shape), idx)
     n_sliced_dims = 0
     for i in idx:
-        if hasattr(i, 'ndim'):
+        if hasattr(i, 'ndim') and i.ndim >= 1:
             n_sliced_dims += i.ndim
         elif i is None:
             continue


Test patch:
diff --git a/dask/array/tests/test_array_core.py b/dask/array/tests/test_array_core.py
--- a/dask/array/tests/test_array_core.py
+++ b/dask/array/tests/test_array_core.py
@@ -1626,13 +1626,27 @@ def test_slice_with_floats():
         d[[1, 1.5]]
 
 
-def test_slice_with_uint():
+def test_slice_with_integer_types():
     x = np.arange(10)
     dx = da.from_array(x, chunks=5)
     inds = np.array([0, 3, 6], dtype='u8')
     assert_eq(dx[inds], x[inds])
     assert_eq(dx[inds.astype('u4')], x[inds.astype('u4')])
 
+    inds = np.array([0, 3, 6], dtype=np.int64)
+    assert_eq(dx[inds], x[inds])
+    assert_eq(dx[inds.astype('u4')], x[inds.astype('u4')])
+
+
+def test_index_with_integer_types():
+    x = np.arange(10)
+    dx = da.from_array(x, chunks=5)
+    inds = int(3)
+    assert_eq(dx[inds], x[inds])
+
+    inds = np.int64(3)
+    assert_eq(dx[inds], x[inds])
+
 
 def test_vindex_basic():
     x = np.arange(56).reshape((7, 8))
Repository: dask/dask

Issue:
dd.read_parquet() column filter not working for categorical columns
All categorical columns are returned by dd.read_parquet(df, columns=[...]) command. On computing this dataframe, the correct columns are returned.

This problem is independent of whether these columns are known/unknown categories.

As a side note, dd.read_parquet(file) always returns unknown categorical columns regardless of their state on saving with dd.to_parquet(df). I realise that this may be a parquet format limitation, and computing the dask dataframe does return the correct and complete set of categories per partition.

[https://gist.github.com/bendruitt/8dfadc03694a37130752229844057b34](https://gist.github.com/bendruitt/8dfadc03694a37130752229844057b34)


Discussion:
Interesting, thank you for the report. I will provide a fix soon.

Resolving patch:
diff --git a/dask/dataframe/io/parquet.py b/dask/dataframe/io/parquet.py
--- a/dask/dataframe/io/parquet.py
+++ b/dask/dataframe/io/parquet.py
@@ -91,8 +91,9 @@ def _read_fastparquet(fs, paths, myopen, columns=None, filters=None,
                              dtypes)
 
     for cat in categories:
-        meta[cat] = pd.Series(pd.Categorical([],
-                              categories=[UNKNOWN_CATEGORIES]))
+        if cat in meta:
+            meta[cat] = pd.Series(pd.Categorical([],
+                                  categories=[UNKNOWN_CATEGORIES]))
 
     if index_col:
         meta = meta.set_index(index_col)


Test patch:
diff --git a/dask/dataframe/io/tests/test_parquet.py b/dask/dataframe/io/tests/test_parquet.py
--- a/dask/dataframe/io/tests/test_parquet.py
+++ b/dask/dataframe/io/tests/test_parquet.py
@@ -536,3 +536,19 @@ def test_drill_scheme(fn):
     out = df.compute()
     assert 'dir0' in out
     assert (np.unique(out.dir0) == ['test_data1', 'test_data2']).all()
+
+
+def test_parquet_select_cats(fn):
+    df = pd.DataFrame({
+        'categories': pd.Series(
+            np.random.choice(['a', 'b', 'c', 'd', 'e', 'f'], size=100),
+            dtype='category'),
+        'ints': pd.Series(list(range(0, 100)), dtype='int'),
+        'floats': pd.Series(list(range(0, 100)), dtype='float')})
+
+    ddf = dd.from_pandas(df, 1)
+    ddf.to_parquet(fn)
+    rddf = dd.read_parquet(fn, columns=['ints'])
+    assert list(rddf.columns) == ['ints']
+    rddf = dd.read_parquet(fn)
+    assert list(rddf.columns) == list(df)
Repository: dask/dask

Issue:
Document asarray
Would be nice to have `asarray` formally documented as part of the public API.


Discussion:

Resolving patch:
diff --git a/dask/array/core.py b/dask/array/core.py
--- a/dask/array/core.py
+++ b/dask/array/core.py
@@ -2414,38 +2414,74 @@ def store(out, x, index, lock, region):
     return dsk
 
 
-def asarray(array):
-    """Coerce argument into a dask array
+def asarray(a):
+    """Convert the input to a dask array.
+
+    Parameters
+    ----------
+    a : array-like
+        Input data, in any form that can be converted to a dask array.
+
+    Returns
+    -------
+    out : dask array
+        Dask array interpretation of a.
 
     Examples
     --------
+    >>> import dask.array as da
+    >>> import numpy as np
     >>> x = np.arange(3)
-    >>> asarray(x)
+    >>> da.asarray(x)
     dask.array<array, shape=(3,), dtype=int64, chunksize=(3,)>
+
+    >>> y = [[1, 2, 3], [4, 5, 6]]
+    >>> da.asarray(y)
+    dask.array<array, shape=(2, 3), dtype=int64, chunksize=(2, 3)>
     """
-    if isinstance(array, Array):
-        return array
-    if not isinstance(getattr(array, 'shape', None), Iterable):
-        array = np.asarray(array)
-    return from_array(array, chunks=array.shape, getitem=getter_inline)
+    if isinstance(a, Array):
+        return a
+    if isinstance(a, (list, tuple)) and any(isinstance(i, Array) for i in a):
+        a = stack(a)
+    elif not isinstance(getattr(a, 'shape', None), Iterable):
+        a = np.asarray(a)
+    return from_array(a, chunks=a.shape, getitem=getter_inline)
 
 
-def asanyarray(array):
-    """Coerce argument into a dask array.
+def asanyarray(a):
+    """Convert the input to a dask array.
 
-    Subclasses of `np.ndarray` will be passed through as chunks unchanged.
+    Subclasses of ``np.ndarray`` will be passed through as chunks unchanged.
+
+    Parameters
+    ----------
+    a : array-like
+        Input data, in any form that can be converted to a dask array.
+
+    Returns
+    -------
+    out : dask array
+        Dask array interpretation of a.
 
     Examples
     --------
+    >>> import dask.array as da
+    >>> import numpy as np
     >>> x = np.arange(3)
-    >>> asanyarray(x)
+    >>> da.asanyarray(x)
     dask.array<array, shape=(3,), dtype=int64, chunksize=(3,)>
+
+    >>> y = [[1, 2, 3], [4, 5, 6]]
+    >>> da.asanyarray(y)
+    dask.array<array, shape=(2, 3), dtype=int64, chunksize=(2, 3)>
     """
-    if isinstance(array, Array):
-        return array
-    if not isinstance(getattr(array, 'shape', None), Iterable):
-        array = np.asanyarray(array)
-    return from_array(array, chunks=array.shape, getitem=getter_inline,
+    if isinstance(a, Array):
+        return a
+    if isinstance(a, (list, tuple)) and any(isinstance(i, Array) for i in a):
+        a = stack(a)
+    elif not isinstance(getattr(a, 'shape', None), Iterable):
+        a = np.asanyarray(a)
+    return from_array(a, chunks=a.shape, getitem=getter_inline,
                       asarray=False)
 
 


Test patch:
Repository: dask/dask

Issue:
dask.dataframe quantile doesn't skip nans
```python
In [52]: import numpy as np

In [53]: import pandas as pd

In [54]: import dask.dataframe as dd

In [55]: s = dd.from_pandas(pd.Series([1, np.nan, 3]), 2)

In [56]: s.compute().quantile([.5])
Out[56]:
0.5    2.0
dtype: float64

In [57]: s.quantile(0.5).compute()
/Users/taugspurger/.virtualenvs/pandas-dev/lib/python3.6/site-packages/numpy/lib/function_base.py:4274: RuntimeWarning: Invalid value encountered in percentile
  interpolation=interpolation)
Out[57]: nan
```


Discussion:
If we filter out the `nan` values [here](https://github.com/dask/dask/blob/master/dask/dataframe/core.py#L3330-L3331) it should work for both series and dataframes, correct?

Resolving patch:
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -1294,7 +1294,8 @@ def quantile(self, q=0.5, axis=0):
             qnames = [(_q._name, 0) for _q in quantiles]
 
             if isinstance(quantiles[0], Scalar):
-                dask[(keyname, 0)] = (pd.Series, qnames, num.columns)
+                dask[(keyname, 0)] = (pd.Series, qnames, num.columns,
+                                      None, meta.name)
                 divisions = (min(num.columns), max(num.columns))
                 return Series(dask, keyname, meta, divisions)
             else:
@@ -3326,6 +3327,7 @@ def quantile(df, q):
     else:
         new_divisions = [np.min(q), np.max(q)]
 
+    df = df.dropna()
     name = 'quantiles-1-' + token
     val_dsk = {(name, i): (_percentile, (getattr, key, 'values'), qs)
                for i, key in enumerate(df.__dask_keys__())}


Test patch:
diff --git a/dask/dataframe/tests/test_dataframe.py b/dask/dataframe/tests/test_dataframe.py
--- a/dask/dataframe/tests/test_dataframe.py
+++ b/dask/dataframe/tests/test_dataframe.py
@@ -690,16 +690,26 @@ def test_quantile():
     assert 4 < result < 6
 
 
+def test_quantile_missing():
+    df = pd.DataFrame({"A": [0, np.nan, 2]})
+    ddf = dd.from_pandas(df, 2)
+    expected = df.quantile()
+    result = ddf.quantile()
+    assert_eq(result, expected)
+
+    expected = df.A.quantile()
+    result = ddf.A.quantile()
+    assert_eq(result, expected)
+
+
 def test_empty_quantile():
     result = d.b.quantile([])
     exp = full.b.quantile([])
     assert result.divisions == (None, None)
 
-    # because of a pandas bug, name is not preserved
-    # https://github.com/pydata/pandas/pull/10881
     assert result.name == 'b'
     assert result.compute().name == 'b'
-    assert_eq(result, exp, check_names=False)
+    assert_eq(result, exp)
 
 
 def test_dataframe_quantile():
@@ -717,6 +727,7 @@ def test_dataframe_quantile():
 
     result = result.compute()
     assert isinstance(result, pd.Series)
+    assert result.name == 0.5
     tm.assert_index_equal(result.index, pd.Index(['A', 'X', 'B']))
     assert (result > pd.Series([16, 36, 26], index=['A', 'X', 'B'])).all()
     assert (result < pd.Series([17, 37, 27], index=['A', 'X', 'B'])).all()
Repository: dask/dask

Issue:
Calling dd.concat([s], axis=1) should return a DataFrame
The behaviour of dask.dataframe.concat() and pandas.concat differ if axis=1 which concatenating a single series:

```
from dask import dataframe as dd
import pandas as pd

s_pd = pd.Series(range(10))
s_dd = dd.from_pandas(s_pd, npartitions=1)

print( type(pd.concat([s_pd], axis=1)) )
print( type(dd.concat([s_dd], axis=1)) )

<class 'pandas.core.frame.DataFrame'>
<class 'dask.dataframe.core.Series'>
```

With axis=1, dd.concat  should return a `<class 'dask.dataframe.core.DataFrame'>` instead.


Discussion:

Resolving patch:
diff --git a/dask/dataframe/multi.py b/dask/dataframe/multi.py
--- a/dask/dataframe/multi.py
+++ b/dask/dataframe/multi.py
@@ -64,7 +64,7 @@
 
 from ..base import tokenize
 from ..compatibility import apply
-from .core import (_Frame, DataFrame, map_partitions, Index,
+from .core import (_Frame, DataFrame, Series, map_partitions, Index,
                    _maybe_from_pandas, new_dd_object, is_broadcastable)
 from .io import from_pandas
 from . import methods
@@ -550,7 +550,10 @@ def concat(dfs, axis=0, join='outer', interleave_partitions=False):
     if len(dfs) == 0:
         raise ValueError('No objects to concatenate')
     if len(dfs) == 1:
-        return dfs[0]
+        if axis == 1 and isinstance(dfs[0], Series):
+            return dfs[0].to_frame()
+        else:
+            return dfs[0]
 
     if join not in ('inner', 'outer'):
         raise ValueError("'join' must be 'inner' or 'outer'")


Test patch:
diff --git a/dask/dataframe/tests/test_multi.py b/dask/dataframe/tests/test_multi.py
--- a/dask/dataframe/tests/test_multi.py
+++ b/dask/dataframe/tests/test_multi.py
@@ -799,6 +799,17 @@ def test_errors_for_merge_on_frame_columns():
         dd.merge(aa, bb, left_on=aa.x, right_on=bb.y)
 
 
+def test_concat_one_series():
+    a = pd.Series([1, 2, 3, 4])
+    aa = dd.from_pandas(a, npartitions=2, sort=False)
+
+    c = dd.concat([aa], axis=0)
+    assert isinstance(c, dd.Series)
+
+    c = dd.concat([aa], axis=1)
+    assert isinstance(c, dd.DataFrame)
+
+
 def test_concat_unknown_divisions():
     a = pd.Series([1, 2, 3, 4])
     b = pd.Series([4, 3, 2, 1])
Repository: dask/dask

Issue:
dask.diagnostics.visualize incompatible with bokeh 0.12.10 (latest)
I am getting this stack trace when calling `dask.diagnostics.visualize` with `dask==0.15.4` and `bokeh==0.12.10`:

```
Traceback (most recent call last):
  File "scripts/compress_volumes_dask.py", line 173, in <module>
    run_pipeline()
  File "scripts/compress_volumes_dask.py", line 170, in run_pipeline
    visualize([prof, rprof, cprof])
  File "/Users/jcwork/.pyenv/versions/capstone/lib/python3.5/site-packages/dask/diagnostics/profile_visualize.py", line 171, in visualize
    from bokeh.io import _state
ImportError: cannot import name '_state'
```

It appears that bokeh.io no longer exports a _state variable. I'm not familiar with any of the code here -- possibly this now wants to be fetched from `bokeh.io.state.curstate()`?

For now, installing the previous version of bokeh, 0.12.9, fixes the problem.


Discussion:

Resolving patch:
diff --git a/dask/diagnostics/profile_visualize.py b/dask/diagnostics/profile_visualize.py
--- a/dask/diagnostics/profile_visualize.py
+++ b/dask/diagnostics/profile_visualize.py
@@ -2,6 +2,7 @@
 
 import random
 from bisect import bisect_left
+from distutils.version import LooseVersion
 from itertools import cycle
 from operator import itemgetter, add
 
@@ -168,9 +169,16 @@ def visualize(profilers, file_path=None, show=True, save=True, **kwargs):
     The completed bokeh plot object.
     """
     bp = import_required('bokeh.plotting', _BOKEH_MISSING_MSG)
-    from bokeh.io import _state
+    import bokeh
 
-    if not _state._notebook:
+    if LooseVersion(bokeh.__version__) >= "0.12.10":
+        from bokeh.io import state
+        in_notebook = state.curstate().notebook
+    else:
+        from bokeh.io import _state
+        in_notebook = _state._notebook
+
+    if not in_notebook:
         file_path = file_path or "profile.html"
         bp.output_file(file_path)
 


Test patch:
Repository: dask/dask

Issue:
Percentile doesn't work on scalars
According to numpy API, percentile can accept a scalar
```
from dask import array
res = array.random.random(10000, chunks=100)
array.percentile(res, 15)
```

traceback:
```

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-57-4f859bf3a205> in <module>()
      1 from dask import array
      2 res = array.random.random(10000, chunks=100)
----> 3 array.percentile(res, 15)

~/anaconda3/envs/hello/lib/python3.6/site-packages/dask/array/percentile.py in percentile(a, q, interpolation)
     40             "Percentiles only implemented for 1-d arrays")
     41     q = np.array(q)
---> 42     token = tokenize(a, list(q), interpolation)
     43     name = 'percentile_chunk-' + token
     44     dsk = dict(((name, i), (_percentile, (key), q, interpolation))

TypeError: iteration over a 0-d array
```

dask version: '0.15.2'


Discussion:

Resolving patch:
diff --git a/dask/array/percentile.py b/dask/array/percentile.py
--- a/dask/array/percentile.py
+++ b/dask/array/percentile.py
@@ -2,6 +2,7 @@
 
 from functools import wraps
 from collections import Iterator
+from numbers import Number
 
 import numpy as np
 from toolz import merge, merge_sorted
@@ -38,6 +39,8 @@ def percentile(a, q, interpolation='linear'):
     if not a.ndim == 1:
         raise NotImplementedError(
             "Percentiles only implemented for 1-d arrays")
+    if isinstance(q, Number):
+        q = [q]
     q = np.array(q)
     token = tokenize(a, list(q), interpolation)
     name = 'percentile_chunk-' + token


Test patch:
diff --git a/dask/array/tests/test_percentiles.py b/dask/array/tests/test_percentiles.py
--- a/dask/array/tests/test_percentiles.py
+++ b/dask/array/tests/test_percentiles.py
@@ -51,3 +51,11 @@ def test_percentile_with_categoricals():
 def test_percentiles_with_empty_arrays():
     x = da.ones(10, chunks=((5, 0, 5),))
     assert_eq(da.percentile(x, [10, 50, 90]), np.array([1, 1, 1], dtype=x.dtype))
+
+
+@pytest.mark.parametrize('q', [5, 5.0, np.int64(5), np.float64(5)])
+def test_percentiles_with_scaler_percentile(q):
+    # Regression test to ensure da.percentile works with scalar percentiles
+    # See #3020
+    d = da.ones((16,), chunks=(4,))
+    assert_eq(da.percentile(d, q), np.array([1], dtype=d.dtype))