Repository: dask/dask

Issue:
BUG: da.percentile silently returns nan for array of unknown length
So the percentile on a normal dask array is fine:

```
In [2]: a = da.from_array(np.random.randn(10), chunks=(10,))

In [3]: a
Out[3]: dask.array<array, shape=(10,), dtype=float64, chunksize=(10,)>

In [4]: da.percentile(a, 50)
Out[4]: dask.array<percentile, shape=(1,), dtype=float64, chunksize=(1,)>

In [5]: da.percentile(a, 50).compute()
Out[5]: array([ 0.27603973])
```

but if it has unknown chunk sizes (eg from getting the values from a dataframe), it doesn't complain about this, but silently returns a dask array consisting of NaNs:
```
In [6]: ddf = dd.from_pandas(pd.DataFrame(np.random.randn(10, 3)), npartitions=2)

In [7]: ddf.values
Out[7]: dask.array<values, shape=(nan, 3), dtype=float64, chunksize=(nan, 3)>

In [8]: a2 = ddf.values[:, 0]

In [9]: a2
Out[9]: dask.array<getitem, shape=(nan,), dtype=float64, chunksize=(nan,)>

In [10]: da.percentile(a2, 50)
Out[10]: dask.array<percentile, shape=(1,), dtype=float64, chunksize=(1,)>

In [11]: da.percentile(a2, 50).compute()
Out[11]: array([ nan])
```

Should this rather raise an error?


Discussion:
It looks like merge_percentile takes in chunks from the dask array.  We can probably change things to compute the lengths in the _percentile function. I'll see if I can get something up quickly.

Resolving patch:
diff --git a/dask/array/percentile.py b/dask/array/percentile.py
--- a/dask/array/percentile.py
+++ b/dask/array/percentile.py
@@ -14,21 +14,22 @@
 
 @wraps(np.percentile)
 def _percentile(a, q, interpolation='linear'):
+    n = len(a)
     if not len(a):
-        return None
+        return None, n
     if isinstance(q, Iterator):
         q = list(q)
     if a.dtype.name == 'category':
         result = np.percentile(a.codes, q, interpolation=interpolation)
         import pandas as pd
-        return pd.Categorical.from_codes(result, a.categories, a.ordered)
+        return pd.Categorical.from_codes(result, a.categories, a.ordered), n
     if np.issubdtype(a.dtype, np.datetime64):
         a2 = a.astype('i8')
         result = np.percentile(a2, q, interpolation=interpolation)
-        return result.astype(a.dtype)
+        return result.astype(a.dtype), n
     if not np.issubdtype(a.dtype, np.number):
         interpolation = 'nearest'
-    return np.percentile(a, q, interpolation=interpolation)
+    return np.percentile(a, q, interpolation=interpolation), n
 
 
 def percentile(a, q, interpolation='linear'):
@@ -49,7 +50,7 @@ def percentile(a, q, interpolation='linear'):
 
     name2 = 'percentile-' + token
     dsk2 = {(name2, 0): (merge_percentiles, q, [q] * len(a.chunks[0]),
-                         sorted(dsk), a.chunks[0], interpolation)}
+                         sorted(dsk), interpolation)}
 
     dtype = a.dtype
     if np.issubdtype(dtype, np.integer):
@@ -60,7 +61,7 @@ def percentile(a, q, interpolation='linear'):
     return Array(dsk, name2, chunks=((len(q),),), dtype=dtype)
 
 
-def merge_percentiles(finalq, qs, vals, Ns, interpolation='lower'):
+def merge_percentiles(finalq, qs, vals, interpolation='lower', Ns=None):
     """ Combine several percentile calculations of different data.
 
     Parameters
@@ -86,7 +87,7 @@ def merge_percentiles(finalq, qs, vals, Ns, interpolation='lower'):
     >>> vals = [np.array([1, 2, 3, 4]), np.array([10, 11, 12, 13])]
     >>> Ns = [100, 100]  # Both original arrays had 100 elements
 
-    >>> merge_percentiles(finalq, qs, vals, Ns)
+    >>> merge_percentiles(finalq, qs, vals, Ns=Ns)
     array([ 1,  2,  3,  4, 10, 11, 12, 13])
     """
     if isinstance(finalq, Iterator):
@@ -94,6 +95,8 @@ def merge_percentiles(finalq, qs, vals, Ns, interpolation='lower'):
     finalq = np.array(finalq)
     qs = list(map(list, qs))
     vals = list(vals)
+    if Ns is None:
+        vals, Ns = zip(*vals)
     Ns = list(Ns)
 
     L = list(zip(*[(q, val, N) for q, val, N in zip(qs, vals, Ns) if N]))
@@ -104,7 +107,7 @@ def merge_percentiles(finalq, qs, vals, Ns, interpolation='lower'):
     # TODO: Perform this check above in percentile once dtype checking is easy
     #       Here we silently change meaning
     if vals[0].dtype.name == 'category':
-        result = merge_percentiles(finalq, qs, [v.codes for v in vals], Ns, interpolation)
+        result = merge_percentiles(finalq, qs, [v.codes for v in vals], interpolation, Ns)
         import pandas as pd
         return pd.Categorical.from_codes(result, vals[0].categories, vals[0].ordered)
     if not np.issubdtype(vals[0].dtype, np.number):
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -3446,14 +3446,12 @@ def quantile(df, q):
     name = 'quantiles-1-' + token
     val_dsk = {(name, i): (_percentile, (getattr, key, 'values'), qs)
                for i, key in enumerate(df.__dask_keys__())}
-    name2 = 'quantiles-2-' + token
-    len_dsk = {(name2, i): (len, key) for i, key in enumerate(df.__dask_keys__())}
 
     name3 = 'quantiles-3-' + token
     merge_dsk = {(name3, 0): finalize_tsk((merge_percentiles, qs,
                                            [qs] * df.npartitions,
-                                           sorted(val_dsk), sorted(len_dsk)))}
-    dsk = merge(df.dask, val_dsk, len_dsk, merge_dsk)
+                                           sorted(val_dsk)))}
+    dsk = merge(df.dask, val_dsk, merge_dsk)
     return return_type(dsk, name3, meta, new_divisions)
 
 
diff --git a/dask/dataframe/partitionquantiles.py b/dask/dataframe/partitionquantiles.py
--- a/dask/dataframe/partitionquantiles.py
+++ b/dask/dataframe/partitionquantiles.py
@@ -410,7 +410,7 @@ def percentiles_summary(df, num_old, num_new, upsample, state):
     if is_categorical_dtype(data):
         data = data.codes
         interpolation = 'nearest'
-    vals = _percentile(data, qs, interpolation=interpolation)
+    vals, n = _percentile(data, qs, interpolation=interpolation)
     if interpolation == 'linear' and np.issubdtype(data.dtype, np.integer):
         vals = np.round(vals).astype(data.dtype)
     vals_and_weights = percentiles_to_weights(qs, vals, length)


Test patch:
diff --git a/dask/array/tests/test_percentiles.py b/dask/array/tests/test_percentiles.py
--- a/dask/array/tests/test_percentiles.py
+++ b/dask/array/tests/test_percentiles.py
@@ -59,3 +59,16 @@ def test_percentiles_with_scaler_percentile(q):
     # See #3020
     d = da.ones((16,), chunks=(4,))
     assert_eq(da.percentile(d, q), np.array([1], dtype=d.dtype))
+
+
+def test_unknown_chunk_sizes():
+    x = da.random.random(1000, chunks=(100,))
+    x._chunks = ((np.nan,) * 10,)
+
+    result = da.percentile(x, 50).compute()
+    assert 0.1 < result < 0.9
+
+    a, b = da.percentile(x, [40, 60]).compute()
+    assert 0.1 < a < 0.9
+    assert 0.1 < b < 0.9
+    assert a < b