Repository: dask/dask

Issue:
dataframe.quantile on empty all missing data raises
```python
In [4]: import dask.dataframe as dd; import pandas as pd; import numpy as np

In [5]: df = pd.DataFrame({"A": []})

In [6]: ddf = dd.from_pandas(df, 2)

In [7]: df.quantile(0.5)
Out[7]:
A   NaN
Name: 0.5, dtype: float64

In [8]: ddf.quantile(0.5)
Out[8]:
Dask Series Structure:
npartitions=1
A    float64
A        ...
Name: 0.5, dtype: float64
Dask Name: quantiles-concat, 7 tasks

In [9]: ddf.quantile(0.5).compute()
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-9-4bed07fb63b8> in <module>()
----> 1 ddf.quantile(0.5).compute()

~/Envs/dask-dev/lib/python3.6/site-packages/dask/dask/base.py in compute(self, **kwargs)
    136         dask.base.compute
    137         """
--> 138         (result,) = compute(self, traverse=False, **kwargs)
    139         return result
    140

~/Envs/dask-dev/lib/python3.6/site-packages/dask/dask/base.py in compute(*args, **kwargs)
    334     postcomputes = [a.__dask_postcompute__() if is_dask_collection(a)
    335                     else (None, a) for a in args]
--> 336     results = get(dsk, keys, **kwargs)
    337     results_iter = iter(results)
    338     return tuple(a if f is None else f(next(results_iter), *a)

~/Envs/dask-dev/lib/python3.6/site-packages/dask/dask/threaded.py in get(dsk, result, cache, num_workers, **kwargs)
     73     results = get_async(pool.apply_async, len(pool._pool), dsk, result,
     74                         cache=cache, get_id=_thread_get_id,
---> 75                         pack_exception=pack_exception, **kwargs)
     76
     77     # Cleanup pools associated to dead threads

~/Envs/dask-dev/lib/python3.6/site-packages/dask/dask/local.py in get_async(apply_async, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, **kwargs)
    519                         _execute_task(task, data)  # Re-execute locally
    520                     else:
--> 521                         raise_exception(exc, tb)
    522                 res, worker_id = loads(res_info)
    523                 state['cache'][key] = res

~/Envs/dask-dev/lib/python3.6/site-packages/dask/dask/compatibility.py in reraise(exc, tb)
     58         if exc.__traceback__ is not tb:
     59             raise exc.with_traceback(tb)
---> 60         raise exc
     61
     62 else:

~/Envs/dask-dev/lib/python3.6/site-packages/dask/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
    288     try:
    289         task, data = loads(task_info)
--> 290         result = _execute_task(task, data)
    291         id = get_id()
    292         result = dumps((result, id))

~/Envs/dask-dev/lib/python3.6/site-packages/dask/dask/local.py in _execute_task(arg, cache, dsk)
    268     elif istask(arg):
    269         func, args = arg[0], arg[1:]
--> 270         args2 = [_execute_task(a, cache) for a in args]
    271         return func(*args2)
    272     elif not ishashable(arg):

~/Envs/dask-dev/lib/python3.6/site-packages/dask/dask/local.py in <listcomp>(.0)
    268     elif istask(arg):
    269         func, args = arg[0], arg[1:]
--> 270         args2 = [_execute_task(a, cache) for a in args]
    271         return func(*args2)
    272     elif not ishashable(arg):

~/Envs/dask-dev/lib/python3.6/site-packages/dask/dask/local.py in _execute_task(arg, cache, dsk)
    269         func, args = arg[0], arg[1:]
    270         args2 = [_execute_task(a, cache) for a in args]
--> 271         return func(*args2)
    272     elif not ishashable(arg):
    273         return arg

~/Envs/dask-dev/lib/python3.6/site-packages/dask/dask/array/percentile.py in merge_percentiles(finalq, qs, vals, Ns, interpolation)
     96     L = list(zip(*[(q, val, N) for q, val, N in zip(qs, vals, Ns) if N]))
     97     if not L:
---> 98         raise ValueError("No non-trivial arrays found")
     99     qs, vals, Ns = L
    100

ValueError: No non-trivial arrays found
```


Discussion:
Any news on this bug? I'm facing the same problem.
Still open. Let me know if you're interested in working on a fix and need
help getting started.

On Mon, Oct 1, 2018 at 12:30 PM Ivan Marin <notifications@github.com> wrote:

> Any news on this bug? I'm facing the same problem.
>
> —
> You are receiving this because you authored the thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/2792#issuecomment-425993550>, or mute
> the thread
> <https://github.com/notifications/unsubscribe-auth/ABQHIhgbjdqqGktlQGI981_2wd59I9EXks5uglElgaJpZM4P97vM>
> .
>

I tried to work around it using dropna, but it seems that there is no way to pass `axis=1` to drop the column with only NA values. What is the suggested way to use describe with columns with only NA values?
I think a constructing a workaround would be difficult. Best to properly
fix this bug.

On Mon, Oct 1, 2018 at 12:36 PM Ivan Marin <notifications@github.com> wrote:

> I tried to work around it using dropna, but it seems that there is no way
> to pass axis=1 to drop the column with only NA values. What is the
> suggested way to use describe with columns with only NA values?
>
> —
> You are receiving this because you authored the thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/2792#issuecomment-425995778>, or mute
> the thread
> <https://github.com/notifications/unsubscribe-auth/ABQHIqQ2bsJ4RWI4g9hFd4Q9OO3e94ziks5uglKtgaJpZM4P97vM>
> .
>

How can this bug be fixed?
Since this is an issue with the values, I think the complexity has to be handled all the way down in `dask/array/percentile/percentile.py` in `merge_percentiles`. I'd recommend creating a new keyword in `percentile` to control the behavior when an all-NA array is encountered. Something like `raise_on_nan=True`. If that's false, which dask.dataframe would pass, an all-NaN block should result in a NaN quantile.

I haven't looked beyond that, so there may be additional complexities getting the result back up to dask dataframe.
I'm happy to take a look at this issue some time this week (self-assigning)
is this bug still not fixed pls?
I also got a similar error msg.

```
 distributed.worker - WARNING -  Compute Failed
Function:  execute_task
args:      ((<class 'pandas.core.series.Series'>, (<function merge_percentiles at 0x7fe34d0c5bf8>, array([25., 50., 75.]), [array([  0.,  25.,  50.,  75., 100.]), array([  0.,  25.,  50.,  75., 100.]), array([  0.,  25.,  50.,  75., 100.])], [(None, 0), (None, 0), (None, 0)]), array([0.25, 0.5 , 0.75]), None, '_precipm'))
kwargs:    {}
Exception: ValueError('No non-trivial arrays found',)

ERROR:ADS Exception
Traceback (most recent call last):
  File "/home/datascience/conda/mlcpuv1/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3343, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-21-ee2cf2aaf2d9>", line 1, in <module>
    ds.ddf.describe(include='all').round(2).compute()
  File "/home/datascience/conda/mlcpuv1/lib/python3.6/site-packages/dask/base.py", line 167, in compute
    (result,) = compute(self, traverse=False, **kwargs)
  File "/home/datascience/conda/mlcpuv1/lib/python3.6/site-packages/dask/base.py", line 452, in compute
    results = schedule(dsk, keys, **kwargs)
  File "/home/datascience/conda/mlcpuv1/lib/python3.6/site-packages/distributed/client.py", line 2725, in get
    results = self.gather(packed, asynchronous=asynchronous, direct=direct)
  File "/home/datascience/conda/mlcpuv1/lib/python3.6/site-packages/distributed/client.py", line 1992, in gather
    asynchronous=asynchronous,
  File "/home/datascience/conda/mlcpuv1/lib/python3.6/site-packages/distributed/client.py", line 833, in sync
    self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
  File "/home/datascience/conda/mlcpuv1/lib/python3.6/site-packages/distributed/utils.py", line 340, in sync
    raise exc.with_traceback(tb)
  File "/home/datascience/conda/mlcpuv1/lib/python3.6/site-packages/distributed/utils.py", line 324, in f
    result[0] = yield future
  File "/home/datascience/conda/mlcpuv1/lib/python3.6/site-packages/tornado/gen.py", line 762, in run
    value = future.result()
  File "/home/datascience/conda/mlcpuv1/lib/python3.6/site-packages/distributed/client.py", line 1851, in _gather
    raise exception.with_traceback(traceback)
  File "/home/datascience/conda/mlcpuv1/lib/python3.6/site-packages/dask/array/percentile.py", line 194, in merge_percentiles
    raise ValueError("No non-trivial arrays found")
ValueError: No non-trivial arrays found
ValueError: No non-trivial arrays found
```
Yes, unfortunately this is still a bug

Resolving patch:
diff --git a/dask/array/percentile.py b/dask/array/percentile.py
--- a/dask/array/percentile.py
+++ b/dask/array/percentile.py
@@ -165,7 +165,9 @@ def percentile(a, q, interpolation="linear", method="default"):
     return Array(graph, name2, chunks=((len(q),),), meta=meta)
 
 
-def merge_percentiles(finalq, qs, vals, interpolation="lower", Ns=None):
+def merge_percentiles(
+    finalq, qs, vals, interpolation="lower", Ns=None, raise_on_nan=True
+):
     """Combine several percentile calculations of different data.
 
     Parameters
@@ -207,14 +209,16 @@ def merge_percentiles(finalq, qs, vals, interpolation="lower", Ns=None):
 
     L = list(zip(*[(q, val, N) for q, val, N in zip(qs, vals, Ns) if N]))
     if not L:
-        raise ValueError("No non-trivial arrays found")
+        if raise_on_nan:
+            raise ValueError("No non-trivial arrays found")
+        return np.full(len(qs[0]) - 2, np.nan)
     qs, vals, Ns = L
 
     # TODO: Perform this check above in percentile once dtype checking is easy
     #       Here we silently change meaning
     if vals[0].dtype.name == "category":
         result = merge_percentiles(
-            finalq, qs, [v.codes for v in vals], interpolation, Ns
+            finalq, qs, [v.codes for v in vals], interpolation, Ns, raise_on_nan
         )
         import pandas as pd
 
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -6052,7 +6052,15 @@ def quantile(df, q, method="default"):
         name2 = "quantiles-2-" + token
         merge_dsk = {
             (name2, 0): finalize_tsk(
-                (merge_percentiles, qs, [calc_qs] * df.npartitions, sorted(val_dsk))
+                (
+                    merge_percentiles,
+                    qs,
+                    [calc_qs] * df.npartitions,
+                    sorted(val_dsk),
+                    "lower",
+                    None,
+                    False,
+                )
             )
         }
     dsk = merge(val_dsk, merge_dsk)


Test patch:
diff --git a/dask/dataframe/tests/test_dataframe.py b/dask/dataframe/tests/test_dataframe.py
--- a/dask/dataframe/tests/test_dataframe.py
+++ b/dask/dataframe/tests/test_dataframe.py
@@ -1408,6 +1408,19 @@ def test_quantile_tiny_partitions():
     assert r == 2
 
 
+def test_quantile_trivial_partitions():
+    """See https://github.com/dask/dask/issues/2792"""
+    df = pd.DataFrame({"A": []})
+    ddf = dd.from_pandas(df, npartitions=2)
+    expected = df.quantile(0.5)
+    assert_eq(ddf.quantile(0.5), expected)
+
+    df = pd.DataFrame({"A": [np.nan, np.nan, np.nan, np.nan]})
+    ddf = dd.from_pandas(df, npartitions=2)
+    expected = df.quantile(0.5)
+    assert_eq(ddf.quantile(0.5), expected)
+
+
 def test_index():
     assert_eq(d.index, full.index)