Repository: dask/dask

Issue:
dask.dataframe.corr() doesn't seem to be working well with booleans
I stumbled across this strange behaviour on the corr() method for the dask.dataframe API. It throws out this error when dealing with a boolean type column. The following code replicates what happened

<details>
<summary>Code</summary>

```python
import dask.dataframe as dd
import pandas as pd
import numpy as np

size = 1000

d = {    
    'dates' : pd.date_range('2015-01-01', periods=size, frequency='1T'),
    'unique_id' : np.arange(0, size),
    'ints' : np.random.randint(0, size, size=size),
    'floats' : np.random.randn(size),
    'bools' : np.random.choice([0, 1], size=size),
    'int_nans' : np.random.choice([0, 1, np.nan], size=size),
    'float_nans' : np.random.choice([0.0, 1.0, np.nan], size=size),
    'constant' : 1, 
    'int_categorical' : np.random.choice([10, 20, 30, 40, 50], size=size) , 
    'categorical_binary' : np.random.choice(['a', 'b'], size=size), 
    'categorical_nans' : np.random.choice(['a', 'b', 'c'], size=size)
}

df = pd.DataFrame(d)
df['hardbools'] = df['bools'] == 1
df['categorical_nans'] = df['categorical_nans'].replace('c', np.nan)
df['categorical_binary'] = df['categorical_binary'].astype('category')
df['unique_id'] = df['unique_id'].astype(str)

df.corr()
```

<div>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>bools</th>
      <th>constant</th>
      <th>float_nans</th>
      <th>floats</th>
      <th>int_categorical</th>
      <th>int_nans</th>
      <th>ints</th>
      <th>hardbools</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>bools</th>
      <td>1.000000</td>
      <td>NaN</td>
      <td>0.061330</td>
      <td>0.027821</td>
      <td>0.015810</td>
      <td>-0.037277</td>
      <td>0.005449</td>
      <td>1.000000</td>
    </tr>
    <tr>
      <th>constant</th>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
    </tr>
    <tr>
      <th>float_nans</th>
      <td>0.061330</td>
      <td>NaN</td>
      <td>1.000000</td>
      <td>-0.021301</td>
      <td>-0.001698</td>
      <td>0.039741</td>
      <td>-0.006224</td>
      <td>0.061330</td>
    </tr>
    <tr>
      <th>floats</th>
      <td>0.027821</td>
      <td>NaN</td>
      <td>-0.021301</td>
      <td>1.000000</td>
      <td>-0.038576</td>
      <td>-0.020085</td>
      <td>0.024252</td>
      <td>0.027821</td>
    </tr>
    <tr>
      <th>int_categorical</th>
      <td>0.015810</td>
      <td>NaN</td>
      <td>-0.001698</td>
      <td>-0.038576</td>
      <td>1.000000</td>
      <td>0.002101</td>
      <td>0.055539</td>
      <td>0.015810</td>
    </tr>
    <tr>
      <th>int_nans</th>
      <td>-0.037277</td>
      <td>NaN</td>
      <td>0.039741</td>
      <td>-0.020085</td>
      <td>0.002101</td>
      <td>1.000000</td>
      <td>0.027783</td>
      <td>-0.037277</td>
    </tr>
    <tr>
      <th>ints</th>
      <td>0.005449</td>
      <td>NaN</td>
      <td>-0.006224</td>
      <td>0.024252</td>
      <td>0.055539</td>
      <td>0.027783</td>
      <td>1.000000</td>
      <td>0.005449</td>
    </tr>
    <tr>
      <th>hardbools</th>
      <td>1.000000</td>
      <td>NaN</td>
      <td>0.061330</td>
      <td>0.027821</td>
      <td>0.015810</td>
      <td>-0.037277</td>
      <td>0.005449</td>
      <td>1.000000</td>
    </tr>
  </tbody>
</table>
</div>

```python
ddf = dd.from_pandas(df, npartitions=10)
ddf.corr().compute()
```

    --------------------------------------------------------------------------

    TypeError                                Traceback (most recent call last)

    <ipython-input-5-313b992845a4> in <module>()
    ----> 1 ddf.corr().compute()
    

    ~/envs/dev/lib/python3.5/site-packages/dask/base.py in compute(self, **kwargs)
         97             Extra keywords to forward to the scheduler ``get`` function.
         98         """
    ---> 99         (result,) = compute(self, traverse=False, **kwargs)
        100         return result
        101 


    ~/envs/dev/lib/python3.5/site-packages/dask/base.py in compute(*args, **kwargs)
        204     dsk = collections_to_dsk(variables, optimize_graph, **kwargs)
        205     keys = [var._keys() for var in variables]
    --> 206     results = get(dsk, keys, **kwargs)
        207 
        208     results_iter = iter(results)


    ~/envs/dev/lib/python3.5/site-packages/dask/threaded.py in get(dsk, result, cache, num_workers, **kwargs)
         73     results = get_async(pool.apply_async, len(pool._pool), dsk, result,
         74                         cache=cache, get_id=_thread_get_id,
    ---> 75                         pack_exception=pack_exception, **kwargs)
         76 
         77     # Cleanup pools associated to dead threads


    ~/envs/dev/lib/python3.5/site-packages/dask/local.py in get_async(apply_async, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, **kwargs)
        519                         _execute_task(task, data)  # Re-execute locally
        520                     else:
    --> 521                         raise_exception(exc, tb)
        522                 res, worker_id = loads(res_info)
        523                 state['cache'][key] = res


    ~/envs/dev/lib/python3.5/site-packages/dask/compatibility.py in reraise(exc, tb)
         58         if exc.__traceback__ is not tb:
         59             raise exc.with_traceback(tb)
    ---> 60         raise exc
         61 
         62 else:


    ~/envs/dev/lib/python3.5/site-packages/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
        288     try:
        289         task, data = loads(task_info)
    --> 290         result = _execute_task(task, data)
        291         id = get_id()
        292         result = dumps((result, id))


    ~/envs/dev/lib/python3.5/site-packages/dask/local.py in _execute_task(arg, cache, dsk)
        269         func, args = arg[0], arg[1:]
        270         args2 = [_execute_task(a, cache) for a in args]
    --> 271         return func(*args2)
        272     elif not ishashable(arg):
        273         return arg


    ~/envs/dev/lib/python3.5/site-packages/dask/dataframe/core.py in cov_corr_chunk(df, corr)
       3395     """Chunk part of a covariance or correlation computation"""
       3396     mat = df.values
    -> 3397     mask = np.isfinite(mat)
       3398     keep = np.bitwise_and(mask[:, None, :], mask[:, :, None])
       3399 


    TypeError: ufunc 'isfinite' not supported for the input types, and the inputs could not be safely coerced to any supported types according to the casting rule ''safe''



```python
# Changin the boolean column to float solves the computation problem.
# The warning doesn't have to do with the booleans I think.
ddf['hardbools'] = ddf['hardbools'].astype(float)
ddf.corr().compute()
```
    /home/sikko/envs/dev/lib/python3.5/site-packages/dask/dataframe/core.py:3460: RuntimeWarning: invalid value encountered in true_divide
      mat = C / den

<div>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>bools</th>
      <th>constant</th>
      <th>float_nans</th>
      <th>floats</th>
      <th>int_categorical</th>
      <th>int_nans</th>
      <th>ints</th>
      <th>hardbools</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>bools</th>
      <td>1.000000</td>
      <td>NaN</td>
      <td>0.061330</td>
      <td>0.027821</td>
      <td>0.015810</td>
      <td>-0.037277</td>
      <td>0.005449</td>
      <td>1.000000</td>
    </tr>
    <tr>
      <th>constant</th>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
      <td>NaN</td>
    </tr>
    <tr>
      <th>float_nans</th>
      <td>0.061330</td>
      <td>NaN</td>
      <td>1.000000</td>
      <td>-0.021301</td>
      <td>-0.001698</td>
      <td>0.039741</td>
      <td>-0.006224</td>
      <td>0.061330</td>
    </tr>
    <tr>
      <th>floats</th>
      <td>0.027821</td>
      <td>NaN</td>
      <td>-0.021301</td>
      <td>1.000000</td>
      <td>-0.038576</td>
      <td>-0.020085</td>
      <td>0.024252</td>
      <td>0.027821</td>
    </tr>
    <tr>
      <th>int_categorical</th>
      <td>0.015810</td>
      <td>NaN</td>
      <td>-0.001698</td>
      <td>-0.038576</td>
      <td>1.000000</td>
      <td>0.002101</td>
      <td>0.055539</td>
      <td>0.015810</td>
    </tr>
    <tr>
      <th>int_nans</th>
      <td>-0.037277</td>
      <td>NaN</td>
      <td>0.039741</td>
      <td>-0.020085</td>
      <td>0.002101</td>
      <td>1.000000</td>
      <td>0.027783</td>
      <td>-0.037277</td>
    </tr>
    <tr>
      <th>ints</th>
      <td>0.005449</td>
      <td>NaN</td>
      <td>-0.006224</td>
      <td>0.024252</td>
      <td>0.055539</td>
      <td>0.027783</td>
      <td>1.000000</td>
      <td>0.005449</td>
    </tr>
    <tr>
      <th>hardbools</th>
      <td>1.000000</td>
      <td>NaN</td>
      <td>0.061330</td>
      <td>0.027821</td>
      <td>0.015810</td>
      <td>-0.037277</td>
      <td>0.005449</td>
      <td>1.000000</td>
    </tr>
  </tbody>
</table>
</div>
</details>

<details>
  <summary>Packages from pip freeze</summary>
  bleach==2.1.1
bokeh==0.12.9
Cartopy==0.15.1
certifi==2017.7.27.1
chardet==3.0.4
click==6.7
cloudpickle==0.4.1
colorcet==0.9.1
cycler==0.10.0
Cython==0.27.3
dask==0.15.4
dask-summary==0.1.2
datashader==0.6.2
datashape==0.5.2
decorator==4.1.2
distributed==1.19.3
entrypoints==0.2.3
fastparquet==0.1.3
geoviews==1.3.2
HeapDict==1.0.0
holoviews==1.8.4
html5lib==1.0b10
idna==2.6
ipykernel==4.6.1
ipython==6.2.1
ipython-genutils==0.2.0
ipywidgets==7.0.3
jedi==0.11.0
Jinja2==2.9.6
jsonschema==2.6.0
jupyter==1.0.0
jupyter-client==5.1.0
jupyter-console==5.2.0
jupyter-core==4.3.0
lancet-ioam==0.9.0
llvmlite==0.20.0
locket==0.2.0
MarkupSafe==1.0
matplotlib==2.1.0
mistune==0.8
msgpack-python==0.4.8
multipledispatch==0.4.9
nbconvert==5.3.1
nbformat==4.4.0
netCDF4==1.3.0
notebook==5.2.0
numba==0.35.0
numpy==1.13.3
olefile==0.44
pandas==0.20.3
pandocfilters==1.4.2
param==1.5.1
parso==0.1.0
partd==0.3.8
pexpect==4.2.1
pickleshare==0.7.4
Pillow==4.3.0
pkg-resources==0.0.0
pprofile==1.11.0
prompt-toolkit==1.0.15
psutil==5.4.0
ptyprocess==0.5.2
py-heat==0.0.2
pyarrow==0.7.1
Pygments==2.2.0
pyparsing==2.2.0
pyshp==1.2.12
python-dateutil==2.6.1
pytz==2017.2
PyYAML==3.12
pyzmq==16.0.2
qtconsole==4.3.1
requests==2.18.4
scikit-learn==0.19.1
scipy==1.0.0
seaborn==0.8.1
selenium==3.6.0
Shapely==1.6.2
simplegeneric==0.8.1
six==1.11.0
sklearn==0.0
sortedcontainers==1.5.7
tblib==1.3.2
terminado==0.6
testpath==0.3.1
thrift==0.10.0
toolz==0.8.2
tornado==4.5.2
traitlets==4.3.2
urllib3==1.22
wcwidth==0.1.7
webencodings==0.5.1
widgetsnbextension==3.0.6
xarray==0.9.6
zict==0.1.3
</details>



Discussion:
Thank you for the excellent bug report.
 It looks like the current per-chunk part of the correlation computation expects homogenous data

```python
def cov_corr_chunk(df, corr=False):
    """Chunk part of a covariance or correlation computation"""
    mat = df.values
    mask = np.isfinite(mat)
    keep = np.bitwise_and(mask[:, None, :], mask[:, :, None])

    x = np.where(keep, mat[:, None, :], np.nan)
    sums = np.nansum(x, 0)
    counts = keep.astype('int').sum(0)
    cov = df.cov().values
    dtype = [('sum', sums.dtype), ('count', counts.dtype), ('cov', cov.dtype)]
    if corr:
        m = np.nansum((x - sums / np.where(counts, counts, np.nan)) ** 2, 0)
        dtype.append(('m', m.dtype))

    out = np.empty(counts.shape, dtype=dtype)
    out['sum'] = sums
    out['count'] = counts
    out['cov'] = cov * (counts - 1)
    if corr:
        out['m'] = m
    return out
```

My guess is that this needs to be rewritten using Pandas operations rather than NumPy ones.  The last people to touch this were @jcrist and @sinhrks , though I suspect that they might be busy.  Is this something that you're interested in looking into further @neuronist ?

Resolving patch:
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -3438,7 +3438,7 @@ def cov_corr(df, min_periods=None, corr=False, scalar=False, split_every=False):
 
 def cov_corr_chunk(df, corr=False):
     """Chunk part of a covariance or correlation computation"""
-    mat = df.values
+    mat = df.astype('float64', copy=False).values
     mask = np.isfinite(mat)
     keep = np.bitwise_and(mask[:, None, :], mask[:, :, None])
 
@@ -3472,10 +3472,10 @@ def cov_corr_combine(data, corr=False):
     s2 = sums[1:]
     n1 = cum_counts[:-1]
     n2 = counts[1:]
-    d = (s2 / n2) - (s1 / n1)
-
-    C = (np.nansum((n1 * n2) / (n1 + n2) * (d * d.transpose((0, 2, 1))), 0) +
-         np.nansum(data['cov'], 0))
+    with np.errstate(invalid='ignore'):
+        d = (s2 / n2) - (s1 / n1)
+        C = (np.nansum((n1 * n2) / (n1 + n2) * (d * d.transpose((0, 2, 1))), 0) +
+             np.nansum(data['cov'], 0))
 
     out = np.empty(C.shape, dtype=data.dtype)
     out['sum'] = cum_sums[-1]
@@ -3502,7 +3502,8 @@ def cov_corr_agg(data, cols, min_periods=2, corr=False, scalar=False):
         den = np.sqrt(m2 * m2.T)
     else:
         den = np.where(counts, counts, np.nan) - 1
-    mat = C / den
+    with np.errstate(invalid='ignore', divide='ignore'):
+        mat = C / den
     if scalar:
         return mat[0, 1]
     return pd.DataFrame(mat, columns=cols, index=cols)


Test patch:
diff --git a/dask/dataframe/tests/test_dataframe.py b/dask/dataframe/tests/test_dataframe.py
--- a/dask/dataframe/tests/test_dataframe.py
+++ b/dask/dataframe/tests/test_dataframe.py
@@ -1938,6 +1938,30 @@ def test_cov_corr_stable():
     assert_eq(ddf.corr(split_every=8), df.corr())
 
 
+def test_cov_corr_mixed():
+    size = 1000
+    d = {'dates' : pd.date_range('2015-01-01', periods=size, frequency='1T'),
+         'unique_id' : np.arange(0, size),
+         'ints' : np.random.randint(0, size, size=size),
+         'floats' : np.random.randn(size),
+         'bools' : np.random.choice([0, 1], size=size),
+         'int_nans' : np.random.choice([0, 1, np.nan], size=size),
+         'float_nans' : np.random.choice([0.0, 1.0, np.nan], size=size),
+         'constant' : 1,
+         'int_categorical' : np.random.choice([10, 20, 30, 40, 50], size=size) ,
+         'categorical_binary' : np.random.choice(['a', 'b'], size=size),
+         'categorical_nans' : np.random.choice(['a', 'b', 'c'], size=size)}
+    df = pd.DataFrame(d)
+    df['hardbools'] = df['bools'] == 1
+    df['categorical_nans'] = df['categorical_nans'].replace('c', np.nan)
+    df['categorical_binary'] = df['categorical_binary'].astype('category')
+    df['unique_id'] = df['unique_id'].astype(str)
+
+    ddf = dd.from_pandas(df, npartitions=20)
+    assert_eq(ddf.corr(split_every=4), df.corr(), check_divisions=False)
+    assert_eq(ddf.cov(split_every=4), df.cov(), check_divisions=False)
+
+
 def test_autocorr():
     x = pd.Series(np.random.random(100))
     dx = dd.from_pandas(x, npartitions=10)