Repository: dask/dask

Issue:
pandas 0.23.0 compatibility
Forgot to check this earlier :/ I'll carve out some time to do CI maintenance soon.

### breaks

- `result_type` arg to http://pandas.pydata.org/pandas-docs/version/0.23/whatsnew.html#changes-to-make-output-of-dataframe-apply-consistent

### warnings from deprecations

- rolling / expanding raw: http://pandas.pydata.org/pandas-docs/version/0.23/whatsnew.html#rolling-expanding-apply-accepts-a-raw-keyword-to-pass-a-series-to-the-function
- str.cat align: http://pandas.pydata.org/pandas-docs/version/0.23/whatsnew.html#series-str-cat-has-gained-the-join-kwarg

...


Discussion:

Resolving patch:
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -38,7 +38,8 @@
 from .optimize import optimize
 from .utils import (meta_nonempty, make_meta, insert_meta_param_description,
                     raise_on_meta_error, clear_known_categories,
-                    is_categorical_dtype, has_known_categories, PANDAS_VERSION)
+                    is_categorical_dtype, has_known_categories, PANDAS_VERSION,
+                    index_summary)
 
 no_default = '__no_default__'
 
@@ -2785,7 +2786,8 @@ def meth(self, other, axis='columns', level=None):
         bind_method(cls, name, meth)
 
     @insert_meta_param_description(pad=12)
-    def apply(self, func, axis=0, args=(), meta=no_default, **kwds):
+    def apply(self, func, axis=0, broadcast=None, raw=False, reduce=None,
+              args=(), meta=no_default, **kwds):
         """ Parallel version of pandas.DataFrame.apply
 
         This mimics the pandas version except for the following:
@@ -2847,6 +2849,17 @@ def apply(self, func, axis=0, args=(), meta=no_default, **kwds):
         """
 
         axis = self._validate_axis(axis)
+        pandas_kwargs = {
+            'axis': axis,
+            'broadcast': broadcast,
+            'raw': raw,
+            'reduce': None,
+        }
+
+        if PANDAS_VERSION >= '0.23.0':
+            kwds.setdefault('result_type', None)
+
+        kwds.update(pandas_kwargs)
 
         if axis == 0:
             msg = ("dd.DataFrame.apply only supports axis=1\n"
@@ -2862,10 +2875,9 @@ def apply(self, func, axis=0, args=(), meta=no_default, **kwds):
             warnings.warn(msg)
 
             meta = _emulate(M.apply, self._meta_nonempty, func,
-                            axis=axis, args=args, udf=True, **kwds)
+                            args=args, udf=True, **kwds)
 
-        return map_partitions(M.apply, self, func, axis,
-                              False, False, None, args, meta=meta, **kwds)
+        return map_partitions(M.apply, self, func, args=args, meta=meta, **kwds)
 
     @derived_from(pd.DataFrame)
     def applymap(self, func, meta='__no_default__'):
@@ -2914,7 +2926,7 @@ def info(self, buf=None, verbose=False, memory_usage=False):
         if verbose:
             index = computations['index']
             counts = computations['count']
-            lines.append(index.summary())
+            lines.append(index_summary(index))
             lines.append('Data columns (total {} columns):'.format(len(self.columns)))
 
             if PANDAS_VERSION >= '0.20.0':
@@ -2926,7 +2938,7 @@ def info(self, buf=None, verbose=False, memory_usage=False):
             column_info = [column_template.format(pprint_thing(x[0]), x[1], x[2])
                            for x in zip(self.columns, counts, self.dtypes)]
         else:
-            column_info = [self.columns.summary(name='Columns')]
+            column_info = [index_summary(self.columns, name='Columns')]
 
         lines.extend(column_info)
         dtype_counts = ['%s(%d)' % k for k in sorted(self.dtypes.value_counts().iteritems(), key=str)]
diff --git a/dask/dataframe/rolling.py b/dask/dataframe/rolling.py
--- a/dask/dataframe/rolling.py
+++ b/dask/dataframe/rolling.py
@@ -8,7 +8,7 @@
 from ..base import tokenize
 from ..utils import M, funcname, derived_from
 from .core import _emulate
-from .utils import make_meta
+from .utils import make_meta, PANDAS_VERSION
 
 
 def overlap_chunk(func, prev_part, current_part, next_part, before, after,
@@ -292,8 +292,19 @@ def quantile(self, quantile):
         return self._call_method('quantile', quantile)
 
     @derived_from(pd_Rolling)
-    def apply(self, func, args=(), kwargs={}):
-        return self._call_method('apply', func, args=args, kwargs=kwargs)
+    def apply(self, func, args=(), kwargs={}, **kwds):
+        # TODO: In a future version of pandas this will change to
+        # raw=False. Think about inspecting the function signature and setting
+        # to that?
+        if PANDAS_VERSION >= '0.23.0':
+            kwds.setdefault("raw", None)
+        else:
+            if kwargs:
+                msg = ("Invalid argument to 'apply'. Keyword arguments "
+                       "should be given as a dict to the 'kwargs' arugment. ")
+                raise TypeError(msg)
+        return self._call_method('apply', func, args=args,
+                                 kwargs=kwargs, **kwds)
 
     def __repr__(self):
 
diff --git a/dask/dataframe/utils.py b/dask/dataframe/utils.py
--- a/dask/dataframe/utils.py
+++ b/dask/dataframe/utils.py
@@ -498,6 +498,22 @@ def equal_dtypes(a, b):
                              errmsg))
 
 
+def index_summary(idx, name=None):
+    """Summarized representation of an Index.
+    """
+    n = len(idx)
+    if name is None:
+        name = idx.__class__.__name__
+    if n:
+        head = idx[0]
+        tail = idx[-1]
+        summary = ', {} to {}'.format(head, tail)
+    else:
+        summary = ''
+
+    return "{}: {} entries{}".format(name, n, summary)
+
+
 ###############################################################
 # Testing
 ###############################################################


Test patch:
diff --git a/dask/dataframe/tests/test_categorical.py b/dask/dataframe/tests/test_categorical.py
--- a/dask/dataframe/tests/test_categorical.py
+++ b/dask/dataframe/tests/test_categorical.py
@@ -119,12 +119,14 @@ def test_is_categorical_dtype():
 
 
 def test_categorize():
-    meta = clear_known_categories(frames4[0])
+    # rename y to y_ to avoid pandas future warning about ambiguous
+    # levels
+    meta = clear_known_categories(frames4[0]).rename(columns={'y': 'y_'})
     ddf = dd.DataFrame({('unknown', i): df for (i, df) in enumerate(frames3)},
-                       'unknown', meta, [None] * 4)
+                       'unknown', meta, [None] * 4).rename(columns={'y': 'y_'})
     ddf = ddf.assign(w=ddf.w.cat.set_categories(['x', 'y', 'z']))
     assert ddf.w.cat.known
-    assert not ddf.y.cat.known
+    assert not ddf.y_.cat.known
     assert not ddf.index.cat.known
     df = ddf.compute()
 
@@ -132,27 +134,27 @@ def test_categorize():
         known_index = index is not False
         # By default categorize object and unknown cat columns
         ddf2 = ddf.categorize(index=index)
-        assert ddf2.y.cat.known
+        assert ddf2.y_.cat.known
         assert ddf2.v.cat.known
         assert ddf2.index.cat.known == known_index
         assert_eq(ddf2, df.astype({'v': 'category'}), check_categorical=False)
 
         # Specifying split_every works
         ddf2 = ddf.categorize(index=index, split_every=2)
-        assert ddf2.y.cat.known
+        assert ddf2.y_.cat.known
         assert ddf2.v.cat.known
         assert ddf2.index.cat.known == known_index
         assert_eq(ddf2, df.astype({'v': 'category'}), check_categorical=False)
 
         # Specifying one column doesn't affect others
         ddf2 = ddf.categorize('v', index=index)
-        assert not ddf2.y.cat.known
+        assert not ddf2.y_.cat.known
         assert ddf2.v.cat.known
         assert ddf2.index.cat.known == known_index
         assert_eq(ddf2, df.astype({'v': 'category'}), check_categorical=False)
 
-        ddf2 = ddf.categorize('y', index=index)
-        assert ddf2.y.cat.known
+        ddf2 = ddf.categorize('y_', index=index)
+        assert ddf2.y_.cat.known
         assert ddf2.v.dtype == 'object'
         assert ddf2.index.cat.known == known_index
         assert_eq(ddf2, df)
@@ -188,7 +190,7 @@ def test_categorize_index():
     assert ddf.categorize(index=False) is ddf
 
     # Non-object dtype
-    ddf = dd.from_pandas(df.set_index(df.A), npartitions=5)
+    ddf = dd.from_pandas(df.set_index(df.A.rename('idx')), npartitions=5)
     df = ddf.compute()
 
     ddf2 = ddf.categorize(index=True)
diff --git a/dask/dataframe/tests/test_dataframe.py b/dask/dataframe/tests/test_dataframe.py
--- a/dask/dataframe/tests/test_dataframe.py
+++ b/dask/dataframe/tests/test_dataframe.py
@@ -1,4 +1,5 @@
 import sys
+import textwrap
 from distutils.version import LooseVersion
 from itertools import product
 from operator import add
@@ -1612,12 +1613,19 @@ def test_select_dtypes(include, exclude):
 
     # count dtypes
     tm.assert_series_equal(a.get_dtype_counts(), df.get_dtype_counts())
-    tm.assert_series_equal(a.get_ftype_counts(), df.get_ftype_counts())
 
     tm.assert_series_equal(result.get_dtype_counts(),
                            expected.get_dtype_counts())
-    tm.assert_series_equal(result.get_ftype_counts(),
-                           expected.get_ftype_counts())
+
+    if PANDAS_VERSION >= '0.23.0':
+        ctx = pytest.warns(FutureWarning)
+    else:
+        ctx = pytest.warns(None)
+
+    with ctx:
+        tm.assert_series_equal(a.get_ftype_counts(), df.get_ftype_counts())
+        tm.assert_series_equal(result.get_ftype_counts(),
+                               expected.get_ftype_counts())
 
 
 def test_deterministic_apply_concat_apply_names():
@@ -2097,7 +2105,7 @@ def test_cov_corr_stable():
 
 def test_cov_corr_mixed():
     size = 1000
-    d = {'dates' : pd.date_range('2015-01-01', periods=size, frequency='1T'),
+    d = {'dates' : pd.date_range('2015-01-01', periods=size, freq='1T'),
          'unique_id' : np.arange(0, size),
          'ints' : np.random.randint(0, size, size=size),
          'floats' : np.random.randn(size),
@@ -2415,9 +2423,11 @@ def test_groupby_multilevel_info():
 
     buf = StringIO()
     g.info(buf, verbose=False)
-    assert buf.getvalue() == unicode("""<class 'dask.dataframe.core.DataFrame'>
-Columns: 2 entries, (C, count) to (C, sum)
-dtypes: int64(2)""")
+    expected = unicode(textwrap.dedent("""\
+    <class 'dask.dataframe.core.DataFrame'>
+    Columns: 2 entries, ('C', 'count') to ('C', 'sum')
+    dtypes: int64(2)"""))
+    assert buf.getvalue() == expected
 
 
 def test_categorize_info():
diff --git a/dask/dataframe/tests/test_indexing.py b/dask/dataframe/tests/test_indexing.py
--- a/dask/dataframe/tests/test_indexing.py
+++ b/dask/dataframe/tests/test_indexing.py
@@ -7,7 +7,7 @@
 import dask.dataframe as dd
 
 from dask.dataframe.indexing import _coerce_loc_index
-from dask.dataframe.utils import assert_eq, make_meta
+from dask.dataframe.utils import assert_eq, make_meta, PANDAS_VERSION
 
 
 dsk = {('x', 0): pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]},
@@ -32,18 +32,30 @@ def test_loc():
     assert_eq(d.loc[:8], full.loc[:8])
     assert_eq(d.loc[3:], full.loc[3:])
     assert_eq(d.loc[[5]], full.loc[[5]])
-    assert_eq(d.loc[[3, 4, 1, 8]], full.loc[[3, 4, 1, 8]])
-    assert_eq(d.loc[[3, 4, 1, 9]], full.loc[[3, 4, 1, 9]])
-    assert_eq(d.loc[np.array([3, 4, 1, 9])], full.loc[np.array([3, 4, 1, 9])])
+
+    if PANDAS_VERSION >= '0.23.0':
+        expected_warning = FutureWarning
+    else:
+        expected_warning = None
+
+    with pytest.warns(expected_warning):
+        assert_eq(d.loc[[3, 4, 1, 8]], full.loc[[3, 4, 1, 8]])
+    with pytest.warns(expected_warning):
+        assert_eq(d.loc[[3, 4, 1, 9]], full.loc[[3, 4, 1, 9]])
+    with pytest.warns(expected_warning):
+        assert_eq(d.loc[np.array([3, 4, 1, 9])], full.loc[np.array([3, 4, 1, 9])])
 
     assert_eq(d.a.loc[5], full.a.loc[5:5])
     assert_eq(d.a.loc[3:8], full.a.loc[3:8])
     assert_eq(d.a.loc[:8], full.a.loc[:8])
     assert_eq(d.a.loc[3:], full.a.loc[3:])
     assert_eq(d.a.loc[[5]], full.a.loc[[5]])
-    assert_eq(d.a.loc[[3, 4, 1, 8]], full.a.loc[[3, 4, 1, 8]])
-    assert_eq(d.a.loc[[3, 4, 1, 9]], full.a.loc[[3, 4, 1, 9]])
-    assert_eq(d.a.loc[np.array([3, 4, 1, 9])], full.a.loc[np.array([3, 4, 1, 9])])
+    with pytest.warns(expected_warning):
+        assert_eq(d.a.loc[[3, 4, 1, 8]], full.a.loc[[3, 4, 1, 8]])
+    with pytest.warns(expected_warning):
+        assert_eq(d.a.loc[[3, 4, 1, 9]], full.a.loc[[3, 4, 1, 9]])
+    with pytest.warns(expected_warning):
+        assert_eq(d.a.loc[np.array([3, 4, 1, 9])], full.a.loc[np.array([3, 4, 1, 9])])
     assert_eq(d.a.loc[[]], full.a.loc[[]])
     assert_eq(d.a.loc[np.array([])], full.a.loc[np.array([])])
 
diff --git a/dask/dataframe/tests/test_rolling.py b/dask/dataframe/tests/test_rolling.py
--- a/dask/dataframe/tests/test_rolling.py
+++ b/dask/dataframe/tests/test_rolling.py
@@ -3,7 +3,7 @@
 import numpy as np
 
 import dask.dataframe as dd
-from dask.dataframe.utils import assert_eq
+from dask.dataframe.utils import assert_eq, PANDAS_VERSION
 
 N = 40
 df = pd.DataFrame({'a': np.random.randn(N).cumsum(),
@@ -122,18 +122,28 @@ def test_rolling_methods(method, args, window, center, check_less_precise):
     # DataFrame
     prolling = df.rolling(window, center=center)
     drolling = ddf.rolling(window, center=center)
-    assert_eq(getattr(prolling, method)(*args),
-              getattr(drolling, method)(*args),
+    if method == 'apply' and PANDAS_VERSION >= '0.23.0':
+        kwargs = {'raw': False}
+    else:
+        kwargs = {}
+    assert_eq(getattr(prolling, method)(*args, **kwargs),
+              getattr(drolling, method)(*args, **kwargs),
               check_less_precise=check_less_precise)
 
     # Series
     prolling = df.a.rolling(window, center=center)
     drolling = ddf.a.rolling(window, center=center)
-    assert_eq(getattr(prolling, method)(*args),
-              getattr(drolling, method)(*args),
+    assert_eq(getattr(prolling, method)(*args, **kwargs),
+              getattr(drolling, method)(*args, **kwargs),
               check_less_precise=check_less_precise)
 
 
+@pytest.mark.skipif(PANDAS_VERSION >= '0.23.0', reason="Raw is allowed.")
+def test_rolling_raw_pandas_lt_0230_raises():
+    with pytest.raises(TypeError):
+        df.rolling(2).apply(mad, raw=True)
+
+
 def test_rolling_raises():
     df = pd.DataFrame({'a': np.random.randn(25).cumsum(),
                        'b': np.random.randint(100, size=(25,))})
@@ -209,17 +219,21 @@ def test_time_rolling_constructor():
 @pytest.mark.parametrize('window', ['1S', '2S', '3S', pd.offsets.Second(5)])
 def test_time_rolling_methods(method, args, window, check_less_precise):
     # DataFrame
+    if method == 'apply' and PANDAS_VERSION >= '0.23.0':
+        kwargs = {"raw": False}
+    else:
+        kwargs = {}
     prolling = ts.rolling(window)
     drolling = dts.rolling(window)
-    assert_eq(getattr(prolling, method)(*args),
-              getattr(drolling, method)(*args),
+    assert_eq(getattr(prolling, method)(*args, **kwargs),
+              getattr(drolling, method)(*args, **kwargs),
               check_less_precise=check_less_precise)
 
     # Series
     prolling = ts.a.rolling(window)
     drolling = dts.a.rolling(window)
-    assert_eq(getattr(prolling, method)(*args),
-              getattr(drolling, method)(*args),
+    assert_eq(getattr(prolling, method)(*args, **kwargs),
+              getattr(drolling, method)(*args, **kwargs),
               check_less_precise=check_less_precise)
 
 
diff --git a/dask/dataframe/tests/test_ufunc.py b/dask/dataframe/tests/test_ufunc.py
--- a/dask/dataframe/tests/test_ufunc.py
+++ b/dask/dataframe/tests/test_ufunc.py
@@ -345,17 +345,17 @@ def test_2args_with_array(ufunc, pandas, darray):
     assert isinstance(dafunc(dask, darray), dask_type)
     assert isinstance(dafunc(darray, dask), dask_type)
 
-    tm.assert_numpy_array_equal(dafunc(dask, darray).compute().as_matrix(),
-                                npfunc(pandas.as_matrix(), darray).compute())
+    tm.assert_numpy_array_equal(dafunc(dask, darray).compute().values,
+                                npfunc(pandas.values, darray).compute())
 
     # applying NumPy ufunc is lazy
     assert isinstance(npfunc(dask, darray), dask_type)
     assert isinstance(npfunc(darray, dask), dask_type)
 
-    tm.assert_numpy_array_equal(npfunc(dask, darray).compute().as_matrix(),
-                                npfunc(pandas.as_matrix(), darray.compute()))
-    tm.assert_numpy_array_equal(npfunc(darray, dask).compute().as_matrix(),
-                                npfunc(darray.compute(), pandas.as_matrix()))
+    tm.assert_numpy_array_equal(npfunc(dask, darray).compute().values,
+                                npfunc(pandas.values, darray.compute()))
+    tm.assert_numpy_array_equal(npfunc(darray, dask).compute().values,
+                                npfunc(darray.compute(), pandas.values))
 
 
 @pytest.mark.parametrize('redfunc', ['sum', 'prod', 'min', 'max', 'mean'])