Repository: dask/dask

Issue:
DataFrame.fillna() not able to take method as a parameter
Hi guys,

Tried to use `dask.dataframe.fillna()` with `method='ffill'`, this gave the following error. 

```python
TypeError: fillna() got an unexpected keyword argument 'method'
```

According to the API docs [here](http://dask.pydata.org/en/latest/dataframe-api.html#dask.dataframe.DataFrame.fillna), `fillna()` should have been able to take a lot more parameters, just like `pandas`'s. 

Is this a function that is yet to be implemented? Would like to use `ffill` or the likes without having to provide a fill value.

Thanks.


Discussion:
`ffill` and `bfill` should be relatively easy to do now that `map_overlap` is done in https://github.com/dask/dask/pull/1769

For the documentation issue, see https://github.com/dask/dask/issues/1815

Resolving patch:
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -807,9 +807,47 @@ def repartition(self, divisions=None, npartitions=None, force=False):
             raise ValueError(
                 "Provide either divisions= or npartitions= to repartition")
 
-    @derived_from(pd.Series)
-    def fillna(self, value):
-        return self.map_partitions(M.fillna, value=value)
+    @derived_from(pd.DataFrame)
+    def fillna(self, value=None, method=None, limit=None, axis=None):
+        axis = self._validate_axis(axis)
+        if method is None and limit is not None:
+            raise NotImplementedError("fillna with set limit and method=None")
+        meta = self._meta_nonempty.fillna(value=value, method=method,
+                                          limit=limit, axis=axis)
+
+        if axis == 1 or method is None:
+            return self.map_partitions(M.fillna, value=value, method=method,
+                                       limit=limit, axis=axis, meta=meta)
+
+        if method in ('pad', 'ffill'):
+            method = 'ffill'
+            skip_check = 0
+            before, after = 1 if limit is None else limit, 0
+        else:
+            method = 'bfill'
+            skip_check = self.npartitions - 1
+            before, after = 0, 1 if limit is None else limit
+
+        if limit is None:
+            name = 'fillna-chunk-' + tokenize(self, method)
+            dsk = {(name, i): (methods.fillna_check, (self._name, i),
+                               method, i != skip_check)
+                   for i in range(self.npartitions)}
+            parts = new_dd_object(merge(dsk, self.dask), name, meta,
+                                  self.divisions)
+        else:
+            parts = self
+
+        return parts.map_overlap(M.fillna, before, after, method=method,
+                                 limit=limit, meta=meta)
+
+    @derived_from(pd.DataFrame)
+    def ffill(self, axis=None, limit=None):
+        return self.fillna(method='ffill', limit=limit, axis=axis)
+
+    @derived_from(pd.DataFrame)
+    def bfill(self, axis=None, limit=None):
+        return self.fillna(method='bfill', limit=limit, axis=axis)
 
     def sample(self, frac, replace=False, random_state=None):
         """ Random sample of items
diff --git a/dask/dataframe/methods.py b/dask/dataframe/methods.py
--- a/dask/dataframe/methods.py
+++ b/dask/dataframe/methods.py
@@ -149,6 +149,15 @@ def drop_columns(df, columns, dtype):
     return df
 
 
+def fillna_check(df, method, check=True):
+    out = df.fillna(method=method)
+    if check and out.isnull().values.all(axis=0).any():
+        raise ValueError("All NaN partition encountered in `fillna`. Try "
+                         "using ``df.repartition`` to increase the partition "
+                         "size, or specify `limit` in `fillna`.")
+    return out
+
+
 # ---------------------------------
 # reshape
 # ---------------------------------


Test patch:
diff --git a/dask/dataframe/tests/test_dataframe.py b/dask/dataframe/tests/test_dataframe.py
--- a/dask/dataframe/tests/test_dataframe.py
+++ b/dask/dataframe/tests/test_dataframe.py
@@ -1548,9 +1548,6 @@ def test_embarrassingly_parallel_operations():
 
     assert_eq(a.x.dropna(), df.x.dropna())
 
-    assert_eq(a.x.fillna(100), df.x.fillna(100))
-    assert_eq(a.fillna(100), df.fillna(100))
-
     assert_eq(a.x.between(2, 4), df.x.between(2, 4))
 
     assert_eq(a.x.clip(2, 4), df.x.clip(2, 4))
@@ -1563,6 +1560,55 @@ def test_embarrassingly_parallel_operations():
     assert len(a.sample(0.5).compute()) < len(df)
 
 
+def test_fillna():
+    df = tm.makeMissingDataframe(0.8, 42)
+    ddf = dd.from_pandas(df, npartitions=5, sort=False)
+
+    assert_eq(ddf.fillna(100), df.fillna(100))
+    assert_eq(ddf.A.fillna(100), df.A.fillna(100))
+
+    assert_eq(ddf.fillna(method='pad'), df.fillna(method='pad'))
+    assert_eq(ddf.A.fillna(method='pad'), df.A.fillna(method='pad'))
+
+    assert_eq(ddf.fillna(method='bfill'), df.fillna(method='bfill'))
+    assert_eq(ddf.A.fillna(method='bfill'), df.A.fillna(method='bfill'))
+
+    assert_eq(ddf.fillna(method='pad', limit=2),
+              df.fillna(method='pad', limit=2))
+    assert_eq(ddf.A.fillna(method='pad', limit=2),
+              df.A.fillna(method='pad', limit=2))
+
+    assert_eq(ddf.fillna(method='bfill', limit=2),
+              df.fillna(method='bfill', limit=2))
+    assert_eq(ddf.A.fillna(method='bfill', limit=2),
+              df.A.fillna(method='bfill', limit=2))
+
+    assert_eq(ddf.fillna(100, axis=1), df.fillna(100, axis=1))
+    assert_eq(ddf.fillna(method='pad', axis=1), df.fillna(method='pad', axis=1))
+    assert_eq(ddf.fillna(method='pad', limit=2, axis=1),
+              df.fillna(method='pad', limit=2, axis=1))
+
+    pytest.raises(ValueError, lambda: ddf.A.fillna(0, axis=1))
+    pytest.raises(NotImplementedError, lambda: ddf.fillna(0, limit=10))
+    pytest.raises(NotImplementedError, lambda: ddf.fillna(0, limit=10, axis=1))
+
+    df = tm.makeMissingDataframe(0.2, 42)
+    ddf = dd.from_pandas(df, npartitions=5, sort=False)
+    pytest.raises(ValueError, lambda: ddf.fillna(method='pad').compute())
+    assert_eq(df.fillna(method='pad', limit=3),
+              ddf.fillna(method='pad', limit=3))
+
+
+def test_ffill_bfill():
+    df = tm.makeMissingDataframe(0.8, 42)
+    ddf = dd.from_pandas(df, npartitions=5, sort=False)
+
+    assert_eq(ddf.ffill(), df.ffill())
+    assert_eq(ddf.bfill(), df.bfill())
+    assert_eq(ddf.ffill(axis=1), df.ffill(axis=1))
+    assert_eq(ddf.bfill(axis=1), df.bfill(axis=1))
+
+
 def test_sample():
     df = pd.DataFrame({'x': [1, 2, 3, 4, None, 6], 'y': list('abdabd')},
                       index=[10, 20, 30, 40, 50, 60])