Repository: databricks/koalas

Issue:
Fillna doesn't get the correct result when arg "method" is "ffill".
Fillna the method of Koalas Series  doesn't get the correct result when method "ffill" not work.

After I execute fillna with method "ffill", print the whole result, it is right. But when I print the value which is filled, it is None.

example:
```python
>>> from databricks.koalas import Series
>>> import numpy as np
>>> a = ["Bob", "John", None]
>>> b = ["Bob", "John", np.nan]
>>> s1 = Series(a)
>>> s2 = Series(b)
>>> r1 = s1.fillna(method="ffill")
>>> r1
20/07/15 15:18:44 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
0     Bob
1    John
2    John
Name: 0, dtype: object
>>> print(r1[2])
20/07/15 15:19:06 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
None
>>> r2 = s2.fillna(method="ffill")
>>> r2
20/07/15 15:19:19 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
0     Bob
1    John
2    John
Name: 0, dtype: object
>>> print(r2[2])
20/07/15 15:19:27 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
None
```

I think r1[2] and r2[2] should be "John", but I got "None".



Discussion:

Resolving patch:
diff --git a/databricks/koalas/frame.py b/databricks/koalas/frame.py
--- a/databricks/koalas/frame.py
+++ b/databricks/koalas/frame.py
@@ -4708,11 +4708,11 @@ def fillna(self, value=None, method=None, axis=None, inplace=False, limit=None):
         2  0.0  1.0  2.0  5
         3  0.0  3.0  1.0  4
         """
+        axis = validate_axis(axis)
+        if axis != 0:
+            raise NotImplementedError("fillna currently only works for axis=0 or axis='index'")
+
         if value is not None:
-            axis = validate_axis(axis)
-            inplace = validate_bool_kwarg(inplace, "inplace")
-            if axis != 0:
-                raise NotImplementedError("fillna currently only works for axis=0 or axis='index'")
             if not isinstance(value, (float, int, str, bool, dict, pd.Series)):
                 raise TypeError("Unsupported type %s" % type(value))
             if limit is not None:
@@ -4729,26 +4729,24 @@ def op(kser):
                     label = kser._internal.column_labels[0]
                     for k, v in value.items():
                         if k == label[: len(k)]:
-                            return kser.fillna(
-                                value=value[k], method=method, axis=axis, inplace=False, limit=limit
+                            return kser._fillna(
+                                value=value[k], method=method, axis=axis, limit=limit
                             )
                     else:
                         return kser
 
             else:
-                op = lambda kser: kser.fillna(
-                    value=value, method=method, axis=axis, inplace=False, limit=limit
-                )
+                op = lambda kser: kser._fillna(value=value, method=method, axis=axis, limit=limit)
         elif method is not None:
-            op = lambda kser: kser.fillna(
-                value=value, method=method, axis=axis, inplace=False, limit=limit
-            )
+            op = lambda kser: kser._fillna(value=value, method=method, axis=axis, limit=limit)
         else:
             raise ValueError("Must specify a fillna 'value' or 'method' parameter.")
 
-        kdf = self._apply_series_op(op)
+        kdf = self._apply_series_op(op, should_resolve=(method is not None))
+
+        inplace = validate_bool_kwarg(inplace, "inplace")
         if inplace:
-            self._update_internal_frame(kdf._internal)
+            self._update_internal_frame(kdf._internal, requires_same_anchor=False)
         else:
             return kdf
 
diff --git a/databricks/koalas/groupby.py b/databricks/koalas/groupby.py
--- a/databricks/koalas/groupby.py
+++ b/databricks/koalas/groupby.py
@@ -1564,8 +1564,9 @@ def fillna(self, value=None, method=None, axis=None, inplace=False, limit=None):
         """
         return self._apply_series_op(
             lambda sg: sg._kser._fillna(
-                value, method, axis, inplace, limit, part_cols=sg._groupkeys_scols,
-            )
+                value=value, method=method, axis=axis, limit=limit, part_cols=sg._groupkeys_scols
+            ),
+            should_resolve=(method is not None),
         )
 
     def bfill(self, limit=None):
diff --git a/databricks/koalas/series.py b/databricks/koalas/series.py
--- a/databricks/koalas/series.py
+++ b/databricks/koalas/series.py
@@ -1605,11 +1605,19 @@ def fillna(self, value=None, method=None, axis=None, inplace=False, limit=None):
         4       c
         Name: x, dtype: object
         """
-        return self._fillna(value, method, axis, inplace, limit)
+        kser = self._fillna(value=value, method=method, axis=axis, limit=limit)
+
+        if method is not None:
+            kser = DataFrame(kser._kdf._internal.resolved_copy)._kser_for(self._column_label)
 
-    def _fillna(self, value=None, method=None, axis=None, inplace=False, limit=None, part_cols=()):
-        axis = validate_axis(axis)
         inplace = validate_bool_kwarg(inplace, "inplace")
+        if inplace:
+            self._kdf._update_internal_frame(kser._kdf._internal, requires_same_anchor=False)
+        else:
+            return kser._with_new_scol(kser.spark.column)
+
+    def _fillna(self, value=None, method=None, axis=None, limit=None, part_cols=()):
+        axis = validate_axis(axis)
         if axis != 0:
             raise NotImplementedError("fillna currently only works for axis=0 or axis='index'")
         if (value is None) and (method is None):
@@ -1623,10 +1631,7 @@ def _fillna(self, value=None, method=None, axis=None, inplace=False, limit=None,
             cond = scol.isNull() | F.isnan(scol)
         else:
             if not self.spark.nullable:
-                if inplace:
-                    return
-                else:
-                    return self
+                return self.copy()
             cond = scol.isNull()
 
         if value is not None:
@@ -1658,14 +1663,11 @@ def _fillna(self, value=None, method=None, axis=None, inplace=False, limit=None,
             )
             scol = F.when(cond, func(scol, True).over(window)).otherwise(scol)
 
-        if inplace:
-            self._kdf._update_internal_frame(
-                self._kdf._internal.with_new_spark_column(
-                    self._column_label, scol.alias(name_like_string(self.name))
-                )
+        return DataFrame(
+            self._kdf._internal.with_new_spark_column(
+                self._column_label, scol.alias(name_like_string(self.name))
             )
-        else:
-            return self._with_new_scol(scol)
+        )._kser_for(self._column_label)
 
     def dropna(self, axis=0, inplace=False, **kwargs):
         """


Test patch:
diff --git a/databricks/koalas/tests/test_dataframe.py b/databricks/koalas/tests/test_dataframe.py
--- a/databricks/koalas/tests/test_dataframe.py
+++ b/databricks/koalas/tests/test_dataframe.py
@@ -2710,31 +2710,55 @@ def test_duplicated(self):
         )
 
     def test_ffill(self):
+        idx = np.random.rand(6)
         pdf = pd.DataFrame(
             {
                 "x": [np.nan, 2, 3, 4, np.nan, 6],
                 "y": [1, 2, np.nan, 4, np.nan, np.nan],
                 "z": [1, 2, 3, 4, np.nan, np.nan],
             },
-            index=np.random.rand(6),
+            index=idx,
         )
         kdf = ks.from_pandas(pdf)
+
         self.assert_eq(kdf.ffill(), pdf.ffill())
         self.assert_eq(kdf.ffill(limit=1), pdf.ffill(limit=1))
 
+        pser = pdf.y
+        kser = kdf.y
+
+        kdf.ffill(inplace=True)
+        pdf.ffill(inplace=True)
+
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kser, pser)
+        self.assert_eq(kser[idx[2]], pser[idx[2]])
+
     def test_bfill(self):
+        idx = np.random.rand(6)
         pdf = pd.DataFrame(
             {
                 "x": [np.nan, 2, 3, 4, np.nan, 6],
                 "y": [1, 2, np.nan, 4, np.nan, np.nan],
                 "z": [1, 2, 3, 4, np.nan, np.nan],
             },
-            index=np.random.rand(6),
+            index=idx,
         )
         kdf = ks.from_pandas(pdf)
+
         self.assert_eq(kdf.bfill(), pdf.bfill())
         self.assert_eq(kdf.bfill(limit=1), pdf.bfill(limit=1))
 
+        pser = pdf.x
+        kser = kdf.x
+
+        kdf.bfill(inplace=True)
+        pdf.bfill(inplace=True)
+
+        self.assert_eq(kdf, pdf)
+        self.assert_eq(kser, pser)
+        self.assert_eq(kser[idx[0]], pser[idx[0]])
+
     def test_filter(self):
         pdf = pd.DataFrame(
             {
diff --git a/databricks/koalas/tests/test_groupby.py b/databricks/koalas/tests/test_groupby.py
--- a/databricks/koalas/tests/test_groupby.py
+++ b/databricks/koalas/tests/test_groupby.py
@@ -1232,6 +1232,7 @@ def test_fillna(self):
         )
 
     def test_ffill(self):
+        idx = np.random.rand(4 * 3)
         pdf = pd.DataFrame(
             {
                 "A": [1, 1, 2, 2] * 3,
@@ -1239,7 +1240,7 @@ def test_ffill(self):
                 "C": [None, None, None, 1] * 3,
                 "D": [0, 1, 5, 4] * 3,
             },
-            index=np.random.rand(4 * 3),
+            index=idx,
         )
         kdf = ks.from_pandas(pdf)
 
@@ -1265,6 +1266,7 @@ def test_ffill(self):
             pdf.groupby("A")["B"].ffill().sort_index(),
             almost=True,
         )
+        self.assert_eq(kdf.groupby("A")["B"].ffill()[idx[6]], pdf.groupby("A")["B"].ffill()[idx[6]])
 
         # multi-index columns
         columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B"), ("Y", "C"), ("Z", "D")])
@@ -1283,6 +1285,7 @@ def test_ffill(self):
             )
 
     def test_bfill(self):
+        idx = np.random.rand(4 * 3)
         pdf = pd.DataFrame(
             {
                 "A": [1, 1, 2, 2] * 3,
@@ -1290,7 +1293,7 @@ def test_bfill(self):
                 "C": [None, None, None, 1] * 3,
                 "D": [0, 1, 5, 4] * 3,
             },
-            index=np.random.rand(4 * 3),
+            index=idx,
         )
         kdf = ks.from_pandas(pdf)
 
@@ -1314,6 +1317,7 @@ def test_bfill(self):
         self.assert_eq(
             kdf.groupby("A")["B"].bfill().sort_index(), pdf.groupby("A")["B"].bfill().sort_index(),
         )
+        self.assert_eq(kdf.groupby("A")["B"].bfill()[idx[6]], pdf.groupby("A")["B"].bfill()[idx[6]])
 
         # multi-index columns
         columns = pd.MultiIndex.from_tuples([("X", "A"), ("X", "B"), ("Y", "C"), ("Z", "D")])
diff --git a/databricks/koalas/tests/test_series.py b/databricks/koalas/tests/test_series.py
--- a/databricks/koalas/tests/test_series.py
+++ b/databricks/koalas/tests/test_series.py
@@ -227,7 +227,7 @@ def test_drop_duplicates(self):
         self.assert_eq(kdf, pdf)
 
     def test_fillna(self):
-        pdf = pd.DataFrame({"x": [np.nan, 2, 3, 4, np.nan, 6]})
+        pdf = pd.DataFrame({"x": [np.nan, 2, 3, 4, np.nan, 6], "y": [np.nan, 2, 3, 4, np.nan, 6]})
         kdf = ks.from_pandas(pdf)
 
         pser = pdf.x
@@ -1781,24 +1781,36 @@ def test_abs(self):
         self.assert_eq(np.abs(kser), np.abs(pser))
 
     def test_bfill(self):
-        pser = pd.Series([np.nan, 2, 3, 4, np.nan, 6], name="x")
-        kser = ks.from_pandas(pser)
+        pdf = pd.DataFrame({"x": [np.nan, 2, 3, 4, np.nan, 6], "y": [np.nan, 2, 3, 4, np.nan, 6]})
+        kdf = ks.from_pandas(pdf)
+
+        pser = pdf.x
+        kser = kdf.x
 
         self.assert_eq(kser.bfill(), pser.bfill())
+        self.assert_eq(kser.bfill()[0], pser.bfill()[0])
 
         kser.bfill(inplace=True)
         pser.bfill(inplace=True)
         self.assert_eq(kser, pser)
+        self.assert_eq(kser[0], pser[0])
+        self.assert_eq(kdf, pdf)
 
     def test_ffill(self):
-        pser = pd.Series([np.nan, 2, 3, 4, np.nan, 6], name="x")
-        kser = ks.from_pandas(pser)
+        pdf = pd.DataFrame({"x": [np.nan, 2, 3, 4, np.nan, 6], "y": [np.nan, 2, 3, 4, np.nan, 6]})
+        kdf = ks.from_pandas(pdf)
 
-        self.assert_eq(repr(kser.ffill()), repr(pser.ffill()))
+        pser = pdf.x
+        kser = kdf.x
+
+        self.assert_eq(kser.ffill(), pser.ffill(), almost=True)
+        self.assert_eq(kser.ffill()[4], pser.ffill()[4])
 
         kser.ffill(inplace=True)
         pser.ffill(inplace=True)
-        self.assert_eq(repr(kser), repr(pser))
+        self.assert_eq(kser, pser, almost=True)
+        self.assert_eq(kser[4], pser[4])
+        self.assert_eq(kdf, pdf)
 
     def test_iteritems(self):
         pser = pd.Series(["A", "B", "C"])