Repository: databricks/koalas

Issue:
cumsum() for Sries/DataFrame works not properly.
assuming that we have dataframe like below.

```python
>>> pdf  # also same for 'kdf'
    a   b  c
7   1   2  3
2   1   3  5
4   1   1  2
1   1   4  5
3   2   6  1
4   2   9  2
9   2   8  6
10  3  10  4
5   3   7  3
6   3   5  6
```

let's do `cumsum()` on this,

pandas

```python
>>> pdf.cumsum()
     a   b   c
7    1   2   3
2    2   5   8
4    3   6  10
1    4  10  15
3    6  16  16
4    8  25  18
9   10  33  24
10  13  43  28
5   16  50  31
6   19  55  37
```

whereas koalas

```python
>>> kdf.cumsum()
     a   b   c
1    1   4   5
2    2   7  10
3    4  13  11
4    5  14  13
4    7  23  15
5   10  30  18
6   13  35  24
7   14  37  27
9   16  45  33
10  19  55  37
```

it seems sort the order by index automatically so that the result are not proper.



Discussion:
i'm fixing this.

Resolving patch:
diff --git a/databricks/koalas/frame.py b/databricks/koalas/frame.py
--- a/databricks/koalas/frame.py
+++ b/databricks/koalas/frame.py
@@ -6737,9 +6737,12 @@ def _cum(self, func, skipna: bool):
             func = "cumsum"
         elif func.__name__ == "cumprod":
             func = "cumprod"
-
+        self = self.copy()
+        columns = self.columns
+        # add a temporal column to keep natural order.
+        self['__natural_order__'] = F.monotonically_increasing_id()
         applied = []
-        for column in self.columns:
+        for column in columns:
             applied.append(getattr(self[column], func)(skipna))
 
         sdf = self._sdf.select(
@@ -6748,6 +6751,8 @@ def _cum(self, func, skipna: bool):
                                        column_index=[c._internal.column_index[0] for c in applied],
                                        column_scols=[scol_for(sdf, c._internal.data_columns[0])
                                                      for c in applied])
+        # add a temporal column to keep natural order.
+        self = self.drop('__natural_order__')
         return DataFrame(internal)
 
     # TODO: implements 'keep' parameters
diff --git a/databricks/koalas/groupby.py b/databricks/koalas/groupby.py
--- a/databricks/koalas/groupby.py
+++ b/databricks/koalas/groupby.py
@@ -1829,8 +1829,9 @@ def _cum(self, func):
             func = "cumprod"
 
         applied = []
-        kdf = self._kdf
-
+        kdf = self._kdf.copy()
+        # add a temporal column to keep natural order.
+        kdf['__natural_order__'] = F.monotonically_increasing_id()
         for column in self._agg_columns:
             # pandas groupby.cumxxx ignores the grouping key itself.
             applied.append(getattr(column.groupby(self._groupkeys), func)())
diff --git a/databricks/koalas/series.py b/databricks/koalas/series.py
--- a/databricks/koalas/series.py
+++ b/databricks/koalas/series.py
@@ -4197,8 +4197,14 @@ def pct_change(self, periods=1):
     def _cum(self, func, skipna, part_cols=()):
         # This is used to cummin, cummax, cumsum, etc.
         index_columns = self._internal.index_columns
+
+        # address temporal column to keep natural order.
+        sdf = self._internal.sdf
+        if '__natural_order__' not in sdf.columns:
+            sdf = sdf.withColumn('__natural_order__', F.monotonically_increasing_id())
+
         window = Window.orderBy(
-            index_columns).partitionBy(*part_cols).rowsBetween(
+            '__natural_order__').partitionBy(*part_cols).rowsBetween(
                 Window.unboundedPreceding, Window.currentRow)
 
         if skipna:
@@ -4273,7 +4279,9 @@ def _cum(self, func, skipna, part_cols=()):
         if func.__name__ == "cumprod":
             scol = F.exp(scol)
 
-        return self._with_new_scol(scol).rename(self.name)
+        internal = self._internal.copy(sdf=sdf, scol=scol)
+
+        return _col(DataFrame(internal))
 
     # ----------------------------------------------------------------------
     # Accessor Methods


Test patch:
diff --git a/databricks/koalas/tests/test_series.py b/databricks/koalas/tests/test_series.py
--- a/databricks/koalas/tests/test_series.py
+++ b/databricks/koalas/tests/test_series.py
@@ -567,24 +567,48 @@ def test_cummin(self):
         self.assertEqual(repr(pser.cummin()), repr(kser.cummin()))
         self.assertEqual(repr(pser.cummin(skipna=False)), repr(kser.cummin(skipna=False)))
 
+        # with reversed index
+        pser.index = [4, 3, 2, 1, 0]
+        kser = ks.from_pandas(pser)
+        self.assertEqual(repr(pser.cummin()), repr(kser.cummin()))
+        self.assertEqual(repr(pser.cummin(skipna=False)), repr(kser.cummin(skipna=False)))
+
     def test_cummax(self):
         pser = pd.Series([1.0, None, 0.0, 4.0, 9.0]).rename("a")
         kser = ks.from_pandas(pser)
         self.assertEqual(repr(pser.cummax()), repr(kser.cummax()))
         self.assertEqual(repr(pser.cummax(skipna=False)), repr(kser.cummax(skipna=False)))
 
+        # with reversed index
+        pser.index = [4, 3, 2, 1, 0]
+        kser = ks.from_pandas(pser)
+        self.assertEqual(repr(pser.cummax()), repr(kser.cummax()))
+        self.assertEqual(repr(pser.cummax(skipna=False)), repr(kser.cummax(skipna=False)))
+
     def test_cumsum(self):
         pser = pd.Series([1.0, None, 0.0, 4.0, 9.0]).rename("a")
         kser = ks.from_pandas(pser)
         self.assertEqual(repr(pser.cumsum()), repr(kser.cumsum()))
         self.assertEqual(repr(pser.cumsum(skipna=False)), repr(kser.cumsum(skipna=False)))
 
+        # with reversed index
+        pser.index = [4, 3, 2, 1, 0]
+        kser = ks.from_pandas(pser)
+        self.assertEqual(repr(pser.cumsum()), repr(kser.cumsum()))
+        self.assertEqual(repr(pser.cumsum(skipna=False)), repr(kser.cumsum(skipna=False)))
+
     def test_cumprod(self):
         pser = pd.Series([1.0, None, 1.0, 4.0, 9.0]).rename("a")
         kser = ks.from_pandas(pser)
         self.assertEqual(repr(pser.cumprod()), repr(kser.cumprod()))
         self.assertEqual(repr(pser.cumprod(skipna=False)), repr(kser.cumprod(skipna=False)))
 
+        # with reversed index
+        pser.index = [4, 3, 2, 1, 0]
+        kser = ks.from_pandas(pser)
+        self.assertEqual(repr(pser.cumprod()), repr(kser.cumprod()))
+        self.assertEqual(repr(pser.cumprod(skipna=False)), repr(kser.cumprod(skipna=False)))
+
         # TODO: due to unknown reason, this test passes in Travis CI. Unable to reproduce in local.
         # with self.assertRaisesRegex(Exception, "values should be bigger than 0"):
         #     repr(ks.Series([0, 1]).cumprod())