Repository: mars-project/mars

Issue:
Support df.append and series.append
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Is your feature request related to a problem? Please describe.**

Add support for append operator in DataFrame module.


Discussion:

Resolving patch:
diff --git a/mars/dataframe/base/standardize_range_index.py b/mars/dataframe/base/standardize_range_index.py
--- a/mars/dataframe/base/standardize_range_index.py
+++ b/mars/dataframe/base/standardize_range_index.py
@@ -33,8 +33,9 @@ def __init__(self, prepare_inputs=None, object_type=None, **kwargs):
     @classmethod
     def execute(cls, ctx, op):
         xdf = cudf if op.gpu else pd
-        in_data = ctx[op.inputs[-1].key]
+        in_data = ctx[op.inputs[-1].key].copy()
         input_keys = [c.key for c in op.inputs[:-1]]
         metas = ctx.get_chunk_metas(input_keys)
         index_start = sum([m.chunk_shape[0] for m in metas])
-        ctx[op.outputs[0].key] = in_data.set_index(xdf.RangeIndex(index_start, index_start + len(in_data)))
+        in_data.index = xdf.RangeIndex(index_start, index_start + len(in_data))
+        ctx[op.outputs[0].key] = in_data
diff --git a/mars/dataframe/merge/__init__.py b/mars/dataframe/merge/__init__.py
--- a/mars/dataframe/merge/__init__.py
+++ b/mars/dataframe/merge/__init__.py
@@ -14,15 +14,19 @@
 
 from .concat import DataFrameConcat
 from .merge import join, merge, DataFrameShuffleMerge, DataFrameMergeAlign
+from .append import DataFrameAppend, append
 
 
 def _install():
-    from ..core import DATAFRAME_TYPE
+    from ..core import DATAFRAME_TYPE, SERIES_TYPE
 
     for cls in DATAFRAME_TYPE:
         setattr(cls, 'join', join)
         setattr(cls, 'merge', merge)
 
+    for cls in DATAFRAME_TYPE + SERIES_TYPE:
+        setattr(cls, 'append', append)
+
 
 _install()
 del _install
diff --git a/mars/dataframe/merge/append.py b/mars/dataframe/merge/append.py
new file mode 100644
--- /dev/null
+++ b/mars/dataframe/merge/append.py
@@ -0,0 +1,198 @@
+# Copyright 1999-2020 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pandas as pd
+
+from ...serialize import BoolField
+from ... import opcodes as OperandDef
+from ..datasource.dataframe import from_pandas
+from ..indexing.iloc import DataFrameIlocGetItem, SeriesIlocGetItem
+from ..utils import parse_index, standardize_range_index
+from ..operands import DataFrameOperand, DataFrameOperandMixin, ObjectType, \
+    DATAFRAME_TYPE, SERIES_TYPE
+
+
+class DataFrameAppend(DataFrameOperand, DataFrameOperandMixin):
+    _op_type_ = OperandDef.APPEND
+
+    _ignore_index = BoolField('ignore_index')
+    _verify_integrity = BoolField('verify_integrity')
+    _sort = BoolField('sort')
+
+    def __init__(self, ignore_index=None, verify_integrity=None, sort=None, object_type=None, **kw):
+        super(DataFrameAppend, self).__init__(_ignore_index=ignore_index,
+                                              _verify_integrity=verify_integrity,
+                                              _sort=sort,
+                                              _object_type=object_type, **kw)
+
+    @property
+    def ignore_index(self):
+        return self._ignore_index
+
+    @property
+    def verify_integrity(self):
+        return self._verify_integrity
+
+    @property
+    def sort(self):
+        return self._sort
+
+    @classmethod
+    def _tile_dataframe(cls, op):
+        out_df = op.outputs[0]
+        inputs = op.inputs
+        first_df, others = inputs[0], inputs[1:]
+        column_splits = first_df.nsplits[1]
+        others = [item.rechunk({1: column_splits})._inplace_tile() for item in others]
+        out_chunks = []
+        nsplits = [[], list(first_df.nsplits[1])]
+        row_index = 0
+        for df in [first_df] + others:
+            for c in df.chunks:
+                index = (c.index[0] + row_index, c.index[1])
+                iloc_op = DataFrameIlocGetItem(indexes=(slice(None),) * 2)
+                out_chunks.append(iloc_op.new_chunk([c], shape=c.shape, index=index,
+                                                    dtypes=c.dtypes,
+                                                    index_value=c.index_value,
+                                                    columns_value=c.columns_value))
+            nsplits[0] += df.nsplits[0]
+            row_index += len(df.nsplits[0])
+        if op.ignore_index:
+            out_chunks = standardize_range_index(out_chunks)
+
+        nsplits = tuple(tuple(n) for n in nsplits)
+        new_op = op.copy()
+        return new_op.new_dataframes(op.inputs, out_df.shape,
+                                     nsplits=nsplits, chunks=out_chunks,
+                                     dtypes=out_df.dtypes,
+                                     index_value=out_df.index_value,
+                                     columns_value=out_df.columns_value)
+
+    @classmethod
+    def _tile_series(cls, op):
+        out_series = op.outputs[0]
+        inputs = op.inputs
+        first_series, others = inputs[0], inputs[1:]
+        out_chunks = []
+        nsplits = ()
+        row_index = 0
+        for series in [first_series] + others:
+            for c in series.chunks:
+                index = (c.index[0] + row_index,)
+                iloc_op = SeriesIlocGetItem(indexes=(slice(None),))
+                out_chunks.append(iloc_op.new_chunk([c], shape=c.shape, index=index,
+                                                    index_value=c.index_value,
+                                                    dtype=c.dtype,
+                                                    name=c.name))
+            nsplits += series.nsplits[0]
+            row_index += len(series.nsplits[0])
+
+        if op.ignore_index:
+            out_chunks = standardize_range_index(out_chunks)
+
+        nsplits = (tuple(nsplits),)
+        new_op = op.copy()
+        return new_op.new_seriess(op.inputs, out_series.shape,
+                                  nsplits=nsplits, chunks=out_chunks,
+                                  dtype=out_series.dtype,
+                                  index_value=out_series.index_value,
+                                  name=out_series.name)
+
+    @classmethod
+    def tile(cls, op):
+        if op.object_type == ObjectType.dataframe:
+            return cls._tile_dataframe(op)
+        else:
+            return cls._tile_series(op)
+
+    def _call_dataframe(self, df, other):
+        if isinstance(other, DATAFRAME_TYPE):
+            shape = (df.shape[0] + other.shape[0], df.shape[1])
+            inputs = [df, other]
+            if self.ignore_index:
+                index_value = parse_index(pd.RangeIndex(shape[0]))
+            else:
+                index_value = parse_index(df.index_value.to_pandas().append(
+                    other.index_value.to_pandas()))
+        elif isinstance(other, list):
+            row_length = df.shape[0]
+            index = df.index_value.to_pandas()
+            for item in other:
+                if not isinstance(item, DATAFRAME_TYPE):  # pragma: no cover
+                    raise ValueError('Invalid type {} to append'.format(type(item)))
+                row_length += item.shape[0]
+                index = index.append(item.index_value.to_pandas())
+            shape = (row_length, df.shape[1])
+            if self.ignore_index:  # pragma: no cover
+                index_value = parse_index(pd.RangeIndex(shape[0]))
+            else:
+                index_value = parse_index(index)
+            inputs = [df] + other
+        else:  # pragma: no cover
+            raise ValueError('Invalid type {} to append'.format(type(other)))
+        return self.new_dataframe(inputs, shape=shape, dtypes=df.dtypes,
+                                  index_value=index_value,
+                                  columns_value=df.columns_value)
+
+    def _call_series(self, df, other):
+        if isinstance(other, SERIES_TYPE):
+            shape = (df.shape[0] + other.shape[0],)
+            inputs = [df, other]
+            if self.ignore_index:
+                index_value = parse_index(pd.RangeIndex(shape[0]))
+            else:
+                index_value = parse_index(df.index_value.to_pandas().append(
+                    other.index_value.to_pandas()))
+        elif isinstance(other, list):
+            row_length = df.shape[0]
+            index = df.index_value.to_pandas()
+            for item in other:
+                if not isinstance(item, SERIES_TYPE):  # pragma: no cover
+                    raise ValueError('Invalid type {} to append'.format(type(item)))
+                row_length += item.shape[0]
+                index = index.append(item.index_value.to_pandas())
+            shape = (row_length,)
+            if self.ignore_index:  # pragma: no cover
+                index_value = parse_index(pd.RangeIndex(shape[0]))
+            else:
+                index_value = parse_index(index)
+            inputs = [df] + other
+        else:  # pragma: no cover
+            raise ValueError('Invalid type {} to append'.format(type(other)))
+        return self.new_series(inputs, shape=shape, dtype=df.dtype,
+                               index_value=index_value, name=df.name)
+
+    @classmethod
+    def execute(cls, ctx, op):
+        first, others = ctx[op.inputs[0].key], [ctx[inp.key] for inp in op.inputs[1:]]
+        r = first.append(others, verify_integrity=op.verify_integrity,
+                         sort=op.sort)
+        ctx[op.outputs[0].key] = r
+
+    def __call__(self, df, other):
+        if isinstance(df, DATAFRAME_TYPE):
+            self._object_type = ObjectType.dataframe
+            return self._call_dataframe(df, other)
+        else:
+            self._object_type = ObjectType.series
+            return self._call_series(df, other)
+
+
+def append(df, other, ignore_index=False, verify_integrity=False, sort=False):
+    if verify_integrity or sort:  # pragma: no cover
+        raise NotImplementedError('verify_integrity and sort are not supported now')
+    if isinstance(other, dict):
+        other = from_pandas(pd.DataFrame(dict((k, [v]) for k, v in other.items())))
+    op = DataFrameAppend(ignore_index=ignore_index, verify_integrity=verify_integrity, sort=sort)
+    return op(df, other)


Test patch:
diff --git a/mars/dataframe/merge/tests/test_merge.py b/mars/dataframe/merge/tests/test_merge.py
--- a/mars/dataframe/merge/tests/test_merge.py
+++ b/mars/dataframe/merge/tests/test_merge.py
@@ -19,6 +19,8 @@
 from mars.executor import Executor
 from mars.tiles import get_tiled
 from mars.tests.core import TestBase
+from mars.dataframe.core import IndexValue
+from mars.dataframe.base.standardize_range_index import ChunkStandardizeRangeIndex
 from mars.dataframe.datasource.dataframe import from_pandas
 from mars.dataframe.merge import DataFrameMergeAlign, DataFrameShuffleMerge
 
@@ -198,3 +200,33 @@ def testMergeOneChunk(self):
         self.assertEqual(tiled.chunks[1].inputs[0].key, get_tiled(mdf1).chunks[1].key)
         self.assertEqual(tiled.chunks[1].inputs[1].key, get_tiled(mdf2).chunks[0].key)
 
+    def testAppend(self):
+        df1 = pd.DataFrame(np.random.rand(10, 4), columns=list('ABCD'))
+        df2 = pd.DataFrame(np.random.rand(10, 4), columns=list('ABCD'))
+
+        mdf1 = from_pandas(df1, chunk_size=3)
+        mdf2 = from_pandas(df2, chunk_size=3)
+        adf = mdf1.append(mdf2)
+
+        self.assertEqual(adf.shape, (20, 4))
+        self.assertIsInstance(adf.index_value.value, IndexValue.Int64Index)
+
+        tiled = adf.tiles()
+        self.assertEqual(tiled.nsplits, ((3, 3, 3, 1, 3, 3, 3, 1), (3, 1)))
+        self.assertEqual(tiled.chunk_shape, (8, 2))
+        for i, c in enumerate(tiled.chunks):
+            index = (i // 2, i % 2)
+            self.assertEqual(c.index, index)
+
+        mdf1 = from_pandas(df1, chunk_size=3)
+        mdf2 = from_pandas(df2, chunk_size=3)
+        adf = mdf1.append(mdf2, ignore_index=True)
+
+        self.assertEqual(adf.shape, (20, 4))
+        self.assertIsInstance(adf.index_value.value, IndexValue.RangeIndex)
+        pd.testing.assert_index_equal(adf.index_value.to_pandas(), pd.RangeIndex(20))
+
+        tiled = adf.tiles()
+        self.assertEqual(tiled.nsplits, ((3, 3, 3, 1, 3, 3, 3, 1), (3, 1)))
+        self.assertEqual(tiled.chunk_shape, (8, 2))
+        self.assertIsInstance(tiled.chunks[0].op, ChunkStandardizeRangeIndex)
diff --git a/mars/dataframe/merge/tests/test_merge_execution.py b/mars/dataframe/merge/tests/test_merge_execution.py
--- a/mars/dataframe/merge/tests/test_merge_execution.py
+++ b/mars/dataframe/merge/tests/test_merge_execution.py
@@ -16,7 +16,9 @@
 import pandas as pd
 
 from mars.tests.core import TestBase, ExecutorForTest
+from mars.session import new_session
 from mars.dataframe.datasource.dataframe import from_pandas
+from mars.dataframe.datasource.series import from_pandas as series_from_pandas
 from mars.dataframe.utils import sort_dataframe_inplace
 
 
@@ -272,3 +274,85 @@ def testMergeOneChunk(self):
 
         pd.testing.assert_frame_equal(expected.sort_values(by=expected.columns[1]).reset_index(drop=True),
                                       result.sort_values(by=result.columns[1]).reset_index(drop=True))
+
+    def testAppendExecution(self):
+        executor = ExecutorForTest(storage=new_session().context)
+
+        df1 = pd.DataFrame(np.random.rand(10, 4), columns=list('ABCD'))
+        df2 = pd.DataFrame(np.random.rand(10, 4), columns=list('ABCD'))
+
+        mdf1 = from_pandas(df1, chunk_size=3)
+        mdf2 = from_pandas(df2, chunk_size=3)
+
+        adf = mdf1.append(mdf2)
+        expected = df1.append(df2)
+        result = self.executor.execute_dataframe(adf, concat=True)[0]
+        pd.testing.assert_frame_equal(expected, result)
+
+        adf = mdf1.append(mdf2, ignore_index=True)
+        expected = df1.append(df2, ignore_index=True)
+        result = executor.execute_dataframe(adf, concat=True)[0]
+        pd.testing.assert_frame_equal(expected, result)
+
+        mdf1 = from_pandas(df1, chunk_size=3)
+        mdf2 = from_pandas(df2, chunk_size=2)
+
+        adf = mdf1.append(mdf2)
+        expected = df1.append(df2)
+        result = self.executor.execute_dataframe(adf, concat=True)[0]
+        pd.testing.assert_frame_equal(expected, result)
+
+        adf = mdf1.append(mdf2, ignore_index=True)
+        expected = df1.append(df2, ignore_index=True)
+        result = executor.execute_dataframe(adf, concat=True)[0]
+        pd.testing.assert_frame_equal(expected, result)
+
+        df3 = pd.DataFrame(np.random.rand(8, 4), columns=list('ABCD'))
+        mdf3 = from_pandas(df3, chunk_size=3)
+        expected = df1.append([df2, df3])
+        adf = mdf1.append([mdf2, mdf3])
+        result = self.executor.execute_dataframe(adf, concat=True)[0]
+        pd.testing.assert_frame_equal(expected, result)
+
+        adf = mdf1.append(dict(A=1, B=2, C=3, D=4), ignore_index=True)
+        expected = df1.append(dict(A=1, B=2, C=3, D=4), ignore_index=True)
+        result = executor.execute_dataframe(adf, concat=True)[0]
+        pd.testing.assert_frame_equal(expected, result)
+
+        # test for series
+        series1 = pd.Series(np.random.rand(10,))
+        series2 = pd.Series(np.random.rand(10,))
+
+        mseries1 = series_from_pandas(series1, chunk_size=3)
+        mseries2 = series_from_pandas(series2, chunk_size=3)
+
+        aseries = mseries1.append(mseries2)
+        expected = series1.append(series2)
+        result = self.executor.execute_dataframe(aseries, concat=True)[0]
+        pd.testing.assert_series_equal(expected, result)
+
+        aseries = mseries1.append(mseries2, ignore_index=True)
+        expected = series1.append(series2, ignore_index=True)
+        result = executor.execute_dataframe(aseries, concat=True)[0]
+        pd.testing.assert_series_equal(expected, result)
+
+        mseries1 = series_from_pandas(series1, chunk_size=3)
+        mseries2 = series_from_pandas(series2, chunk_size=2)
+
+        aseries = mseries1.append(mseries2)
+        expected = series1.append(series2)
+        result = self.executor.execute_dataframe(aseries, concat=True)[0]
+        pd.testing.assert_series_equal(expected, result)
+
+        aseries = mseries1.append(mseries2, ignore_index=True)
+        expected = series1.append(series2, ignore_index=True)
+        result = executor.execute_dataframe(aseries, concat=True)[0]
+        pd.testing.assert_series_equal(expected, result)
+
+        series3 = pd.Series(np.random.rand(4,))
+        mseries3 = series_from_pandas(series3, chunk_size=2)
+        expected = series1.append([series2, series3])
+        aseries = mseries1.append([mseries2, mseries3])
+        result = self.executor.execute_dataframe(aseries, concat=True)[0]
+        pd.testing.assert_series_equal(expected, result)
+