Repository: mars-project/mars

Issue:
[BUG] `md.concat` costs too much memory for tileable building phase
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**

`md.concat` costs too much memory for tileable building phase.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version
2. The version of Mars you use
3. Versions of crucial packages, such as numpy, scipy and protobuf
4. Full stack of the error.
5. Minimized code to reproduce the error.

```python
In [1]: import mars.tensor as mt                                                                                                                                                                            

In [2]: import mars.dataframe as md                                                                                                                                                                         

In [3]: df = md.DataFrame(mt.random.rand(1000_0000, 10))                                                                                                                                                    

In [4]: df2 = md.DataFrame(mt.random.rand(1000_0000, 10))                                                                                                                                                   

In [5]: import ipython_memory_usage                                                                                                                                                                         

In [6]: %ipython_memory_usage_start                                                                                                                                                                         
Out[6]: 'memory profile enabled'
In [6] used 0.4336 MiB RAM in 3.64s, peaked 0.00 MiB above current, total RAM usage 126.13 MiB

In [7]: md.concat([df, df2])                                                                                                                                                                                
Out[7]: DataFrame <op=DataFrameConcat, key=570c63e8d6d1e426717fad9a6ae57d0d>
In [7] used 491.3359 MiB RAM in 1.76s, peaked 152.72 MiB above current, total RAM usage 617.46 MiB
```

During the phase of building tileable, the memory costs ~500M, it's too much.



Discussion:

Resolving patch:
diff --git a/mars/_version.py b/mars/_version.py
--- a/mars/_version.py
+++ b/mars/_version.py
@@ -15,7 +15,7 @@
 import subprocess
 import os
 
-version_info = (0, 6, 0, 'a3')
+version_info = (0, 6, 0, 'b1')
 _num_index = max(idx if isinstance(v, int) else 0
                  for idx, v in enumerate(version_info))
 __version__ = '.'.join(map(str, version_info[:_num_index + 1])) + \
diff --git a/mars/dataframe/merge/concat.py b/mars/dataframe/merge/concat.py
--- a/mars/dataframe/merge/concat.py
+++ b/mars/dataframe/merge/concat.py
@@ -287,6 +287,28 @@ def _auto_concat_categorical_chunks(_, inputs):
         else:
             ctx[chunk.key] = _base_concat(chunk, inputs)
 
+    @classmethod
+    def _concat_index(cls, prev_index: pd.Index, cur_index: pd.Index):
+        if isinstance(prev_index, pd.RangeIndex) and \
+                isinstance(cur_index, pd.RangeIndex):
+            # handle RangeIndex that append may generate huge amount of data
+            # e.g. pd.RangeIndex(10_000) and pd.RangeIndex(10_000)
+            # will generate a Int64Index full of data
+            # for details see GH#1647
+            prev_stop = prev_index.start + prev_index.size * prev_index.step
+            cur_start = cur_index.start
+            if prev_stop == cur_start and prev_index.step == cur_index.step:
+                # continuous RangeIndex, still return RangeIndex
+                return prev_index.append(cur_index)
+            else:
+                # otherwise, return an empty index
+                return pd.Index([], dtype=prev_index.dtype)
+        elif isinstance(prev_index, pd.RangeIndex):
+            return pd.Index([], prev_index.dtype).append(cur_index)
+        elif isinstance(cur_index, pd.RangeIndex):
+            return prev_index.append(pd.Index([], cur_index.dtype))
+        return prev_index.append(cur_index)
+
     def _call_series(self, objs):
         if self.axis == 0:
             row_length = 0
@@ -295,12 +317,12 @@ def _call_series(self, objs):
                 if index is None:
                     index = series.index_value.to_pandas()
                 else:
-                    index = index.append(series.index_value.to_pandas())
+                    index = self._concat_index(index, series.index_value.to_pandas())
                 row_length += series.shape[0]
             if self.ignore_index:  # pragma: no cover
                 index_value = parse_index(pd.RangeIndex(row_length))
             else:
-                index_value = parse_index(index)
+                index_value = parse_index(index, objs)
             return self.new_series(objs, shape=(row_length,), dtype=objs[0].dtype,
                                    index_value=index_value, name=objs[0].name)
         else:
@@ -335,7 +357,7 @@ def _call_dataframes(self, objs):
                 if index is None:
                     index = df.index_value.to_pandas()
                 else:
-                    index = index.append(df.index_value.to_pandas())
+                    index = self._concat_index(index, df.index_value.to_pandas())
                 row_length += df.shape[0]
                 empty_dfs.append(build_empty_df(df.dtypes))
 
@@ -349,7 +371,7 @@ def _call_dataframes(self, objs):
             if self.ignore_index:  # pragma: no cover
                 index_value = parse_index(pd.RangeIndex(row_length))
             else:
-                index_value = parse_index(index)
+                index_value = parse_index(index, objs)
             return self.new_dataframe(objs, shape=shape, dtypes=emtpy_result.dtypes,
                                       index_value=index_value, columns_value=columns_value)
         else:


Test patch:
diff --git a/mars/dataframe/merge/tests/test_merge.py b/mars/dataframe/merge/tests/test_merge.py
--- a/mars/dataframe/merge/tests/test_merge.py
+++ b/mars/dataframe/merge/tests/test_merge.py
@@ -247,6 +247,38 @@ def testConcat(self):
         for i, c in enumerate(tiled.chunks):
             self.assertEqual(c.index, (i, 0))
 
+        df3 = pd.DataFrame(np.random.rand(10, 4), columns=list('ABCD'),
+                           index=pd.RangeIndex(10, 20))
+
+        mdf3 = from_pandas(df3, chunk_size=4)
+        r = concat([mdf1, mdf3], axis='index')
+
+        self.assertEqual(r.shape, (20, 4))
+        pd.testing.assert_series_equal(r.dtypes, df1.dtypes)
+        pd.testing.assert_index_equal(r.index_value.to_pandas(), pd.RangeIndex(20))
+
+        df4 = pd.DataFrame(np.random.rand(10, 4), columns=list('ABCD'),
+                           index=np.random.permutation(np.arange(10)))
+
+        mdf4 = from_pandas(df4, chunk_size=4)
+        r = concat([mdf1, mdf4], axis='index')
+
+        self.assertEqual(r.shape, (20, 4))
+        pd.testing.assert_series_equal(r.dtypes, df1.dtypes)
+        pd.testing.assert_index_equal(r.index_value.to_pandas(), pd.Index([], dtype=np.int64))
+
+        r = concat([mdf4, mdf1], axis='index')
+
+        self.assertEqual(r.shape, (20, 4))
+        pd.testing.assert_series_equal(r.dtypes, df1.dtypes)
+        pd.testing.assert_index_equal(r.index_value.to_pandas(), pd.Index([], dtype=np.int64))
+
+        r = concat([mdf4, mdf4], axis='index')
+
+        self.assertEqual(r.shape, (20, 4))
+        pd.testing.assert_series_equal(r.dtypes, df1.dtypes)
+        pd.testing.assert_index_equal(r.index_value.to_pandas(), pd.Index([], dtype=np.int64))
+
         mdf1 = from_pandas(df1, chunk_size=3)
         mdf2 = from_pandas(df2, chunk_size=4)
         r = concat([mdf1, mdf2], axis='columns')