Repository: mars-project/mars

Issue:
[BUG] `merge_index_value` takes too long time to run
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**

`merge_index_value` is used to merge index values of chunks, but it takes too long time to run.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version
2. The version of Mars you use
3. Versions of crucial packages, such as numpy, scipy and pandas
4. Full stack of the error.
5. Minimized code to reproduce the error.

```
In [1]: from mars.dataframe.utils import merge_index_value, parse_index

In [2]: import pandas as pd

In [3]: index_values = {i: parse_index(pd.RangeIndex(1e7)) for i in range(20)}

In [4]: %%time
   ...: merge_index_value(index_values)
   ...: 
   ...: 
CPU times: user 16.8 s, sys: 7.12 s, total: 24 s
Wall time: 24.4 s
Out[4]: IndexValue(_FIELD_VALUES={'int64_index': Int64Index(_FIELD_VALUES={'min_val': 0, 'max_val': 9999999, 'min_val_close': True, 'max_val_close': True, 'key': 'f339241d19ec6b2e1e39c614724dd227', 'is_monotonic_increasing': False, 'is_monotonic_decreasing': False, 'is_unique': False, 'dtype': dtype('int64'), 'name': None})})
```





Discussion:

Resolving patch:
diff --git a/mars/dataframe/utils.py b/mars/dataframe/utils.py
--- a/mars/dataframe/utils.py
+++ b/mars/dataframe/utils.py
@@ -714,24 +714,44 @@ def indexing_index_value(index_value, indexes, store_data=False):
             return parse_index(pd_index[indexes], store_data=store_data)
 
 
-def merge_index_value(to_merge_index_values, store_data=False):
+def merge_index_value(to_merge_index_values: dict, store_data: bool = False):
     """
     Merge index value according to their chunk index.
-    :param to_merge_index_values: Dict object. {index: index_value}
-    :return: Merged index_value
+
+    Parameters
+    ----------
+    to_merge_index_values : dict
+        index to index_value
+    store_data : bool
+        store data in index_value
+
+    Returns
+    -------
+    merged_index_value
     """
-    index_value = None
+
+    pd_index = None
     min_val, min_val_close, max_val, max_val_close = None, None, None, None
     for _, chunk_index_value in sorted(to_merge_index_values.items()):
-        if index_value is None:
-            index_value = chunk_index_value.to_pandas()
+        if pd_index is None:
+            pd_index = chunk_index_value.to_pandas()
             min_val, min_val_close, max_val, max_val_close = \
                 chunk_index_value.min_val, \
                 chunk_index_value.min_val_close, \
                 chunk_index_value.max_val, \
                 chunk_index_value.max_val_close
         else:
-            index_value = index_value.append(chunk_index_value.to_pandas())
+            cur_pd_index = chunk_index_value.to_pandas()
+            if store_data or (
+                    isinstance(pd_index, pd.RangeIndex) and
+                    isinstance(cur_pd_index, pd.RangeIndex) and
+                    cur_pd_index.step == pd_index.step and
+                    cur_pd_index.start == pd_index.stop
+            ):
+                # range index that is continuous
+                pd_index = pd_index.append(cur_pd_index)
+            else:
+                pd_index = pd.Index([], dtype=pd_index.dtype)
             if chunk_index_value.min_val is not None:
                 try:
                     if min_val is None or min_val > chunk_index_value.min_val:
@@ -746,13 +766,13 @@ def merge_index_value(to_merge_index_values, store_data=False):
                     max_val = chunk_index_value.max_val
                     max_val_close = chunk_index_value.max_val_close
 
-    new_index_value = parse_index(index_value, store_data=store_data)
-    if not new_index_value.has_value():
-        new_index_value._index_value._min_val = min_val
-        new_index_value._index_value._min_val_close = min_val_close
-        new_index_value._index_value._max_val = max_val
-        new_index_value._index_value._max_val_close = max_val_close
-    return new_index_value
+    index_value = parse_index(pd_index, store_data=store_data)
+    if not index_value.has_value():
+        index_value._index_value._min_val = min_val
+        index_value._index_value._min_val_close = min_val_close
+        index_value._index_value._max_val = max_val
+        index_value._index_value._max_val_close = max_val_close
+    return index_value
 
 
 def infer_dtypes(left_dtypes, right_dtypes, operator):
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -164,11 +164,11 @@ def run(cls):
         web_src_path = os.path.join(repo_root, *cls._web_src_path.split('/'))
         web_dest_path = os.path.join(repo_root, *cls._web_dest_path.split('/'))
 
-        if os.path.exists(web_dest_path):
-            if npm_path is None:
-                warnings.warn('Cannot find NPM, may affect displaying Mars Web')
+        if not os.path.exists(web_src_path):
             return
-        elif not os.path.exists(web_src_path):
+        elif npm_path is None:
+            if not os.path.exists(web_dest_path):
+                warnings.warn('Cannot find NPM, may affect displaying Mars Web')
             return
 
         replacements = {'npm': npm_path}


Test patch:
diff --git a/mars/dataframe/tests/test_utils.py b/mars/dataframe/tests/test_utils.py
--- a/mars/dataframe/tests/test_utils.py
+++ b/mars/dataframe/tests/test_utils.py
@@ -28,8 +28,9 @@
     decide_series_chunk_size, split_monotonic_index_min_max, \
     build_split_idx_to_origin_idx, parse_index, filter_index_value, \
     infer_dtypes, infer_index_value, validate_axis, fetch_corner_data, \
-    make_dtypes, build_concatenated_rows_frame
+    make_dtypes, build_concatenated_rows_frame, merge_index_value
 from mars.tests import setup
+from mars.utils import Timer
 
 
 setup = setup
@@ -255,6 +256,26 @@ def test_filter_index_value():
     assert isinstance(filtered.value, IndexValue.Int64Index)
 
 
+def test_merge_index_value():
+    with Timer() as timer:
+        index_values = {i: parse_index(pd.RangeIndex(1e7)) for i in range(20)}
+        index_value = merge_index_value(index_values)
+        pd.testing.assert_index_equal(index_value.to_pandas(),
+                                      pd.Index([], dtype=np.int64))
+        assert index_value.min_val == 0
+        assert index_value.max_val == 1e7 - 1
+
+        # range indexes that are continuous
+        index_values = {i: parse_index(pd.RangeIndex(i * 1e7, (i + 1) * 1e7))
+                        for i in range(20)}
+        index_value = merge_index_value(index_values)
+        pd.testing.assert_index_equal(index_value.to_pandas(),
+                                      pd.RangeIndex(1e7 * 20))
+        assert index_value.min_val == 0
+        assert index_value.max_val == 1e7 * 20 - 1
+    assert timer.duration < 1
+
+
 def test_infer_dtypes():
     data1 = pd.DataFrame([[1, 'a', False]], columns=[2.0, 3.0, 4.0])
     data2 = pd.DataFrame([[1, 3.0, 'b']], columns=[1, 2, 3])