Repository: mars-project/mars

Issue:
[BUG] concat Series with DataFrame or Series Failed
**Describe the bug**
concat Series with DataFrame or Series Failed.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version: 3.8.3
2. The version of Mars you use: pymars[distributed]==0.6.0
3. Versions of crucial packages, such as numpy, scipy and protobuf: numpy==1.19.4, scipy==1.5.4, protobuf==3.14.0, pyarrow==2.0.0
4. Minimized code and full stack of the error.
concat dataframe with series shown as below:
```python
import mars.dataframe as md
df = md.read_parquet("data.parquet")
df.execute()
md.concat([df[["id", "var_1"]], df["var_0"]], axis=1).execute()
```
```bash
Traceback (most recent call last):
  File "demo.py", line 5, in <module>
    md.concat([df["var_0"], df[["id", "var_1"]]], axis=1).execute()
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/dataframe/merge/concat.py", line 476, in concat
    return op(objs)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/utils.py", line 451, in _inner
    return func(*args, **kwargs)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/dataframe/merge/concat.py", line 399, in __call__
    return self._call_dataframes(objs)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/dataframe/merge/concat.py", line 380, in _call_dataframes
    col_length += df.shape[1]
IndexError: tuple index out of range
```
concat series with series shown as below
```python
import mars.dataframe as md
df = md.read_parquet("data.parquet")
df.execute()
print(md.concat([df["id"], df["var_0"]], axis=1).execute())
```
```bash
Traceback (most recent call last):
  File "demo.py", line 5, in <module>
    print(md.concat([df["id"], df["var_0"]], axis=1).execute())
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/core.py", line 131, in __str__
    return self._data.__str__()
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/dataframe/core.py", line 1083, in __str__
    return self._to_str(representation=False)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/dataframe/core.py", line 1058, in _to_str
    corner_data = fetch_corner_data(
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/dataframe/utils.py", line 899, in fetch_corner_data
    ExecutableTuple([head, tail]).fetch(session=session)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/core.py", line 738, in fetch
    return super().fetch(session=session, **kw)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/core.py", line 408, in fetch
    return session.fetch(self, **kw)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/session.py", line 528, in fetch
    result = self._sess.fetch(*tileables, **kw)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/session.py", line 125, in fetch
    return self._executor.fetch_tileables(tileables, **kw)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/utils.py", line 451, in _inner
    return func(*args, **kwargs)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/executor.py", line 973, in fetch_tileables
    result = self.execute_tileables(tileables, **kw)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/utils.py", line 451, in _inner
    return func(*args, **kwargs)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/executor.py", line 860, in execute_tileables
    chunk_graph = chunk_graph_builder.build(
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/utils.py", line 451, in _inner
    return func(*args, **kwargs)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tiles.py", line 347, in build
    chunk_graph = super().build(
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/utils.py", line 451, in _inner
    return func(*args, **kwargs)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tiles.py", line 262, in build
    self._on_tile_failure(tileable_data.op, exc_info)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tiles.py", line 301, in inner
    raise exc_info[1].with_traceback(exc_info[2]) from None
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tiles.py", line 242, in build
    tiled = self._tile(tileable_data, tileable_graph)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tiles.py", line 337, in _tile
    return super()._tile(tileable_data, tileable_graph)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tiles.py", line 201, in _tile
    tds[0]._inplace_tile()
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/core.py", line 168, in _inplace_tile
    return handler.inplace_tile(self)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tiles.py", line 136, in inplace_tile
    dispatched = self.dispatch(to_tile.op)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/utils.py", line 451, in _inner
    return func(*args, **kwargs)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tiles.py", line 119, in dispatch
    tiled = op_cls.tile(op)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/dataframe/indexing/iloc.py", line 344, in tile
    return [handler.handle(op)]
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tensor/indexing/index_lib.py", line 898, in handle
    self._process(context, index_infos)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tensor/indexing/index_lib.py", line 918, in _process
    index_info.handler.process(index_info, context)
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/tensor/indexing/index_lib.py", line 331, in process
    self.set_chunk_index_info(context, index_info, chunk_index, chunk_index_info,
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/dataframe/indexing/index_lib.py", line 178, in set_chunk_index_info
    index = chunk_input.columns_value.to_pandas()
  File "/home/smartguo/lib/anaconda3/lib/python3.8/site-packages/mars/core.py", line 171, in __getattr__
    return getattr(self._data, attr)
AttributeError: 'SeriesChunkData' object has no attribute 'columns_value'
```


**Expected behavior**
concat series with dataframe or series and return a dataframe




Discussion:
Thank you for your report, will fix it ASAP.

Resolving patch:
diff --git a/mars/dataframe/core.py b/mars/dataframe/core.py
--- a/mars/dataframe/core.py
+++ b/mars/dataframe/core.py
@@ -891,11 +891,10 @@ def to_frame(self, name=None):
         1    b
         2    c
         """
-        from ..tensor import tensor as astensor
         from . import dataframe_from_tensor
 
         name = name or self.name or 0
-        return dataframe_from_tensor(astensor(self), columns=[name])
+        return dataframe_from_tensor(self, columns=[name])
 
 
 class BaseDataFrameChunkData(ChunkData):
diff --git a/mars/dataframe/datasource/from_tensor.py b/mars/dataframe/datasource/from_tensor.py
--- a/mars/dataframe/datasource/from_tensor.py
+++ b/mars/dataframe/datasource/from_tensor.py
@@ -26,7 +26,7 @@
 from ...tensor.utils import unify_chunks
 from ...tiles import TilesError
 from ...utils import check_chunks_unknown_shape
-from ..core import INDEX_TYPE
+from ..core import INDEX_TYPE, SERIES_TYPE, SERIES_CHUNK_TYPE
 from ..operands import DataFrameOperand, DataFrameOperandMixin
 from ..utils import parse_index
 
@@ -150,6 +150,8 @@ def _call_input_tensor(self, input_tensor, index, columns):
                 raise ValueError(
                     f'index {index} should have the same shape with tensor: {input_tensor.shape[0]}')
             index_value = self._process_index(index, inputs)
+        elif isinstance(input_tensor, SERIES_TYPE):
+            index_value = input_tensor.index_value
         else:
             index_value = parse_index(pd.RangeIndex(start=0, stop=input_tensor.shape[0]))
 
@@ -278,6 +280,8 @@ def _tile_input_tensor(cls, op):
                 index_chunk = index_tensor.chunks[i]
                 index_value = index_chunk.index_value
                 chunk_inputs.append(index_chunk)
+            elif isinstance(in_chunk, SERIES_CHUNK_TYPE):
+                index_value = in_chunk.index_value
             elif out_df.index_value.has_value():
                 pd_index = out_df.index_value.to_pandas()
                 chunk_pd_index = pd_index[index_stop - in_chunk.shape[0]:index_stop]
@@ -326,13 +330,16 @@ def execute(cls, ctx, op):
                                           columns=chunk.columns_value.to_pandas())
         else:
             tensor_data = ctx[op.inputs[0].key]
-            if op.index is not None:
-                # index is a tensor
-                index_data = ctx[op.inputs[1].key]
+            if isinstance(tensor_data, pd.Series):
+                ctx[chunk.key] = tensor_data.to_frame(name=chunk.dtypes.index[0])
             else:
-                index_data = chunk.index_value.to_pandas()
-            ctx[chunk.key] = pd.DataFrame(tensor_data, index=index_data,
-                                          columns=chunk.columns_value.to_pandas())
+                if op.index is not None:
+                    # index is a tensor
+                    index_data = ctx[op.inputs[1].key]
+                else:
+                    index_data = chunk.index_value.to_pandas()
+                ctx[chunk.key] = pd.DataFrame(tensor_data, index=index_data,
+                                              columns=chunk.columns_value.to_pandas())
 
 
 def dataframe_from_tensor(tensor, index=None, columns=None, gpu=None, sparse=False):
diff --git a/mars/dataframe/merge/concat.py b/mars/dataframe/merge/concat.py
--- a/mars/dataframe/merge/concat.py
+++ b/mars/dataframe/merge/concat.py
@@ -20,7 +20,8 @@
 from ...serialize import ValueType, ListField, StringField, BoolField, AnyField
 from ...tiles import TilesError
 from ...utils import lazy_import, check_chunks_unknown_shape
-from ..utils import parse_index, build_empty_df, standardize_range_index, validate_axis
+from ..utils import parse_index, build_empty_df, build_empty_series, \
+    standardize_range_index, validate_axis
 from ..operands import DataFrameOperand, DataFrameOperandMixin, SERIES_TYPE
 
 cudf = lazy_import('cudf', globals=globals())
@@ -342,6 +343,7 @@ def _call_series(self, objs):
                 columns_value = parse_index(pd.RangeIndex(col_length))
             else:
                 columns_value = parse_index(pd.Index(columns), store_data=True)
+
             shape = (objs[0].shape[0], col_length)
             return self.new_dataframe(objs, shape=shape, dtypes=pd.Series(dtypes),
                                       index_value=objs[0].index_value,
@@ -358,7 +360,10 @@ def _call_dataframes(self, objs):
                 else:
                     index = self._concat_index(index, df.index_value.to_pandas())
                 row_length += df.shape[0]
-                empty_dfs.append(build_empty_df(df.dtypes))
+                if df.ndim == 2:
+                    empty_dfs.append(build_empty_df(df.dtypes))
+                else:
+                    empty_dfs.append(build_empty_series(df.dtype, name=df.name))
 
             emtpy_result = pd.concat(empty_dfs, join=self.join, sort=True)
             shape = (row_length, emtpy_result.shape[1])
@@ -371,22 +376,50 @@ def _call_dataframes(self, objs):
                 index_value = parse_index(pd.RangeIndex(row_length))
             else:
                 index_value = parse_index(index, objs)
-            return self.new_dataframe(objs, shape=shape, dtypes=emtpy_result.dtypes,
+
+            new_objs = []
+            for obj in objs:
+                if obj.ndim != 2:
+                    # series
+                    new_obj = obj.to_frame().reindex(columns=emtpy_result.dtypes.index)
+                else:
+                    # dataframe
+                    if list(obj.dtypes.index) != list(emtpy_result.dtypes.index):
+                        new_obj = obj.reindex(columns=emtpy_result.dtypes.index)
+                    else:
+                        new_obj = obj
+                new_objs.append(new_obj)
+
+            return self.new_dataframe(new_objs, shape=shape, dtypes=emtpy_result.dtypes,
                                       index_value=index_value, columns_value=columns_value)
         else:
             col_length = 0
             empty_dfs = []
             for df in objs:
-                col_length += df.shape[1]
-                empty_dfs.append(build_empty_df(df.dtypes))
+                if df.ndim == 2:
+                    # DataFrame
+                    col_length += df.shape[1]
+                    empty_dfs.append(build_empty_df(df.dtypes))
+                else:
+                    # Series
+                    col_length += 1
+                    empty_dfs.append(build_empty_series(df.dtype, name=df.name))
 
             emtpy_result = pd.concat(empty_dfs, join=self.join, axis=1, sort=True)
             if self.ignore_index:
                 columns_value = parse_index(pd.RangeIndex(col_length))
             else:
                 columns_value = parse_index(pd.Index(emtpy_result.columns), store_data=True)
+
+            if self.ignore_index or len({o.index_value.key for o in objs}) == 1:
+                new_objs = [obj if obj.ndim == 2 else obj.to_frame()
+                            for obj in objs]
+            else:  # pragma: no cover
+                raise NotImplementedError('Does not support concat dataframes '
+                                          'which has different index')
+
             shape = (objs[0].shape[0], col_length)
-            return self.new_dataframe(objs, shape=shape, dtypes=emtpy_result.dtypes,
+            return self.new_dataframe(new_objs, shape=shape, dtypes=emtpy_result.dtypes,
                                       index_value=objs[0].index_value,
                                       columns_value=columns_value)
 


Test patch:
diff --git a/mars/dataframe/merge/tests/test_merge_execution.py b/mars/dataframe/merge/tests/test_merge_execution.py
--- a/mars/dataframe/merge/tests/test_merge_execution.py
+++ b/mars/dataframe/merge/tests/test_merge_execution.py
@@ -477,3 +477,27 @@ def testConcat(self):
         expected = pd.concat([series1, series2], axis=1)
         result = self.executor.execute_dataframe(r, concat=True)[0]
         pd.testing.assert_frame_equal(result, expected)
+
+        # test merge dataframe and series
+        r = concat([mdf1, mseries2], ignore_index=True)
+        expected = pd.concat([df1, series2], ignore_index=True)
+        result = executor.execute_dataframe(r, concat=True)[0]
+        pd.testing.assert_frame_equal(result, expected)
+
+        # test merge series and dataframe
+        r = concat([mseries1, mdf2], ignore_index=True)
+        expected = pd.concat([series1, df2], ignore_index=True)
+        result = executor.execute_dataframe(r, concat=True)[0]
+        pd.testing.assert_frame_equal(result, expected)
+
+        # test merge dataframe and series, axis=1
+        r = concat([mdf1, mseries2], axis=1)
+        expected = pd.concat([df1, series2], axis=1)
+        result = executor.execute_dataframe(r, concat=True)[0]
+        pd.testing.assert_frame_equal(result, expected)
+
+        # test merge series and dataframe, axis=1
+        r = concat([mseries1, mdf2], axis=1)
+        expected = pd.concat([series1, df2], axis=1)
+        result = executor.execute_dataframe(r, concat=True)[0]
+        pd.testing.assert_frame_equal(result, expected)
diff --git a/mars/dataframe/tests/test_core.py b/mars/dataframe/tests/test_core.py
--- a/mars/dataframe/tests/test_core.py
+++ b/mars/dataframe/tests/test_core.py
@@ -42,6 +42,11 @@ def testToFrameOrSeries(self):
         result = self.executor.execute_dataframe(r, concat=True)[0]
         pd.testing.assert_frame_equal(raw.to_frame(name='new_name'), result)
 
+        series = series[series > 0.1]
+        r = series.to_frame(name='new_name')
+        result = self.executor.execute_dataframes([r])[0]
+        pd.testing.assert_frame_equal(raw[raw > 0.1].to_frame(name='new_name'), result)
+
         raw = pd.Index(np.random.rand(10), name='col')
         index = Index(raw)