Repository: mars-project/mars

Issue:
[BUG] Failed for multiple md.merge
1、code run as follow,run error。If the times of running merge is greater than or equal to 5, an error will occur。of course,I can run every **merge** with **reset_index(name='params')**。but,
it seems troublesome。I'd like to assignment column name finally together.

```
import mars.dataframe as pd
data = pd.DataFrame({'a':[1,2,3,4],'b':[11,12,13,14],'c':[11,12,13,14],'d':[11,12,13,14],'e':[11,12,13,14]})
data = pd.merge(data,data.groupby(['a']).size().reset_index(),on = ['a'],how='left')
data = pd.merge(data,data.groupby(['b']).size().reset_index(),on = ['b'],how='left')
data = pd.merge(data,data.groupby(['c']).size().reset_index(),on = ['c'],how='left')
data = pd.merge(data,data.groupby(['d']).size().reset_index(),on = ['d'],how='left')
data = pd.merge(data,data.groupby(['e']).size().reset_index(),on = ['e'],how='left')
print(data.columns.execute())
```

error_description:

```
KeyError                                  
Traceback (most recent call last)
<ipython-input-73-3d10a0dadb7d> in <module>
      5 data = pd.merge(data,data.groupby(['c']).size().reset_index(),on = ['c'],how='left')
      6 data = pd.merge(data,data.groupby(['d']).size().reset_index(),on = ['d'],how='left')
----> 7 data = pd.merge(data,data.groupby(['e']).size().reset_index(),on = ['3'],how='left')
      8 print(data.columns.execute())

D:\kinggsoft\anaconda\lib\site-packages\mars\dataframe\merge\merge.py in merge(df, right, how, on, left_on, right_on, left_index, right_index, sort, suffixes, copy, indicator, strategy, validate)
    357         left_index=left_index, right_index=right_index, sort=sort, suffixes=suffixes,
    358         copy=copy, indicator=indicator, validate=validate, object_type=ObjectType.dataframe)
--> 359     return op(df, right)
    360 
    361 

D:\kinggsoft\anaconda\lib\site-packages\mars\dataframe\merge\merge.py in __call__(self, left, right)
    175 
    176     def __call__(self, left, right):
--> 177         empty_left, empty_right = build_df(left), build_df(right)
    178         # this `merge` will check whether the combination of those arguments is valid
    179         merged = empty_left.merge(empty_right, how=self.how, on=self.on,

D:\kinggsoft\anaconda\lib\site-packages\mars\dataframe\utils.py in build_df(df_obj, fill_value, size)
    442     empty_df = pd.concat([empty_df] * size)
    443     # make sure dtypes correct for MultiIndex
--> 444     empty_df = empty_df.astype(dtypes, copy=False)
    445     return empty_df
    446 

D:\kinggsoft\anaconda\lib\site-packages\pandas\core\generic.py in astype(self, dtype, copy, errors, **kwargs)
   5863                     results.append(
   5864                         col.astype(
-> 5865                             dtype=dtype[col_name], copy=copy, errors=errors, **kwargs
   5866                         )
   5867                     )

D:\kinggsoft\anaconda\lib\site-packages\pandas\core\generic.py in astype(self, dtype, copy, errors, **kwargs)
   5846                 if len(dtype) > 1 or self.name not in dtype:
   5847                     raise KeyError(
-> 5848                         "Only the Series name can be used for "
   5849                         "the key in Series dtype mappings."
   5850                     )

KeyError: 'Only the Series name can be used for the key in Series dtype mappings.'
```


Discussion:
Thank you again, still, we will fix it ASAP.
excuse me,I just really want to try Mars instead of pandas.Then I find..  but this method is available in the official API
> data.drop_duplicates(keep='first')
err_description:
> AttributeError: 'DataFrameData' object has no attribute 'drop_duplicates'
> excuse me,I just really want to try Mars instead of pandas.Then I find.. but this method is available in the official API
> 
> > data.drop_duplicates(keep='first')
> > err_description:
> > AttributeError: 'DataFrameData' object has no attribute 'drop_duplicates'

Yeah, it's already implemented in the Master branch, but we have not released version that includes the implementation.

You can wait for the next release next week, or try to install Mars from source code.
BTW, the issue about `drop_duplicates` is #1241 , the related PR is implementation.

Resolving patch:
diff --git a/mars/dataframe/utils.py b/mars/dataframe/utils.py
--- a/mars/dataframe/utils.py
+++ b/mars/dataframe/utils.py
@@ -435,9 +435,12 @@ def _generate_value(dtype, fill_value):
 
 def build_empty_df(dtypes, index=None):
     columns = dtypes.index
-    df = pd.DataFrame(columns=columns, index=index)
-    for c, d in zip(columns, dtypes):
-        df[c] = pd.Series(dtype=d, index=index)
+    # duplicate column may exist,
+    # so use RangeIndex first
+    df = pd.DataFrame(columns=pd.RangeIndex(len(columns)), index=index)
+    for i, d in enumerate(dtypes):
+        df[i] = pd.Series(dtype=d, index=index)
+    df.columns = columns
     return df
 
 
@@ -454,7 +457,10 @@ def build_df(df_obj, fill_value=1, size=1):
 
     empty_df = pd.concat([empty_df] * size)
     # make sure dtypes correct for MultiIndex
-    empty_df = empty_df.astype(dtypes, copy=False)
+    for i, dtype in enumerate(dtypes.tolist()):
+        s = empty_df.iloc[:, i]
+        if s.dtype != dtype:
+            empty_df.iloc[:, i] = s.astype(dtype)
     return empty_df
 
 


Test patch:
diff --git a/mars/dataframe/merge/tests/test_merge_execution.py b/mars/dataframe/merge/tests/test_merge_execution.py
--- a/mars/dataframe/merge/tests/test_merge_execution.py
+++ b/mars/dataframe/merge/tests/test_merge_execution.py
@@ -300,6 +300,23 @@ def testMergeOneChunk(self):
         pd.testing.assert_frame_equal(expected.sort_values(by=expected.columns[1]).reset_index(drop=True),
                                       result.sort_values(by=result.columns[1]).reset_index(drop=True))
 
+    def testMergeOnDuplicateColumns(self):
+        raw1 = pd.DataFrame([['foo', 1, 'bar'],
+                             ['bar', 2, 'foo'],
+                             ['baz', 3, 'foo']],
+                            columns=['lkey', 'value', 'value'],
+                            index=['a1', 'a2', 'a3'])
+        raw2 = pd.DataFrame({'rkey': ['foo', 'bar', 'baz', 'foo'],
+                             'value': [5, 6, 7, 8]}, index=['a1', 'a2', 'a3', 'a4'])
+
+        df1 = from_pandas(raw1, chunk_size=2)
+        df2 = from_pandas(raw2, chunk_size=3)
+
+        r = df1.merge(df2, left_on='lkey', right_on='rkey')
+        result = self.executor.execute_dataframe(r, concat=True)[0]
+        expected = raw1.merge(raw2, left_on='lkey', right_on='rkey')
+        pd.testing.assert_frame_equal(expected, result)
+
     def testAppendExecution(self):
         executor = ExecutorForTest(storage=new_session().context)