Repository: dask/dask

Issue:
Issue with casting during concatenation of dataframe
I got an issue with the following code which suprised me:

```python
import numpy as np
import pandas as pd

import dask.dataframe as dd

pd.DataFrame(np.random.randint(10, size=100000)).to_csv('df_a.csv',
                                                        index=False)
pd.DataFrame(np.random.randn(100000)).to_csv('df_b.csv', index=False)

df_a = dd.read_csv('df_a.csv')
df_b = dd.read_csv('df_b.csv')
df = dd.concat([df_a, df_b], axis=0)

df.to_parquet('df.parquet')
df = dd.read_parquet('df.parquet')
```

The parquet parts dumped on the disk are of different dtype while I would have expect `dd.concat` to make the casting without the need to explicitly call `.compute()`.

Is it a desired behavior?


Discussion:
No, it's not desired.  I don't think that there is any dtype handling in `dd.concat`, but there could be.

Assuming Pandas handles this correctly, one solution might be to look at the dtypes of all the arguments, if they are different then compute `new_dtypes = pd.concat([df._meta for df in dfs]).dtypes` and then call `df.astype(new_dtypes)` on each input argument before proceeding in `dd.concat`.  Any interest in pushing up a PR @glemaitre ?

Resolving patch:
diff --git a/dask/dataframe/multi.py b/dask/dataframe/multi.py
--- a/dask/dataframe/multi.py
+++ b/dask/dataframe/multi.py
@@ -69,7 +69,7 @@
 from .io import from_pandas
 from . import methods
 from .shuffle import shuffle, rearrange_by_divisions
-from .utils import strip_unknown_categories
+from .utils import strip_unknown_categories, is_dataframe_like, is_series_like
 
 
 def align_partitions(*dfs):
@@ -470,13 +470,25 @@ def concat_indexed_dataframes(dfs, axis=0, join='outer'):
 
 def stack_partitions(dfs, divisions, join='outer'):
     """Concatenate partitions on axis=0 by doing a simple stack"""
-    meta = methods.concat([df._meta for df in dfs], join=join, filter_warning=False)
+    meta = methods.concat([df._meta for df in dfs], join=join,
+                          filter_warning=False)
     empty = strip_unknown_categories(meta)
 
     name = 'concat-{0}'.format(tokenize(*dfs))
     dsk = {}
     i = 0
     for df in dfs:
+        # dtypes of all dfs need to be coherent
+        # refer to https://github.com/dask/dask/issues/4685
+        if is_dataframe_like(df):
+            shared = df.columns.intersection(meta)
+            if not df._meta[shared].dtypes.equals(meta[shared].dtypes):
+                df = df.astype(meta[shared].dtypes)
+        elif is_series_like(df) and is_series_like(meta):
+            if not df.dtype == meta.dtype and str(df.dtype) != 'category':
+                df = df.astype(meta.dtype)
+        else:
+            pass  # TODO: there are other non-covered cases here
         dsk.update(df.dask)
         # An error will be raised if the schemas or categories don't match. In
         # this case we need to pass along the meta object to transform each


Test patch:
diff --git a/dask/dataframe/tests/test_multi.py b/dask/dataframe/tests/test_multi.py
--- a/dask/dataframe/tests/test_multi.py
+++ b/dask/dataframe/tests/test_multi.py
@@ -1,5 +1,6 @@
 import warnings
 
+import dask
 import dask.dataframe as dd
 import numpy as np
 import pandas as pd
@@ -323,6 +324,27 @@ def test_concat(join):
         assert_eq(result, expected)
 
 
+@pytest.mark.parametrize('join', ['inner', 'outer'])
+def test_concat_different_dtypes(join):
+    # check that the resulting dataframe has coherent dtypes
+    # refer to https://github.com/dask/dask/issues/4685
+    pdf1 = pd.DataFrame({'x': [1, 2, 3, 4, 6, 7],
+                         'y': list('abcdef')},
+                        index=[1, 2, 3, 4, 6, 7])
+    ddf1 = dd.from_pandas(pdf1, 2)
+    pdf2 = pd.DataFrame({'x': [1.0, 2.0, 3.0, 4.0, 6.0, 7.0],
+                         'y': list('abcdef')},
+                        index=[8, 9, 10, 11, 12, 13])
+    ddf2 = dd.from_pandas(pdf2, 2)
+
+    expected = pd.concat([pdf1, pdf2], join=join)
+    result = dd.concat([ddf1, ddf2], join=join)
+    assert_eq(expected, result)
+
+    dtypes_list = dask.compute([part.dtypes for part in result.to_delayed()])
+    assert len(set(map(str, dtypes_list))) == 1  # all the same
+
+
 @pytest.mark.parametrize('how', ['inner', 'outer', 'left', 'right'])
 @pytest.mark.parametrize('shuffle', ['disk', 'tasks'])
 def test_merge(how, shuffle):