Repository: dask/dask
Pull request: Correct dtypes when `concat` with an empty dataframe
- [x] Closes #9046
- [x] Tests added / passed
- [x] Passes `pre-commit run --all-files`


Problem statement:
Implicit data type conversion after concat
**What happened**: 
Dask implicitly converts column datatypes after DF's concat, when one of DF's is empty.

**What you expected to happen**:
Warning (at-least) or exception if dtype converted or same dtype for columns (this how it works in Pandas)

**Minimal Complete Verifiable Example**:

```python

import pandas as pd
import dask.dataframe as dd

sample_data = [
  {'col_int64': 100},
  {'col_int64': 200},
  {'col_int64': 300},
]

def main():

  # let's dummy pandas df with int64 col
  pandas_df = pd.DataFrame(sample_data, dtype='int64')
  assert pandas_df.dtypes['col_int64'] == 'int64'

  print('\nPandas :: Types before DF concat:')
  print(pandas_df.dtypes)
  print()

  # init empty pandas dataframe
  empty_pandas_df = pd.DataFrame([])

  # concat dummy dataframe with empty
  df = pd.concat([pandas_df, empty_pandas_df])
  assert df.dtypes['col_int64'] == 'int64'

  print('\nPandas :: after DF concat:')
  print(df.dtypes)
  print()

  # now let's try Dask
  dask_df = dd.from_pandas(pandas_df, npartitions=1)
  assert dask_df.dtypes['col_int64'] == 'int64'

  print('\nDask :: Types before DF concat:')
  print(dask_df.dtypes)
  print()

  # init empty dask
  empty_dask_df = dd.from_pandas(empty_pandas_df, npartitions=1)

  # concat with empty
  df = dd.concat([dask_df, empty_dask_df])
  print('\nDask :: Types after DF concat:')
  print(df.dtypes)
  print()
  if df.dtypes['col_int64'] != 'int64':
    print(f'Silently converted to {df.dtypes["col_int64"]}\n\n')

main()

### output:

# Pandas :: Types before DF concat:
# col_int64    int64
# dtype: object


# Pandas :: after DF concat:
# col_int64    int64
# dtype: object


# Dask :: Types before DF concat:
# col_int64    int64
# dtype: object


# Dask :: Types after DF concat:
# col_int64    float64
# dtype: object

# Silently converted to float64

```

**Anything else we need to know?**:

**Environment**:

- Dask version:  2022.5.0
- Python version: 3.9.10
- Operating System: OSX
- Install method (conda, pip, source): pip

Hints from issue discussion:
@patefon Thanks for reporting this! I'm able to reproduce this, and it looks like a bug.

Commit message: use assert_eq

Patch to review:
diff --git a/dask/dataframe/multi.py b/dask/dataframe/multi.py
index 169c1c76b67..9ca3ddb9e03 100644
--- a/dask/dataframe/multi.py
+++ b/dask/dataframe/multi.py
@@ -1237,6 +1237,12 @@ def concat(
         raise ValueError("'join' must be 'inner' or 'outer'")
 
     axis = DataFrame._validate_axis(axis)
+    try:
+        # remove any empty DataFrames
+        dfs = [df for df in dfs if bool(len(df.columns))]
+    except AttributeError:
+        # 'Series' object has no attribute 'columns'
+        pass
     dasks = [df for df in dfs if isinstance(df, _Frame)]
     dfs = _maybe_from_pandas(dfs)
 
diff --git a/dask/dataframe/tests/test_multi.py b/dask/dataframe/tests/test_multi.py
index 09876e50851..657dcf18790 100644
--- a/dask/dataframe/tests/test_multi.py
+++ b/dask/dataframe/tests/test_multi.py
@@ -796,6 +796,23 @@ def test_concat_with_operation_remains_hlg():
     assert_eq(result, expected)
 
 
+def test_concat_dataframe_empty():
+    df = pd.DataFrame({"a": [100, 200, 300]}, dtype="int64")
+    empty_df = pd.DataFrame([], dtype="int64")
+    df_concat = pd.concat([df, empty_df])
+
+    ddf = dd.from_pandas(df, npartitions=1)
+    empty_ddf = dd.from_pandas(empty_df, npartitions=1)
+    ddf_concat = dd.concat([ddf, empty_ddf])
+    assert_eq(df_concat, ddf_concat)
+
+    empty_df_with_col = pd.DataFrame([], columns=["x"], dtype="int64")
+    df_concat_with_col = pd.concat([df, empty_df_with_col])
+    empty_ddf_with_col = dd.from_pandas(empty_df_with_col, npartitions=1)
+    ddf_concat_with_col = dd.concat([ddf, empty_ddf_with_col])
+    assert_eq(df_concat_with_col, ddf_concat_with_col)
+
+
 @pytest.mark.parametrize(
     "value_1, value_2",
     [


Code review comments:
diff_hunk: @@ -796,6 +796,23 @@ def test_concat_with_operation_remains_hlg():
     assert_eq(result, expected)
 
 
+def test_concat_dataframe_empty():
+    df = pd.DataFrame({"a": [100, 200, 300]}, dtype="int64")
+    empty_df = pd.DataFrame([], dtype="int64")
+    df_concat = pd.concat([df, empty_df])
+
+    ddf = dd.from_pandas(df, npartitions=1)
+    empty_ddf = dd.from_pandas(empty_df, npartitions=1)
+    ddf_concat = dd.concat([ddf, empty_ddf])
+    assert_eq(df_concat, ddf_concat)
+
+    empty_df_with_col = pd.DataFrame([], columns=["x"], dtype="int64")
+    df_concat_with_col = pd.concat([df, empty_df_with_col])
+    empty_ddf_with_col = dd.from_pandas(empty_df_with_col, npartitions=1)
+    ddf_concat_with_col = dd.concat([ddf, empty_ddf_with_col])
+    assert_eq(df_concat_with_col, ddf_concat_with_col)
path: dask/dataframe/tests/test_multi.py
text: @user1:
This suggestion makes the test pass, but there is a workaround due to subtleties explained above about `assert_eq` and how it check dtypes of the index. I tried to explain it into the note. 

I'm basically avoiding to check for dtypes in the assert and handling that separately. It's not the nicest but it works. what do we think? 

```suggestion
    # NOTE: check_dtypes compares dtypes of the index and for the case
    # of ddf_concat_with_col this is an Object, while the pandas version
    # df_concat_with_col is int64, pandas already knows the rows are empty while
    # dask is not aware. check_index=False doesn't avoid checking the dtype of
    # the index, hence the workaround below.
    assert_eq(df_concat_with_col, ddf_concat_with_col, check_dtype=False)

    assert (df_concat_with_col.index == ddf_concat_with_col.compute().index).all()
    assert df_concat_with_col.index.dtype == ddf_concat_with_col.compute().index.dtype
    assert (df_concat_with_col.dtypes == ddf_concat_with_col.dtypes).all()

```

@user2:
I think I would prefer to just change `ddf._meta.index.dtype` and then use `assert_eq` as regular.

diff_hunk: @@ -796,6 +796,23 @@ def test_concat_with_operation_remains_hlg():
     assert_eq(result, expected)
 
 
+def test_concat_dataframe_empty():
+    df = pd.DataFrame({"a": [100, 200, 300]}, dtype="int64")
+    empty_df = pd.DataFrame([], dtype="int64")
+    df_concat = pd.concat([df, empty_df])
+
+    ddf = dd.from_pandas(df, npartitions=1)
+    empty_ddf = dd.from_pandas(empty_df, npartitions=1)
+    ddf_concat = dd.concat([ddf, empty_ddf])
+    assert_eq(df_concat, ddf_concat)
+
+    empty_df_with_col = pd.DataFrame([], columns=["x"], dtype="int64")
+    df_concat_with_col = pd.concat([df, empty_df_with_col])
+    empty_ddf_with_col = dd.from_pandas(empty_df_with_col, npartitions=1)
+    ddf_concat_with_col = dd.concat([ddf, empty_ddf_with_col])
+    assert_eq(df_concat_with_col, ddf_concat_with_col)
path: dask/dataframe/tests/test_multi.py
text: @author:
This is currently failing, still figuring out why :/

@user2:
I've been trying to figure out what's going on here, and it seems that when we create the empty dataframe with a column name, `ddf._meta.dtype` in https://github.com/dask/dask/blob/78452267da9683fe637148fd7a667724c3ff7713/dask/dataframe/utils.py#L648-L651

returns dtype('O') but I can't seem to find how this gets assigned. 

One solution would be to compare the dtypes without using assert_eq, but I'm not sure that's the path we want to go. 

@user1 do you have any suggestions on how to proceed here?

@user1:
This is certainly a weird one. If you inspect the objects created by the test you see that it's the index that is causing this issue. In particular for the case of `empty_df_with_col` you'll see that `empty_df_with_col.index.dtype == dtype('O')` that is just on the pandas object. On the other hand `df.index.dtype == dtype('int64')`. 

It seems odd to me that pandas would use a different default index dtype based on whether there are rows or not. But keeping going with the test, when `df` and `empty_df_with_col` are concatted, the output has `index.dtype == dtype('int64')`. In dask it does does not. Instead it has `index.dtype == dtype('O')`. 

I don't think there is actually a good way to handle this since the `_meta` can't depend on the len of the index. So I think you should just explicitly set the dtype of the index on `_meta` before doing `assert_eq`


@author:
Thank you @user2 and @user1!

diff_hunk: @@ -796,6 +796,23 @@ def test_concat_with_operation_remains_hlg():
     assert_eq(result, expected)
 
 
+def test_concat_dataframe_empty():
+    df = pd.DataFrame({"a": [100, 200, 300]}, dtype="int64")
+    empty_df = pd.DataFrame([], dtype="int64")
+    df_concat = pd.concat([df, empty_df])
+
+    ddf = dd.from_pandas(df, npartitions=1)
+    empty_ddf = dd.from_pandas(empty_df, npartitions=1)
+    ddf_concat = dd.concat([ddf, empty_ddf])
+    assert_eq(df_concat, ddf_concat)
+
+    empty_df_with_col = pd.DataFrame([], columns=["x"], dtype="int64")
+    df_concat_with_col = pd.concat([df, empty_df_with_col])
+    empty_ddf_with_col = dd.from_pandas(empty_df_with_col, npartitions=1)
+    ddf_concat_with_col = dd.concat([ddf, empty_ddf_with_col])
+    assert_eq(df_concat_with_col, ddf_concat_with_col)
path: dask/dataframe/tests/test_multi.py
text: @user1:
Something more like this? (Test passes)
 
```suggestion
    # NOTE: empty_ddf_with_col.index.dtype == dtype('O') while ddf.index.dtype == dtype('int64')
    # when we concatenate the resulting ddf_concat_with_col.index.dtype == dtype('O'). However,
    # the pandas version is smart enough to identify the empty rows and assign dtype('int64'),
    # which causes assert_eq to fail when checking dtype. Hence the follwoing line.
    ddf_concat_with_col._meta.index = ddf_concat_with_col._meta.index.astype("int64")

    assert_eq(df_concat_with_col, ddf_concat_with_col, check_dtype=False)
```