Repository: dask/dask

Issue:
Outer or right merges with missing values can fail with meta mismatches
When merging two dataframes on a specific column, it is sometimes that case that individual chunks of the left or right frame are empty. This can happen, for instance, if the values in the right frame do not exist in the left (for a given set of divisions).

As is sometimes the case, pandas can give a different answer when one of the two frames are empty (here, the left frame). There is an upstream bug report here: https://github.com/pandas-dev/pandas/issues/9937. So you can wind up with a case where some partitions have the wrong shape due to an empty partition for a given set of divisions, even though the overall merge is non-empty. A minimal reproducer:

```python
import pandas
import dask.dataframe as dd

# Two dataframes, both with "a" columns. By design, not all values in one
# exist in the other, so some of the chunk-wise merges will be with an empty frame.
left = pandas.DataFrame({"a": [1, 1, 2, 2], "val": [5, 6, 7, 8]})
right = pandas.DataFrame({"a": [2, 2, 3, 3], "val": [11, 12, 13, 14]})

# More partitions than values, ensure empty frames.
dd_left = dd.from_pandas(left, npartitions=4)
dd_right = dd.from_pandas(right, npartitions=4)

merged = dd_left.merge(dd_right, on="a", how="right")
merged.map_partitions(lambda x: x, meta = merged._meta).compute()  # Fails due to meta mismatch!
```

**Environment**:

- Dask version: `main`
- Python version: 3.8
- Operating System: ubuntu
- Install method (conda, pip, source): source

Outer or right merges with missing values can fail with meta mismatches
When merging two dataframes on a specific column, it is sometimes that case that individual chunks of the left or right frame are empty. This can happen, for instance, if the values in the right frame do not exist in the left (for a given set of divisions).

As is sometimes the case, pandas can give a different answer when one of the two frames are empty (here, the left frame). There is an upstream bug report here: https://github.com/pandas-dev/pandas/issues/9937. So you can wind up with a case where some partitions have the wrong shape due to an empty partition for a given set of divisions, even though the overall merge is non-empty. A minimal reproducer:

```python
import pandas
import dask.dataframe as dd

# Two dataframes, both with "a" columns. By design, not all values in one
# exist in the other, so some of the chunk-wise merges will be with an empty frame.
left = pandas.DataFrame({"a": [1, 1, 2, 2], "val": [5, 6, 7, 8]})
right = pandas.DataFrame({"a": [2, 2, 3, 3], "val": [11, 12, 13, 14]})

# More partitions than values, ensure empty frames.
dd_left = dd.from_pandas(left, npartitions=4)
dd_right = dd.from_pandas(right, npartitions=4)

merged = dd_left.merge(dd_right, on="a", how="right")
merged.map_partitions(lambda x: x, meta = merged._meta).compute()  # Fails due to meta mismatch!
```

**Environment**:

- Dask version: `main`
- Python version: 3.8
- Operating System: ubuntu
- Install method (conda, pip, source): source



Discussion:

Resolving patch:
diff --git a/dask/dataframe/multi.py b/dask/dataframe/multi.py
--- a/dask/dataframe/multi.py
+++ b/dask/dataframe/multi.py
@@ -238,11 +238,18 @@ def require(divisions, parts, required=None):
 allowed_right = ("inner", "right")
 
 
-def merge_chunk(lhs, *args, empty_index_dtype=None, categorical_columns=None, **kwargs):
+def merge_chunk(
+    lhs,
+    *args,
+    result_meta,
+    **kwargs,
+):
 
     rhs, *args = args
     left_index = kwargs.get("left_index", False)
     right_index = kwargs.get("right_index", False)
+    empty_index_dtype = result_meta.index.dtype
+    categorical_columns = result_meta.select_dtypes(include="category").columns
 
     if categorical_columns is not None:
         for col in categorical_columns:
@@ -280,6 +287,12 @@ def merge_chunk(lhs, *args, empty_index_dtype=None, categorical_columns=None, **
 
     out = lhs.merge(rhs, *args, **kwargs)
 
+    # Workaround for pandas bug where if the left frame of a merge operation is
+    # empty, the resulting dataframe can have columns in the wrong order.
+    # https://github.com/pandas-dev/pandas/issues/9937
+    if len(lhs) == 0:
+        out = out[result_meta.columns]
+
     # Workaround pandas bug where if the output result of a merge operation is
     # an empty dataframe, the output index is `int64` in all cases, regardless
     # of input dtypes.
@@ -300,8 +313,7 @@ def merge_indexed_dataframes(lhs, rhs, left_index=True, right_index=True, **kwar
     name = "join-indexed-" + tokenize(lhs, rhs, **kwargs)
 
     meta = lhs._meta_nonempty.merge(rhs._meta_nonempty, **kwargs)
-    kwargs["empty_index_dtype"] = meta.index.dtype
-    kwargs["categorical_columns"] = meta.select_dtypes(include="category").columns
+    kwargs["result_meta"] = meta
 
     dsk = dict()
     for i, (a, b) in enumerate(parts):
@@ -376,8 +388,7 @@ def hash_join(
     if isinstance(right_on, list):
         right_on = (list, tuple(right_on))
 
-    kwargs["empty_index_dtype"] = meta.index.dtype
-    kwargs["categorical_columns"] = meta.select_dtypes(include="category").columns
+    kwargs["result_meta"] = meta
 
     joined = map_partitions(
         merge_chunk,
@@ -414,8 +425,7 @@ def single_partition_join(left, right, **kwargs):
         else:
             meta.index = meta.index.astype("int64")
 
-    kwargs["empty_index_dtype"] = meta.index.dtype
-    kwargs["categorical_columns"] = meta.select_dtypes(include="category").columns
+    kwargs["result_meta"] = meta
 
     if right.npartitions == 1 and kwargs["how"] in allowed_left:
         if use_left:
@@ -616,7 +626,6 @@ def merge(
             suffixes=suffixes,
             indicator=indicator,
         )
-        categorical_columns = meta.select_dtypes(include="category").columns
 
         if merge_indexed_left and left.known_divisions:
             right = rearrange_by_divisions(
@@ -641,8 +650,7 @@ def merge(
             right_index=right_index,
             suffixes=suffixes,
             indicator=indicator,
-            empty_index_dtype=meta.index.dtype,
-            categorical_columns=categorical_columns,
+            result_meta=meta,
         )
     # Catch all hash join
     else:
@@ -1508,8 +1516,7 @@ def broadcast_join(
 
     # dummy result
     meta = lhs._meta_nonempty.merge(rhs._meta_nonempty, **merge_kwargs)
-    merge_kwargs["empty_index_dtype"] = meta.index.dtype
-    merge_kwargs["categorical_columns"] = meta.select_dtypes(include="category").columns
+    merge_kwargs["result_meta"] = meta
 
     # Assume the output partitions/divisions
     # should correspond to the collection that


Test patch:
diff --git a/dask/dataframe/tests/test_multi.py b/dask/dataframe/tests/test_multi.py
--- a/dask/dataframe/tests/test_multi.py
+++ b/dask/dataframe/tests/test_multi.py
@@ -1001,6 +1001,26 @@ def test_merge(how, shuffle_method):
     #         pd.merge(A, B, left_index=True, right_on='y'))
 
 
+@pytest.mark.parametrize("how", ["right", "outer"])
+def test_merge_empty_left_df(shuffle_method, how):
+    # We're merging on "a", but in a situation where the left
+    # frame is missing some values, so some of the blocked merges
+    # will involve an empty left frame.
+    # https://github.com/pandas-dev/pandas/issues/9937
+    left = pd.DataFrame({"a": [1, 1, 2, 2], "val": [5, 6, 7, 8]})
+    right = pd.DataFrame({"a": [0, 0, 3, 3], "val": [11, 12, 13, 14]})
+
+    dd_left = dd.from_pandas(left, npartitions=4)
+    dd_right = dd.from_pandas(right, npartitions=4)
+
+    merged = dd_left.merge(dd_right, on="a", how=how)
+    expected = left.merge(right, on="a", how=how)
+    assert_eq(merged, expected, check_index=False)
+
+    # Check that the individual partitions have the expected shape
+    merged.map_partitions(lambda x: x, meta=merged._meta).compute()
+
+
 def test_merge_how_raises():
     left = pd.DataFrame(
         {