Repository: dask/dask

Issue:
dd.merge indicator support/alternative
Hi there,

I want to do outer join of two dataframes (each has only one column).
In pandas I would do it like this:

``` python
df1 = pd.DataFrame({'v0':[1,2,3,4,5]})
df2 = pd.DataFrame({'v0':[2,3,4,6,7]})
out=pd.merge(df1,df2, how='outer', indicator=True)
out[out['_merge']!='both'].drop('_merge', axis=1)
    v0
0   1.0
4   5.0
5   6.0
6   7.0
```

From dask.pdf docs it looks like indicator keyword is not supported. Is there an alternative to 'indicator' in my use case?

``` python
import dask.dataframe as dd
import dask
df1 = pd.DataFrame({'v0':[1,2,3,4,5]})
df2 = pd.DataFrame({'v0':[2,3,4,6,7]})
ddf1 = dd.from_pandas(df1, npartitions=1)
ddf2 = dd.from_pandas(df2, npartitions=1)
dask.compute(dd.merge(df1,df2, on='v0', how='outer'))

(    v0
 0  1.0
 1  2.0
 2  3.0
 3  4.0
 4  5.0
 5  6.0
 6  7.0,)
```

Thank you,
Anton



Discussion:
Seems like we should be able to pass this down through the pd.merge calls.  Any interest in trying this out and submitting a pull request?


Resolving patch:
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -2172,17 +2172,18 @@ def drop(self, labels, axis=0, dtype=None):
 
     @derived_from(pd.DataFrame)
     def merge(self, right, how='inner', on=None, left_on=None, right_on=None,
-              left_index=False, right_index=False,
-              suffixes=('_x', '_y'), npartitions=None, shuffle=None):
+              left_index=False, right_index=False, suffixes=('_x', '_y'),
+              indicator=False, npartitions=None, shuffle=None):
 
         if not isinstance(right, (DataFrame, pd.DataFrame)):
             raise ValueError('right must be DataFrame')
 
         from .multi import merge
-        return merge(self, right, how=how, on=on,
-                     left_on=left_on, right_on=right_on,
-                     left_index=left_index, right_index=right_index,
-                     suffixes=suffixes, npartitions=npartitions, shuffle=shuffle)
+        return merge(self, right, how=how, on=on, left_on=left_on,
+                     right_on=right_on, left_index=left_index,
+                     right_index=right_index, suffixes=suffixes,
+                     npartitions=npartitions, indicator=indicator,
+                     shuffle=shuffle)
 
     @derived_from(pd.DataFrame)
     def join(self, other, on=None, how='left',
diff --git a/dask/dataframe/multi.py b/dask/dataframe/multi.py
--- a/dask/dataframe/multi.py
+++ b/dask/dataframe/multi.py
@@ -65,7 +65,6 @@
 
 from ..base import tokenize
 from ..compatibility import apply
-from ..utils import M
 from .core import (_Frame, DataFrame, map_partitions,
                    Index, _maybe_from_pandas, new_dd_object)
 from .io import from_pandas
@@ -201,15 +200,17 @@ def require(divisions, parts, required=None):
 required = {'left': [0], 'right': [1], 'inner': [0, 1], 'outer': []}
 
 
-def join_indexed_dataframes(lhs, rhs, how='left', lsuffix='', rsuffix=''):
+def merge_indexed_dataframes(lhs, rhs, how='left', lsuffix='', rsuffix='',
+                            indicator=False):
     """ Join two partitioned dataframes along their index """
     (lhs, rhs), divisions, parts = align_partitions(lhs, rhs)
     divisions, parts = require(divisions, parts, required[how])
 
-    left_empty = lhs._meta_nonempty
-    right_empty = rhs._meta_nonempty
+    left_empty = lhs._meta
+    right_empty = rhs._meta
 
-    name = 'join-indexed-' + tokenize(lhs, rhs, how, lsuffix, rsuffix)
+    name = 'join-indexed-' + tokenize(lhs, rhs, how, lsuffix, rsuffix,
+                                      indicator)
 
     dsk = dict()
     for i, (a, b) in enumerate(parts):
@@ -218,38 +219,38 @@ def join_indexed_dataframes(lhs, rhs, how='left', lsuffix='', rsuffix=''):
         if b is None and how in ('left', 'outer'):
             b = right_empty
 
-        dsk[(name, i)] = (M.join, a, b, None, how,
-                          lsuffix, rsuffix)
+        dsk[(name, i)] = (_pdmerge, a, b, how, None, None, True, True,
+                          indicator, (lsuffix, rsuffix), left_empty,
+                          right_empty)
 
-    # dummy result
-    meta = left_empty.join(right_empty, on=None, how=how,
-                           lsuffix=lsuffix, rsuffix=rsuffix)
+    meta = pd.merge(lhs._meta_nonempty, rhs._meta_nonempty, how=how,
+                    left_index=True, right_index=True,
+                    suffixes=(lsuffix, rsuffix), indicator=indicator)
     return DataFrame(toolz.merge(lhs.dask, rhs.dask, dsk),
                      name, meta, divisions)
 
 
 def _pdmerge(left, right, how, left_on, right_on,
-             left_index, right_index, suffixes,
-             default_left_columns, default_right_columns):
+             left_index, right_index, indicator, suffixes,
+             default_left, default_right):
 
     if not len(left):
-        left = pd.DataFrame(columns=default_left_columns)
+        left = default_left
 
     if not len(right):
-        right = pd.DataFrame(columns=default_right_columns)
+        right = default_right
 
-    result = pd.merge(left, right, how=how,
-                      left_on=left_on, right_on=right_on,
-                      left_index=left_index, right_index=right_index,
-                      suffixes=suffixes)
-    return result
+    return pd.merge(left, right, how=how, left_on=left_on, right_on=right_on,
+                    left_index=left_index, right_index=right_index,
+                    suffixes=suffixes, indicator=indicator)
 
 
 shuffle_func = shuffle  # name sometimes conflicts with keyword argument
 
 
 def hash_join(lhs, left_on, rhs, right_on, how='inner',
-              npartitions=None, suffixes=('_x', '_y'), shuffle=None):
+              npartitions=None, suffixes=('_x', '_y'), shuffle=None,
+              indicator=False):
     """ Join two DataFrames on particular columns with hash join
 
     This shuffles both datasets on the joined column and then performs an
@@ -276,14 +277,14 @@ def hash_join(lhs, left_on, rhs, right_on, how='inner',
         right_index = False
 
     # dummy result
-    meta = pd.merge(lhs._meta_nonempty, rhs._meta_nonempty, how, None,
+    meta = pd.merge(lhs._meta_nonempty, rhs._meta_nonempty, how=how,
                     left_on=left_on, right_on=right_on,
                     left_index=left_index, right_index=right_index,
-                    suffixes=suffixes)
+                    suffixes=suffixes, indicator=indicator)
 
     merger = partial(_pdmerge, suffixes=suffixes,
-                     default_left_columns=list(lhs.columns),
-                     default_right_columns=list(rhs.columns))
+                     default_left=lhs._meta,
+                     default_right=rhs._meta)
 
     if isinstance(left_on, list):
         left_on = (list, tuple(left_on))
@@ -291,12 +292,12 @@ def hash_join(lhs, left_on, rhs, right_on, how='inner',
         right_on = (list, tuple(right_on))
 
     token = tokenize(lhs2, left_on, rhs2, right_on, left_index, right_index,
-                     how, npartitions, suffixes, shuffle)
+                     how, npartitions, suffixes, shuffle, indicator)
     name = 'hash-join-' + token
 
     dsk = dict(((name, i), (merger, (lhs2._name, i), (rhs2._name, i),
                             how, left_on, right_on,
-                            left_index, right_index))
+                            left_index, right_index, indicator))
                 for i in range(npartitions))
 
     divisions = [None] * (npartitions + 1)
@@ -403,7 +404,7 @@ def concat_indexed_dataframes(dfs, axis=0, join='outer'):
 
 def merge(left, right, how='inner', on=None, left_on=None, right_on=None,
           left_index=False, right_index=False, suffixes=('_x', '_y'),
-          npartitions=None, shuffle=None, max_branch=None):
+          indicator=False, npartitions=None, shuffle=None, max_branch=None):
     for o in [on, left_on, right_on]:
         if isinstance(o, _Frame):
             raise NotImplementedError(
@@ -421,7 +422,8 @@ def merge(left, right, how='inner', on=None, left_on=None, right_on=None,
             isinstance(right, (pd.Series, pd.DataFrame))):
         return pd.merge(left, right, how=how, on=on, left_on=left_on,
                         right_on=right_on, left_index=left_index,
-                        right_index=right_index, suffixes=suffixes)
+                        right_index=right_index, suffixes=suffixes,
+                        indicator=indicator)
 
     # Transform pandas objects into dask.dataframe objects
     if isinstance(left, (pd.Series, pd.DataFrame)):
@@ -443,15 +445,18 @@ def merge(left, right, how='inner', on=None, left_on=None, right_on=None,
     # Both sides indexed
     if (left_index and left.known_divisions and
             right_index and right.known_divisions):  # Do indexed join
-        return join_indexed_dataframes(left, right, how=how,
-                                       lsuffix=suffixes[0], rsuffix=suffixes[1])
+        return merge_indexed_dataframes(left, right, how=how,
+                                       lsuffix=suffixes[0],
+                                       rsuffix=suffixes[1],
+                                       indicator=indicator)
 
     # Single partition on one side
     elif (left.npartitions == 1 and how in ('inner', 'right') or
           right.npartitions == 1 and how in ('inner', 'left')):
         return single_partition_join(left, right, how=how, right_on=right_on,
-                left_on=left_on, left_index=left_index,
-                right_index=right_index, suffixes=suffixes)
+                                     left_on=left_on, left_index=left_index,
+                                     right_index=right_index,
+                                     suffixes=suffixes, indicator=indicator)
 
     # One side is indexed, the other not
     elif (left_index and left.known_divisions and not right_index or
@@ -461,23 +466,25 @@ def merge(left, right, how='inner', on=None, left_on=None, right_on=None,
         meta = pd.merge(left_empty, right_empty, how=how, on=on,
                         left_on=left_on, right_on=right_on,
                         left_index=left_index, right_index=right_index,
-                        suffixes=suffixes)
+                        suffixes=suffixes, indicator=indicator)
         if left_index and left.known_divisions:
             right = rearrange_by_divisions(right, right_on, left.divisions,
-                    max_branch, shuffle=shuffle)
+                                           max_branch, shuffle=shuffle)
             left = left.clear_divisions()
         elif right_index and right.known_divisions:
             left = rearrange_by_divisions(left, left_on, right.divisions,
-                    max_branch, shuffle=shuffle)
+                                          max_branch, shuffle=shuffle)
             right = right.clear_divisions()
         return map_partitions(pd.merge, left, right, meta=meta, how=how, on=on,
-                left_on=left_on, right_on=right_on, left_index=left_index,
-                right_index=right_index, suffixes=suffixes)
+                              left_on=left_on, right_on=right_on,
+                              left_index=left_index, right_index=right_index,
+                              suffixes=suffixes, indicator=indicator)
     # Catch all hash join
     else:
         return hash_join(left, left.index if left_index else left_on,
                          right, right.index if right_index else right_on,
-                         how, npartitions, suffixes, shuffle=shuffle)
+                         how, npartitions, suffixes, shuffle=shuffle,
+                         indicator=indicator)
 
 
 ###############################################################


Test patch:
diff --git a/dask/dataframe/tests/test_multi.py b/dask/dataframe/tests/test_multi.py
--- a/dask/dataframe/tests/test_multi.py
+++ b/dask/dataframe/tests/test_multi.py
@@ -1,7 +1,7 @@
 import dask.dataframe as dd
 import numpy as np
 import pandas as pd
-from dask.dataframe.multi import (align_partitions, join_indexed_dataframes,
+from dask.dataframe.multi import (align_partitions, merge_indexed_dataframes,
         hash_join, concat_indexed_dataframes, _maybe_align_partitions)
 import pandas.util.testing as tm
 from dask.async import get_sync
@@ -147,7 +147,7 @@ def test__maybe_align_partitions():
         _maybe_align_partitions([ddf, ddf2])
 
 
-def test_join_indexed_dataframe_to_indexed_dataframe():
+def test_merge_indexed_dataframe_to_indexed_dataframe():
     A = pd.DataFrame({'x': [1, 2, 3, 4, 5, 6]},
                      index=[1, 2, 3, 4, 6, 7])
     a = dd.repartition(A, [1, 4, 7])
@@ -156,30 +156,30 @@ def test_join_indexed_dataframe_to_indexed_dataframe():
                      index=[1, 2, 4, 5, 6, 8])
     b = dd.repartition(B, [1, 2, 5, 8])
 
-    c = join_indexed_dataframes(a, b, how='left')
+    c = merge_indexed_dataframes(a, b, how='left')
     assert c.divisions[0] == a.divisions[0]
     assert c.divisions[-1] == max(a.divisions + b.divisions)
     assert eq(c, A.join(B))
 
-    c = join_indexed_dataframes(a, b, how='right')
+    c = merge_indexed_dataframes(a, b, how='right')
     assert c.divisions[0] == b.divisions[0]
     assert c.divisions[-1] == b.divisions[-1]
     assert eq(c, A.join(B, how='right'))
 
-    c = join_indexed_dataframes(a, b, how='inner')
+    c = merge_indexed_dataframes(a, b, how='inner')
     assert c.divisions[0] == 1
     assert c.divisions[-1] == max(a.divisions + b.divisions)
     assert eq(c.compute(), A.join(B, how='inner'))
 
-    c = join_indexed_dataframes(a, b, how='outer')
+    c = merge_indexed_dataframes(a, b, how='outer')
     assert c.divisions[0] == 1
     assert c.divisions[-1] == 8
     assert eq(c.compute(), A.join(B, how='outer'))
 
-    assert sorted(join_indexed_dataframes(a, b, how='inner').dask) == \
-           sorted(join_indexed_dataframes(a, b, how='inner').dask)
-    assert sorted(join_indexed_dataframes(a, b, how='inner').dask) != \
-           sorted(join_indexed_dataframes(a, b, how='outer').dask)
+    assert sorted(merge_indexed_dataframes(a, b, how='inner').dask) == \
+           sorted(merge_indexed_dataframes(a, b, how='inner').dask)
+    assert sorted(merge_indexed_dataframes(a, b, how='inner').dask) != \
+           sorted(merge_indexed_dataframes(a, b, how='outer').dask)
 
 
 def list_eq(aa, bb):
@@ -193,18 +193,13 @@ def list_eq(aa, bb):
         b = bb
     tm.assert_index_equal(a.columns, b.columns)
 
-    # ToDo: As of pandas 0,17, tm.assert_numpy_array_equal can
-    # compare arrays include NaN. This logic can be replaced
-    for c in a.columns:
-        if a[c].dtype in (int, float):
-            a[c] = a[c].fillna(100)
-            b[c] = b[c].fillna(100)
-        else:
-            a[c] = a[c].fillna('NAN')
-            b[c] = b[c].fillna('NAN')
-    av = sorted(a.values.tolist())
-    bv = sorted(b.values.tolist())
-    assert av == bv, (av, bv)
+    if isinstance(a, pd.DataFrame):
+        av = a.sort_values(list(a.columns)).values
+        bv = b.sort_values(list(b.columns)).values
+    else:
+        av = a.sort_values().values
+        bv = b.sort_values().values
+    tm.assert_numpy_array_equal(av, bv)
 
 
 @pytest.mark.parametrize('how', ['inner', 'left', 'right', 'outer'])
@@ -301,8 +296,8 @@ def test_merge(how, shuffle):
     B = pd.DataFrame({'y': [1, 3, 4, 4, 5, 6], 'z': [6, 5, 4, 3, 2, 1]})
     b = dd.repartition(B, [0, 2, 5])
 
-    eq(dd.merge(a, b, left_index=True, right_index=True, shuffle=shuffle),
-       pd.merge(A, B, left_index=True, right_index=True))
+    eq(dd.merge(a, b, left_index=True, right_index=True, how=how, shuffle=shuffle),
+       pd.merge(A, B, left_index=True, right_index=True, how=how))
 
     result = dd.merge(a, b, on='y', how=how)
     list_eq(result, pd.merge(A, B, on='y', how=how))
@@ -421,6 +416,15 @@ def test_merge_by_index_patterns(how, shuffle):
                         shuffle=shuffle),
                pd.merge(pdr, pdl, how=how, left_index=True, right_index=True))
 
+            eq(dd.merge(ddl, ddr, how=how, left_index=True, right_index=True,
+                        shuffle=shuffle, indicator=True),
+               pd.merge(pdl, pdr, how=how, left_index=True, right_index=True,
+                        indicator=True))
+            eq(dd.merge(ddr, ddl, how=how, left_index=True, right_index=True,
+                        shuffle=shuffle, indicator=True),
+               pd.merge(pdr, pdl, how=how, left_index=True, right_index=True,
+                        indicator=True))
+
             eq(ddr.merge(ddl, how=how, left_index=True, right_index=True,
                          shuffle=shuffle),
                pdr.merge(pdl, how=how, left_index=True, right_index=True))
@@ -436,6 +440,15 @@ def test_merge_by_index_patterns(how, shuffle):
                              shuffle=shuffle),
                     pd.merge(pdl, pdr, how=how, left_on='b', right_on='d'))
 
+            list_eq(dd.merge(ddr, ddl, how=how, left_on='c', right_on='a',
+                             shuffle=shuffle, indicator=True),
+                    pd.merge(pdr, pdl, how=how, left_on='c', right_on='a',
+                             indicator=True))
+            list_eq(dd.merge(ddr, ddl, how=how, left_on='d', right_on='b',
+                             shuffle=shuffle, indicator=True),
+                    pd.merge(pdr, pdl, how=how, left_on='d', right_on='b',
+                             indicator=True))
+
             list_eq(dd.merge(ddr, ddl, how=how, left_on='c', right_on='a',
                              shuffle=shuffle),
                     pd.merge(pdr, pdl, how=how, left_on='c', right_on='a'))