Repository: pandas-dev/pandas

Issue:
Feature Request: row-level Merge Status Variable
Hello All!

Moving into Pandas from R/Matlab/Stata. One feature I'm finding I really miss: a field generated during a merge that reports, for each row, whether that row came from the left dataset, the right dataset, or was successfully merged. 

I do a lot of work in social science where our data is VERY dirty. For example, I'm often merging transliterated names from Pakistan, and so I just want to see after a merge how many records successfully merged, and then easily pull out the records of each time to compare. I'm including a kludge I'm using below, but an in-line option would be so nice, and I think others in my area would also appreciate it. 

Thanks!

``` python
    df1['left'] = 1
    df2['right'] = 1

    mergedDF = pd.merge(df1,df2, how='outer', left_on='key1', right_on='key2')

    def mergeVar(x):
        if x['left'] == 1 and x['right'] == 1 :
            return 'both'
        elif x['left'] == 1 and x['right'] != 1:
            return 'leftOnly'
        else: return 'rightOnly'

    mergedDF['mergeReport'] = mergedDF.apply(mergeVar, axis=1)
    mergedDF.drop(['left', 'right'], axis = 1)
```

(I'd also be happy to help make it, but I'm relatively new to python, so would need some hand-holding to learn how to integrate a new option into an existing function...)



Discussion:
mentioned here originall: https://github.com/pydata/pandas/issues/7412

but that's closed. Yep this could be an option. I'll put it on the list. Its not hard, but just takes a bit of work.

Thanks!

As an aside, that would also fill another gap in Pandas, which is a lack of a "difference" function that reports what keys aren't shared between dataframes. 

look at set operations on Index objects: http://pandas.pydata.org/pandas-docs/stable/indexing.html#set-operations-on-index-objects

`.difference()/.union()` etc. 

Brilliant! Thanks! You guys are the best!
On Tue, Nov 11, 2014 at 3:32 PM jreback notifications@github.com wrote:

> look at set operations on Index objects:
> http://pandas.pydata.org/pandas-docs/stable/indexing.html#set-operations-on-index-objects
> 
> .difference()/.union() etc.
> 
> —
> Reply to this email directly or view it on GitHub
> https://github.com/pydata/pandas/issues/8790#issuecomment-62641624.


Resolving patch:
diff --git a/pandas/core/frame.py b/pandas/core/frame.py
--- a/pandas/core/frame.py
+++ b/pandas/core/frame.py
@@ -115,6 +115,17 @@
     side, respectively
 copy : boolean, default True
     If False, do not copy data unnecessarily
+indicator : boolean or string, default False
+    If True, adds a column to output DataFrame called "_merge" with 
+    information on the source of each row. 
+    If string, column with information on source of each row will be added to 
+    output DataFrame, and column will be named value of string. 
+    Information column is Categorical-type and takes on a value of "left_only" 
+    for observations whose merge key only appears in 'left' DataFrame, 
+    "right_only" for observations whose merge key only appears in 'right' 
+    DataFrame, and "both" if the observation's merge key is found in both. 
+
+    .. versionadded:: 0.17.0
 
 Examples
 --------
diff --git a/pandas/tools/merge.py b/pandas/tools/merge.py
--- a/pandas/tools/merge.py
+++ b/pandas/tools/merge.py
@@ -27,11 +27,11 @@
 @Appender(_merge_doc, indents=0)
 def merge(left, right, how='inner', on=None, left_on=None, right_on=None,
           left_index=False, right_index=False, sort=False,
-          suffixes=('_x', '_y'), copy=True):
+          suffixes=('_x', '_y'), copy=True, indicator=False):
     op = _MergeOperation(left, right, how=how, on=on, left_on=left_on,
                          right_on=right_on, left_index=left_index,
                          right_index=right_index, sort=sort, suffixes=suffixes,
-                         copy=copy)
+                         copy=copy, indicator=indicator)
     return op.get_result()
 if __debug__:
     merge.__doc__ = _merge_doc % '\nleft : DataFrame'
@@ -157,7 +157,7 @@ class _MergeOperation(object):
     def __init__(self, left, right, how='inner', on=None,
                  left_on=None, right_on=None, axis=1,
                  left_index=False, right_index=False, sort=True,
-                 suffixes=('_x', '_y'), copy=True):
+                 suffixes=('_x', '_y'), copy=True, indicator=False):
         self.left = self.orig_left = left
         self.right = self.orig_right = right
         self.how = how
@@ -174,12 +174,25 @@ def __init__(self, left, right, how='inner', on=None,
         self.left_index = left_index
         self.right_index = right_index
 
+        self.indicator = indicator
+
+        if isinstance(self.indicator, compat.string_types):
+            self.indicator_name = self.indicator
+        elif isinstance(self.indicator, bool):
+            self.indicator_name = '_merge' if self.indicator else None
+        else:
+            raise ValueError('indicator option can only accept boolean or string arguments')
+
+
         # note this function has side effects
         (self.left_join_keys,
          self.right_join_keys,
          self.join_names) = self._get_merge_keys()
 
     def get_result(self):
+        if self.indicator:
+            self.left, self.right = self._indicator_pre_merge(self.left, self.right)
+
         join_index, left_indexer, right_indexer = self._get_join_info()
 
         ldata, rdata = self.left._data, self.right._data
@@ -199,10 +212,46 @@ def get_result(self):
         typ = self.left._constructor
         result = typ(result_data).__finalize__(self, method='merge')
 
+        if self.indicator:
+            result = self._indicator_post_merge(result)
+
         self._maybe_add_join_keys(result, left_indexer, right_indexer)
 
         return result
 
+    def _indicator_pre_merge(self, left, right):
+                
+        columns = left.columns.union(right.columns)  
+
+        for i in ['_left_indicator', '_right_indicator']:
+            if i in columns:
+                raise ValueError("Cannot use `indicator=True` option when data contains a column named {}".format(i))
+        if self.indicator_name in columns:
+            raise ValueError("Cannot use name of an existing column for indicator column")
+
+        left = left.copy()
+        right = right.copy()
+
+        left['_left_indicator'] = 1  
+        left['_left_indicator'] = left['_left_indicator'].astype('int8')  
+        
+        right['_right_indicator'] = 2     
+        right['_right_indicator'] = right['_right_indicator'].astype('int8') 
+        
+        return left, right
+
+    def _indicator_post_merge(self, result):
+
+        result['_left_indicator'] = result['_left_indicator'].fillna(0)
+        result['_right_indicator'] = result['_right_indicator'].fillna(0)
+
+        result[self.indicator_name] = Categorical((result['_left_indicator'] + result['_right_indicator']), categories=[1,2,3])
+        result[self.indicator_name] = result[self.indicator_name].cat.rename_categories(['left_only', 'right_only', 'both'])        
+ 
+        result = result.drop(labels=['_left_indicator', '_right_indicator'], axis=1)
+
+        return result
+
     def _maybe_add_join_keys(self, result, left_indexer, right_indexer):
         # insert group keys
 


Test patch:
diff --git a/pandas/tools/tests/test_merge.py b/pandas/tools/tests/test_merge.py
--- a/pandas/tools/tests/test_merge.py
+++ b/pandas/tools/tests/test_merge.py
@@ -946,6 +946,85 @@ def test_overlapping_columns_error_message(self):
         df2.columns = ['key1', 'foo', 'foo']
         self.assertRaises(ValueError, merge, df, df2)
 
+    def test_indicator(self):
+        # PR #10054. xref #7412 and closes #8790.
+        df1 = pd.DataFrame({'col1':[0,1], 'col_left':['a','b'], 'col_conflict':[1,2]})
+        df1_copy = df1.copy()
+
+        df2 = pd.DataFrame({'col1':[1,2,3,4,5],'col_right':[2,2,2,2,2], 
+                            'col_conflict':[1,2,3,4,5]})
+        df2_copy = df2.copy()
+        
+        df_result = pd.DataFrame({'col1':[0,1,2,3,4,5], 
+                'col_conflict_x':[1,2,np.nan,np.nan,np.nan,np.nan],
+                'col_left':['a','b', np.nan,np.nan,np.nan,np.nan], 
+                'col_conflict_y':[np.nan,1,2,3,4,5], 
+                'col_right':[np.nan, 2,2,2,2,2]},
+                dtype='float64')
+        df_result['_merge'] = pd.Categorical(['left_only','both','right_only',
+            'right_only','right_only','right_only']
+            , categories=['left_only', 'right_only', 'both'])
+
+        df_result = df_result[['col1', 'col_conflict_x', 'col_left', 
+                               'col_conflict_y', 'col_right', '_merge' ]]
+
+        test = pd.merge(df1, df2, on='col1', how='outer', indicator=True)
+        assert_frame_equal(test, df_result)
+
+        # No side effects
+        assert_frame_equal(df1, df1_copy)
+        assert_frame_equal(df2, df2_copy)
+
+        # Check with custom name
+        df_result_custom_name = df_result
+        df_result_custom_name = df_result_custom_name.rename(columns={'_merge':'custom_name'})
+
+        test_custom_name = pd.merge(df1, df2, on='col1', how='outer', indicator='custom_name')
+        assert_frame_equal(test_custom_name, df_result_custom_name)
+
+        # Check only accepts strings and booleans
+        with tm.assertRaises(ValueError):
+            pd.merge(df1, df2, on='col1', how='outer', indicator=5)
+
+        # Check result integrity
+    
+        test2 = pd.merge(df1, df2, on='col1', how='left', indicator=True)
+        self.assertTrue((test2._merge != 'right_only').all())
+
+        test3 = pd.merge(df1, df2, on='col1', how='right', indicator=True)
+        self.assertTrue((test3._merge != 'left_only').all())
+
+        test4 = pd.merge(df1, df2, on='col1', how='inner', indicator=True)
+        self.assertTrue((test4._merge == 'both').all())
+
+        # Check if working name in df
+        for i in ['_right_indicator', '_left_indicator', '_merge']:
+            df_badcolumn = pd.DataFrame({'col1':[1,2], i:[2,2]})
+        
+            with tm.assertRaises(ValueError):
+                pd.merge(df1, df_badcolumn, on='col1', how='outer', indicator=True)
+
+        # Check for name conflict with custom name
+        df_badcolumn = pd.DataFrame({'col1':[1,2], 'custom_column_name':[2,2]})
+        
+        with tm.assertRaises(ValueError):
+            pd.merge(df1, df_badcolumn, on='col1', how='outer', indicator='custom_column_name')
+
+        # Merge on multiple columns
+        df3 = pd.DataFrame({'col1':[0,1], 'col2':['a','b']})
+
+        df4 = pd.DataFrame({'col1':[1,1,3], 'col2':['b','x','y']})
+
+        hand_coded_result = pd.DataFrame({'col1':[0,1,1,3.0], 
+                                         'col2':['a','b','x','y']})
+        hand_coded_result['_merge'] = pd.Categorical(
+            ['left_only','both','right_only','right_only']
+            , categories=['left_only', 'right_only', 'both'])
+ 
+        test5 = pd.merge(df3, df4, on=['col1', 'col2'], how='outer', indicator=True)
+        assert_frame_equal(test5, hand_coded_result)
+    
+
 def _check_merge(x, y):
     for how in ['inner', 'left', 'outer']:
         result = x.join(y, how=how)