Repository: pandas-dev/pandas

Issue:
REGR: merge is failing with right_index=True and how='right'
GeoPandas tests started failing, and this is due to a regression in the merge functionality:

```
In [13]: left = pd.DataFrame({'a': [1, 2, 3], 'key': [0, 1, 1]})                                                                                                                                                    

In [14]: right = pd.DataFrame({'b': [1, 2, 3]})                                                                                                                                                                     

In [15]: left.merge(right, left_on='key', right_index=True)                                                                                                                                                         
Out[15]: 
   a  key  b
0  1    0  1
1  2    1  2
2  3    1  2

In [16]: left.merge(right, left_on='key', right_index=True, how='right')                                                                                                                                            
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-16-cb931ec12982> in <module>
----> 1 left.merge(right, left_on='key', right_index=True, how='right')

~/scipy/pandas/pandas/core/frame.py in merge(self, right, how, on, left_on, right_on, left_index, right_index, sort, suffixes, copy, indicator, validate)
   6875                      right_on=right_on, left_index=left_index,
   6876                      right_index=right_index, sort=sort, suffixes=suffixes,
-> 6877                      copy=copy, indicator=indicator, validate=validate)
   6878 
   6879     def round(self, decimals=0, *args, **kwargs):

~/scipy/pandas/pandas/core/reshape/merge.py in merge(left, right, how, on, left_on, right_on, left_index, right_index, sort, suffixes, copy, indicator, validate)
     46                          copy=copy, indicator=indicator,
     47                          validate=validate)
---> 48     return op.get_result()
     49 
     50 

~/scipy/pandas/pandas/core/reshape/merge.py in get_result(self)
    544                 self.left, self.right)
    545 
--> 546         join_index, left_indexer, right_indexer = self._get_join_info()
    547 
    548         ldata, rdata = self.left._data, self.right._data

~/scipy/pandas/pandas/core/reshape/merge.py in _get_join_info(self)
    761                                                          self.right.index,
    762                                                          left_indexer,
--> 763                                                          how='right')
    764                 else:
    765                     join_index = self.right.index.take(right_indexer)

~/scipy/pandas/pandas/core/reshape/merge.py in _create_join_index(self, index, other_index, indexer, how)
    807                 # take from other_index instead
    808                 join_list = join_index.to_numpy()
--> 809                 join_list[mask] = other_index.to_numpy()[mask]
    810                 join_index = Index(join_list, dtype=join_index.dtype,
    811                                    name=join_index.name)

IndexError: boolean index did not match indexed array along dimension 0; dimension is 3 but corresponding boolean dimension is 4
```

This seems to be related to https://github.com/pandas-dev/pandas/pull/24733, which was merged couple of days ago (after the RC).


Discussion:

Resolving patch:
diff --git a/pandas/core/reshape/merge.py b/pandas/core/reshape/merge.py
--- a/pandas/core/reshape/merge.py
+++ b/pandas/core/reshape/merge.py
@@ -757,13 +757,21 @@ def _get_join_info(self):
 
             if self.right_index:
                 if len(self.left) > 0:
-                    join_index = self.left.index.take(left_indexer)
+                    join_index = self._create_join_index(self.left.index,
+                                                         self.right.index,
+                                                         left_indexer,
+                                                         right_indexer,
+                                                         how='right')
                 else:
                     join_index = self.right.index.take(right_indexer)
                     left_indexer = np.array([-1] * len(join_index))
             elif self.left_index:
                 if len(self.right) > 0:
-                    join_index = self.right.index.take(right_indexer)
+                    join_index = self._create_join_index(self.right.index,
+                                                         self.left.index,
+                                                         right_indexer,
+                                                         left_indexer,
+                                                         how='left')
                 else:
                     join_index = self.left.index.take(left_indexer)
                     right_indexer = np.array([-1] * len(join_index))
@@ -774,6 +782,39 @@ def _get_join_info(self):
             join_index = join_index.astype(object)
         return join_index, left_indexer, right_indexer
 
+    def _create_join_index(self, index, other_index, indexer,
+                           other_indexer, how='left'):
+        """
+        Create a join index by rearranging one index to match another
+
+        Parameters
+        ----------
+        index: Index being rearranged
+        other_index: Index used to supply values not found in index
+        indexer: how to rearrange index
+        how: replacement is only necessary if indexer based on other_index
+
+        Returns
+        -------
+        join_index
+        """
+        join_index = index.take(indexer)
+        if (self.how in (how, 'outer') and
+                not isinstance(other_index, MultiIndex)):
+            # if final index requires values in other_index but not target
+            # index, indexer may hold missing (-1) values, causing Index.take
+            # to take the final value in target index
+            mask = indexer == -1
+            if np.any(mask):
+                # if values missing (-1) from target index,
+                # take from other_index instead
+                join_list = join_index.to_numpy()
+                other_list = other_index.take(other_indexer).to_numpy()
+                join_list[mask] = other_list[mask]
+                join_index = Index(join_list, dtype=join_index.dtype,
+                                   name=join_index.name)
+        return join_index
+
     def _get_merge_keys(self):
         """
         Note: has side effects (copy/delete key columns)


Test patch:
diff --git a/pandas/tests/reshape/merge/test_merge.py b/pandas/tests/reshape/merge/test_merge.py
--- a/pandas/tests/reshape/merge/test_merge.py
+++ b/pandas/tests/reshape/merge/test_merge.py
@@ -939,25 +939,22 @@ def test_merge_two_empty_df_no_division_error(self):
         with np.errstate(divide='raise'):
             merge(a, a, on=('a', 'b'))
 
-    @pytest.mark.parametrize('how', ['left', 'outer'])
-    @pytest.mark.xfail(reason="GH-24897")
+    @pytest.mark.parametrize('how', ['right', 'outer'])
     def test_merge_on_index_with_more_values(self, how):
         # GH 24212
-        # pd.merge gets [-1, -1, 0, 1] as right_indexer, ensure that -1 is
-        # interpreted as a missing value instead of the last element
-        df1 = pd.DataFrame([[1, 2], [2, 4], [3, 6], [4, 8]],
-                           columns=['a', 'b'])
-        df2 = pd.DataFrame([[3, 30], [4, 40]],
-                           columns=['a', 'c'])
-        df1.set_index('a', drop=False, inplace=True)
-        df2.set_index('a', inplace=True)
-        result = pd.merge(df1, df2, left_index=True, right_on='a', how=how)
-        expected = pd.DataFrame([[1, 2, np.nan],
-                                 [2, 4, np.nan],
-                                 [3, 6, 30.0],
-                                 [4, 8, 40.0]],
-                                columns=['a', 'b', 'c'])
-        expected.set_index('a', drop=False, inplace=True)
+        # pd.merge gets [0, 1, 2, -1, -1, -1] as left_indexer, ensure that
+        # -1 is interpreted as a missing value instead of the last element
+        df1 = pd.DataFrame({'a': [1, 2, 3], 'key': [0, 2, 2]})
+        df2 = pd.DataFrame({'b': [1, 2, 3, 4, 5]})
+        result = df1.merge(df2, left_on='key', right_index=True, how=how)
+        expected = pd.DataFrame([[1.0, 0, 1],
+                                 [2.0, 2, 3],
+                                 [3.0, 2, 3],
+                                 [np.nan, 1, 2],
+                                 [np.nan, 3, 4],
+                                 [np.nan, 4, 5]],
+                                columns=['a', 'key', 'b'])
+        expected.set_index(Int64Index([0, 1, 2, 1, 3, 4]), inplace=True)
         assert_frame_equal(result, expected)
 
     def test_merge_right_index_right(self):