Repository: pandas-dev/pandas

Issue:
Merge on categorical type columns gives wrong results
#### Code Sample, a copy-pastable example if possible

```python
import pandas as pd
from pandas.api.types import CategoricalDtype

# Setup CategoricalDtype causing issue
cat_type1 = CategoricalDtype(categories=['A', 'B', 'C'], ordered=False)
cat_type2 = CategoricalDtype(categories=['C', 'B', 'A'], ordered=False)

print('Check dtypes are equivalent:', cat_type1 == cat_type2)
print()

# Test Data
df1 = pd.DataFrame({
    'Foo': pd.Series(['A', 'B', 'C']).astype(cat_type1),
    'Left': ['A0', 'B0', 'C0'],
})

df2 = pd.DataFrame({
    'Foo': pd.Series(['C', 'B', 'A']).astype(cat_type2),
    'Right': ['C1', 'B1', 'A1'],
})

print('df1:\n', df1)
print()

print('df2:\n', df2)
print()

# Issue happens here. Merges on codes instead of value.
# Notice, data from df2 isn't merged correctly.
df_merge = df1.merge(df2, on=['Foo'])
print('df_merge:\n', df_merge)
print()

results = """
Check dtypes are equivalent: True

df1:
   Foo Left
0   A   A0
1   B   B0
2   C   C0

df2:
   Foo Right
0   C    C1
1   B    B1
2   A    A1

df_merge:
   Foo Left Right
0   A   A0    C1
1   B   B0    B1
2   C   C0    A1
"""

```
#### Problem description

Since upgrading from `v0.20.3` to `v0.22.0` I noticed data missing on my datasets. After a few hours debugging I narrowed it down to an issue involving **merges that involve `Categoricals`**. I downgraded to `v0.20.3` to test my original code and didn't have the issue. I then tested on `v.0.21.0` and noticed the issue was first introduced on that version. 

While the example provided doesn't run in `v0.20.3`, it does highlight the bug and I think shows why its happening. Notice that the merge should give a result of 

```
   Foo Left Right
0   A   A0    A1
1   B   B0    B1
2   C   C0    C1
```

instead of 

```
   Foo Left Right
0   A   A0    C1  # Wrong
1   B   B0    B1
2   C   C0    A1  # Wrong
```

#### Expected Output
```
   Foo Left Right
0   A   A0    A1
1   B   B0    B1
2   C   C0    C1
```

#### Output of ``pd.show_versions()``

<details>

[paste the output of ``pd.show_versions()`` here below this line]
INSTALLED VERSIONS
------------------
commit: None
python: 3.6.3.final.0
python-bits: 64
OS: Windows
OS-release: 10
machine: AMD64
processor: Intel64 Family 6 Model 79 Stepping 1, GenuineIntel
byteorder: little
LC_ALL: None
LANG: None
LOCALE: None.None

pandas: 0.22.0
pytest: 3.2.1
pip: 9.0.1
setuptools: 36.5.0.post20170921
Cython: 0.26.1
numpy: 1.12.1
scipy: 0.19.1
pyarrow: 0.7.1
xarray: None
IPython: 6.1.0
sphinx: 1.6.3
patsy: 0.4.1
dateutil: 2.6.1
pytz: 2017.2
blosc: None
bottleneck: 1.2.1
tables: 3.4.2
numexpr: 2.6.2
feather: None
matplotlib: 2.1.0
openpyxl: 2.4.8
xlrd: 1.1.0
xlwt: 1.3.0
xlsxwriter: 1.0.2
lxml: 4.1.0
bs4: 4.6.0
html5lib: 0.999999999
sqlalchemy: 1.1.13
pymysql: None
psycopg2: None
jinja2: 2.9.6
s3fs: None
fastparquet: None
pandas_gbq: None
pandas_datareader: None
</details>



Discussion:
This looks similar to  https://github.com/pandas-dev/pandas/issues/19096 / https://github.com/pandas-dev/pandas/pull/19097, taking a closer look now.

Resolving patch:
diff --git a/pandas/core/indexes/category.py b/pandas/core/indexes/category.py
--- a/pandas/core/indexes/category.py
+++ b/pandas/core/indexes/category.py
@@ -553,6 +553,8 @@ def _reindex_non_unique(self, target):
 
     @Appender(_index_shared_docs['get_indexer'] % _index_doc_kwargs)
     def get_indexer(self, target, method=None, limit=None, tolerance=None):
+        from pandas.core.arrays.categorical import _recode_for_categories
+
         method = missing.clean_reindex_fill_method(method)
         target = ibase._ensure_index(target)
 
@@ -568,8 +570,13 @@ def get_indexer(self, target, method=None, limit=None, tolerance=None):
 
         if (isinstance(target, CategoricalIndex) and
                 self.values.is_dtype_equal(target)):
-            # we have the same codes
-            codes = target.codes
+            if self.values.equals(target.values):
+                # we have the same codes
+                codes = target.codes
+            else:
+                codes = _recode_for_categories(target.codes,
+                                               target.categories,
+                                               self.values.categories)
         else:
             if isinstance(target, CategoricalIndex):
                 code_indexer = self.categories.get_indexer(target.categories)
diff --git a/pandas/core/reshape/merge.py b/pandas/core/reshape/merge.py
--- a/pandas/core/reshape/merge.py
+++ b/pandas/core/reshape/merge.py
@@ -12,6 +12,7 @@
 
 from pandas import (Categorical, DataFrame,
                     Index, MultiIndex, Timedelta)
+from pandas.core.arrays.categorical import _recode_for_categories
 from pandas.core.frame import _merge_doc
 from pandas.core.dtypes.common import (
     is_datetime64tz_dtype,
@@ -1540,8 +1541,15 @@ def _factorize_keys(lk, rk, sort=True):
             is_categorical_dtype(rk) and
             lk.is_dtype_equal(rk)):
         klass = libhashtable.Int64Factorizer
+
+        if lk.categories.equals(rk.categories):
+            rk = rk.codes
+        else:
+            # Same categories in different orders -> recode
+            rk = _recode_for_categories(rk.codes, rk.categories, lk.categories)
+
         lk = _ensure_int64(lk.codes)
-        rk = _ensure_int64(rk.codes)
+        rk = _ensure_int64(rk)
     elif is_int_or_datetime_dtype(lk) and is_int_or_datetime_dtype(rk):
         klass = libhashtable.Int64Factorizer
         lk = _ensure_int64(com._values_from_object(lk))


Test patch:
diff --git a/pandas/tests/indexing/test_categorical.py b/pandas/tests/indexing/test_categorical.py
--- a/pandas/tests/indexing/test_categorical.py
+++ b/pandas/tests/indexing/test_categorical.py
@@ -432,6 +432,23 @@ def test_get_indexer_array(self):
         expected = np.array([0, 1], dtype='intp')
         tm.assert_numpy_array_equal(result, expected)
 
+    def test_get_indexer_same_categories_same_order(self):
+        ci = CategoricalIndex(['a', 'b'], categories=['a', 'b'])
+
+        result = ci.get_indexer(CategoricalIndex(['b', 'b'],
+                                                 categories=['a', 'b']))
+        expected = np.array([1, 1], dtype='intp')
+        tm.assert_numpy_array_equal(result, expected)
+
+    def test_get_indexer_same_categories_different_order(self):
+        # https://github.com/pandas-dev/pandas/issues/19551
+        ci = CategoricalIndex(['a', 'b'], categories=['a', 'b'])
+
+        result = ci.get_indexer(CategoricalIndex(['b', 'b'],
+                                                 categories=['b', 'a']))
+        expected = np.array([1, 1], dtype='intp')
+        tm.assert_numpy_array_equal(result, expected)
+
     def test_getitem_with_listlike(self):
         # GH 16115
         cats = Categorical([Timestamp('12-31-1999'),
diff --git a/pandas/tests/reshape/merge/test_merge.py b/pandas/tests/reshape/merge/test_merge.py
--- a/pandas/tests/reshape/merge/test_merge.py
+++ b/pandas/tests/reshape/merge/test_merge.py
@@ -1643,6 +1643,25 @@ def test_merge_categorical(self):
         result = pd.merge(cleft, cright, how='left', left_on='b', right_on='c')
         tm.assert_frame_equal(result, expected)
 
+    def tests_merge_categorical_unordered_equal(self):
+        # GH-19551
+        df1 = DataFrame({
+            'Foo': Categorical(['A', 'B', 'C'], categories=['A', 'B', 'C']),
+            'Left': ['A0', 'B0', 'C0'],
+        })
+
+        df2 = DataFrame({
+            'Foo': Categorical(['C', 'B', 'A'], categories=['C', 'B', 'A']),
+            'Right': ['C1', 'B1', 'A1'],
+        })
+        result = pd.merge(df1, df2, on=['Foo'])
+        expected = DataFrame({
+            'Foo': pd.Categorical(['A', 'B', 'C']),
+            'Left': ['A0', 'B0', 'C0'],
+            'Right': ['A1', 'B1', 'C1'],
+        })
+        assert_frame_equal(result, expected)
+
     def test_other_columns(self, left, right):
         # non-merge columns should preserve if possible
         right = right.assign(Z=right.Z.astype('category'))