Repository: pandas-dev/pandas

Issue:
Index.get_indexer_non_unique misbehaves when passed a CategoricalIndex
#### Code Sample, a copy-pastable example if possible

```python
In [2]: idx = pd.CategoricalIndex(['a', 'a', 'b', 'b', 'c', 'a'], categories=['c', 'a', 'b', 'e
   ...: '], dtype='category')

In [3]: key = ['a', 'b']

In [4]: idx.get_indexer_non_unique(key)
Out[4]: (array([0, 1, 5, 2, 3]), array([], dtype=int64))

In [5]: idx.get_indexer_non_unique(pd.CategoricalIndex(key, categories=idx.categories))
Out[5]: (array([ 4,  0,  1,  5,  2,  3, -1]), array([3]))

In [6]: pd.Index(['a', 'a', 'b', 'b', 'c', 'a']).get_indexer_non_unique(pd.CategoricalIndex(key
   ...: , categories=idx.categories))
Out[6]: (array([-1, -1]), array([0, 1]))
```

#### Problem description

``Out[5]`` and ``Out[6]`` are both wrong, and apparently in pretty different ways.

#### Expected Output

``Out[3]``

#### Output of ``pd.show_versions()``

<details>

INSTALLED VERSIONS
------------------
commit: defdb34bafa3900069d399ce597c0abbd4a2b0cc
python: 3.5.3.final.0
python-bits: 64
OS: Linux
OS-release: 4.9.0-6-amd64
machine: x86_64
processor: 
byteorder: little
LC_ALL: None
LANG: it_IT.UTF-8
LOCALE: it_IT.UTF-8

pandas: 0.24.0.dev0+88.gdefdb34ba
pytest: 3.5.0
pip: 9.0.1
setuptools: 39.2.0
Cython: 0.25.2
numpy: 1.14.3
scipy: 0.19.0
pyarrow: None
xarray: None
IPython: 6.2.1
sphinx: 1.5.6
patsy: 0.5.0
dateutil: 2.7.3
pytz: 2018.4
blosc: None
bottleneck: 1.2.0dev
tables: 3.3.0
numexpr: 2.6.1
feather: 0.3.1
matplotlib: 2.2.2.post1153+gff6786446
openpyxl: 2.3.0
xlrd: 1.0.0
xlwt: 1.3.0
xlsxwriter: 0.9.6
lxml: 4.1.1
bs4: 4.5.3
html5lib: 0.999999999
sqlalchemy: 1.0.15
pymysql: None
psycopg2: None
jinja2: 2.10
s3fs: None
fastparquet: None
pandas_gbq: None
pandas_datareader: 0.2.1


</details>



Discussion:

Resolving patch:
diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -31,6 +31,7 @@
     is_dtype_equal,
     is_dtype_union_equal,
     is_object_dtype,
+    is_categorical,
     is_categorical_dtype,
     is_interval_dtype,
     is_period_dtype,
@@ -3300,6 +3301,8 @@ def _filter_indexer_tolerance(self, target, indexer, tolerance):
     @Appender(_index_shared_docs['get_indexer_non_unique'] % _index_doc_kwargs)
     def get_indexer_non_unique(self, target):
         target = _ensure_index(target)
+        if is_categorical(target):
+            target = target.astype(target.dtype.categories.dtype)
         pself, ptarget = self._maybe_promote(target)
         if pself is not self or ptarget is not target:
             return pself.get_indexer_non_unique(ptarget)
diff --git a/pandas/core/indexes/category.py b/pandas/core/indexes/category.py
--- a/pandas/core/indexes/category.py
+++ b/pandas/core/indexes/category.py
@@ -598,7 +598,12 @@ def get_indexer_non_unique(self, target):
         target = ibase._ensure_index(target)
 
         if isinstance(target, CategoricalIndex):
-            target = target.categories
+            # Indexing on codes is more efficient if categories are the same:
+            if target.categories is self.categories:
+                target = target.codes
+                indexer, missing = self._engine.get_indexer_non_unique(target)
+                return _ensure_platform_int(indexer), missing
+            target = target.values
 
         codes = self.categories.get_indexer(target)
         indexer, missing = self._engine.get_indexer_non_unique(codes)


Test patch:
diff --git a/pandas/tests/categorical/test_indexing.py b/pandas/tests/categorical/test_indexing.py
--- a/pandas/tests/categorical/test_indexing.py
+++ b/pandas/tests/categorical/test_indexing.py
@@ -5,7 +5,7 @@
 import numpy as np
 
 import pandas.util.testing as tm
-from pandas import Categorical, Index, PeriodIndex
+from pandas import Categorical, Index, CategoricalIndex, PeriodIndex
 from pandas.tests.categorical.common import TestCategorical
 
 
@@ -103,3 +103,21 @@ def f():
             s.categories = [1, 2]
 
         pytest.raises(ValueError, f)
+
+    # Combinations of sorted/unique:
+    @pytest.mark.parametrize("idx_values", [[1, 2, 3, 4], [1, 3, 2, 4],
+                                            [1, 3, 3, 4], [1, 2, 2, 4]])
+    # Combinations of missing/unique
+    @pytest.mark.parametrize("key_values", [[1, 2], [1, 5], [1, 1], [5, 5]])
+    @pytest.mark.parametrize("key_class", [Categorical, CategoricalIndex])
+    def test_get_indexer_non_unique(self, idx_values, key_values, key_class):
+        # GH 21448
+        key = key_class(key_values, categories=range(1, 5))
+        # Test for flat index and CategoricalIndex with same/different cats:
+        for dtype in None, 'category', key.dtype:
+            idx = Index(idx_values, dtype=dtype)
+            expected, exp_miss = idx.get_indexer_non_unique(key_values)
+            result, res_miss = idx.get_indexer_non_unique(key)
+
+            tm.assert_numpy_array_equal(expected, result)
+            tm.assert_numpy_array_equal(exp_miss, res_miss)