Repository: pandas-dev/pandas

Issue:
PERF: make Categorical.searchsorted not require ordered=True
``searchsorted`` requires that the searched object is (monotonically) sorted to produce correct results. Orderedness is neither a necessary or sufficient condition to make searchsorted work correctly.

``Categorical.searchsorted`` has a hard check for if the Categorical is ordered:

https://github.com/pandas-dev/pandas/blob/e0f978d43c960e0af7944a9d7d410662e071c0c7/pandas/core/arrays/categorical.py#L1382-L1388

This is too strict, as unordered but sorted Categoricals could also benefit from using searchsorted.

I propose removing this check and (like for non-categoricals) let the user have the responsibility to ensure that the Categorical is sorted correctly. This would allow very quick lookups in all sorted Categoricals, whether they're ordered or not.


Discussion:
Since the underlying binary search is actually being done on the codes, I wonder if the thought may have have been preventing surprising behavior if code ordering isn't what you expected, e.g. you ended up with something like this that 'feels' sorted, but isn't.  At least with `as_ordered()` it becomes more clear the data is not sorted?  
```python
c = pd.Categorical(['a', 'a', 'b', 'b', 'c'], categories=pd.Index(['b', 'a', 'c']))

c
Out[140]: 
[a, a, b, b, c]
Categories (3, object): [b, a, c]

c.as_ordered()
Out[142]: 
[a, a, b, b, c]
Categories (3, object): [b < a < c]
```

Not necessarily opposed to the change though.

Resolving patch:
diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py
--- a/pandas/core/arrays/categorical.py
+++ b/pandas/core/arrays/categorical.py
@@ -1399,13 +1399,6 @@ def memory_usage(self, deep=False):
     @Substitution(klass="Categorical")
     @Appender(_shared_docs["searchsorted"])
     def searchsorted(self, value, side="left", sorter=None):
-        if not self.ordered:
-            raise ValueError(
-                "Categorical not ordered\nyou can use "
-                ".as_ordered() to change the Categorical to an "
-                "ordered one"
-            )
-
         from pandas.core.series import Series
 
         codes = _get_codes_for_values(Series(value).values, self.categories)
diff --git a/pandas/core/base.py b/pandas/core/base.py
--- a/pandas/core/base.py
+++ b/pandas/core/base.py
@@ -1515,6 +1515,12 @@ def factorize(self, sort=False, na_sentinel=-1):
         corresponding elements in `value` were inserted before the indices,
         the order of `self` would be preserved.
 
+        .. note::
+
+            The %(klass)s *must* be monotonically sorted, otherwise
+            wrong locations will likely be returned. Pandas does *not*
+            check this for you.
+
         Parameters
         ----------
         value : array_like
@@ -1540,6 +1546,7 @@ def factorize(self, sort=False, na_sentinel=-1):
 
         See Also
         --------
+        sort_values
         numpy.searchsorted
 
         Notes
@@ -1578,6 +1585,13 @@ def factorize(self, sort=False, na_sentinel=-1):
 
         >>> x.searchsorted(['bread'], side='right')
         array([3])
+
+        If the values are not monotonically sorted, wrong locations
+        may be returned:
+
+        >>> x = pd.Series([2, 1, 3])
+        >>> x.searchsorted(1)
+        0  # wrong result, correct would be 1
         """
 
     @Substitution(klass="Index")


Test patch:
diff --git a/pandas/tests/arrays/categorical/test_analytics.py b/pandas/tests/arrays/categorical/test_analytics.py
--- a/pandas/tests/arrays/categorical/test_analytics.py
+++ b/pandas/tests/arrays/categorical/test_analytics.py
@@ -78,42 +78,36 @@ def test_mode(self, values, categories, exp_mode):
         exp = Categorical(exp_mode, categories=categories, ordered=True)
         tm.assert_categorical_equal(res, exp)
 
-    def test_searchsorted(self):
+    def test_searchsorted(self, ordered_fixture):
         # https://github.com/pandas-dev/pandas/issues/8420
         # https://github.com/pandas-dev/pandas/issues/14522
 
-        c1 = Categorical(
-            ["cheese", "milk", "apple", "bread", "bread"],
-            categories=["cheese", "milk", "apple", "bread"],
-            ordered=True,
-        )
-        s1 = Series(c1)
-        c2 = Categorical(
+        cat = Categorical(
             ["cheese", "milk", "apple", "bread", "bread"],
             categories=["cheese", "milk", "apple", "bread"],
-            ordered=False,
+            ordered=ordered_fixture,
         )
-        s2 = Series(c2)
+        ser = Series(cat)
 
         # Searching for single item argument, side='left' (default)
-        res_cat = c1.searchsorted("apple")
+        res_cat = cat.searchsorted("apple")
         assert res_cat == 2
         assert is_scalar(res_cat)
 
-        res_ser = s1.searchsorted("apple")
+        res_ser = ser.searchsorted("apple")
         assert res_ser == 2
         assert is_scalar(res_ser)
 
         # Searching for single item array, side='left' (default)
-        res_cat = c1.searchsorted(["bread"])
-        res_ser = s1.searchsorted(["bread"])
+        res_cat = cat.searchsorted(["bread"])
+        res_ser = ser.searchsorted(["bread"])
         exp = np.array([3], dtype=np.intp)
         tm.assert_numpy_array_equal(res_cat, exp)
         tm.assert_numpy_array_equal(res_ser, exp)
 
         # Searching for several items array, side='right'
-        res_cat = c1.searchsorted(["apple", "bread"], side="right")
-        res_ser = s1.searchsorted(["apple", "bread"], side="right")
+        res_cat = cat.searchsorted(["apple", "bread"], side="right")
+        res_ser = ser.searchsorted(["apple", "bread"], side="right")
         exp = np.array([3, 5], dtype=np.intp)
         tm.assert_numpy_array_equal(res_cat, exp)
         tm.assert_numpy_array_equal(res_ser, exp)
@@ -121,22 +115,15 @@ def test_searchsorted(self):
         # Searching for a single value that is not from the Categorical
         msg = r"Value\(s\) to be inserted must be in categories"
         with pytest.raises(KeyError, match=msg):
-            c1.searchsorted("cucumber")
+            cat.searchsorted("cucumber")
         with pytest.raises(KeyError, match=msg):
-            s1.searchsorted("cucumber")
+            ser.searchsorted("cucumber")
 
         # Searching for multiple values one of each is not from the Categorical
         with pytest.raises(KeyError, match=msg):
-            c1.searchsorted(["bread", "cucumber"])
+            cat.searchsorted(["bread", "cucumber"])
         with pytest.raises(KeyError, match=msg):
-            s1.searchsorted(["bread", "cucumber"])
-
-        # searchsorted call for unordered Categorical
-        msg = "Categorical not ordered"
-        with pytest.raises(ValueError, match=msg):
-            c2.searchsorted("apple")
-        with pytest.raises(ValueError, match=msg):
-            s2.searchsorted("apple")
+            ser.searchsorted(["bread", "cucumber"])
 
     def test_unique(self):
         # categories are reordered based on value when ordered=False