Repository: pandas-dev/pandas

Issue:
.str.contains(..., na=False) consistency between categorical and object series
#### Code Sample, a copy-pastable example if possible

```python
>>> import pandas as pd
>>> import numpy as np
>>> s = pd.Series(["a","b","c","a", np.nan], dtype="category")
>>> s
0      a
1      b
2      c
3      a
4    NaN
dtype: category
Categories (3, object): [a, b, c]
# `na=False` kwarg should make all missing values False, but this seems broken for categoricals 
>>> s.str.contains("a", na=False)                                                                                                                                                                                               
0     True
1    False
2    False
3     True
4      NaN
dtype: object
# compare that to object series
>>> s2 = pd.Series(["a","b","c","a", np.nan])
>>> s2
0      a
1      b
2      c
3      a
4    NaN
dtype: object
>>> s2.str.contains("a", na=False)
0     True
1    False
2    False
3     True
4    False
dtype: bool

```
#### Problem description

`.str.contains(..., na=False)` should make missing values False when the calling series is of type categorical just like it does for object series.

#### Expected Output

```python
>>> s = pd.Series(["a","b","c","a", np.nan], dtype="category")
>>> s2 = pd.Series(["a","b","c","a", np.nan])
>>> (s.str.contains("a", na=False) == s2.str.contains("a", na=False)).all()
True
```

#### Output of ``pd.show_versions()``

<details>

INSTALLED VERSIONS
------------------
commit: None
python: 3.7.0.final.0
python-bits: 64
OS: Darwin
OS-release: 17.6.0
machine: x86_64
processor: i386
byteorder: little
LC_ALL: None
LANG: en_US.UTF-8
LOCALE: en_US.UTF-8

pandas: 0.23.3
pytest: None
pip: 18.0
setuptools: 40.0.0
Cython: None
numpy: 1.15.0
scipy: None
pyarrow: None
xarray: None
IPython: None
sphinx: None
patsy: None
dateutil: 2.7.3
pytz: 2018.5
blosc: None
bottleneck: None
tables: None
numexpr: None
feather: None
matplotlib: None
openpyxl: None
xlrd: None
xlwt: None
xlsxwriter: None
lxml: None
bs4: None
html5lib: None
sqlalchemy: None
pymysql: None
psycopg2: None
jinja2: None
s3fs: None
fastparquet: None

</details>



Discussion:
I think it'll require updating `StringMethods._wrap_result` to be aware of how NA values should be handled. Might be a bit tricky, not sure ahead of time.
I'd like to work on this issue.

Resolving patch:
diff --git a/pandas/core/strings.py b/pandas/core/strings.py
--- a/pandas/core/strings.py
+++ b/pandas/core/strings.py
@@ -1857,7 +1857,7 @@ def __iter__(self):
             g = self.get(i)
 
     def _wrap_result(self, result, use_codes=True,
-                     name=None, expand=None):
+                     name=None, expand=None, fill_value=np.nan):
 
         from pandas.core.index import Index, MultiIndex
 
@@ -1867,7 +1867,8 @@ def _wrap_result(self, result, use_codes=True,
         # so make it possible to skip this step as the method already did this
         # before the transformation...
         if use_codes and self._is_categorical:
-            result = take_1d(result, self._orig.cat.codes)
+            result = take_1d(result, self._orig.cat.codes,
+                             fill_value=fill_value)
 
         if not hasattr(result, 'ndim') or not hasattr(result, 'dtype'):
             return result
@@ -2515,12 +2516,12 @@ def join(self, sep):
     def contains(self, pat, case=True, flags=0, na=np.nan, regex=True):
         result = str_contains(self._parent, pat, case=case, flags=flags, na=na,
                               regex=regex)
-        return self._wrap_result(result)
+        return self._wrap_result(result, fill_value=na)
 
     @copy(str_match)
     def match(self, pat, case=True, flags=0, na=np.nan):
         result = str_match(self._parent, pat, case=case, flags=flags, na=na)
-        return self._wrap_result(result)
+        return self._wrap_result(result, fill_value=na)
 
     @copy(str_replace)
     def replace(self, pat, repl, n=-1, case=None, flags=0, regex=True):


Test patch:
diff --git a/pandas/tests/test_strings.py b/pandas/tests/test_strings.py
--- a/pandas/tests/test_strings.py
+++ b/pandas/tests/test_strings.py
@@ -512,10 +512,28 @@ def test_contains(self):
         assert result.dtype == np.bool_
         tm.assert_numpy_array_equal(result, expected)
 
-        # na
-        values = Series(['om', 'foo', np.nan])
-        res = values.str.contains('foo', na="foo")
-        assert res.loc[2] == "foo"
+    def test_contains_for_object_category(self):
+        # gh 22158
+
+        # na for category
+        values = Series(["a", "b", "c", "a", np.nan], dtype="category")
+        result = values.str.contains('a', na=True)
+        expected = Series([True, False, False, True, True])
+        tm.assert_series_equal(result, expected)
+
+        result = values.str.contains('a', na=False)
+        expected = Series([True, False, False, True, False])
+        tm.assert_series_equal(result, expected)
+
+        # na for objects
+        values = Series(["a", "b", "c", "a", np.nan])
+        result = values.str.contains('a', na=True)
+        expected = Series([True, False, False, True, True])
+        tm.assert_series_equal(result, expected)
+
+        result = values.str.contains('a', na=False)
+        expected = Series([True, False, False, True, False])
+        tm.assert_series_equal(result, expected)
 
     def test_startswith(self):
         values = Series(['om', NA, 'foo_nom', 'nom', 'bar_foo', NA, 'foo'])
@@ -2875,7 +2893,7 @@ def test_get_complex_nested(self, to_type):
         expected = Series([np.nan])
         tm.assert_series_equal(result, expected)
 
-    def test_more_contains(self):
+    def test_contains_moar(self):
         # PR #1179
         s = Series(['A', 'B', 'C', 'Aaba', 'Baca', '', NA,
                     'CABA', 'dog', 'cat'])
@@ -2925,7 +2943,7 @@ def test_contains_nan(self):
         expected = Series([np.nan, np.nan, np.nan], dtype=np.object_)
         assert_series_equal(result, expected)
 
-    def test_more_replace(self):
+    def test_replace_moar(self):
         # PR #1179
         s = Series(['A', 'B', 'C', 'Aaba', 'Baca', '', NA, 'CABA',
                     'dog', 'cat'])