Repository: pandas-dev/pandas

Issue:
df.sort_values() not respecting na_position with categoricals
#### Problem description
`DataFrame.sort_values()` appears not to respect the `na_position` parameter when sorting by a categorical series:

```python
>>> import pandas as pd
>>> c = pd.Categorical(['A', np.nan, 'B'], categories=['A','B'], ordered=True)
>>> df = pd.DataFrame({'c': c})
>>> df.sort_values(by='c', na_position='first')
     c
1  NaN
0    A
2    B
>>> df.sort_values(by='c', na_position='last')
     c
1  NaN
0    A
2    B
```
Unexpectedly, the NaNs always come first regardless of `na_position`.

#### Additional information
`Series.sort_values()` works as expected:
```python
>>> c.sort_values(na_position='first')
[NaN, A, B]
Categories (2, object): [A < B]
>>> c.sort_values(na_position='last')
[A, B, NaN]
Categories (2, object): [A < B]
```

Strangely, `df.sort_values()` does seem to respect `na_position` if you sort by more than one column (even the same column):
```python
>>> df.sort_values(by=['c','c'], na_position='first')
     c
1  NaN
0    A
2    B
>>> df.sort_values(by=['c','c'], na_position='last')
     c
0    A
2    B
1  NaN
```

#### Output of ``pd.show_versions()``

<details>

```
INSTALLED VERSIONS
------------------
commit: None
python: 3.7.0.final.0
python-bits: 64
OS: Windows
OS-release: 7
machine: AMD64
processor: Intel64 Family 6 Model 94 Stepping 3, GenuineIntel
byteorder: little
LC_ALL: None
LANG: None
LOCALE: None.None

pandas: 0.23.4
pytest: None
pip: 10.0.1
setuptools: 40.0.0
Cython: None
numpy: 1.15.0
scipy: None
pyarrow: None
xarray: None
IPython: 6.5.0
sphinx: None
patsy: None
dateutil: 2.7.3
pytz: 2018.5
blosc: None
bottleneck: None
tables: None
numexpr: None
feather: None
matplotlib: None
openpyxl: None
xlrd: None
xlwt: None
xlsxwriter: None
lxml: None
bs4: None
html5lib: None
sqlalchemy: None
pymysql: None
psycopg2: None
jinja2: None
s3fs: None
fastparquet: None
pandas_gbq: None
pandas_datareader: None
```
</details>



Discussion:
How odd!  Investigation and PR are welcome!
c = pd.Series(['A', np.nan, 'B'])
using Series instead of Categorical, this problem can be tackled.

Resolving patch:
diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py
--- a/pandas/core/arrays/categorical.py
+++ b/pandas/core/arrays/categorical.py
@@ -45,6 +45,8 @@
 
 import pandas.core.algorithms as algorithms
 
+from pandas.core.sorting import nargsort
+
 from pandas.io.formats import console
 from pandas.io.formats.terminal import get_terminal_size
 from pandas.util._validators import validate_bool_kwarg, validate_fillna_kwargs
@@ -1605,32 +1607,15 @@ def sort_values(self, inplace=False, ascending=True, na_position='last'):
             msg = 'invalid na_position: {na_position!r}'
             raise ValueError(msg.format(na_position=na_position))
 
-        codes = np.sort(self._codes)
-        if not ascending:
-            codes = codes[::-1]
-
-        # NaN handling
-        na_mask = (codes == -1)
-        if na_mask.any():
-            n_nans = len(codes[na_mask])
-            if na_position == "first":
-                # in this case sort to the front
-                new_codes = codes.copy()
-                new_codes[0:n_nans] = -1
-                new_codes[n_nans:] = codes[~na_mask]
-                codes = new_codes
-            elif na_position == "last":
-                # ... and to the end
-                new_codes = codes.copy()
-                pos = len(codes) - n_nans
-                new_codes[0:pos] = codes[~na_mask]
-                new_codes[pos:] = -1
-                codes = new_codes
+        sorted_idx = nargsort(self,
+                              ascending=ascending,
+                              na_position=na_position)
+
         if inplace:
-            self._codes = codes
-            return
+            self._codes = self._codes[sorted_idx]
         else:
-            return self._constructor(values=codes, dtype=self.dtype,
+            return self._constructor(values=self._codes[sorted_idx],
+                                     dtype=self.dtype,
                                      fastpath=True)
 
     def _values_for_rank(self):
diff --git a/pandas/core/sorting.py b/pandas/core/sorting.py
--- a/pandas/core/sorting.py
+++ b/pandas/core/sorting.py
@@ -241,7 +241,19 @@ def nargsort(items, kind='quicksort', ascending=True, na_position='last'):
 
     # specially handle Categorical
     if is_categorical_dtype(items):
-        return items.argsort(ascending=ascending, kind=kind)
+        if na_position not in {'first', 'last'}:
+            raise ValueError('invalid na_position: {!r}'.format(na_position))
+
+        mask = isna(items)
+        cnt_null = mask.sum()
+        sorted_idx = items.argsort(ascending=ascending, kind=kind)
+        if ascending and na_position == 'last':
+            # NaN is coded as -1 and is listed in front after sorting
+            sorted_idx = np.roll(sorted_idx, -cnt_null)
+        elif not ascending and na_position == 'first':
+            # NaN is coded as -1 and is listed in the end after sorting
+            sorted_idx = np.roll(sorted_idx, cnt_null)
+        return sorted_idx
 
     items = np.asanyarray(items)
     idx = np.arange(len(items))


Test patch:
diff --git a/pandas/tests/frame/test_sorting.py b/pandas/tests/frame/test_sorting.py
--- a/pandas/tests/frame/test_sorting.py
+++ b/pandas/tests/frame/test_sorting.py
@@ -10,7 +10,7 @@
 from pandas.compat import lrange
 from pandas.api.types import CategoricalDtype
 from pandas import (DataFrame, Series, MultiIndex, Timestamp,
-                    date_range, NaT, IntervalIndex)
+                    date_range, NaT, IntervalIndex, Categorical)
 
 from pandas.util.testing import assert_series_equal, assert_frame_equal
 
@@ -161,7 +161,7 @@ def test_sort_nan(self):
              'B': [5, 9, 2, nan, 5, 5, 4]},
             index=[2, 0, 3, 1, 6, 4, 5])
         sorted_df = df.sort_values(['A', 'B'], ascending=[
-                                   1, 0], na_position='first')
+            1, 0], na_position='first')
         assert_frame_equal(sorted_df, expected)
 
         # na_position='last', not order
@@ -170,7 +170,7 @@ def test_sort_nan(self):
              'B': [4, 5, 5, nan, 2, 9, 5]},
             index=[5, 4, 6, 1, 3, 0, 2])
         sorted_df = df.sort_values(['A', 'B'], ascending=[
-                                   0, 1], na_position='last')
+            0, 1], na_position='last')
         assert_frame_equal(sorted_df, expected)
 
         # Test DataFrame with nan label
@@ -514,7 +514,7 @@ def test_sort_index_categorical_index(self):
 
         df = (DataFrame({'A': np.arange(6, dtype='int64'),
                          'B': Series(list('aabbca'))
-                         .astype(CategoricalDtype(list('cab')))})
+                        .astype(CategoricalDtype(list('cab')))})
               .set_index('B'))
 
         result = df.sort_index()
@@ -598,3 +598,81 @@ def test_sort_index_intervalindex(self):
             closed='right')
         result = result.columns.levels[1].categories
         tm.assert_index_equal(result, expected)
+
+    def test_sort_index_na_position_with_categories(self):
+        # GH 22556
+        # Positioning missing value properly when column is Categorical.
+        categories = ['A', 'B', 'C']
+        category_indices = [0, 2, 4]
+        list_of_nans = [np.nan, np.nan]
+        na_indices = [1, 3]
+        na_position_first = 'first'
+        na_position_last = 'last'
+        column_name = 'c'
+
+        reversed_categories = sorted(categories, reverse=True)
+        reversed_category_indices = sorted(category_indices, reverse=True)
+        reversed_na_indices = sorted(na_indices, reverse=True)
+
+        df = pd.DataFrame({
+            column_name: pd.Categorical(['A', np.nan, 'B', np.nan, 'C'],
+                                        categories=categories,
+                                        ordered=True)})
+        # sort ascending with na first
+        result = df.sort_values(by=column_name,
+                                ascending=True,
+                                na_position=na_position_first)
+        expected = DataFrame({
+            column_name: Categorical(list_of_nans + categories,
+                                     categories=categories,
+                                     ordered=True)
+        }, index=na_indices + category_indices)
+
+        assert_frame_equal(result, expected)
+
+        # sort ascending with na last
+        result = df.sort_values(by=column_name,
+                                ascending=True,
+                                na_position=na_position_last)
+        expected = DataFrame({
+            column_name: Categorical(categories + list_of_nans,
+                                     categories=categories,
+                                     ordered=True)
+        }, index=category_indices + na_indices)
+
+        assert_frame_equal(result, expected)
+
+        # sort descending with na first
+        result = df.sort_values(by=column_name,
+                                ascending=False,
+                                na_position=na_position_first)
+        expected = DataFrame({
+            column_name: Categorical(list_of_nans + reversed_categories,
+                                     categories=categories,
+                                     ordered=True)
+        }, index=reversed_na_indices + reversed_category_indices)
+
+        assert_frame_equal(result, expected)
+
+        # sort descending with na last
+        result = df.sort_values(by=column_name,
+                                ascending=False,
+                                na_position=na_position_last)
+        expected = DataFrame({
+            column_name: Categorical(reversed_categories + list_of_nans,
+                                     categories=categories,
+                                     ordered=True)
+        }, index=reversed_category_indices + reversed_na_indices)
+
+        assert_frame_equal(result, expected)
+
+    def test_sort_index_na_position_with_categories_raises(self):
+        df = pd.DataFrame({
+            'c': pd.Categorical(['A', np.nan, 'B', np.nan, 'C'],
+                                categories=['A', 'B', 'C'],
+                                ordered=True)})
+
+        with pytest.raises(ValueError):
+            df.sort_values(by='c',
+                           ascending=False,
+                           na_position='bad_position')
diff --git a/pandas/tests/io/test_stata.py b/pandas/tests/io/test_stata.py
--- a/pandas/tests/io/test_stata.py
+++ b/pandas/tests/io/test_stata.py
@@ -995,7 +995,7 @@ def test_categorical_sorting(self, file):
         parsed = read_stata(getattr(self, file))
 
         # Sort based on codes, not strings
-        parsed = parsed.sort_values("srh")
+        parsed = parsed.sort_values("srh", na_position='first')
 
         # Don't sort index
         parsed.index = np.arange(parsed.shape[0])