Repository: pandas-dev/pandas

Issue:
BUG:  Calling Series.astype('category') on a categorical series loaded using pd.read_pickle errors on pandas-0.25.0rc0 
#### Code Sample, a copy-pastable example if possible

```python
import os

import pandas as pd

s = pd.Series(["a", "b", "c", "a"], dtype="category")
s.astype('category')

FILEPATH = 'example.pickle'
s.to_pickle(FILEPATH)

s = pd.read_pickle(FILEPATH)
os.remove(FILEPATH)
s.astype('category')
```

Output
```python-traceback
Traceback (most recent call last):
  File "mre.py", line 13, in <module>
    s.astype('category')
  File "/Users/roy/pandas/pandas/core/generic.py", line 5935, in astype
    dtype=dtype, copy=copy, errors=errors, **kwargs
  File "/Users/roy/pandas/pandas/core/internals/managers.py", line 581, in astype
    return self.apply("astype", dtype=dtype, **kwargs)
  File "/Users/roy/pandas/pandas/core/internals/managers.py", line 438, in apply
    applied = getattr(b, f)(**kwargs)
  File "/Users/roy/pandas/pandas/core/internals/blocks.py", line 555, in astype
    return self._astype(dtype, copy=copy, errors=errors, values=values, **kwargs)
  File "/Users/roy/pandas/pandas/core/internals/blocks.py", line 606, in _astype
    return self.make_block(self.values.astype(dtype, copy=copy))
  File "/Users/roy/pandas/pandas/core/arrays/categorical.py", line 524, in astype
    self = self.copy() if copy else self
  File "/Users/roy/pandas/pandas/core/arrays/categorical.py", line 503, in copy
    values=self._codes.copy(), dtype=self.dtype, fastpath=True
  File "/Users/roy/pandas/pandas/core/arrays/categorical.py", line 353, in __init__
    self._dtype = self._dtype.update_dtype(dtype)
  File "/Users/roy/pandas/pandas/core/dtypes/dtypes.py", line 556, in update_dtype
    new_ordered_from_sentinel = dtype._ordered_from_sentinel
AttributeError: 'CategoricalDtype' object has no attribute '_ordered_from_sentinel'
```

#### Problem description

Calling `Series.astype('category')` on a categorical series loaded using `pd.read_pickle` errors  with pandas 0.25.0rc0.  The example code ran without error using pandas 0.24.2

#### Expected Output
```
0    a
1    b
2    c
3    a
dtype: category
```
#### Output of ``pd.show_versions()``

<details>

INSTALLED VERSIONS
------------------
commit           : c64c9cb44222a42f7b02d4d6007919cd0645f1be
python           : 3.7.3.final.0
python-bits      : 64
OS               : Darwin
OS-release       : 18.6.0
machine          : x86_64
processor        : i386
byteorder        : little
LC_ALL           : None
LANG             : en_US.UTF-8
LOCALE           : en_US.UTF-8

pandas           : 0.25.0rc0+23.gc64c9cb44
numpy            : 1.16.4
pytz             : 2019.1
dateutil         : 2.8.0
pip              : 19.1.1
setuptools       : 41.0.1
Cython           : 0.29.12
pytest           : None
hypothesis       : None
sphinx           : None
blosc            : None
feather          : None
xlsxwriter       : None
lxml.etree       : None
html5lib         : None
pymysql          : None
psycopg2         : None
jinja2           : None
IPython          : None
pandas_datareader: None
bs4              : None
bottleneck       : None
fastparquet      : None
gcsfs            : None
lxml.etree       : None
matplotlib       : None
numexpr          : None
odfpy            : None
openpyxl         : None
pandas_gbq       : None
pyarrow          : None
pytables         : None
s3fs             : None
scipy            : None
sqlalchemy       : None
tables           : None
xarray           : None
xlrd             : None
xlwt             : None
xlsxwriter       : None

</details>



Discussion:
Thanks for trying the RC.

cc @jschendel
Looks like it's just a matter of adding `_ordered_from_sentinel` to `CategoricalDtype.__setstate__`:

```diff
diff --git a/pandas/core/dtypes/dtypes.py b/pandas/core/dtypes/dtypes.py
index d8d910a16..54f2c6551 100644
--- a/pandas/core/dtypes/dtypes.py
+++ b/pandas/core/dtypes/dtypes.py
@@ -360,6 +360,7 @@ class CategoricalDtype(PandasExtensionDtype, ExtensionDtype):
         # pickle -> need to set the settable private ones here (see GH26067)
         self._categories = state.pop('categories', None)
         self._ordered = state.pop('ordered', False)
+        self._ordered_from_sentinel = state.pop('ordered_from_sentinel', False)
 
     def __hash__(self) -> int:
         # _hash_categories returns a uint64, so use the negative
```

Should be able to get a fix in tonight or tomorrow.

Resolving patch:
diff --git a/pandas/core/dtypes/dtypes.py b/pandas/core/dtypes/dtypes.py
--- a/pandas/core/dtypes/dtypes.py
+++ b/pandas/core/dtypes/dtypes.py
@@ -219,7 +219,7 @@ class CategoricalDtype(PandasExtensionDtype, ExtensionDtype):
     kind = "O"  # type: str_type
     str = "|O08"
     base = np.dtype("O")
-    _metadata = ("categories", "ordered")
+    _metadata = ("categories", "ordered", "_ordered_from_sentinel")
     _cache = {}  # type: Dict[str_type, PandasExtensionDtype]
 
     def __init__(self, categories=None, ordered: OrderedType = ordered_sentinel):
@@ -356,6 +356,7 @@ def __setstate__(self, state: Dict[str_type, Any]) -> None:
         # pickle -> need to set the settable private ones here (see GH26067)
         self._categories = state.pop("categories", None)
         self._ordered = state.pop("ordered", False)
+        self._ordered_from_sentinel = state.pop("_ordered_from_sentinel", False)
 
     def __hash__(self) -> int:
         # _hash_categories returns a uint64, so use the negative


Test patch:
diff --git a/pandas/tests/dtypes/test_dtypes.py b/pandas/tests/dtypes/test_dtypes.py
--- a/pandas/tests/dtypes/test_dtypes.py
+++ b/pandas/tests/dtypes/test_dtypes.py
@@ -903,6 +903,19 @@ def test_ordered_none_default_deprecated(self, ordered):
         with tm.assert_produces_warning(warning):
             dtype.ordered
 
+    @pytest.mark.parametrize("ordered", [True, False, None, ordered_sentinel])
+    def test_pickle_ordered_from_sentinel(self, ordered):
+        # GH 27295: can remove test when _ordered_from_sentinel is removed (GH 26403)
+        dtype = CategoricalDtype(categories=list("abc"), ordered=ordered)
+
+        warning = FutureWarning if ordered is ordered_sentinel else None
+        with tm.assert_produces_warning(warning, check_stacklevel=False):
+            dtype_from_pickle = tm.round_trip_pickle(dtype)
+
+        result = dtype_from_pickle._ordered_from_sentinel
+        expected = ordered is ordered_sentinel
+        assert result is expected
+
 
 @pytest.mark.parametrize(
     "dtype", [CategoricalDtype, IntervalDtype, DatetimeTZDtype, PeriodDtype]
diff --git a/pandas/tests/series/test_io.py b/pandas/tests/series/test_io.py
--- a/pandas/tests/series/test_io.py
+++ b/pandas/tests/series/test_io.py
@@ -226,6 +226,15 @@ def test_pickle_preserve_name(self):
             unpickled = self._pickle_roundtrip_name(tm.makeTimeSeries(name=n))
             assert unpickled.name == n
 
+    def test_pickle_categorical_ordered_from_sentinel(self):
+        # GH 27295: can remove test when _ordered_from_sentinel is removed (GH 26403)
+        s = Series(["a", "b", "c", "a"], dtype="category")
+        result = tm.round_trip_pickle(s)
+        result = result.astype("category")
+
+        tm.assert_series_equal(result, s)
+        assert result.dtype._ordered_from_sentinel is False
+
     def _pickle_roundtrip_name(self, obj):
 
         with ensure_clean() as path: