Repository: pandas-dev/pandas

Issue:
`pandas.Categorical.from_codes` incorrectly converts NaN codes to 0.
#### Code Sample, a copy-pastable example if possible

```python
import pandas


def test_categorical_from_codes():
    "Demonstrate issue with pandas.Categorical.from_codes."
    codes = pandas.Series([1, 2, float('NaN')])
    categories = ['foo', 'bar', 'baz']
    c = pandas.Categorical.from_codes(codes, categories, ordered=True)

    expected = pandas.Categorical.from_codes([1, 2, -1], categories,
                                             ordered=True)

    pandas.util.testing.assert_categorical_equal(c, expected)
```
#### Problem description

`pandas.Categorical.from_codes` is incorrectly coercing `NaN` code values into `0` values. I think it should be raising a `ValueError`. I believe this is because `pandas._libs.algos.ensure_int8/16/32/64` is behaving in an unanticipated manner on `numpy` arrays.

The call chain is:

1. `pandas.core.arrays.categorical.Categorical.from_codes`
1. `pandas.core.dtypes.cast.coerce_indexer_dtype`
1. `pandas._libs.algos.ensure_int8/16/32/64` (aliased to `pandas.core.dtypes.common.ensure_int8/16/32/64`)

When given a single value all the ensure functions behave correctly. **But when given a `numpy.array` they do not**.

```python
import pandas
import pytest

def test_ensure_on_value():
    """
    Test that ensure_int functions raise a ValueError when converting a NaN.

    These tests ALL PASS, and should pass.
    """
    with pytest.raises(ValueError):
        pandas._libs.algos.ensure_int8(float('NaN'))

    with pytest.raises(ValueError):
        pandas._libs.algos.ensure_int16(float('NaN'))

    with pytest.raises(ValueError):
        pandas._libs.algos.ensure_int32(float('NaN'))

    with pytest.raises(ValueError):
        pandas._libs.algos.ensure_int64(float('NaN'))


def test_ensure_on_array():
    """
    These tests ALL PASS.

    I assume all of these are supposed to raise ValueErrors.

    Interestingly I did not get 0 values with ensure_int32 and ensure_int64.
    """
    a8 = numpy.array([float('NaN')])
    c8 = pandas._libs.algos.ensure_int8(a8)
    numpy.testing.assert_array_equal(c8, numpy.array([0]))

    a16 = numpy.array([float('NaN')])
    c16 = pandas._libs.algos.ensure_int16(a16)
    numpy.testing.assert_array_equal(c16, numpy.array([0]))

    a32 = numpy.array([float('NaN')])
    c32 = pandas._libs.algos.ensure_int32(a32)
    numpy.testing.assert_array_equal(c32, numpy.array([-2147483648]))

    a64 = numpy.array([float('NaN')])
    c64 = pandas._libs.algos.ensure_int64(a64)
    numpy.testing.assert_array_equal(c64, numpy.array([-9223372036854775808]))
```

#### Expected Output

`test_categorical_from_codes` should pass. Instead it contains the codes `[1, 2, 0]`.

#### Output of ``pd.show_versions()``

<details>

[paste the output of ``pd.show_versions()`` here below this line]
>>> pandas.show_versions()
/Users/miker985/miniconda3/envs/general3/lib/python3.6/site-packages/psycopg2/__init__.py:144: UserWarning: The psycopg2 wheel package will be renamed from release 2.8; in order to keep installing from binary please use "pip install psycopg2-binary" instead. For details see: <http://initd.org/psycopg/docs/install.html#binary-install-from-pypi>.
  """)

INSTALLED VERSIONS
------------------
commit: None
python: 3.6.0.final.0
python-bits: 64
OS: Darwin
OS-release: 17.3.0
machine: x86_64
processor: i386
byteorder: little
LC_ALL: None
LANG: en_US.UTF-8
LOCALE: en_US.UTF-8

pandas: 0.23.2
pytest: 3.6.3
pip: 10.0.1
setuptools: 27.2.0
Cython: None
numpy: 1.12.1
scipy: None
pyarrow: None
xarray: None
IPython: None
sphinx: None
patsy: None
dateutil: 2.6.0
pytz: 2017.2
blosc: None
bottleneck: None
tables: None
numexpr: None
feather: None
matplotlib: 2.0.0
openpyxl: None
xlrd: None
xlwt: None
xlsxwriter: None
lxml: None
bs4: 4.6.0
html5lib: None
sqlalchemy: 1.2.6
pymysql: None
psycopg2: 2.7.4 (dt dec pq3 ext lo64)
jinja2: 2.10
s3fs: None
fastparquet: None
pandas_gbq: None
pandas_datareader: None

</details>



Discussion:
I'm not familiar enough with pandas `algos` to know whether it's appropriate to make changes to it.

If you agree this is a problem, I'm happy to submit a patch to either `Categorical.from_codes` or `coerce_indexer_dtype`
I think `from_codes` should raise when it's `np.asarray(codes)` isn't an integer dtype.

NaN isn't a valid code. It should be -1.
That will break the obvious fix (using `.fillna()`). It also breaks 8 existing tests.

```python
>>> codes = pandas.Series([1, 2, float('NaN')])
>>> numpy.asarray(codes.fillna(-1))
array([ 1.,  2., -1.])
>>> numpy.asarray(codes.fillna(-1)).dtype
dtype('float64')
```

Will a simple `if isna(codes).any()` check suffice for raising an error?
Right, we *may* want to warn on that for a version. But `codes` should be integers, not floats.

Resolving patch:
diff --git a/pandas/core/arrays/categorical.py b/pandas/core/arrays/categorical.py
--- a/pandas/core/arrays/categorical.py
+++ b/pandas/core/arrays/categorical.py
@@ -27,6 +27,8 @@
     is_timedelta64_dtype,
     is_categorical,
     is_categorical_dtype,
+    is_float_dtype,
+    is_integer_dtype,
     is_list_like, is_sequence,
     is_scalar, is_iterator,
     is_dict_like)
@@ -629,8 +631,21 @@ def from_codes(cls, codes, categories, ordered=False):
             categorical. If not given, the resulting categorical will be
             unordered.
         """
+        codes = np.asarray(codes)  # #21767
+        if not is_integer_dtype(codes):
+            msg = "codes need to be array-like integers"
+            if is_float_dtype(codes):
+                icodes = codes.astype('i8')
+                if (icodes == codes).all():
+                    msg = None
+                    codes = icodes
+                    warn(("float codes will be disallowed in the future and "
+                          "raise a ValueError"), FutureWarning, stacklevel=2)
+            if msg:
+                raise ValueError(msg)
+
         try:
-            codes = coerce_indexer_dtype(np.asarray(codes), categories)
+            codes = coerce_indexer_dtype(codes, categories)
         except (ValueError, TypeError):
             raise ValueError(
                 "codes need to be convertible to an arrays of integers")


Test patch:
diff --git a/pandas/tests/arrays/categorical/test_constructors.py b/pandas/tests/arrays/categorical/test_constructors.py
--- a/pandas/tests/arrays/categorical/test_constructors.py
+++ b/pandas/tests/arrays/categorical/test_constructors.py
@@ -468,6 +468,26 @@ def test_from_codes_with_categorical_categories(self):
         with pytest.raises(ValueError):
             Categorical.from_codes([0, 1], Categorical(['a', 'b', 'a']))
 
+    def test_from_codes_with_nan_code(self):
+        # GH21767
+        codes = [1, 2, np.nan]
+        categories = ['a', 'b', 'c']
+        with pytest.raises(ValueError):
+            Categorical.from_codes(codes, categories)
+
+    def test_from_codes_with_float(self):
+        # GH21767
+        codes = [1.0, 2.0, 0]  # integer, but in float dtype
+        categories = ['a', 'b', 'c']
+
+        with tm.assert_produces_warning(FutureWarning):
+            cat = Categorical.from_codes(codes, categories)
+        tm.assert_numpy_array_equal(cat.codes, np.array([1, 2, 0], dtype='i1'))
+
+        codes = [1.1, 2.0, 0]  # non-integer
+        with pytest.raises(ValueError):
+            Categorical.from_codes(codes, categories)
+
     @pytest.mark.parametrize('dtype', [None, 'category'])
     def test_from_inferred_categories(self, dtype):
         cats = ['a', 'b']