Repository: pandas-dev/pandas

Issue:
read_csv ignores dtype for bool columns with missing values
#### Code Sample, a copy-pastable example if possible
```python
import pandas as pd
from pandas.compat import StringIO
data = "false,1\n,1\ntrue,"
df = pd.read_csv(StringIO(data), header=None, names=['a','b'], dtype={'a': 'bool'}, engine='c')
print(df['a'].dtype)
print(df)
```
##### outputs:
object
       a    b
0  False  1.0
1    NaN  1.0
2   True  NaN

````python
import pandas as pd
from pandas.compat import StringIO
data = "false,1\n,1\ntrue,"
df = pd.read_csv(StringIO(data), header=None, names=['a','b'], dtype={'a': 'bool'},engine='python')
print (df['a'].dtype)
print(df)
````
##### outputs:
bool
       a    b
0  False  1.0
1   True  1.0
2   True  NaN

#### Problem description
Missing values cannot be coerced into a column of dtype bool. The expected behaviour, to be consistent with the analogous case for integers, is to throw a ValueError. The user is then aware of the issue and can specify a converter or parse the boolean as a float (subject to https://github.com/pandas-dev/pandas/issues/16698).

Instead the c engine silently ignores the requested dtype and returns a column of type object.  This behaviour is especially confusing on large datasets with low_memory=True as you get the warning: Column 1 has mixed types. Specify dtype option on import or set low_memory=False.

The python engine silently converts all missing values to True and returns a column of dtype bool. 

#### Expected Output
ValueError('Boolean column has NA values in column 1')
#### Output of ``pd.show_versions()``

<details>

INSTALLED VERSIONS
------------------
commit: None
python: 3.6.5.final.0
python-bits: 64
OS: Darwin
OS-release: 14.5.0
machine: x86_64
processor: i386
byteorder: little
LC_ALL: None
LANG: None
LOCALE: en_US.UTF-8

pandas: 0.23.0.dev0+725.gf67c6fa80
pytest: 3.5.0
pip: 9.0.3
setuptools: 39.0.1
Cython: 0.28.1
numpy: 1.14.2
scipy: 1.0.1
pyarrow: 0.9.0
xarray: 0.10.2
IPython: 6.3.0
sphinx: 1.7.2
patsy: 0.5.0
dateutil: 2.7.2
pytz: 2018.3
blosc: None
bottleneck: 1.2.1
tables: 3.4.2
numexpr: 2.6.4
feather: 0.4.0
matplotlib: 2.2.2
openpyxl: 2.5.1
xlrd: 1.1.0
xlwt: 1.2.0
xlsxwriter: 1.0.2
lxml: 4.2.1
bs4: 4.6.0
html5lib: 1.0.1
sqlalchemy: 1.2.6
pymysql: 0.8.0
psycopg2: None
jinja2: 2.10
s3fs: 0.1.4
fastparquet: 0.1.5
pandas_gbq: None
pandas_datareader: None

</details>



Discussion:
Yes, I agree this should match the integer behavior, thanks for the report, PR welcome!

```
df = pd.read_csv(StringIO(data), header=None, names=['a','b'], dtype={'a': 'bool', 'b': 'int64'}, engine='c')

ValueError                                Traceback (most recent call last)
<snip>
pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader.read()
pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._read_low_memory()
pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._read_rows()
pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._convert_column_data()
pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._convert_tokens()
pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._convert_with_dtype()
ValueError: Integer column has NA values in column 1
```
Here is the current behavior

Bool Value | Engine = C | Engine = Python
-----------|------------|-----------------
NA | NaN | True
0 | False | False
1 | True | True
-1 | error | True
100 | error | True
abc | error | True

What is expected behavior for above cases? @chris-b1 

Resolving patch:
diff --git a/pandas/io/parsers.py b/pandas/io/parsers.py
--- a/pandas/io/parsers.py
+++ b/pandas/io/parsers.py
@@ -27,9 +27,9 @@
 
 from pandas.core.dtypes.cast import astype_nansafe
 from pandas.core.dtypes.common import (
-    ensure_object, is_categorical_dtype, is_dtype_equal, is_float, is_integer,
-    is_integer_dtype, is_list_like, is_object_dtype, is_scalar,
-    is_string_dtype)
+    ensure_object, is_bool_dtype, is_categorical_dtype, is_dtype_equal,
+    is_float, is_integer, is_integer_dtype, is_list_like, is_object_dtype,
+    is_scalar, is_string_dtype)
 from pandas.core.dtypes.dtypes import CategoricalDtype
 from pandas.core.dtypes.missing import isna
 
@@ -1669,6 +1669,16 @@ def _convert_to_ndarrays(self, dct, na_values, na_fvalues, verbose=False,
 
                 # type specified in dtype param
                 if cast_type and not is_dtype_equal(cvals, cast_type):
+                    try:
+                        if (is_bool_dtype(cast_type) and
+                                not is_categorical_dtype(cast_type)
+                                and na_count > 0):
+                            raise ValueError("Bool column has NA values in "
+                                             "column {column}"
+                                             .format(column=c))
+                    except (AttributeError, TypeError):
+                        # invalid input to is_bool_dtype
+                        pass
                     cvals = self._cast_types(cvals, cast_type, c)
 
             result[c] = cvals


Test patch:
diff --git a/pandas/tests/io/parser/test_na_values.py b/pandas/tests/io/parser/test_na_values.py
--- a/pandas/tests/io/parser/test_na_values.py
+++ b/pandas/tests/io/parser/test_na_values.py
@@ -421,3 +421,21 @@ def test_na_values_with_dtype_str_and_na_filter(all_parsers, na_filter):
 
     result = parser.read_csv(StringIO(data), na_filter=na_filter, dtype=str)
     tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.parametrize("data, na_values", [
+    ("false,1\n,1\ntrue", None),
+    ("false,1\nnull,1\ntrue", None),
+    ("false,1\nnan,1\ntrue", None),
+    ("false,1\nfoo,1\ntrue", 'foo'),
+    ("false,1\nfoo,1\ntrue", ['foo']),
+    ("false,1\nfoo,1\ntrue", {'a': 'foo'}),
+])
+def test_cast_NA_to_bool_raises_error(all_parsers, data, na_values):
+    parser = all_parsers
+    msg = ("(Bool column has NA values in column [0a])|"
+           "(cannot safely convert passed user dtype of "
+           "bool for object dtyped data in column 0)")
+    with pytest.raises(ValueError, match=msg):
+        parser.read_csv(StringIO(data), header=None, names=['a', 'b'],
+                        dtype={'a': 'bool'}, na_values=na_values)