Repository: dask/dask

Issue:
Read_csv dtypes= for integer NA columns
A common problem with dask.dataframes is that we think a column is of integer dtype, when it actually has missing values and needs to be represented as a float.  Because no missing values occur in the first few megabytes we incorrectly call it an integer dtype.  

Normally we tell people to identify this situation and then redefine their dtypes as float explicitly

```python
df = dd.read_csv(..., dtype={col1: float, col2: float})
```

This seems to not always work, and output the dtype remains integer.  Generally this is a pain for new users.  It has come up many times.  How can we help this experience?

Some options:

1.  Always use floats
2.  Provide a keyword option that turns all integers into floats
3.  Improve the error handling around the integer/float situation to point to a longer explanation
4.  Do some sort of flexible computation where we allow columns of different dtypes


cc @moody-marlin cc @jcrist 


Discussion:
> This seems to not always work, and output the dtype remains integer.

Can you explain/provide a reproducer for this? If a dtype is specified, the output dtype should always match the specification. If not, then that's a bug.

Of your options above, I'd prefer either the keyword option (`assume_missing=True`, default of False)? or improving error messages.

Flexible dtypes may also be doable. We already have a hard time predicting between int and float for *some* operations due to operations that involve missing values - allowing flexible dtypes between partitions for numbers in `read_csv` may not be that tricky to add. The options above would be easier though, and maybe less confusing. 
> Can you explain/provide a reproducer for this? If a dtype is specified, the output dtype should always match the specification. If not, then that's a bug.

I experienced the buggy behavior.  I can work to produce a reproducer later, but not today.

I'm in favor of a keyword argument.  I would suggest that we either make it true (permissive) by default or that we make it false by default but improve the exception and point users to it (preferred).
Cool. Fixing now.

Resolving patch:
diff --git a/dask/dataframe/io/csv.py b/dask/dataframe/io/csv.py
--- a/dask/dataframe/io/csv.py
+++ b/dask/dataframe/io/csv.py
@@ -9,8 +9,8 @@
 except ImportError:
     psutil = None
 
-import numpy as np
 import pandas as pd
+from pandas.types.common import is_integer_dtype, is_float_dtype
 
 from ...bytes import read_bytes
 from ...bytes.core import write_bytes
@@ -75,13 +75,27 @@ def coerce_dtypes(df, dtypes):
     """
     for c in df.columns:
         if c in dtypes and df.dtypes[c] != dtypes[c]:
-            if (np.issubdtype(df.dtypes[c], np.floating) and
-                    np.issubdtype(dtypes[c], np.integer)):
-                if (df[c] % 1).any():
-                    msg = ("Runtime type mismatch. "
-                           "Add {'%s': float} to dtype= keyword in "
-                           "read_csv/read_table")
-                    raise TypeError(msg % c)
+            if is_float_dtype(df.dtypes[c]) and is_integer_dtype(dtypes[c]):
+                # There is a mismatch between floating and integer columns.
+                # Determine all mismatched and error.
+                mismatched = sorted(c for c in df.columns if
+                                    is_float_dtype(df.dtypes[c]) and
+                                    is_integer_dtype(dtypes[c]))
+
+                msg = ("Mismatched dtypes found.\n"
+                       "Expected integers, but found floats for columns:\n"
+                       "%s\n\n"
+                       "To fix, specify dtypes manually by adding:\n\n"
+                       "%s\n\n"
+                       "to the call to `read_csv`/`read_table`.\n\n"
+                       "Alternatively, provide `assume_missing=True` to "
+                       "interpret all unspecified integer columns as floats.")
+
+                missing_list = '\n'.join('- %r' % c for c in mismatched)
+                dtype_list = ('%r: float' % c for c in mismatched)
+                missing_dict = 'dtype={%s}' % ',\n       '.join(dtype_list)
+                raise ValueError(msg % (missing_list, missing_dict))
+
             df[c] = df[c].astype(dtypes[c])
 
 
@@ -155,7 +169,8 @@ def auto_blocksize(total_memory, cpu_count):
 
 def read_pandas(reader, urlpath, blocksize=AUTO_BLOCKSIZE, collection=True,
                 lineterminator=None, compression=None, sample=256000,
-                enforce=False, storage_options=None, **kwargs):
+                enforce=False, assume_missing=False, storage_options=None,
+                **kwargs):
     reader_name = reader.__name__
     if lineterminator is not None and len(lineterminator) == 1:
         kwargs['lineterminator'] = lineterminator
@@ -208,6 +223,16 @@ def read_pandas(reader, urlpath, blocksize=AUTO_BLOCKSIZE, collection=True,
 
     head = reader(BytesIO(sample), **kwargs)
 
+    specified_dtypes = kwargs.get('dtype', {})
+    if specified_dtypes is None:
+        specified_dtypes = {}
+    # If specified_dtypes is a single type, then all columns were specified
+    if assume_missing and isinstance(specified_dtypes, dict):
+        # Convert all non-specified integer columns to floats
+        for c in head.columns:
+            if is_integer_dtype(head[c].dtype) and c not in specified_dtypes:
+                head[c] = head[c].astype(float)
+
     return text_blocks_to_pandas(reader, values, header, head, kwargs,
                                  collection=collection, enforce=enforce)
 
@@ -254,6 +279,9 @@ def read_pandas(reader, urlpath, blocksize=AUTO_BLOCKSIZE, collection=True,
     Return a dask.dataframe if True or list of dask.delayed objects if False
 sample : int
     Number of bytes to use when determining dtypes
+assume_missing : bool, optional
+    If True, all integer columns that aren't specified in ``dtype`` are assumed
+    to contain missing values, and are converted to floats. Default is False.
 storage_options : dict
     Extra options that make sense to a particular storage connection, e.g.
     host, port, username, password, etc.
@@ -265,12 +293,14 @@ def read_pandas(reader, urlpath, blocksize=AUTO_BLOCKSIZE, collection=True,
 def make_reader(reader, reader_name, file_type):
     def read(urlpath, blocksize=AUTO_BLOCKSIZE, collection=True,
              lineterminator=None, compression=None, sample=256000,
-             enforce=False, storage_options=None, **kwargs):
+             enforce=False, assume_missing=False, storage_options=None,
+             **kwargs):
         return read_pandas(reader, urlpath, blocksize=blocksize,
                            collection=collection,
                            lineterminator=lineterminator,
                            compression=compression, sample=sample,
-                           enforce=enforce, storage_options=storage_options,
+                           enforce=enforce, assume_missing=assume_missing,
+                           storage_options=storage_options,
                            **kwargs)
     read.__doc__ = READ_DOC_TEMPLATE.format(reader=reader_name,
                                             file_type=file_type)


Test patch:
diff --git a/dask/dataframe/io/tests/test_csv.py b/dask/dataframe/io/tests/test_csv.py
--- a/dask/dataframe/io/tests/test_csv.py
+++ b/dask/dataframe/io/tests/test_csv.py
@@ -434,24 +434,6 @@ def test_windows_line_terminator():
         assert df.a.sum().compute() == 1 + 2 + 3 + 4 + 5 + 6
 
 
-def test_late_dtypes():
-    text = 'a,b\n1,2\n2,3\n3,4\n4,5\n5.5,6\n6,7.5'
-    with filetext(text) as fn:
-        df = dd.read_csv(fn, blocksize=5, sample=10)
-        try:
-            df.b.sum().compute()
-            assert False
-        except TypeError as e:
-            assert ("'b': float" in str(e) or
-                    "'a': float" in str(e))
-
-        df = dd.read_csv(fn, blocksize=5, sample=10,
-                         dtype={'a': float, 'b': float})
-
-        assert df.a.sum().compute() == 1 + 2 + 3 + 4 + 5.5 + 6
-        assert df.b.sum().compute() == 2 + 3 + 4 + 5 + 6 + 7.5
-
-
 def test_header_None():
     with filetexts({'.tmp.1.csv': '1,2',
                     '.tmp.2.csv': '',
@@ -559,16 +541,71 @@ def test_read_csv_of_modified_file_has_different_name():
         assert sorted(a.dask, key=str) != sorted(b.dask, key=str)
 
 
-@pytest.mark.xfail(reason='we might want permissive behavior here')
-def test_report_dtype_correction_on_csvs():
-    text = 'numbers,names\n'
+def test_late_dtypes():
+    text = 'numbers,names,more_numbers,integers\n'
     for i in range(1000):
-        text += '1,foo\n'
-    text += '1.5,bar\n'
+        text += '1,foo,2,3\n'
+    text += '1.5,bar,2.5,3\n'
     with filetext(text) as fn:
+        sol = pd.read_csv(fn)
         with pytest.raises(ValueError) as e:
-            dd.read_csv(fn).compute(get=get_sync)
-        assert "'numbers': 'float64'" in str(e)
+            dd.read_csv(fn, sample=50).compute(get=get_sync)
+
+        msg = ("Mismatched dtypes found.\n"
+               "Expected integers, but found floats for columns:\n"
+               "- 'more_numbers'\n"
+               "- 'numbers'\n"
+               "\n"
+               "To fix, specify dtypes manually by adding:\n"
+               "\n"
+               "dtype={'more_numbers': float,\n"
+               "       'numbers': float}\n"
+               "\n"
+               "to the call to `read_csv`/`read_table`.\n"
+               "\n"
+               "Alternatively, provide `assume_missing=True` to interpret "
+               "all unspecified integer columns as floats.")
+
+        assert str(e.value) == msg
+
+        # Specifying dtypes works
+        res = dd.read_csv(fn, sample=50,
+                          dtype={'more_numbers': float, 'numbers': float})
+        assert_eq(res, sol)
+
+
+def test_assume_missing():
+    text = 'numbers,names,more_numbers,integers\n'
+    for i in range(1000):
+        text += '1,foo,2,3\n'
+    text += '1.5,bar,2.5,3\n'
+    with filetext(text) as fn:
+        sol = pd.read_csv(fn)
+
+        # assume_missing affects all columns
+        res = dd.read_csv(fn, sample=50, assume_missing=True)
+        assert_eq(res, sol.astype({'integers': float}))
+
+        # assume_missing doesn't override specified dtypes
+        res = dd.read_csv(fn, sample=50, assume_missing=True,
+                          dtype={'integers': 'int64'})
+        assert_eq(res, sol)
+
+        # assume_missing works with dtype=None
+        res = dd.read_csv(fn, sample=50, assume_missing=True, dtype=None)
+        assert_eq(res, sol.astype({'integers': float}))
+
+    text = 'numbers,integers\n'
+    for i in range(1000):
+        text += '1,2\n'
+    text += '1.5,2\n'
+
+    with filetext(text) as fn:
+        sol = pd.read_csv(fn)
+
+        # assume_missing ignored when all dtypes specifed
+        df = dd.read_csv(fn, sample=30, dtype='int64', assume_missing=True)
+        assert df.numbers.dtype == 'int64'
 
 
 def test_index_col():