Repository: dask/dask

Issue:
Cache issues with dtypes
Not sure if this should be classified as user error.

``` python
In [1]: import dask.dataframe as dd

In [2]: import pandas as pd

In [3]: import numpy as np

In [4]: df1 = pd.DataFrame({'A': [1, 2]})  # int dtype

In [5]: df2 = pd.DataFrame({'A': [np.nan, 3]})  # cast to float

In [6]: df1.to_csv('csv1.csv')

In [7]: df2.to_csv('csv2.csv')

In [8]: df = dd.read_csv('csv*.csv')

In [9]: df.sum().compute()
```

raises with a long traceback, including `cannot safely convert passed user dtype of <i8 for float64 dtyped data in column 1`.

This is just no integer-NA biting me, I can work around it by just reading in my data as floats I think.



Discussion:
Yeah, this came up recently with someone else.  We currently determine dtypes by looking at the first 1000 rows of the first CSV file.  This can and often does lead to incorrect results.

We used to look at many of the CSV files but this made `dd.read_csv` take a fairly long time on the pile of csvs case.  

I'm coming around to the idea that the right solution here is to catch the error at computation time and interpret it very clearly to the user  

_Hey, by looking at the first few lines of your first csv file we guessed that column: COLUMN had dtype DTYPE.  Now once we're actually reading all of the data we've found some values that don't match this dtype, probably you should add the following keyword argument to your `read_csv` call and compute again_

```
dtype={'column-name':'dtype'}
```

Or something like that, possibly more concise.

What are some other solutions?

This is something I've ran into with pandas when streaming chunks for data from Redshift to HDF5. The best I've been able to do is what you've proposed, specifying the dtypes at IO time or `.astyping`.

The other option (in this case) is to try the computation, and then retry with the type upcast. I don't think dask should do this though, since it could lead to very surprising / hard to debug performance issues.

Yeah, I'd prefer to raise an error and then have them make the change in their code explicitly.  I suppose that this needs to be added to a read_csv variant function that we call instead of `pd.read_csv`.


Resolving patch:
diff --git a/dask/dataframe/io.py b/dask/dataframe/io.py
--- a/dask/dataframe/io.py
+++ b/dask/dataframe/io.py
@@ -25,6 +25,48 @@
 
 csv_defaults = {'compression': None}
 
+
+
+def _read_csv(fn, i, chunkbytes, compression, kwargs):
+    block = textblock(fn, i*chunkbytes, (i+1) * chunkbytes, compression)
+    block = BytesIO(block)
+    try:
+        return pd.read_csv(block, **kwargs)
+    except ValueError as e:
+        msg = """
+    Dask dataframe inspected the first 1,000 rows of your csv file to guess the
+    data types of your columns.  These first 1,000 rows led us to an incorrect
+    guess.
+
+    For example a column may have had integers in the first 1000
+    rows followed by a float or missing value in the 1,001-st row.
+
+    You will need to specify some dtype information explicitly using the
+    ``dtype=`` keyword argument for the right column names and dtypes.
+
+        df = dd.read_csv(..., dtype={'my-column': float})
+
+    Pandas has given us the following error when trying to parse the file:
+
+      "%s"
+        """ % e.args[0]
+        match = re.match('cannot safely convert passed user dtype of (?P<old_dtype>\S+) for (?P<new_dtype>\S+) dtyped data in column (?P<column_number>\d+)', e.args[0])
+        if match:
+            d = match.groupdict()
+            d['column'] = kwargs['names'][int(d['column_number'])]
+            msg += """
+    From this we think that you should probably add the following column/dtype
+    pair to your dtype= dictionary
+
+    '%(column)s': '%(new_dtype)s'
+        """ % d
+
+        # TODO: add more regexes and msg logic here for other pandas errors
+        #       as apporpriate
+
+        raise ValueError(msg)
+
+
 def fill_kwargs(fn, args, kwargs):
     """ Read a csv file and fill up kwargs
 
@@ -77,15 +119,24 @@ def fill_kwargs(fn, args, kwargs):
     if 'parse_dates' not in kwargs:
         kwargs['parse_dates'] = [col for col in head.dtypes.index
                            if np.issubdtype(head.dtypes[col], np.datetime64)]
-    if 'dtype' not in kwargs:
-        kwargs['dtype'] = dict(head.dtypes)
+
+    new_dtype = dict(head.dtypes)
+    dtype = kwargs.get('dtype', dict())
+    for k, v in dict(head.dtypes).items():
+        if k not in dtype:
+            dtype[k] = v
+
+    if kwargs.get('parse_dates'):
         for col in kwargs['parse_dates']:
-            del kwargs['dtype'][col]
+            del dtype[col]
+
+    kwargs['dtype'] = dtype
 
     kwargs['columns'] = list(head.columns)
 
     return kwargs
 
+
 @wraps(pd.read_csv)
 def read_csv(fn, *args, **kwargs):
     chunkbytes = kwargs.pop('chunkbytes', 2**25)  # 50 MB
@@ -117,19 +168,16 @@ def read_csv(fn, *args, **kwargs):
         nchunks = int(ceil(total_bytes / chunkbytes))
         divisions = [None] * (nchunks + 1)
 
-        first_read_csv = partial(pd.read_csv, *args, header=header,
-                               **dissoc(kwargs, 'compression'))
-        rest_read_csv = partial(pd.read_csv, *args, header=None,
-                              **dissoc(kwargs, 'compression'))
+        first_kwargs = merge(kwargs, dict(header=header, compression=None))
+        rest_kwargs = merge(kwargs, dict(header=None, compression=None))
 
         # Create dask graph
-        dsk = dict(((name, i), (rest_read_csv, (BytesIO,
-                                   (textblock, fn,
-                                       i*chunkbytes, (i+1) * chunkbytes,
-                                       kwargs['compression']))))
+        dsk = dict(((name, i), (_read_csv, fn, i, chunkbytes,
+                                           kwargs['compression'], rest_kwargs))
                    for i in range(1, nchunks))
-        dsk[(name, 0)] = (first_read_csv, (BytesIO,
-                           (textblock, fn, 0, chunkbytes, kwargs['compression'])))
+
+        dsk[(name, 0)] = (_read_csv, fn, 0, chunkbytes, kwargs['compression'],
+                                     first_kwargs)
 
         result = DataFrame(dsk, name, columns, divisions)
 


Test patch:
diff --git a/dask/dataframe/tests/test_io.py b/dask/dataframe/tests/test_io.py
--- a/dask/dataframe/tests/test_io.py
+++ b/dask/dataframe/tests/test_io.py
@@ -639,3 +639,32 @@ def test_to_bag():
     assert ddf.to_bag(True).compute(get=get_sync) == list(a.itertuples(True))
     assert ddf.x.to_bag(True).compute(get=get_sync) == list(a.x.iteritems())
     assert ddf.x.to_bag().compute(get=get_sync) == list(a.x)
+
+
+def test_csv_expands_dtypes():
+    with filetext(text) as fn:
+        a = read_csv(fn, chunkbytes=30, dtype={})
+        a_kwargs = list(a.dask.values())[0][-1]
+
+        b = read_csv(fn, chunkbytes=30)
+        b_kwargs = list(b.dask.values())[0][-1]
+
+        assert a_kwargs['dtype'] == b_kwargs['dtype']
+
+        a = read_csv(fn, chunkbytes=30, dtype={'amount': float})
+        a_kwargs = list(a.dask.values())[0][-1]
+
+        assert a_kwargs['dtype']['amount'] == float
+
+
+def test_report_dtype_correction_on_csvs():
+    text = 'numbers,names\n'
+    for i in range(1000):
+        text += '1,foo\n'
+    text += '1.5,bar\n'
+    with filetext(text) as fn:
+        try:
+            dd.read_csv(fn).compute(get=get_sync)
+            assert False
+        except ValueError as e:
+            assert "'numbers': 'float64'" in str(e)