Repository: dask/dask

Issue:
Incorrect parsing/coercing of invalid timestamps
Dask does not appear to handle out of bounds dates very well:

```
# minimal-0.csv
id,date
1,2017-10-31 00:00:00

# minimal-1.csv - out of bounds date/timestamp
id,date
2,4998-01-01 00:00:00
```

```python
    import dask.dataframe as ddf

    df = ddf.read_csv('minimal-*.csv', dtype={'id': 'int',
                                              'date': 'str'},
                      parse_dates=['date']).compute()
```

For some reason, this doesn't fail, but produces incorrect timestamps
```
df
Out[1]: 
   id                          date
0   1 2017-10-31 00:00:00.000000000
0   2 2075-03-26 02:07:11.452241920
df.dtypes
Out[2]: 
id               int64
date    datetime64[ns]
dtype: object
```

Pandas will ignore the failing `parse_dates` directive and return an `object` dtype. `pd.to_datetime` will throw a `pandas._libs.tslib.OutOfBoundsDatetime` error as expected. 

Is it possible to use `pd.to_datetime(.., error='coerce')` with Dask (which would return `NaT` in this case?).

```
pandas (0.21.0rc1)
distributed (1.19.3)
numpy (1.13.3)
dask (0.15.4) 
```


Discussion:
`ddf.read_csv('minimal-*.csv', parse_dates=['date']).head(n=2)` seems to just drop the row. How is this impacted by the distributed scheduler? The source CSV (not the minimal example) has ~300M rows with ~100 dodgy dates.
Unfortunately, I don't think there's much dask can. It'd be nice if pandas had an option to throw an error when the CSV parser encounters an out of bounds value.

As a workaround:

```python
df = pd.read_csv("minimal-*.csv', dtype={'id': int})
df['date'] = df['date'].map_partitions(pd.to_datetime, meta=('date', 'datetime64[ns]'))
```

will (correctly) thrown an exception. And you can include `errors='coerce'` to coerce to NaT and then drop those.

Resolving patch:
diff --git a/dask/dataframe/io/csv.py b/dask/dataframe/io/csv.py
--- a/dask/dataframe/io/csv.py
+++ b/dask/dataframe/io/csv.py
@@ -23,9 +23,11 @@
 from .io import from_delayed
 
 if PANDAS_VERSION >= '0.20.0':
-    from pandas.api.types import is_integer_dtype, is_float_dtype
+    from pandas.api.types import (is_integer_dtype, is_float_dtype,
+                                  is_object_dtype, is_datetime64_any_dtype)
 else:
-    from pandas.types.common import is_integer_dtype, is_float_dtype
+    from pandas.types.common import (is_integer_dtype, is_float_dtype,
+                                     is_object_dtype, is_datetime64_any_dtype)
 
 
 delayed = delayed(pure=True)
@@ -78,22 +80,29 @@ def coerce_dtypes(df, dtypes):
     df: Pandas DataFrame
     dtypes: dict like {'x': float}
     """
-    bad = []
+    bad_dtypes = []
+    bad_dates = []
     errors = []
     for c in df.columns:
         if c in dtypes and df.dtypes[c] != dtypes[c]:
             actual = df.dtypes[c]
             desired = dtypes[c]
             if is_float_dtype(actual) and is_integer_dtype(desired):
-                bad.append((c, actual, desired))
+                bad_dtypes.append((c, actual, desired))
+            elif is_object_dtype(actual) and is_datetime64_any_dtype(desired):
+                # This can only occur when parse_dates is specified, but an
+                # invalid date is encountered. Pandas then silently falls back
+                # to object dtype. Since `object_array.astype(datetime)` will
+                # silently overflow, error here and report.
+                bad_dates.append(c)
             else:
                 try:
                     df[c] = df[c].astype(dtypes[c])
                 except Exception as e:
-                    bad.append((c, actual, desired))
+                    bad_dtypes.append((c, actual, desired))
                     errors.append((c, e))
 
-    if bad:
+    if bad_dtypes:
         if errors:
             ex = '\n'.join("- %s\n  %r" % (c, e) for c, e in
                            sorted(errors, key=lambda x: str(x[0])))
@@ -107,21 +116,41 @@ def coerce_dtypes(df, dtypes):
                      "to interpret\n"
                      "all unspecified integer columns as floats.")
 
-        bad = sorted(bad, key=lambda x: str(x[0]))
-        table = asciitable(['Column', 'Found', 'Expected'], bad)
+        bad_dtypes = sorted(bad_dtypes, key=lambda x: str(x[0]))
+        table = asciitable(['Column', 'Found', 'Expected'], bad_dtypes)
         dtype_kw = ('dtype={%s}' % ',\n'
-                    '       '.join("%r: '%s'" % (k, v) for (k, v, _) in bad))
-
-        msg = ("Mismatched dtypes found in `pd.read_csv`/`pd.read_table`.\n"
-               "\n"
-               "{table}\n"
-               "\n{exceptions}"
-               "Usually this is due to dask's dtype inference failing, and\n"
-               "*may* be fixed by specifying dtypes manually by adding:\n\n"
-               "{dtype_kw}\n\n"
-               "to the call to `read_csv`/`read_table`."
-               "{extra}").format(table=table, exceptions=exceptions,
-                                 dtype_kw=dtype_kw, extra=extra)
+                    '       '.join("%r: '%s'" % (k, v)
+                                   for (k, v, _) in bad_dtypes))
+
+        dtype_msg = (
+            "{table}\n\n"
+            "{exceptions}"
+            "Usually this is due to dask's dtype inference failing, and\n"
+            "*may* be fixed by specifying dtypes manually by adding:\n\n"
+            "{dtype_kw}\n\n"
+            "to the call to `read_csv`/`read_table`."
+            "{extra}").format(table=table, exceptions=exceptions,
+                              dtype_kw=dtype_kw, extra=extra)
+    else:
+        dtype_msg = None
+
+    if bad_dates:
+        also = " also " if bad_dtypes else " "
+        cols = '\n'.join("- %s" % c for c in bad_dates)
+        date_msg = (
+            "The following columns{also}failed to properly parse as dates:\n\n"
+            "{cols}\n\n"
+            "This is usually due to an invalid value in that column. To\n"
+            "diagnose and fix it's recommended to drop these columns from the\n"
+            "`parse_dates` keyword, and manually convert them to dates later\n"
+            "using `dd.to_datetime`.").format(also=also, cols=cols)
+    else:
+        date_msg = None
+
+    if bad_dtypes or bad_dates:
+        rule = "\n\n%s\n\n" % ('-' * 61)
+        msg = ("Mismatched dtypes found in `pd.read_csv`/`pd.read_table`.\n\n"
+               "%s" % (rule.join(filter(None, [dtype_msg, date_msg]))))
         raise ValueError(msg)
 
 


Test patch:
diff --git a/dask/dataframe/io/tests/test_csv.py b/dask/dataframe/io/tests/test_csv.py
--- a/dask/dataframe/io/tests/test_csv.py
+++ b/dask/dataframe/io/tests/test_csv.py
@@ -564,15 +564,26 @@ def test_read_csv_of_modified_file_has_different_name():
 
 
 def test_late_dtypes():
-    text = 'numbers,names,more_numbers,integers\n'
+    text = 'numbers,names,more_numbers,integers,dates\n'
     for i in range(1000):
-        text += '1,,2,3\n'
-    text += '1.5,bar,2.5,3\n'
+        text += '1,,2,3,2017-10-31 00:00:00\n'
+    text += '1.5,bar,2.5,3,4998-01-01 00:00:00\n'
+
+    date_msg = ("\n"
+                "\n"
+                "-------------------------------------------------------------\n"
+                "\n"
+                "The following columns also failed to properly parse as dates:\n"
+                "\n"
+                "- dates\n"
+                "\n"
+                "This is usually due to an invalid value in that column. To\n"
+                "diagnose and fix it's recommended to drop these columns from the\n"
+                "`parse_dates` keyword, and manually convert them to dates later\n"
+                "using `dd.to_datetime`.")
+
     with filetext(text) as fn:
         sol = pd.read_csv(fn)
-        with pytest.raises(ValueError) as e:
-            dd.read_csv(fn, sample=50).compute(get=dask.get)
-
         msg = ("Mismatched dtypes found in `pd.read_csv`/`pd.read_table`.\n"
                "\n"
                "+--------------+---------+----------+\n"
@@ -593,13 +604,16 @@ def test_late_dtypes():
                "       'names': 'object',\n"
                "       'numbers': 'float64'}\n"
                "\n"
-               "to the call to `read_csv`/`read_table`.\n")
-
-        e.match(msg)
+               "to the call to `read_csv`/`read_table`.")
 
         with pytest.raises(ValueError) as e:
             dd.read_csv(fn, sample=50,
-                        dtype={'names': 'O'}).compute(get=dask.get)
+                        parse_dates=['dates']).compute(get=dask.get)
+        assert e.match(msg + date_msg)
+
+        with pytest.raises(ValueError) as e:
+            dd.read_csv(fn, sample=50).compute(get=dask.get)
+        assert e.match(msg)
 
         msg = ("Mismatched dtypes found in `pd.read_csv`/`pd.read_table`.\n"
                "\n"
@@ -621,6 +635,31 @@ def test_late_dtypes():
                "Alternatively, provide `assume_missing=True` to interpret\n"
                "all unspecified integer columns as floats.")
 
+        with pytest.raises(ValueError) as e:
+            dd.read_csv(fn, sample=50,
+                        dtype={'names': 'O'}).compute(get=dask.get)
+        assert str(e.value) == msg
+
+        with pytest.raises(ValueError) as e:
+            dd.read_csv(fn, sample=50, parse_dates=['dates'],
+                        dtype={'names': 'O'}).compute(get=dask.get)
+        assert str(e.value) == msg + date_msg
+
+        msg = ("Mismatched dtypes found in `pd.read_csv`/`pd.read_table`.\n"
+               "\n"
+               "The following columns failed to properly parse as dates:\n"
+               "\n"
+               "- dates\n"
+               "\n"
+               "This is usually due to an invalid value in that column. To\n"
+               "diagnose and fix it's recommended to drop these columns from the\n"
+               "`parse_dates` keyword, and manually convert them to dates later\n"
+               "using `dd.to_datetime`.")
+
+        with pytest.raises(ValueError) as e:
+            dd.read_csv(fn, sample=50, parse_dates=['dates'],
+                        dtype={'more_numbers': float, 'names': object,
+                               'numbers': float}).compute(get=dask.get)
         assert str(e.value) == msg
 
         # Specifying dtypes works