Repository: dask/dask

Issue:
read_parquet does not handle timezones
Dask DataFrame method `read_parquet` does not read parquet timezone metadata (despite fastparquet handling this metadata), except if the file was written with fastparquet _and_ is being read with pyarrow.

This works:
```
import dask.dataframe as dd
import pandas as pd

df = pd.DataFrame([[0]], columns=['a'], dtype='datetime64[ns, UTC]')
df.to_parquet('tmp1.parquet', engine='fastparquet')
df = dd.read_parquet('tmp1.parquet', engine='pyarrow')
df.dtypes
# a    datetime64[ns, UTC]
# dtype: object

# But any other permutation of engines results in:
# a    datetime64[ns]
# dtype: object
```

`to_parquet` does correctly write timezones, for any permutation of read/write engines:
```
df.astype('datetime64[ns, UTC]').to_parquet('tmp2.parquet', engine='fastparquet')
pd.read_parquet('tmp2.parquet/part.0.parquet', engine='fastparquet').dtypes
# a    datetime64[ns, UTC]
# dtype: object
```


Discussion:
@martindurant any thoughts here?

@bchu does this also happen when using `engine='pyarrow'`?
I've updated the comment. `dd.to_parquet` actually does work. `dd.read_parquet` only works with pyarrow, and only when the original Pandas dataframe is written with fastparquet.
Hah, that is a surprise! Metadata handling for parquet is something we've been talking about elsewhere also and could probably do with an overhaul. In the meantime, could you please check that the timezone information does roundtrip successfully when using fastparquet directly? I seem to think there is such a test.
Do you mean without using Dask? Using fastparquet directly does correctly roundtrip (both as an engine in pandas, and if I do `fastparquet.write` / `fastparquet.ParquetFile(...).to_pandas()`).
OK, thanks for testing that - then the bug is in Dask alone. I'll try to look into it.
For the fastparquet arm, I can fix this in fastparquet alone. That PR: https://github.com/dask/fastparquet/pull/411
(and fastparquet is due for a release anyway)

Resolving patch:
diff --git a/dask/bytes/http.py b/dask/bytes/http.py
--- a/dask/bytes/http.py
+++ b/dask/bytes/http.py
@@ -165,7 +165,7 @@ def read(self, length=-1):
             # asked for no data, so supply no data and shortcut doing work
             return b''
         if self.size is None:
-            if length >= 0:
+            if length >= 0:  # pragma: no cover
                 # asked for specific amount of data, but we don't know how
                 # much is available
                 raise ValueError('File size is unknown, must read all data')
diff --git a/dask/dataframe/io/parquet.py b/dask/dataframe/io/parquet.py
--- a/dask/dataframe/io/parquet.py
+++ b/dask/dataframe/io/parquet.py
@@ -213,7 +213,8 @@ def _read_fastparquet(fs, fs_token, paths, columns=None, filters=None,
     dsk = {(name, i): (_read_parquet_row_group, fs, pf.row_group_filename(rg),
                        index_names, all_columns, rg, out_type == Series,
                        categories, pf.schema, pf.cats, pf.dtypes,
-                       pf.file_scheme, storage_name_mapping)
+                       pf.file_scheme, storage_name_mapping,
+                       getattr(pf, 'tz', {}))
            for i, rg in enumerate(rgs)}
     if not dsk:
         # empty dataframe
@@ -483,7 +484,8 @@ def _read_parquet_file(fs, base, fn, index, columns, series, categories,
 
 
 def _read_parquet_row_group(fs, fn, index, columns, rg, series, categories,
-                            schema, cs, dt, scheme, storage_name_mapping, *args):
+                            schema, cs, dt, scheme, storage_name_mapping, tz,
+                            *args):
     from fastparquet.api import _pre_allocate
     from fastparquet.core import read_row_group_file
     from collections import OrderedDict
@@ -501,7 +503,8 @@ def _read_parquet_row_group(fs, fn, index, columns, rg, series, categories,
     index = name_storage_mapping.get(index, index)
     cs = OrderedDict([(k, v) for k, v in cs.items() if k in columns])
 
-    df, views = _pre_allocate(rg.num_rows, columns, categories, index, cs, dt)
+    df, views = _pre_allocate(rg.num_rows, columns, categories, index, cs, dt,
+                              tz)
     read_row_group_file(fn, rg, columns, categories, schema, cs,
                         open=fs.open, assign=views, scheme=scheme)
 
diff --git a/dask/dataframe/io/utils.py b/dask/dataframe/io/utils.py
--- a/dask/dataframe/io/utils.py
+++ b/dask/dataframe/io/utils.py
@@ -10,8 +10,15 @@ def _get_pyarrow_dtypes(schema, categories):
     has_pandas_metadata = schema.metadata is not None and b'pandas' in schema.metadata
     if has_pandas_metadata:
         pandas_metadata = json.loads(schema.metadata[b'pandas'].decode('utf8'))
-        pandas_metadata_dtypes = {c.get('field_name', c.get('name', None)): c['numpy_type']
-                                  for c in pandas_metadata.get('columns', [])}
+        pandas_metadata_dtypes = {
+            c.get('field_name', c.get('name', None)):
+                c['numpy_type']
+            for c in pandas_metadata.get('columns', [])}
+        tz = {
+            c.get('field_name', c.get('name', None)):
+                c['metadata'].get('timezone', None)
+            for c in pandas_metadata.get('columns', []) if c['metadata']
+        }
     else:
         pandas_metadata_dtypes = {}
 
@@ -21,7 +28,11 @@ def _get_pyarrow_dtypes(schema, categories):
 
         # Get numpy_dtype from pandas metadata if available
         if field.name in pandas_metadata_dtypes:
-            numpy_dtype = pandas_metadata_dtypes[field.name]
+            if field.name in tz:
+                numpy_dtype = pd.Series([], dtype='M8[ns]').dt.tz_localize(
+                    tz[field.name]).dtype
+            else:
+                numpy_dtype = pandas_metadata_dtypes[field.name]
         else:
             numpy_dtype = field.type.to_pandas_dtype()
 


Test patch:
diff --git a/dask/dataframe/io/tests/test_parquet.py b/dask/dataframe/io/tests/test_parquet.py
--- a/dask/dataframe/io/tests/test_parquet.py
+++ b/dask/dataframe/io/tests/test_parquet.py
@@ -1322,6 +1322,20 @@ def test_select_partitioned_column(tmpdir, engine):
     df_partitioned[df_partitioned.fake_categorical1 == 'A'].compute()
 
 
+def test_with_tz(tmpdir, engine):
+    if engine == 'pyarrow' and pa.__version__ < LooseVersion('0.11.0'):
+        pytest.skip("pyarrow<0.11.0 did not support this")
+    if engine == 'fastparquet' and fastparquet.__version__ < LooseVersion(
+            '0.3.0'):
+        pytest.skip("fastparquet<0.3.0 did not support this")
+    fn = str(tmpdir)
+    df = pd.DataFrame([[0]], columns=['a'], dtype='datetime64[ns, UTC]')
+    df = dd.from_pandas(df, 1)
+    df.to_parquet(fn, engine=engine)
+    df2 = dd.read_parquet(fn, engine=engine)
+    assert_eq(df, df2, check_divisions=False, check_index=False)
+
+
 def test_arrow_partitioning(tmpdir):
     # Issue #3518
     pytest.importorskip('pyarrow')