Repository: dask/dask

Issue:
ENH: fastparquet read of many files without scanning
(I intend to implement this, this issue is a placeholder until I have the time)

Currently, reading from a list of filenames, i.e., without a `_metadata` file, with fastparquet requires scanning of the footers of all of the files in the client, which can be a substantial time burden. It occurs to me that it is possible simply to assume that all of the filenames will have the same schema; the required row-group objects could then be constructed from the filenames either in the client (this is the minimum code solution) or in the delayed functions (this will be more efficient).

The cost is: loss of index ordering and divisions, and the ability to do any row-group statistics-based filtering. The total row count is not used by Dask anyway. Also, if there is a schema mismatch, we would only error upon compute, not early as now.


Discussion:

Resolving patch:
diff --git a/dask/dataframe/io/parquet.py b/dask/dataframe/io/parquet.py
--- a/dask/dataframe/io/parquet.py
+++ b/dask/dataframe/io/parquet.py
@@ -1,6 +1,7 @@
 from __future__ import absolute_import, division, print_function
 
 import re
+from collections import OrderedDict
 import copy
 import json
 import os
@@ -176,12 +177,16 @@ def _normalize_index_columns(user_columns, data_columns, user_index, data_index)
 def _read_fastparquet(fs, fs_token, paths, columns=None, filters=None,
                       categories=None, index=None, infer_divisions=None):
     import fastparquet
-    from fastparquet.util import check_column_names
 
     if isinstance(paths,fastparquet.api.ParquetFile):
         pf = paths
     elif len(paths) > 1:
-        pf = fastparquet.ParquetFile(paths, open_with=fs.open, sep=fs.sep)
+        if infer_divisions:
+            # this scans all the files, allowing index/divisions and filtering
+            pf = fastparquet.ParquetFile(paths, open_with=fs.open, sep=fs.sep)
+        else:
+            return _read_fp_multifile(fs, fs_token, paths, columns=None,
+                                      categories=None, index=None)
     else:
         try:
             pf = fastparquet.ParquetFile(paths[0] + fs.sep + '_metadata',
@@ -192,16 +197,80 @@ def _read_fastparquet(fs, fs_token, paths, columns=None, filters=None,
 
     # Validate infer_divisions
     if os.path.split(pf.fn)[-1] != '_metadata' and infer_divisions is True:
-        raise NotImplementedError("infer_divisions=True is not supported by the fastparquet engine for datasets "
-                                  "that do not contain a global '_metadata' file")
+        raise NotImplementedError("infer_divisions=True is not supported by the "
+                                  "fastparquet engine for datasets that "
+                                  "do not contain a global '_metadata' file")
 
+    (meta, filters, index_name, out_type, all_columns, index_names,
+     storage_name_mapping) = _pf_validation(
+        pf, columns, index, categories, filters)
+    rgs = [rg for rg in pf.row_groups if
+           not (fastparquet.api.filter_out_stats(rg, filters, pf.schema)) and
+           not (fastparquet.api.filter_out_cats(rg, filters))]
+
+    name = 'read-parquet-' + tokenize(fs_token, paths, all_columns, filters,
+                                      categories)
+    dsk = {(name, i): (_read_parquet_row_group, fs, pf.row_group_filename(rg),
+                       index_names, all_columns, rg, out_type == Series,
+                       categories, pf.schema, pf.cats, pf.dtypes,
+                       pf.file_scheme, storage_name_mapping)
+           for i, rg in enumerate(rgs)}
+    if not dsk:
+        # empty dataframe
+        dsk = {(name, 0): meta}
+        divisions = (None, None)
+        return out_type(dsk, name, meta, divisions)
+
+    if index_names and infer_divisions is not False:
+        index_name = meta.index.name
+        try:
+            # is https://github.com/dask/fastparquet/pull/371 available in
+            # current fastparquet installation?
+            minmax = fastparquet.api.sorted_partitioned_columns(pf, filters)
+        except TypeError:
+            minmax = fastparquet.api.sorted_partitioned_columns(pf)
+        if index_name in minmax:
+            divisions = minmax[index_name]
+            divisions = divisions['min'] + [divisions['max'][-1]]
+        else:
+            if infer_divisions is True:
+                raise ValueError(
+                    ("Unable to infer divisions for index of '{index_name}'"
+                     " because it is not known to be "
+                     "sorted across partitions").format(index_name=index_name))
+
+            divisions = (None,) * (len(rgs) + 1)
+    else:
+        if infer_divisions is True:
+            raise ValueError(
+                'Unable to infer divisions for because no index column'
+                ' was discovered')
+
+        divisions = (None,) * (len(rgs) + 1)
+
+    if isinstance(divisions[0], np.datetime64):
+        divisions = [pd.Timestamp(d) for d in divisions]
+
+    return out_type(dsk, name, meta, divisions)
+
+
+def _pf_validation(pf, columns, index, categories, filters):
+    """Validate user options against metadata in dataset
+
+     columns, index and categories must be in the list of columns available
+     (both data columns and path-based partitioning - subject to possible
+     renaming, if pandas metadata is present). The output index will
+     be inferred from any available pandas metadata, if not given.
+     """
+    from fastparquet.util import check_column_names
     check_column_names(pf.columns, categories)
     if isinstance(columns, tuple):
         # ensure they tokenize the same
         columns = list(columns)
 
     if pf.fmd.key_value_metadata:
-        pandas_md = [x.value for x in pf.fmd.key_value_metadata if x.key == 'pandas']
+        pandas_md = [x.value for x in pf.fmd.key_value_metadata
+                     if x.key == 'pandas']
     else:
         pandas_md = []
 
@@ -239,14 +308,11 @@ def _read_fastparquet(fs, fs_token, paths, columns=None, filters=None,
     all_columns = list(column_names)
     all_columns.extend(x for x in index_names if x not in column_names)
 
-    rgs = [rg for rg in pf.row_groups if
-           not (fastparquet.api.filter_out_stats(rg, filters, pf.schema)) and
-           not (fastparquet.api.filter_out_cats(rg, filters))]
-
     dtypes = pf._dtypes(categories)
     dtypes = {storage_name_mapping.get(k, k): v for k, v in dtypes.items()}
 
     meta = _meta_from_dtypes(all_columns, dtypes, index_names, [None])
+
     # fastparquet doesn't handle multiindex
     if len(index_names) > 1:
         raise ValueError("Cannot read DataFrame with MultiIndex.")
@@ -268,50 +334,150 @@ def _read_fastparquet(fs, fs_token, paths, columns=None, filters=None,
     if out_type == Series:
         assert len(meta.columns) == 1
         meta = meta[meta.columns[0]]
-
-    name = 'read-parquet-' + tokenize(fs_token, paths, all_columns, filters,
+    return (meta, filters, index_names, out_type, all_columns, index_names,
+            storage_name_mapping)
+
+
+def _read_fp_multifile(fs, fs_token, paths, columns=None,
+                       categories=None, index=None):
+    """Read dataset with fastparquet by assuming metadata from first file"""
+    from fastparquet import ParquetFile
+    from fastparquet.util import analyse_paths, get_file_scheme
+    base, fns = analyse_paths(paths)
+    scheme = get_file_scheme(fns)
+    pf = ParquetFile(paths[0], open_with=fs.open)
+    pf.file_scheme = scheme
+    pf.cats = _paths_to_cats(fns, scheme)
+    (meta, _, index_name, out_type, all_columns, index_names,
+     storage_name_mapping) = _pf_validation(
+        pf, columns, index, categories, [])
+    name = 'read-parquet-' + tokenize(fs_token, paths, all_columns,
                                       categories)
-
-    dsk = {(name, i): (_read_parquet_row_group, fs, pf.row_group_filename(rg),
-                       index_names, all_columns, rg, out_type == Series,
-                       categories, pf.schema, pf.cats, pf.dtypes,
+    dsk = {(name, i): (_read_pf_simple, fs, path, base,
+                       index_names, all_columns, out_type == Series,
+                       categories, pf.cats,
                        pf.file_scheme, storage_name_mapping)
-           for i, rg in enumerate(rgs)}
-    if not dsk:
-        # empty dataframe
-        dsk = {(name, 0): meta}
-        divisions = (None, None)
-        return out_type(dsk, name, meta, divisions)
+           for i, path in enumerate(paths)}
+    divisions = (None, ) * (len(paths) + 1)
+    return out_type(dsk, name, meta, divisions)
 
-    if index_names and infer_divisions is not False:
-        index_name = meta.index.name
-        try:
-            # is https://github.com/dask/fastparquet/pull/371 available in
-            # current fastparquet installation?
-            minmax = fastparquet.api.sorted_partitioned_columns(pf, filters)
-        except TypeError:
-            minmax = fastparquet.api.sorted_partitioned_columns(pf)
-        if index_name in minmax:
-            divisions = minmax[index_name]
-            divisions = divisions['min'] + [divisions['max'][-1]]
-        else:
-            if infer_divisions is True:
-                raise ValueError(
-                    ("Unable to infer divisions for index of '{index_name}' because it is not known to be "
-                     "sorted across partitions").format(index_name=index_name))
 
-            divisions = (None,) * (len(rgs) + 1)
+def _read_pf_simple(fs, path, base, index_names, all_columns, is_series,
+                    categories, cats, scheme, storage_name_mapping):
+    """Read dataset with fastparquet using ParquetFile machinery"""
+    from fastparquet import ParquetFile
+    pf = ParquetFile(path, open_with=fs.open)
+    relpath = path.replace(base, '').lstrip('/')
+    for rg in pf.row_groups:
+        for ch in rg.columns:
+            ch.file_path = relpath
+    pf.file_scheme = scheme
+    pf.cats = cats
+    pf.fn = base
+    df = pf.to_pandas(all_columns, categories, index=index_names)
+    if df.index.nlevels == 1:
+        if index_names:
+            df.index.name = storage_name_mapping.get(index_names[0],
+                                                     index_names[0])
     else:
-        if infer_divisions is True:
-            raise ValueError(
-                'Unable to infer divisions for because no index column was discovered')
+        if index_names:
+            df.index.names = [storage_name_mapping.get(name, name)
+                              for name in index_names]
+    df.columns = [storage_name_mapping.get(col, col)
+                  for col in all_columns
+                  if col not in (index_names or [])]
 
-        divisions = (None,) * (len(rgs) + 1)
+    if is_series:
+        return df[df.columns[0]]
+    else:
+        return df
 
-    if isinstance(divisions[0], np.datetime64):
-        divisions = [pd.Timestamp(d) for d in divisions]
 
-    return out_type(dsk, name, meta, divisions)
+def _paths_to_cats(paths, scheme):
+    """Extract out fields and labels from directory names"""
+    # can be factored out in fastparquet
+    from fastparquet.util import ex_from_sep, val_to_num, groupby_types
+    cats = OrderedDict()
+    raw_cats = OrderedDict()
+
+    for path in paths:
+        s = ex_from_sep('/')
+        if scheme == 'hive':
+            partitions = s.findall(path)
+            for key, val in partitions:
+                cats.setdefault(key, set()).add(val_to_num(val))
+                raw_cats.setdefault(key, set()).add(val)
+        else:
+            for i, val in enumerate(path.split('/')[:-1]):
+                key = 'dir%i' % i
+                cats.setdefault(key, set()).add(val_to_num(val))
+                raw_cats.setdefault(key, set()).add(val)
+
+    for key, v in cats.items():
+        # Check that no partition names map to the same value after
+        # transformation by val_to_num
+        raw = raw_cats[key]
+        if len(v) != len(raw):
+            conflicts_by_value = OrderedDict()
+            for raw_val in raw_cats[key]:
+                conflicts_by_value.setdefault(val_to_num(raw_val),
+                                              set()).add(raw_val)
+            conflicts = [c for k in conflicts_by_value.values()
+                         if len(k) > 1 for c in k]
+            raise ValueError("Partition names map to the same value: %s"
+                             % conflicts)
+        vals_by_type = groupby_types(v)
+
+        # Check that all partition names map to the same type after
+        # transformation by val_to_num
+        if len(vals_by_type) > 1:
+            import warnings
+            examples = [x[0] for x in vals_by_type.values()]
+            warnings.warn("Partition names coerce to values of different"
+                          " types, e.g. %s" % examples)
+    return {k: list(v) for k, v in cats.items()}
+
+
+def _read_parquet_file(fs, base, fn, index, columns, series, categories,
+                       cs, dt, scheme, storage_name_mapping, *args):
+    """Read a single file with fastparquet, to be used in a task"""
+    from fastparquet.api import ParquetFile
+    from collections import OrderedDict
+
+    name_storage_mapping = {v: k for k, v in storage_name_mapping.items()}
+    if not isinstance(columns, (tuple, list)):
+        columns = [columns,]
+        series = True
+    if index:
+        index, = index
+        if index not in columns:
+            columns = columns + [index]
+    columns = [name_storage_mapping.get(col, col) for col in columns]
+    index = name_storage_mapping.get(index, index)
+    cs = OrderedDict([(k, v) for k, v in cs.items() if k in columns])
+    pf = ParquetFile(fn, open_with=fs.open)
+    pf.file_scheme = scheme
+    for rg in pf.row_groups:
+        for ch in rg.columns:
+            ch.file_path = fn.replace(base, "").lstrip('/')
+    pf.fn = base
+    df = pf.to_pandas(columns=columns, index=index, categories=categories)
+
+    if df.index.nlevels == 1:
+        if index:
+            df.index.name = storage_name_mapping.get(index, index)
+    else:
+        if index:
+            df.index.names = [storage_name_mapping.get(name, name)
+                              for name in index]
+    df.columns = [storage_name_mapping.get(col, col)
+                  for col in columns
+                  if col != index]
+
+    if series:
+        return df[df.columns[0]]
+    else:
+        return df
 
 
 def _read_parquet_row_group(fs, fn, index, columns, rg, series, categories,


Test patch:
diff --git a/dask/dataframe/io/tests/test_parquet.py b/dask/dataframe/io/tests/test_parquet.py
--- a/dask/dataframe/io/tests/test_parquet.py
+++ b/dask/dataframe/io/tests/test_parquet.py
@@ -187,12 +187,13 @@ def test_read_glob(tmpdir, write_engine, read_engine):
 
     # Infer divisions for engines/versions that support it
 
-    ddf2 = dd.read_parquet(os.path.join(fn, '*'), engine=read_engine,
+    ddf2 = dd.read_parquet(os.path.join(fn, '*.parquet'), engine=read_engine,
                            infer_divisions=should_check_divs(write_engine) and should_check_divs(read_engine))
     assert_eq(ddf, ddf2, check_divisions=should_check_divs(write_engine) and should_check_divs(read_engine))
 
     # No divisions
-    ddf2_no_divs = dd.read_parquet(os.path.join(fn, '*'), engine=read_engine, infer_divisions=False)
+    ddf2_no_divs = dd.read_parquet(os.path.join(fn, '*.parquet'),
+                                   engine=read_engine, infer_divisions=False)
     assert_eq(ddf.clear_divisions(), ddf2_no_divs, check_divisions=True)