Repository: dask/dask

Issue:
read_parquet fails with given filters but no common metadata for dataset
Reading in multiple Parquet files with a single call to Dask 2.22.0's `read_parquet` fails unless we read in common metadata. If we call `read_parquet` on a partitioned dataset, reading works when the directory containing the dataset and metadata file is supplied.
```python
In [1]: import pandas as pd                                                                                                               

In [2]: import dask.dataframe as dd

In [3]: df = pd.DataFrame({"x": range(10), "y": list("aabbccddee")})
   ...: ddf = dd.from_pandas(df, npartitions=5)

In [4]: ddf.to_parquet("t.parquet")
/datasets/cwinston/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/pyarrow/compat.py:24: FutureWarning: pyarrow.compat has been deprecated and will be removed in a future release
  warnings.warn("pyarrow.compat has been deprecated and will be removed in a "

In [5]: dd.read_parquet("t.parquet/*parquet")
Out[5]: 
Dask DataFrame Structure:
                   x       y
npartitions=5               
               int64  object
                 ...     ...
...              ...     ...
                 ...     ...
                 ...     ...
Dask Name: read-parquet, 5 tasks
```
However, when we read in a set of files without common metadata, either using a wildcard or using a list of files, we get an error.
```python
In [6]: dd.read_parquet("t.parquet/*parquet", filters=[("x",">","4")])                                                                                 
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-6-c9834dcbe354> in <module>
----> 1 dd.read_parquet("t.parquet/*parquet", filters=[("x",">","4")])

~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/dask/dataframe/io/parquet/core.py in read_parquet(path, columns, filters, categories, index, storage_options, engine, gather_statistics, split_row_groups, chunksize, **kwargs)
    229         index = [index]
    230 
--> 231     meta, statistics, parts, index = engine.read_metadata(
    232         fs,
    233         paths,

~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/dask/dataframe/io/parquet/arrow.py in read_metadata(cls, fs, paths, categories, index, gather_statistics, filters, split_row_groups, **kwargs)
    648             split_row_groups,
    649             gather_statistics,
--> 650         ) = _gather_metadata(
    651             paths, fs, split_row_groups, gather_statistics, filters, dataset_kwargs
    652         )

~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/dask/dataframe/io/parquet/arrow.py in _gather_metadata(paths, fs, split_row_groups, gather_statistics, filters, dataset_kwargs)
    159 
    160     # Step 1: Create a ParquetDataset object
--> 161     dataset, base, fns = _get_dataset_object(paths, fs, filters, dataset_kwargs)
    162     if fns == [None]:
    163         # This is a single file. No danger in gathering statistics

~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/dask/dataframe/io/parquet/arrow.py in _get_dataset_object(paths, fs, filters, dataset_kwargs)
    118         # Create our dataset from the list of data files.
    119         # Note that this will not parse all the files (yet)
--> 120         dataset = pq.ParquetDataset(
    121             paths, filesystem=fs, filters=filters, **dataset_kwargs
    122         )

~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/pyarrow/parquet.py in __init__(self, path_or_paths, filesystem, schema, metadata, split_row_groups, validate_schema, filters, metadata_nthreads, read_dictionary, memory_map, buffer_size, partitioning, use_legacy_dataset)
   1195         if filters is not None:
   1196             filters = _check_filters(filters)
-> 1197             self._filter(filters)
   1198 
   1199         if validate_schema:

~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/pyarrow/parquet.py in _filter(self, filters)
   1309 
   1310     def _filter(self, filters):
-> 1311         accepts_filter = self.partitions.filter_accepts_partition
   1312 
   1313         def one_filter_accepts(piece, filter):

AttributeError: 'NoneType' object has no attribute 'filter_accepts_partition'
```
This also fails when a list of files is provided.
```python
In [8]: dd.read_parquet(["t.parquet/part.0.parquet", "t.parquet/part.1.parquet"], filters=[("x",">","4")])                                             
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-8-d3b740206c89> in <module>
----> 1 dd.read_parquet(["t.parquet/part.0.parquet", "t.parquet/part.1.parquet"], filters=[("x",">","4")])

~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/dask/dataframe/io/parquet/core.py in read_parquet(path, columns, filters, categories, index, storage_options, engine, gather_statistics, split_row_groups, chunksize, **kwargs)
    229         index = [index]
    230 
--> 231     meta, statistics, parts, index = engine.read_metadata(
    232         fs,
    233         paths,

~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/dask/dataframe/io/parquet/arrow.py in read_metadata(cls, fs, paths, categories, index, gather_statistics, filters, split_row_groups, **kwargs)
    648             split_row_groups,
    649             gather_statistics,
--> 650         ) = _gather_metadata(
    651             paths, fs, split_row_groups, gather_statistics, filters, dataset_kwargs
    652         )

~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/dask/dataframe/io/parquet/arrow.py in _gather_metadata(paths, fs, split_row_groups, gather_statistics, filters, dataset_kwargs)
    159 
    160     # Step 1: Create a ParquetDataset object
--> 161     dataset, base, fns = _get_dataset_object(paths, fs, filters, dataset_kwargs)
    162     if fns == [None]:
    163         # This is a single file. No danger in gathering statistics

~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/dask/dataframe/io/parquet/arrow.py in _get_dataset_object(paths, fs, filters, dataset_kwargs)
    118         # Create our dataset from the list of data files.
    119         # Note that this will not parse all the files (yet)
--> 120         dataset = pq.ParquetDataset(
    121             paths, filesystem=fs, filters=filters, **dataset_kwargs
    122         )

~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/pyarrow/parquet.py in __init__(self, path_or_paths, filesystem, schema, metadata, split_row_groups, validate_schema, filters, metadata_nthreads, read_dictionary, memory_map, buffer_size, partitioning, use_legacy_dataset)
   1195         if filters is not None:
   1196             filters = _check_filters(filters)
-> 1197             self._filter(filters)
   1198 
   1199         if validate_schema:

~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/pyarrow/parquet.py in _filter(self, filters)
   1309 
   1310     def _filter(self, filters):
-> 1311         accepts_filter = self.partitions.filter_accepts_partition
   1312 
   1313         def one_filter_accepts(piece, filter):

AttributeError: 'NoneType' object has no attribute 'filter_accepts_partition'
```
It seems like the error is in `ArrowEngine.read_metadata`'s call to `_gather_metadata`.


Discussion:
Thanks for the detailed bug report! I think this might be a pyarrow bug, but @martindurant or @rjzamora are likely to know for sure. It seems like `pq.ParquetDataset` is meant to be able to handle single partitions.

In doing some investigations I found that using the directory name as the path kind of works:

```python
dd.read_parquet("t.parquet", filters=[("x",">", 4)], engine='pyarrow').compute()
```

Although it wasn't giving me precisely the right answer...

```
   x  y
3  3  b
4  4  c
5  5  c
6  6  d
7  7  d
8  8  e
9  9  e
```

which is troubling.
> Although it wasn't giving me precisely the right answer...

That probably *is* the right answer, because the filters are only used to avoid partitions which cannot meet the requirement at all, i.e., no good values, but you get all the values from any partition which has any good values.
Ah! Ok phew. So just a mismatch of expectations. So that should be a good workaround for now. 
@jsignell @martindurant Thanks for the response.

Sorry for not specifying. I meant to list that case in my original post as well - the case where you provide the directory and filters. That does indeed work. Unfortunately, that isn't sufficient for a workflow we are trying to support [1]. We need to be able to apply filters when we don't have a common metadata file.

[1] https://github.com/rapidsai/cudf/pull/5843#issuecomment-673596927
This is slightly crazy, but putting them in a dir even without a common metadata file seems to work fine. I deleted the metadata files from my parquet directory and  this works as expected:

```python
print(dd.read_parquet("t.parquet", filters=[("x",">", 4)], engine='pyarrow', gather_statistics=True).compute())
```
To be clear there is definitely a bug. But this is possibly a workaround.
Interesting, it seems like without a common metadata file, however, it is required that all files in the directory be `*.parquet` files. This may or may not be the case with the workflow @VibhuJawa is working with.
cc @rjzamora in case you have time

On Thu, Aug 13, 2020 at 1:08 PM caleb winston <notifications@github.com>
wrote:

> Reading in multiple Parquet files with a single call to Dask 2.22.0's
> read_parquet fails unless we read in common metadata. If we call
> read_parquet on a partitioned dataset, reading works when the directory
> containing the dataset and metadata file is supplied.
>
> In [1]: import pandas as pd
> In [2]: import dask.dataframe as dd
> In [3]: df = pd.DataFrame({"x": range(10), "y": list("aabbccddee")})
>    ...: ddf = dd.from_pandas(df, npartitions=5)
> In [4]: ddf.to_parquet("t.parquet")/datasets/cwinston/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/pyarrow/compat.py:24: FutureWarning: pyarrow.compat has been deprecated and will be removed in a future release
>   warnings.warn("pyarrow.compat has been deprecated and will be removed in a "
> In [5]: dd.read_parquet("t.parquet/*parquet")Out[5]: Dask DataFrame Structure:
>                    x       ynpartitions=5
>                int64  object
>                  ...     ...
> ...              ...     ...
>                  ...     ...
>                  ...     ...Dask Name: read-parquet, 5 tasks
>
> However, when we read in a set of files without common metadata, either
> using a wildcard or using a list of files, we get an error.
>
> In [6]: dd.read_parquet("t.parquet/*parquet", filters=[("x",">","4")])                                                                                 ---------------------------------------------------------------------------AttributeError                            Traceback (most recent call last)<ipython-input-6-c9834dcbe354> in <module>----> 1 dd.read_parquet("t.parquet/*parquet", filters=[("x",">","4")])
> ~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/dask/dataframe/io/parquet/core.py in read_parquet(path, columns, filters, categories, index, storage_options, engine, gather_statistics, split_row_groups, chunksize, **kwargs)
>     229         index = [index]
>     230 --> 231     meta, statistics, parts, index = engine.read_metadata(
>     232         fs,
>     233         paths,
> ~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/dask/dataframe/io/parquet/arrow.py in read_metadata(cls, fs, paths, categories, index, gather_statistics, filters, split_row_groups, **kwargs)
>     648             split_row_groups,
>     649             gather_statistics,--> 650         ) = _gather_metadata(
>     651             paths, fs, split_row_groups, gather_statistics, filters, dataset_kwargs
>     652         )
> ~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/dask/dataframe/io/parquet/arrow.py in _gather_metadata(paths, fs, split_row_groups, gather_statistics, filters, dataset_kwargs)
>     159
>     160     # Step 1: Create a ParquetDataset object--> 161     dataset, base, fns = _get_dataset_object(paths, fs, filters, dataset_kwargs)
>     162     if fns == [None]:
>     163         # This is a single file. No danger in gathering statistics
> ~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/dask/dataframe/io/parquet/arrow.py in _get_dataset_object(paths, fs, filters, dataset_kwargs)
>     118         # Create our dataset from the list of data files.
>     119         # Note that this will not parse all the files (yet)--> 120         dataset = pq.ParquetDataset(
>     121             paths, filesystem=fs, filters=filters, **dataset_kwargs
>     122         )
> ~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/pyarrow/parquet.py in __init__(self, path_or_paths, filesystem, schema, metadata, split_row_groups, validate_schema, filters, metadata_nthreads, read_dictionary, memory_map, buffer_size, partitioning, use_legacy_dataset)
>    1195         if filters is not None:
>    1196             filters = _check_filters(filters)-> 1197             self._filter(filters)
>    1198
>    1199         if validate_schema:
> ~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/pyarrow/parquet.py in _filter(self, filters)
>    1309
>    1310     def _filter(self, filters):-> 1311         accepts_filter = self.partitions.filter_accepts_partition
>    1312
>    1313         def one_filter_accepts(piece, filter):
> AttributeError: 'NoneType' object has no attribute 'filter_accepts_partition'
>
> This also fails when a list of files is provided.
>
> In [8]: dd.read_parquet(["t.parquet/part.0.parquet", "t.parquet/part.1.parquet"], filters=[("x",">","4")])                                             ---------------------------------------------------------------------------AttributeError                            Traceback (most recent call last)<ipython-input-8-d3b740206c89> in <module>----> 1 dd.read_parquet(["t.parquet/part.0.parquet", "t.parquet/part.1.parquet"], filters=[("x",">","4")])
> ~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/dask/dataframe/io/parquet/core.py in read_parquet(path, columns, filters, categories, index, storage_options, engine, gather_statistics, split_row_groups, chunksize, **kwargs)
>     229         index = [index]
>     230 --> 231     meta, statistics, parts, index = engine.read_metadata(
>     232         fs,
>     233         paths,
> ~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/dask/dataframe/io/parquet/arrow.py in read_metadata(cls, fs, paths, categories, index, gather_statistics, filters, split_row_groups, **kwargs)
>     648             split_row_groups,
>     649             gather_statistics,--> 650         ) = _gather_metadata(
>     651             paths, fs, split_row_groups, gather_statistics, filters, dataset_kwargs
>     652         )
> ~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/dask/dataframe/io/parquet/arrow.py in _gather_metadata(paths, fs, split_row_groups, gather_statistics, filters, dataset_kwargs)
>     159
>     160     # Step 1: Create a ParquetDataset object--> 161     dataset, base, fns = _get_dataset_object(paths, fs, filters, dataset_kwargs)
>     162     if fns == [None]:
>     163         # This is a single file. No danger in gathering statistics
> ~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/dask/dataframe/io/parquet/arrow.py in _get_dataset_object(paths, fs, filters, dataset_kwargs)
>     118         # Create our dataset from the list of data files.
>     119         # Note that this will not parse all the files (yet)--> 120         dataset = pq.ParquetDataset(
>     121             paths, filesystem=fs, filters=filters, **dataset_kwargs
>     122         )
> ~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/pyarrow/parquet.py in __init__(self, path_or_paths, filesystem, schema, metadata, split_row_groups, validate_schema, filters, metadata_nthreads, read_dictionary, memory_map, buffer_size, partitioning, use_legacy_dataset)
>    1195         if filters is not None:
>    1196             filters = _check_filters(filters)-> 1197             self._filter(filters)
>    1198
>    1199         if validate_schema:
> ~/compose/etc/conda/cuda_10.2/envs/arrow_1.0_playground/lib/python3.8/site-packages/pyarrow/parquet.py in _filter(self, filters)
>    1309
>    1310     def _filter(self, filters):-> 1311         accepts_filter = self.partitions.filter_accepts_partition
>    1312
>    1313         def one_filter_accepts(piece, filter):
> AttributeError: 'NoneType' object has no attribute 'filter_accepts_partition'
>
> It seems like the error is in ArrowEngine.read_metadata's call to
> _gather_metadata.
>
> —
> You are receiving this because you are subscribed to this thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/6512>, or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AACKZTDZ5BOHYG5Z7VP35ODSARB3TANCNFSM4P6ZUMJA>
> .
>

I'll try to experiment if I have time before I'm back next week, but I suspect this is just related to pyarrow 1.0 changes.  When not using the pyarrow.dataset API, filtering will only "work" for partitioned datasets. Historically, we always passed in the filters argument, but pyarrow would just ignore it when the data was not partitioned.  It seems that parrow may have made some changes that are failing for non-partitioned datasets when a list of files is provided.
It's perhaps related to https://github.com/dask/dask/issues/6504
Yeah I think you are right @quasiben this is probably the same change in arrow.

Resolving patch:
diff --git a/dask/dataframe/io/parquet/arrow.py b/dask/dataframe/io/parquet/arrow.py
--- a/dask/dataframe/io/parquet/arrow.py
+++ b/dask/dataframe/io/parquet/arrow.py
@@ -120,21 +120,14 @@ def _get_dataset_object(paths, fs, filters, dataset_kwargs):
             # open "_metadata" separately.
             paths.remove(fs.sep.join([base, "_metadata"]))
             fns.remove("_metadata")
-            proxy_metadata = (
-                pq.ParquetDataset(
-                    fs.sep.join([base, "_metadata"]),
-                    filesystem=fs,
-                    filters=filters,
-                    **dataset_kwargs,
-                )
-                .pieces[0]
-                .get_metadata()
-            )
+            with fs.open(fs.sep.join([base, "_metadata"]), mode="rb") as fil:
+                proxy_metadata = pq.ParquetFile(fil).metadata
         # Create our dataset from the list of data files.
-        # Note that this will not parse all the files (yet)
-        dataset = pq.ParquetDataset(
-            paths, filesystem=fs, filters=filters, **dataset_kwargs
-        )
+        # Note #1: that this will not parse all the files (yet)
+        # Note #2: Cannot pass filters for legacy pyarrow API (see issue#6512).
+        #          We can handle partitions + filtering for list input after
+        #          adopting new pyarrow.dataset API.
+        dataset = pq.ParquetDataset(paths, filesystem=fs, **dataset_kwargs)
         if proxy_metadata:
             dataset.metadata = proxy_metadata
     elif fs.isdir(paths[0]):
@@ -155,9 +148,7 @@ def _get_dataset_object(paths, fs, filters, dataset_kwargs):
         # and/or splitting row-groups without a "_metadata" file
         base = paths[0]
         fns = [None]
-        dataset = pq.ParquetDataset(
-            paths[0], filesystem=fs, filters=filters, **dataset_kwargs
-        )
+        dataset = pq.ParquetDataset(paths[0], filesystem=fs, **dataset_kwargs)
 
     return dataset, base, fns
 


Test patch:
diff --git a/dask/dataframe/io/tests/test_parquet.py b/dask/dataframe/io/tests/test_parquet.py
--- a/dask/dataframe/io/tests/test_parquet.py
+++ b/dask/dataframe/io/tests/test_parquet.py
@@ -1296,6 +1296,29 @@ def test_filters_v0(tmpdir, write_engine, read_engine):
     assert len(ddf2) > 0
 
 
+def test_fiters_file_list(tmpdir, engine):
+    df = pd.DataFrame({"x": range(10), "y": list("aabbccddee")})
+    ddf = dd.from_pandas(df, npartitions=5)
+
+    ddf.to_parquet(str(tmpdir), engine=engine)
+    fils = str(tmpdir.join("*.parquet"))
+    ddf_out = dd.read_parquet(
+        fils, gather_statistics=True, engine=engine, filters=[("x", ">", 3)]
+    )
+
+    assert ddf_out.npartitions == 3
+    assert_eq(df[df["x"] > 3], ddf_out.compute(), check_index=False)
+
+    # Check that first parition gets filtered for single-path input
+    ddf2 = dd.read_parquet(
+        str(tmpdir.join("part.0.parquet")),
+        gather_statistics=True,
+        engine=engine,
+        filters=[("x", ">", 3)],
+    )
+    assert len(ddf2) == 0
+
+
 def test_divisions_read_with_filters(tmpdir):
     pytest.importorskip("fastparquet", minversion="0.3.1")
     tmpdir = str(tmpdir)