Repository: dask/dask

Issue:
Regression: `read_parquet` fails on main (not on 2021.10.0)
There is a regression on main that causes the following to fail. I discovered this while looking at https://github.com/dask/dask/issues/8323

```python
import numpy as np
import pandas as pd
import dask.dataframe as dd

np.random.seed(1)
df = pd.DataFrame(np.random.randint(0, 10, size=(10, 4)), columns=['a', 'b', 'c', 'd']).set_index(['a', 'b']).sort_index()
df.to_parquet("multi.pq", engine='fastparquet')
dd.read_parquet("multi.pq").compute()
```

```python-traceback
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-4-79a798c099ae> in <module>
      6 df = pd.DataFrame(np.random.randint(0, 10, size=(10, 4)), columns=['a', 'b', 'c', 'd']).set_index(['a', 'b']).sort_index()
      7 df.to_parquet("multi.pq", engine='fastparquet')
----> 8 dd.read_parquet("multi.pq").compute()

~/dask/dask/dataframe/io/parquet/core.py in read_parquet(path, columns, filters, categories, index, storage_options, engine, gather_statistics, ignore_metadata_file, metadata_task_size, split_row_groups, chunksize, aggregate_files, **kwargs)
    324         gather_statistics = True
    325 
--> 326     read_metadata_result = engine.read_metadata(
    327         fs,
    328         paths,

~/dask/dask/dataframe/io/parquet/fastparquet.py in read_metadata(cls, fs, paths, categories, index, gather_statistics, filters, split_row_groups, chunksize, aggregate_files, ignore_metadata_file, metadata_task_size, **kwargs)
    849 
    850         # Stage 1: Collect general dataset information
--> 851         dataset_info = cls._collect_dataset_info(
    852             paths,
    853             fs,

~/dask/dask/dataframe/io/parquet/fastparquet.py in _collect_dataset_info(cls, paths, fs, categories, index, gather_statistics, filters, split_row_groups, chunksize, aggregate_files, ignore_metadata_file, metadata_task_size, require_extension, **kwargs)
    455                 # Will need to pass a list of paths to read_partition
    456                 scheme = get_file_scheme(fns)
--> 457                 pf = ParquetFile(paths[:1], open_with=fs.open, root=base, **kwargs)
    458                 pf.file_scheme = scheme
    459                 pf.cats = paths_to_cats(fns, scheme)

~/conda/envs/dask-dev/lib/python3.9/site-packages/fastparquet/api.py in __init__(self, fn, verify, open_with, root, sep, fs, pandas_nulls)
    103             fs = getattr(open_with, "__self__", None)
    104         if isinstance(fn, (tuple, list)):
--> 105             basepath, fmd = metadata_from_many(fn, verify_schema=verify,
    106                                                open_with=open_with, root=root,
    107                                                fs=fs)

~/conda/envs/dask-dev/lib/python3.9/site-packages/fastparquet/util.py in metadata_from_many(file_list, verify_schema, open_with, root, fs)
    177                     raise ValueError('Incompatible schemas')
    178 
--> 179         fmd = copy.copy(pfs[0].fmd)  # we inherit "created by" field
    180         fmd.row_groups = []
    181 

IndexError: list index out of range
```

ping @martindurant and @rjzamora


Discussion:
Detail: 
fastparquet alone reads this fine, but I see that in the dask code in _collect_dataset_info we are passing an empty list of paths for fastparquet to handle 
Note that the data is a single file, not a directory. If it is renamed "multi.parquet", it works OK - so we are failing the file names filter, which should not apply to a single file input.
Thanks for raising @jsignell , and @martindurant for taking a look. I can work on a fix later today
sorry if the title was too alarmist :grimacing: seems like it is kind of an edge case.

Resolving patch:
diff --git a/dask/dataframe/io/parquet/arrow.py b/dask/dataframe/io/parquet/arrow.py
--- a/dask/dataframe/io/parquet/arrow.py
+++ b/dask/dataframe/io/parquet/arrow.py
@@ -807,9 +807,16 @@ def _collect_dataset_info(
                     gather_statistics = True
             elif require_extension:
                 # Need to materialize all paths if we are missing the _metadata file
+                # Raise error if all files have been filtered by extension
+                len0 = len(paths)
                 paths = [
                     path for path in fs.find(paths) if path.endswith(require_extension)
                 ]
+                if len0 and paths == []:
+                    raise ValueError(
+                        "No files satisfy the `require_extension` criteria "
+                        f"(files must end with {require_extension})."
+                    )
 
         elif len(paths) > 1:
             paths, base, fns = _sort_and_analyze_paths(paths, fs)
@@ -833,13 +840,7 @@ def _collect_dataset_info(
                 # Populate valid_paths, since the original path list
                 # must be used to filter the _metadata-based dataset
                 fns.remove("_metadata")
-                valid_paths = (
-                    [fn for fn in fns if fn.endswith(require_extension)]
-                    if require_extension
-                    else fns
-                )
-            elif require_extension:
-                paths = [path for path in paths if path.endswith(require_extension)]
+                valid_paths = fns
 
         # Final "catch-all" pyarrow.dataset call
         if ds is None:
diff --git a/dask/dataframe/io/parquet/fastparquet.py b/dask/dataframe/io/parquet/fastparquet.py
--- a/dask/dataframe/io/parquet/fastparquet.py
+++ b/dask/dataframe/io/parquet/fastparquet.py
@@ -407,9 +407,6 @@ def _collect_dataset_info(
                 if _update_paths:
                     paths = [fs.sep.join([base, fn]) for fn in fns]
                 _metadata_exists = False
-
-            if require_extension:
-                paths = [path for path in paths if path.endswith(require_extension)]
             if _metadata_exists:
                 # Using _metadata file (best-case scenario)
                 pf = ParquetFile(
@@ -423,6 +420,15 @@ def _collect_dataset_info(
                 # Use 0th file
                 # Note that "_common_metadata" can cause issues for
                 # partitioned datasets.
+                if require_extension:
+                    # Raise error if all files have been filtered by extension
+                    len0 = len(paths)
+                    paths = [path for path in paths if path.endswith(require_extension)]
+                    if len0 and paths == []:
+                        raise ValueError(
+                            "No files satisfy the `require_extension` criteria "
+                            f"(files must end with {require_extension})."
+                        )
                 pf = ParquetFile(paths[:1], open_with=fs.open, root=base, **kwargs)
                 scheme = get_file_scheme(fns)
                 pf.file_scheme = scheme
@@ -439,8 +445,6 @@ def _collect_dataset_info(
             if _metadata_exists and ignore_metadata_file:
                 fns.remove("_metadata")
                 _metadata_exists = False
-            if require_extension:
-                fns = [fn for fn in fns if fn.endswith(require_extension)]
             paths = [fs.sep.join([base, fn]) for fn in fns]
 
             if _metadata_exists:


Test patch:
diff --git a/dask/dataframe/io/tests/test_parquet.py b/dask/dataframe/io/tests/test_parquet.py
--- a/dask/dataframe/io/tests/test_parquet.py
+++ b/dask/dataframe/io/tests/test_parquet.py
@@ -3762,12 +3762,39 @@ def test_extra_file(tmpdir, engine):
         )
         assert_eq(out, df)
 
-        # Should Fail
+        # Should Fail (for not capturing the _SUCCESS and crc files)
         with pytest.raises((OSError, pa.lib.ArrowInvalid)):
             dd.read_parquet(
                 tmpdir, engine=engine, dataset={"require_extension": False}
             ).compute()
 
+        # Should Fail (for filtering out all files)
+        # (Related to: https://github.com/dask/dask/issues/8349)
+        with pytest.raises(ValueError):
+            dd.read_parquet(
+                tmpdir, engine=engine, dataset={"require_extension": ".foo"}
+            ).compute()
+
+
+def test_unsupported_extension_file(tmpdir, engine):
+    # File extension shouldn't matter when we are only
+    # reading a single file.
+    # (See: https://github.com/dask/dask/issues/8349)
+    fn = os.path.join(str(tmpdir), "multi.foo")
+    df0 = pd.DataFrame({"a": range(10)})
+    df0.to_parquet(fn, engine=engine.split("-")[0])
+    assert_eq(df0, dd.read_parquet(fn, engine=engine, index=False))
+
+
+def test_unsupported_extension_dir(tmpdir, engine):
+    # File extensions shouldn't matter when we have
+    # a _metadata file
+    # (Related to: https://github.com/dask/dask/issues/8349)
+    path = str(tmpdir)
+    ddf0 = dd.from_pandas(pd.DataFrame({"a": range(10)}), 1)
+    ddf0.to_parquet(path, engine=engine, name_function=lambda i: f"part.{i}.foo")
+    assert_eq(ddf0, dd.read_parquet(path, engine=engine))
+
 
 def test_custom_filename(tmpdir, engine):
     fn = str(tmpdir)