Repository: dask/dask

Issue:
read_parquet column projection not working as expected
<!-- Please include a self-contained copy-pastable example that generates the issue if possible.

Please be concise with code posted. See guidelines below on how to provide a good bug report:

- Craft Minimal Bug Reports http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports
- Minimal Complete Verifiable Examples https://stackoverflow.com/help/mcve

Bug reports that follow these guidelines are easier to diagnose, and so are often handled much more quickly.
-->

**What happened**:
Filtering out columns after calling `read_parquet` still causes the entire dataset to be loaded for downstream operations:
```
%%time
dd.read_parquet(
    "s3://nyc-tlc/trip data/fhv_tripdata_2015*.parquet",
)[[]].sum().compute()
CPU times: user 24 s, sys: 8.82 s, total: 32.8 s
Wall time: 1min 15s
Out[4]: Series([], dtype: float64)
```

**What you expected to happen**:
From my interpretation of #8692 we should get the same performance as
```
%%time
dd.read_parquet(
     "s3://nyc-tlc/trip data/fhv_tripdata_2015*.parquet",
      columns=[]
).sum().compute()
CPU times: user 147 ms, sys: 20.6 ms, total: 167 ms
Wall time: 706 ms
Out[5]: Series([], dtype: float64)
```

**Anything else we need to know?**:

**Environment**:

- Dask version: 2022.05.0
- Python version: 3.9.12
- PyArrow version: 8.0.0

Have not yet checked this behavior for any past dask versions.

cc: @rjzamora @martindurant @jcrist from #8692


Discussion:
It looks like the "no columns" case isn't covered - but you get much faster performance if you include any one column. This probably also benefits a lot from the parquet pre-cache strategy.

Resolving patch:
diff --git a/dask/dataframe/optimize.py b/dask/dataframe/optimize.py
--- a/dask/dataframe/optimize.py
+++ b/dask/dataframe/optimize.py
@@ -203,7 +203,6 @@ def _walk_deps(dependents, key, success):
         old = layers[io_layer_name]
         new = old.project_columns(columns)
         if new.name != old.name:
-            columns = list(columns)
             assert len(update_blocks)
             for block_key, block in update_blocks.items():
                 # (('read-parquet-old', (.,)), ( ... )) ->
diff --git a/dask/layers.py b/dask/layers.py
--- a/dask/layers.py
+++ b/dask/layers.py
@@ -1219,18 +1219,20 @@ def project_columns(self, columns):
         """
         from dask.dataframe.io.utils import DataFrameIOFunction
 
-        if columns and (self.columns is None or columns < set(self.columns)):
+        columns = list(columns)
+
+        if self.columns is None or set(self.columns).issuperset(columns):
 
             # Apply column projection in IO function.
             # Must satisfy `DataFrameIOFunction` protocol
             if isinstance(self.io_func, DataFrameIOFunction):
-                io_func = self.io_func.project_columns(list(columns))
+                io_func = self.io_func.project_columns(columns)
             else:
                 io_func = self.io_func
 
             layer = DataFrameIOLayer(
                 (self.label or "subset") + "-" + tokenize(self.name, columns),
-                list(columns),
+                columns,
                 self.inputs,
                 io_func,
                 label=self.label,


Test patch:
diff --git a/dask/dataframe/io/tests/test_parquet.py b/dask/dataframe/io/tests/test_parquet.py
--- a/dask/dataframe/io/tests/test_parquet.py
+++ b/dask/dataframe/io/tests/test_parquet.py
@@ -2433,17 +2433,18 @@ def test_getitem_optimization(tmpdir, engine, preserve_index, index):
 
 def test_getitem_optimization_empty(tmpdir, engine):
     df = pd.DataFrame({"A": [1] * 100, "B": [2] * 100, "C": [3] * 100, "D": [4] * 100})
-    ddf = dd.from_pandas(df, 2)
+    ddf = dd.from_pandas(df, 2, sort=False)
     fn = os.path.join(str(tmpdir))
     ddf.to_parquet(fn, engine=engine)
 
-    df2 = dd.read_parquet(fn, columns=[], engine=engine)
-    dsk = optimize_dataframe_getitem(df2.dask, keys=[df2._name])
+    ddf2 = dd.read_parquet(fn, engine=engine)[[]]
+    dsk = optimize_dataframe_getitem(ddf2.dask, keys=[ddf2._name])
 
-    subgraph = next(iter(dsk.layers.values()))
-    assert isinstance(subgraph, DataFrameIOLayer)
+    subgraph = next(l for l in dsk.layers.values() if isinstance(l, DataFrameIOLayer))
     assert subgraph.columns == []
 
+    assert_eq(ddf2, ddf[[]])
+
 
 def test_getitem_optimization_multi(tmpdir, engine):
     df = pd.DataFrame({"A": [1] * 100, "B": [2] * 100, "C": [3] * 100, "D": [4] * 100})
@@ -2677,7 +2678,7 @@ def test_split_row_groups_int_aggregate_files(tmpdir, engine, split_row_groups):
 
     # Read back with both `split_row_groups>1` and
     # `aggregate_files=True`
-    with pytest.warns(match="argument will be deprecated"):
+    with pytest.warns(FutureWarning, match="argument will be deprecated"):
         ddf2 = dd.read_parquet(
             str(tmpdir),
             engine=engine,
@@ -2764,7 +2765,7 @@ def test_chunksize_empty(tmpdir, write_engine, read_engine):
     df = pd.DataFrame({"a": pd.Series(dtype="int"), "b": pd.Series(dtype="float")})
     ddf1 = dd.from_pandas(df, npartitions=1)
     ddf1.to_parquet(tmpdir, engine=write_engine, write_metadata_file=True)
-    with pytest.warns(match="argument will be deprecated"):
+    with pytest.warns(FutureWarning, match="argument will be deprecated"):
         ddf2 = dd.read_parquet(tmpdir, engine=read_engine, chunksize="1MiB")
     assert_eq(ddf1, ddf2, check_index=False)
 
@@ -2799,7 +2800,7 @@ def test_chunksize_files(
         write_index=False,
     )
 
-    with pytest.warns(match="argument will be deprecated"):
+    with pytest.warns(FutureWarning, match="argument will be deprecated"):
         ddf2 = dd.read_parquet(
             str(tmpdir),
             engine=read_engine,
@@ -2848,7 +2849,7 @@ def test_chunksize_aggregate_files(tmpdir, write_engine, read_engine, aggregate_
         partition_on=partition_on,
         write_index=False,
     )
-    with pytest.warns(match="argument will be deprecated"):
+    with pytest.warns(FutureWarning, match="argument will be deprecated"):
         ddf2 = dd.read_parquet(
             str(tmpdir),
             engine=read_engine,
@@ -2900,7 +2901,7 @@ def test_chunksize(tmpdir, chunksize, engine, metadata):
         assert "_metadata" not in files
         path = os.path.join(dirname, "*.parquet")
 
-    with pytest.warns(match="argument will be deprecated"):
+    with pytest.warns(FutureWarning, match="argument will be deprecated"):
         ddf2 = dd.read_parquet(
             path,
             engine=engine,
@@ -2934,7 +2935,7 @@ def test_roundtrip_pandas_chunksize(tmpdir, write_engine, read_engine):
         path, engine="pyarrow" if write_engine.startswith("pyarrow") else "fastparquet"
     )
 
-    with pytest.warns(match="argument will be deprecated"):
+    with pytest.warns(FutureWarning, match="argument will be deprecated"):
         ddf_read = dd.read_parquet(
             path,
             engine=read_engine,