Repository: modin-project/modin

Issue:
Filters in read_parquet
Currently, `pd.read_parquet(..., filters=<filters>)` defaults to Pandas. If I understand correctly, this means that there is no way to read a small part of a large Parquet file in Modin without loading the whole file into memory, and _then_ performing filters.

I was surprised to see this default to Pandas, because the docs for `read_parquet` [say the Modin implementation is complete](https://modin.readthedocs.io/en/stable/supported_apis/io_supported.html) and the docstring even says "Any additional kwargs are passed to the engine" which seems like it would be enough to make this feature work.

In case this is somehow a bug, here's my full reproducible example:

I downloaded userdata1.parquet from [this GitHub repo](https://github.com/Teradata/kylo/tree/master/samples/sample-data/parquet) to use as sample data.

Then I ran:

```
import modin.pandas as pd
pd.read_parquet('./userdata1.parquet', filters=[('id', '==', 1)])
```


Discussion:
This is a recent check added in the code in https://github.com/modin-project/modin/pull/5911, which was fixing an issue somewhat resembling what you say here (https://github.com/modin-project/modin/issues/5509). I think we should amend the documentation for now, and try to implement distributed `read_parquet` sometime in the future.

cc @anmyachev 
@vnlitvinov https://github.com/modin-project/modin/pull/6420

Resolving patch:
diff --git a/modin/core/io/column_stores/parquet_dispatcher.py b/modin/core/io/column_stores/parquet_dispatcher.py
--- a/modin/core/io/column_stores/parquet_dispatcher.py
+++ b/modin/core/io/column_stores/parquet_dispatcher.py
@@ -492,7 +492,7 @@ def build_partition(cls, partition_ids, column_widths):
         )
 
     @classmethod
-    def build_index(cls, dataset, partition_ids, index_columns):
+    def build_index(cls, dataset, partition_ids, index_columns, filters):
         """
         Compute index and its split sizes of resulting Modin DataFrame.
 
@@ -504,6 +504,8 @@ def build_index(cls, dataset, partition_ids, index_columns):
             Array with references to the partitions data.
         index_columns : list
             List of index columns specified by pandas metadata.
+        filters : list
+            List of filters to be used in reading the Parquet file/files.
 
         Returns
         -------
@@ -530,9 +532,22 @@ def build_index(cls, dataset, partition_ids, index_columns):
             elif column["name"] is not None:
                 column_names_to_read.append(column["name"])
 
+        # When the index has meaningful values, stored in a column, we will replicate those
+        # exactly in the Modin dataframe's index. This index may have repeated values, be unsorted,
+        # etc. This is all fine.
+        # A range index is the special case: we want the Modin dataframe to have a single range,
+        # not a range that keeps restarting. i.e. if the partitions have index 0-9, 0-19, 0-29,
+        # we want our Modin dataframe to have 0-59.
+        # When there are no filters, it is relatively trivial to construct the index by
+        # actually reading in the necessary data, here in the main process.
+        # When there are filters, we let the workers materialize the indices before combining to
+        # get a single range.
+
         # For the second check, let us consider the case where we have an empty dataframe,
         # that has a valid index.
-        if range_index or (len(partition_ids) == 0 and len(column_names_to_read) != 0):
+        if (range_index and filters is None) or (
+            len(partition_ids) == 0 and len(column_names_to_read) != 0
+        ):
             complete_index = dataset.to_pandas_dataframe(
                 columns=column_names_to_read
             ).index
@@ -542,7 +557,17 @@ def build_index(cls, dataset, partition_ids, index_columns):
         else:
             index_ids = [part_id[0][1] for part_id in partition_ids if len(part_id) > 0]
             index_objs = cls.materialize(index_ids)
-            complete_index = index_objs[0].append(index_objs[1:])
+            if range_index:
+                # There are filters, so we had to materialize in order to
+                # determine how many items there actually are
+                start = index_objs[0].start
+                total_length = sum(len(index_part) for index_part in index_objs)
+                complete_index = pandas.RangeIndex(
+                    start=start,
+                    stop=start + total_length,
+                )
+            else:
+                complete_index = index_objs[0].append(index_objs[1:])
         return complete_index, range_index or (len(index_columns) == 0)
 
     @classmethod
@@ -567,11 +592,15 @@ def build_query_compiler(cls, dataset, columns, index_columns, **kwargs):
             Query compiler with imported data for further processing.
         """
         storage_options = kwargs.pop("storage_options", {}) or {}
+        filters = kwargs.get("filters", None)
+
         col_partitions, column_widths = cls.build_columns(columns)
         partition_ids = cls.call_deploy(
             dataset, col_partitions, storage_options, **kwargs
         )
-        index, sync_index = cls.build_index(dataset, partition_ids, index_columns)
+        index, sync_index = cls.build_index(
+            dataset, partition_ids, index_columns, filters
+        )
         remote_parts = cls.build_partition(partition_ids, column_widths)
         if len(partition_ids) > 0:
             row_lengths = [part.length() for part in remote_parts.T[0]]
@@ -618,7 +647,7 @@ def _read(cls, path, engine, columns, use_nullable_dtypes, dtype_backend, **kwar
         https://arrow.apache.org/docs/python/parquet.html
         """
         if (
-            any(arg not in ("storage_options",) for arg in kwargs)
+            any(arg not in ("storage_options", "filters") for arg in kwargs)
             or use_nullable_dtypes != lib.no_default
         ):
             return cls.single_worker_read(
@@ -630,6 +659,7 @@ def _read(cls, path, engine, columns, use_nullable_dtypes, dtype_backend, **kwar
                 reason="Parquet options that are not currently supported",
                 **kwargs,
             )
+
         path = stringify_path(path)
         if isinstance(path, list):
             # TODO(https://github.com/modin-project/modin/issues/5723): read all
diff --git a/modin/core/storage_formats/pandas/parsers.py b/modin/core/storage_formats/pandas/parsers.py
--- a/modin/core/storage_formats/pandas/parsers.py
+++ b/modin/core/storage_formats/pandas/parsers.py
@@ -40,6 +40,7 @@
 """
 
 import os
+import json
 from collections import OrderedDict
 from io import BytesIO, TextIOWrapper, IOBase
 import fsspec
@@ -784,28 +785,81 @@ class ParquetFileToRead(NamedTuple):
 class PandasParquetParser(PandasParser):
     @staticmethod
     def _read_row_group_chunk(
-        f, row_group_start, row_group_end, columns, engine, to_pandas_kwargs
+        f, row_group_start, row_group_end, columns, filters, engine, to_pandas_kwargs
     ):  # noqa: GL08
         if engine == "pyarrow":
-            from pyarrow.parquet import ParquetFile
+            if filters is not None:
+                import pyarrow.dataset as ds
+                from pyarrow.parquet import filters_to_expression
 
-            return (
-                ParquetFile(f)
-                .read_row_groups(
-                    range(
+                parquet_format = ds.ParquetFileFormat()
+                fragment = parquet_format.make_fragment(
+                    f,
+                    row_groups=range(
                         row_group_start,
                         row_group_end,
                     ),
+                )
+                dataset = ds.FileSystemDataset(
+                    [fragment],
+                    schema=fragment.physical_schema,
+                    format=parquet_format,
+                    filesystem=fragment.filesystem,
+                )
+
+                # This lower-level API doesn't have the ability to automatically handle pandas metadata
+                # The following code is based on
+                # https://github.com/apache/arrow/blob/f44e28fa03a64ae5b3d9352d21aee2cc84f9af6c/python/pyarrow/parquet/core.py#L2619-L2628
+
+                # if use_pandas_metadata, we need to include index columns in the
+                # column selection, to be able to restore those in the pandas DataFrame
+                metadata = dataset.schema.metadata or {}
+
+                if b"pandas" in metadata and columns is not None:
+                    index_columns = json.loads(metadata[b"pandas"].decode("utf8"))[
+                        "index_columns"
+                    ]
+                    # In the pandas metadata, the index columns can either be string column names,
+                    # or a dictionary that describes a RangeIndex.
+                    # Here, we are finding the real data columns that need to be read to become part
+                    # of the pandas Index, so we can skip the RangeIndex.
+                    # Not only can a RangeIndex be trivially reconstructed later, but we actually
+                    # ignore partition-level range indices, because we want to have a single Modin
+                    # RangeIndex that spans all partitions.
+                    index_columns = [
+                        col for col in index_columns if not isinstance(col, dict)
+                    ]
+                    columns = list(columns) + list(set(index_columns) - set(columns))
+
+                return dataset.to_table(
                     columns=columns,
-                    use_pandas_metadata=True,
+                    filter=filters_to_expression(filters),
+                ).to_pandas(**to_pandas_kwargs)
+            else:
+                from pyarrow.parquet import ParquetFile
+
+                return (
+                    ParquetFile(f)
+                    .read_row_groups(
+                        range(
+                            row_group_start,
+                            row_group_end,
+                        ),
+                        columns=columns,
+                        use_pandas_metadata=True,
+                    )
+                    .to_pandas(**to_pandas_kwargs)
                 )
-                .to_pandas(**to_pandas_kwargs)
-            )
         elif engine == "fastparquet":
             from fastparquet import ParquetFile
 
             return ParquetFile(f)[row_group_start:row_group_end].to_pandas(
-                columns=columns
+                columns=columns,
+                filters=filters,
+                # Setting row_filter=True would perform filtering at the row level, which is more correct
+                # (in line with pyarrow)
+                # However, it doesn't work: https://github.com/dask/fastparquet/issues/873
+                # Also, this would create incompatibility with pandas
             )
         else:
             # We shouldn't ever come to this case, so something went wrong
@@ -824,6 +878,7 @@ def _read_row_group_chunk(
     )
     def parse(files_for_parser, engine, **kwargs):
         columns = kwargs.get("columns", None)
+        filters = kwargs.get("filters", None)
         storage_options = kwargs.get("storage_options", {})
         chunks = []
         # `single_worker_read` just passes in a string path or path-like object
@@ -843,6 +898,7 @@ def parse(files_for_parser, engine, **kwargs):
                     file_for_parser.row_group_start,
                     file_for_parser.row_group_end,
                     columns,
+                    filters,
                     engine,
                     to_pandas_kwargs,
                 )


Test patch:
diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py
--- a/modin/pandas/test/test_io.py
+++ b/modin/pandas/test/test_io.py
@@ -42,6 +42,7 @@
 from modin.pandas.utils import from_arrow
 from modin.test.test_utils import warns_that_defaulting_to_pandas
 import pyarrow as pa
+import fastparquet
 import os
 from scipy import sparse
 import sys
@@ -1397,6 +1398,36 @@ def comparator(df1, df2):
                 comparator=comparator,
             )
 
+    @pytest.mark.parametrize(
+        "filters",
+        [None, [], [("col1", "==", 5)], [("col1", "<=", 215), ("col2", ">=", 35)]],
+    )
+    def test_read_parquet_filters(self, engine, make_parquet_file, filters):
+        if engine == "pyarrow" and filters == []:
+            # pyarrow, and therefore pandas using pyarrow, errors in this case.
+            # Modin correctly replicates this behavior; however error cases
+            # cause race conditions with ensure_clean on Windows.
+            # TODO: Remove this once https://github.com/modin-project/modin/issues/6460 is fixed.
+            pytest.xfail(
+                "Skipping empty filters error case to avoid race condition - see #6460"
+            )
+
+        with ensure_clean(".parquet") as unique_filename:
+            make_parquet_file(filename=unique_filename, row_group_size=100)
+
+            def comparator(df1, df2):
+                df_equals(df1, df2)
+                df_equals(df1.dtypes, df2.dtypes)
+
+            eval_io(
+                fn_name="read_parquet",
+                # read_parquet kwargs
+                engine=engine,
+                path=unique_filename,
+                filters=filters,
+                comparator=comparator,
+            )
+
     def test_read_parquet_list_of_files_5698(self, engine, make_parquet_file):
         if engine == "fastparquet" and os.name == "nt":
             pytest.xfail(reason="https://github.com/pandas-dev/pandas/issues/51720")
@@ -1460,8 +1491,12 @@ def test_read_parquet_directory(
         )
 
     @pytest.mark.parametrize("columns", [None, ["col1"]])
+    @pytest.mark.parametrize(
+        "filters",
+        [None, [], [("col1", "==", 5)], [("col1", "<=", 215), ("col2", ">=", 35)]],
+    )
     def test_read_parquet_partitioned_directory(
-        self, tmp_path, make_parquet_file, columns, engine
+        self, tmp_path, make_parquet_file, columns, filters, engine
     ):
         unique_filename = get_unique_filename(extension=None, data_dir=tmp_path)
         make_parquet_file(filename=unique_filename, partitioned_columns=["col1"])
@@ -1472,9 +1507,24 @@ def test_read_parquet_partitioned_directory(
             engine=engine,
             path=unique_filename,
             columns=columns,
+            filters=filters,
         )
 
-    def test_read_parquet_pandas_index(self, engine):
+    @pytest.mark.parametrize(
+        "filters",
+        [
+            None,
+            [],
+            [("B", "==", "a")],
+            [
+                ("B", "==", "a"),
+                ("A", ">=", 50_000),
+                ("idx", "<=", 30_000),
+                ("idx_categorical", "==", "y"),
+            ],
+        ],
+    )
+    def test_read_parquet_pandas_index(self, engine, filters):
         if (
             version.parse(pa.__version__) >= version.parse("12.0.0")
             and version.parse(pd.__version__) < version.parse("2.0.0")
@@ -1515,6 +1565,18 @@ def test_read_parquet_pandas_index(self, engine):
 
         for col in pandas_df.columns:
             if col.startswith("idx"):
+                # Before this commit, first released in version 2023.1.0, fastparquet relied
+                # on pandas private APIs to handle Categorical indices.
+                # These private APIs broke in pandas 2.
+                # https://github.com/dask/fastparquet/commit/cf60ae0e9a9ca57afc7a8da98d8c0423db1c0c53
+                if (
+                    col == "idx_categorical"
+                    and engine == "fastparquet"
+                    and version.parse(fastparquet.__version__)
+                    < version.parse("2023.1.0")
+                ):
+                    continue
+
                 with ensure_clean(".parquet") as unique_filename:
                     pandas_df.set_index(col).to_parquet(unique_filename)
                     # read the same parquet using modin.pandas
@@ -1523,6 +1585,7 @@ def test_read_parquet_pandas_index(self, engine):
                         # read_parquet kwargs
                         path=unique_filename,
                         engine=engine,
+                        filters=filters,
                     )
 
         with ensure_clean(".parquet") as unique_filename:
@@ -1532,9 +1595,19 @@ def test_read_parquet_pandas_index(self, engine):
                 # read_parquet kwargs
                 path=unique_filename,
                 engine=engine,
+                filters=filters,
             )
 
-    def test_read_parquet_pandas_index_partitioned(self, tmp_path, engine):
+    @pytest.mark.parametrize(
+        "filters",
+        [
+            None,
+            [],
+            [("B", "==", "a")],
+            [("B", "==", "a"), ("A", ">=", 5), ("idx", "<=", 30_000)],
+        ],
+    )
+    def test_read_parquet_pandas_index_partitioned(self, tmp_path, engine, filters):
         # Ensure modin can read parquet files written by pandas with a non-RangeIndex object
         pandas_df = pandas.DataFrame(
             {
@@ -1552,6 +1625,7 @@ def test_read_parquet_pandas_index_partitioned(self, tmp_path, engine):
             # read_parquet kwargs
             path=unique_filename,
             engine=engine,
+            filters=filters,
         )
 
     def test_read_parquet_hdfs(self, engine):
@@ -1589,7 +1663,11 @@ def test_read_parquet_s3(self, path_type, engine):
                 engine=engine,
             )
 
-    def test_read_parquet_without_metadata(self, tmp_path, engine):
+    @pytest.mark.parametrize(
+        "filters",
+        [None, [], [("idx", "<=", 30_000)], [("idx", "<=", 30_000), ("A", ">=", 5)]],
+    )
+    def test_read_parquet_without_metadata(self, tmp_path, engine, filters):
         """Test that Modin can read parquet files not written by pandas."""
         from pyarrow import csv
         from pyarrow import parquet
@@ -1614,6 +1692,7 @@ def test_read_parquet_without_metadata(self, tmp_path, engine):
             # read_parquet kwargs
             path=parquet_fname,
             engine=engine,
+            filters=filters,
         )
 
     def test_read_empty_parquet_file(self, tmp_path, engine):
@@ -1701,22 +1780,6 @@ def test_read_parquet_5767(self, tmp_path, engine):
         # both Modin and pandas read column "b" as a category
         df_equals(test_df, read_df.astype("int64"))
 
-    def test_read_parquet_5509(self, tmp_path, engine):
-        test_df = pandas.DataFrame({"col_a": [1, 2, 3], "col_b": ["a", "b", "c"]})
-
-        path = tmp_path / "data"
-        path.mkdir()
-        file_name = "5509.parquet"
-        test_df.to_parquet(path / file_name)
-        with warns_that_defaulting_to_pandas():
-            eval_io(
-                fn_name="read_parquet",
-                path=str(path / file_name),
-                columns=["col_b"],
-                engine=engine,
-                filters=[[("col_a", "==", 1)]],
-            )
-
     def test_read_parquet_s3_with_column_partitioning(self, engine):
         # This test case comes from
         # https://github.com/modin-project/modin/issues/4636