Repository: modin-project/modin

Issue:
modin pandas read_parquet() failed on ETag KeyError trying to read a partitioned parquet from s3
The source comes from https://stackoverflow.com/questions/68103357/modin-pandas-read-parquet-failed-on-etag-keyerror-trying-to-read-a-partitioned.



Discussion:
@YarShev will you comment there that you opened an issue here and link here so others can follow?
@devin-petersohn, yes, done.

Resolving patch:
diff --git a/modin/backends/pandas/parsers.py b/modin/backends/pandas/parsers.py
--- a/modin/backends/pandas/parsers.py
+++ b/modin/backends/pandas/parsers.py
@@ -599,17 +599,6 @@ class PandasParquetParser(PandasParser):
     def parse(fname, **kwargs):
         num_splits = kwargs.pop("num_splits", None)
         columns = kwargs.get("columns", None)
-        if fname.startswith("s3://"):
-            from botocore.exceptions import NoCredentialsError
-            import s3fs
-
-            try:
-                fs = s3fs.S3FileSystem()
-                fname = fs.open(fname)
-            except NoCredentialsError:
-                fs = s3fs.S3FileSystem(anon=True)
-                fname = fs.open(fname)
-
         if num_splits is None:
             return pandas.read_parquet(fname, **kwargs)
         kwargs["use_pandas_metadata"] = True
diff --git a/modin/engines/base/io/column_stores/parquet_dispatcher.py b/modin/engines/base/io/column_stores/parquet_dispatcher.py
--- a/modin/engines/base/io/column_stores/parquet_dispatcher.py
+++ b/modin/engines/base/io/column_stores/parquet_dispatcher.py
@@ -55,12 +55,11 @@ def _read(cls, path, engine, columns, **kwargs):
         ParquetFile API is used. Please refer to the documentation here
         https://arrow.apache.org/docs/python/parquet.html
         """
-        from pyarrow.parquet import ParquetFile, ParquetDataset
+        from pyarrow.parquet import ParquetDataset
         from modin.pandas.io import PQ_INDEX_REGEX
 
         if isinstance(path, str) and os.path.isdir(path):
             partitioned_columns = set()
-            directory = True
             # We do a tree walk of the path directory because partitioned
             # parquet directories have a unique column at each directory level.
             # Thus, we can use os.walk(), which does a dfs search, to walk
@@ -79,58 +78,22 @@ def _read(cls, path, engine, columns, **kwargs):
                 return cls.single_worker_read(
                     path, engine=engine, columns=columns, **kwargs
                 )
-        else:
-            directory = False
-        if not columns:
-            import s3fs
-
-            if directory:
-                # Path of the sample file that we will read to get the remaining columns
-                pd = ParquetDataset(path)
-                meta = pd.metadata
-                column_names = pd.schema.to_arrow_schema().names
-            elif isinstance(path, str) and path.startswith("hdfs://"):
-                import fsspec.core
 
-                fs, path = fsspec.core.url_to_fs(path)
-                pd = ParquetDataset(path, filesystem=fs)
-                meta = pd.metadata
-                column_names = pd.schema.to_arrow_schema().names
-            elif isinstance(path, s3fs.S3File) or (
-                isinstance(path, str) and path.startswith("s3://")
-            ):
-                from botocore.exceptions import NoCredentialsError
+        if not columns:
+            import fsspec.core
+            from pandas.io.common import is_fsspec_url
 
-                if isinstance(path, s3fs.S3File):
-                    bucket_path = path.url().split(".s3.amazonaws.com")
-                    path = "s3://" + bucket_path[0].split("://")[1] + bucket_path[1]
-                try:
-                    fs = s3fs.S3FileSystem()
-                    pd = ParquetDataset(path, filesystem=fs)
-                except NoCredentialsError:
-                    fs = s3fs.S3FileSystem(anon=True)
-                    pd = ParquetDataset(path, filesystem=fs)
-                meta = pd.metadata
-                column_names = pd.schema.to_arrow_schema().names
-            else:
-                meta = ParquetFile(path).metadata
-                column_names = meta.schema.to_arrow_schema().names
+            fs, path_ = (
+                fsspec.core.url_to_fs(path, **(kwargs.get("storage_options") or {}))
+                if is_fsspec_url(path)
+                else (None, path)
+            )
 
-            if meta is not None and meta.metadata is not None:
-                pandas_metadata = meta.metadata.get(b"pandas", None)
-                if pandas_metadata is not None:
-                    import json
+            dataset = ParquetDataset(path_, filesystem=fs, use_legacy_dataset=False)
+            column_names = dataset.schema.names
 
-                    # This is how we convert the metadata from pyarrow to a python
-                    # dictionary, from which we then get the index columns.
-                    # We use these to filter out from the columns in the metadata since
-                    # the pyarrow storage has no concept of row labels/index.
-                    # This ensures that our metadata lines up with the partitions without
-                    # extra communication steps once we have done all the remote
-                    # computation.
-                    index_columns = json.loads(pandas_metadata.decode("utf8")).get(
-                        "index_columns", []
-                    )
-                    column_names = [c for c in column_names if c not in index_columns]
+            if dataset.schema.pandas_metadata is not None:
+                index_columns = dataset.schema.pandas_metadata.get("index_columns", [])
+                column_names = [c for c in column_names if c not in index_columns]
             columns = [name for name in column_names if not PQ_INDEX_REGEX.match(name)]
         return cls.build_query_compiler(path, columns, **kwargs)


Test patch:
diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py
--- a/modin/pandas/test/test_io.py
+++ b/modin/pandas/test/test_io.py
@@ -1301,24 +1301,21 @@ def test_read_parquet_hdfs(self):
             path="modin/pandas/test/data/hdfs.parquet",
         )
 
+    @pytest.mark.parametrize("path_type", ["url", "object"])
     @pytest.mark.xfail(
-        Engine.get() == "Python",
-        reason="S3-like path doesn't support in pandas with anonymous credentials. See issue #2301.",
+        condition="config.getoption('--simulate-cloud').lower() != 'off'",
+        reason="The reason of tests fail in `cloud` mode is unknown for now - issue #3264",
     )
-    def test_read_parquet_s3(self):
-        import s3fs
-
-        # Pandas currently supports only default credentials for boto therefore
-        # we use S3FileSystem with `anon=True` for  to make testing possible.
+    def test_read_parquet_s3(self, path_type):
         dataset_url = "s3://aws-roda-hcls-datalake/chembl_27/chembl_27_public_tissue_dictionary/part-00000-66508102-96fa-4fd9-a0fd-5bc072a74293-c000.snappy.parquet"
-        fs = s3fs.S3FileSystem(anon=True)
-        pandas_df = pandas.read_parquet(fs.open(dataset_url, "rb"))
-        modin_df_s3fs = pd.read_parquet(fs.open(dataset_url, "rb"))
-        df_equals(pandas_df, modin_df_s3fs)
-
-        # Modin supports default and anonymous credentials and resolves this internally.
-        modin_df_s3 = pd.read_parquet(dataset_url)
-        df_equals(pandas_df, modin_df_s3)
+        if path_type == "object":
+            import s3fs
+
+            fs = s3fs.S3FileSystem(anon=True)
+            with fs.open(dataset_url, "rb") as file_obj:
+                eval_io("read_parquet", path=file_obj)
+        else:
+            eval_io("read_parquet", path=dataset_url, storage_options={"anon": True})
 
     @pytest.mark.xfail(
         condition="config.getoption('--simulate-cloud').lower() != 'off'",