Repository: modin-project/modin

Issue:
Modin issue to cause the read_parquet file from Minio with error
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**:
- **Modin version** (`modin.__version__`):
- **Python version**:
- **Code we can use to reproduce**:

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
Error:  i can use the latest modin to read_csv from Modin/Minio and can not read_parquet from Modin/Minio with below error message. But if i used with standard pandas i still can read_parquet from Minio without modin. this is the Modin problem with minio and read_parquet function.

Error message as  below:

<details>

<summary>Stack trace</summary>

```
Caught schedule exception
Caught schedule exception
Caught schedule exception
Caught schedule exception
Caught schedule exception
Caught schedule exception
Caught schedule exception
Caught schedule exception
Caught schedule exception
Caught schedule exception
Caught schedule exception
Caught schedule exception
Caught schedule exception
---------------------------------------------------------------------------
OSError                                   Traceback (most recent call last)
/tmp/ipykernel_309/3390206043.py in <module>
     42 else:
     43     minio_file_name = 's3://test/part-00017.parquet' #files[i] #
---> 44     df = pd.read_parquet(
     45             #'s3://test/part-00015.parquet',
     46             minio_file_name,

/opt/conda/lib/python3.9/site-packages/modin/logging/logger_decorator.py in run_and_log(*args, **kwargs)
    126             """
    127             if LogMode.get() == "disable":
--> 128                 return obj(*args, **kwargs)
    129 
    130             logger = get_logger()

/opt/conda/lib/python3.9/site-packages/modin/_compat/pandas_api/latest/io.py in read_parquet(path, engine, columns, storage_options, use_nullable_dtypes, **kwargs)
    207 
    208     return DataFrame(
--> 209         query_compiler=FactoryDispatcher.read_parquet(
    210             path=path,
    211             engine=engine,

/opt/conda/lib/python3.9/site-packages/modin/core/execution/dispatching/factories/dispatcher.py in read_parquet(cls, **kwargs)
    178     @_inherit_docstrings(factories.BaseFactory._read_parquet)
    179     def read_parquet(cls, **kwargs):
--> 180         return cls.__factory._read_parquet(**kwargs)
    181 
    182     @classmethod

/opt/conda/lib/python3.9/site-packages/modin/core/execution/dispatching/factories/factories.py in _read_parquet(cls, **kwargs)
    205     )
    206     def _read_parquet(cls, **kwargs):
--> 207         return cls.io_cls.read_parquet(**kwargs)
    208 
    209     @classmethod

/opt/conda/lib/python3.9/site-packages/modin/logging/logger_decorator.py in run_and_log(*args, **kwargs)
    126             """
    127             if LogMode.get() == "disable":
--> 128                 return obj(*args, **kwargs)
    129 
    130             logger = get_logger()

/opt/conda/lib/python3.9/site-packages/modin/core/io/file_dispatcher.py in read(cls, *args, **kwargs)
    154         postprocessing work on the resulting query_compiler object.
    155         """
--> 156         query_compiler = cls._read(*args, **kwargs)
    157         # TODO (devin-petersohn): Make this section more general for non-pandas kernel
    158         # implementations.

/opt/conda/lib/python3.9/site-packages/modin/logging/logger_decorator.py in run_and_log(*args, **kwargs)
    126             """
    127             if LogMode.get() == "disable":
--> 128                 return obj(*args, **kwargs)
    129 
    130             logger = get_logger()

/opt/conda/lib/python3.9/site-packages/modin/core/io/column_stores/parquet_dispatcher.py in _read(cls, path, engine, columns, **kwargs)
    403         ]
    404 
--> 405         return cls.build_query_compiler(path, columns, index_columns, **kwargs)

/opt/conda/lib/python3.9/site-packages/modin/logging/logger_decorator.py in run_and_log(*args, **kwargs)
    126             """
    127             if LogMode.get() == "disable":
--> 128                 return obj(*args, **kwargs)
    129 
    130             logger = get_logger()

/opt/conda/lib/python3.9/site-packages/modin/core/io/column_stores/parquet_dispatcher.py in build_query_compiler(cls, path, columns, index_columns, **kwargs)
    309         col_partitions, column_widths = cls.build_columns(columns)
    310         partition_ids = cls.call_deploy(path, col_partitions, **kwargs)
--> 311         index, sync_index = cls.build_index(path, partition_ids, index_columns)
    312         remote_parts = cls.build_partition(partition_ids, column_widths)
    313         if len(partition_ids) > 0:

/opt/conda/lib/python3.9/site-packages/modin/logging/logger_decorator.py in run_and_log(*args, **kwargs)
    126             """
    127             if LogMode.get() == "disable":
--> 128                 return obj(*args, **kwargs)
    129 
    130             logger = get_logger()

/opt/conda/lib/python3.9/site-packages/modin/core/io/column_stores/parquet_dispatcher.py in build_index(cls, path, partition_ids, index_columns)
    275         if range_index or (len(partition_ids) == 0 and len(column_names_to_read) != 0):
    276             complete_index = (
--> 277                 read_table(path, columns=column_names_to_read).to_pandas().index
    278             )
    279         # Empty DataFrame case

/opt/conda/lib/python3.9/site-packages/pyarrow/parquet/__init__.py in read_table(source, columns, use_threads, metadata, schema, use_pandas_metadata, memory_map, read_dictionary, filesystem, filters, buffer_size, partitioning, use_legacy_dataset, ignore_prefixes, pre_buffer, coerce_int96_timestamp_unit, decryption_properties)
   2735             )
   2736         try:
-> 2737             dataset = _ParquetDatasetV2(
   2738                 source,
   2739                 schema=schema,

/opt/conda/lib/python3.9/site-packages/pyarrow/parquet/__init__.py in __init__(self, path_or_paths, filesystem, filters, partitioning, read_dictionary, buffer_size, memory_map, ignore_prefixes, pre_buffer, coerce_int96_timestamp_unit, schema, decryption_properties, **kwargs)
   2327                     except ValueError:
   2328                         filesystem = LocalFileSystem(use_mmap=memory_map)
-> 2329                 if filesystem.get_file_info(path_or_paths).is_file:
   2330                     single_file = path_or_paths
   2331             else:

/opt/conda/lib/python3.9/site-packages/pyarrow/_fs.pyx in pyarrow._fs.FileSystem.get_file_info()

/opt/conda/lib/python3.9/site-packages/pyarrow/error.pxi in pyarrow.lib.pyarrow_internal_check_status()

/opt/conda/lib/python3.9/site-packages/pyarrow/error.pxi in pyarrow.lib.check_status()

OSError: When getting information for key 'part-00017.parquet' in bucket 'test': AWS Error [code 15]: No response body.


===============================================
```

</details>



### Source code / logs

<details>

<summary>Show source</summary>

```python

 import os
       
        
        MODIN_ENGINE = 'ray'
        RAY_URL = 'ray://ray-ray-head.ray:10001'
        #RAY_URL = 'ray://ray-ray-head.ray.svc.cluster.local:10001'
        AWS_ACCESS_KEY_ID = 'x1miniouser'
        AWS_SECRET_ACCESS_KEY = 'x1miniopass'
        
        import time

        import modin.pandas as pd
        import modin.config as cfg
        
        #import pandas as pd2

        import ray
        
        ray.init(RAY_URL)
        cfg.NPartitions.put(16)        

        select_csv = False
        if select_csv:
            minio_file_name = 's3://test/' + 'df3.csv'
            df = pd.read_csv(
                    #'s3://test/part-00015.parquet',
                    minio_file_name,
                    storage_options={
                        'key': AWS_ACCESS_KEY_ID,
                        'secret': AWS_SECRET_ACCESS_KEY,
                        'client_kwargs': {
                            'endpoint_url': 'https://minio.minio',
                            'verify': False,
                        },
                        'config_kwargs': {
                            'region_name': 'us-west-1',
                            'signature_version': 's3v4',
                        }
                    }
            )
        else:
            minio_file_name = 's3://test/part-00017.parquet' #files[i] #
            df = pd.read_parquet(
                    #'s3://test/part-00015.parquet',
                    minio_file_name,
                    storage_options={
                        'key': AWS_ACCESS_KEY_ID,
                        'secret': AWS_SECRET_ACCESS_KEY,
                        'client_kwargs': {
                            'endpoint_url': 'https://minio.minio',
                            'verify': False,
                        },
                        'config_kwargs': {
                            'region_name': 'us-west-1',
                            'signature_version': 's3v4',
                        }
                    }
            ). 

```

</details>


Discussion:
cc @mvashishtha 
@ZhengHongming888 thanks for opening the issue! It seems that the problem is that `storage_options` is not handled properly for the `read_table` call internally. We should handle this accordingly. I'll make a fix and open a PR for this. 
@ZhengHongming888 I'm working on a fix right now, but you can get around this issue by opening a path with `s3fs.S3FileSystem()`.

Resolving patch:
diff --git a/modin/core/io/column_stores/parquet_dispatcher.py b/modin/core/io/column_stores/parquet_dispatcher.py
--- a/modin/core/io/column_stores/parquet_dispatcher.py
+++ b/modin/core/io/column_stores/parquet_dispatcher.py
@@ -83,7 +83,30 @@ def _unstrip_protocol(protocol, path):
         ]
 
     @classmethod
-    def call_deploy(cls, fname, col_partitions, **kwargs):
+    def _get_fs_and_fs_path(cls, path, storage_options):
+        """
+        Retrieve filesystem interface and filesystem-specific path.
+
+        Parameters
+        ----------
+        path : str, path object or file-like object
+            Path to dataset.
+        storage_options : dict
+            Parameters for specific storage engine.
+
+        Returns
+        -------
+        filesystem : Any
+            Protocol implementation of registry.
+        fs_path : list
+            Filesystem's specific path.
+        """
+        return (
+            url_to_fs(path, **storage_options) if is_fsspec_url(path) else (None, path)
+        )
+
+    @classmethod
+    def call_deploy(cls, fname, col_partitions, storage_options, **kwargs):
         """
         Deploy remote tasks to the workers with passed parameters.
 
@@ -94,6 +117,8 @@ def call_deploy(cls, fname, col_partitions, **kwargs):
         col_partitions : list
             List of arrays with columns names that should be read
             by each partition.
+        storage_options : dict
+            Parameters for specific storage engine.
         **kwargs : dict
             Parameters of deploying read_* function.
 
@@ -110,8 +135,6 @@ def call_deploy(cls, fname, col_partitions, **kwargs):
         if len(col_partitions) == 0:
             return []
 
-        storage_options = kwargs.pop("storage_options", {}) or {}
-
         filesystem, parquet_files = cls.get_fsspec_files(fname, storage_options)
 
         row_groups_per_file = []
@@ -230,7 +253,7 @@ def build_partition(cls, partition_ids, column_widths):
         )
 
     @classmethod
-    def build_index(cls, path, partition_ids, index_columns):
+    def build_index(cls, path, partition_ids, index_columns, storage_options):
         """
         Compute index and its split sizes of resulting Modin DataFrame.
 
@@ -242,6 +265,8 @@ def build_index(cls, path, partition_ids, index_columns):
             Array with references to the partitions data.
         index_columns : list
             List of index columns specified by pandas metadata.
+        storage_options : dict
+            Parameters for specific storage engine.
 
         Returns
         -------
@@ -273,8 +298,11 @@ def build_index(cls, path, partition_ids, index_columns):
         # For the second check, let us consider the case where we have an empty dataframe,
         # that has a valid index.
         if range_index or (len(partition_ids) == 0 and len(column_names_to_read) != 0):
+            fs, fs_path = cls._get_fs_and_fs_path(path, storage_options)
             complete_index = (
-                read_table(path, columns=column_names_to_read).to_pandas().index
+                read_table(fs_path, columns=column_names_to_read, filesystem=fs)
+                .to_pandas()
+                .index
             )
         # Empty DataFrame case
         elif len(partition_ids) == 0:
@@ -306,9 +334,12 @@ def build_query_compiler(cls, path, columns, index_columns, **kwargs):
         new_query_compiler : BaseQueryCompiler
             Query compiler with imported data for further processing.
         """
+        storage_options = kwargs.pop("storage_options", {}) or {}
         col_partitions, column_widths = cls.build_columns(columns)
-        partition_ids = cls.call_deploy(path, col_partitions, **kwargs)
-        index, sync_index = cls.build_index(path, partition_ids, index_columns)
+        partition_ids = cls.call_deploy(path, col_partitions, storage_options, **kwargs)
+        index, sync_index = cls.build_index(
+            path, partition_ids, index_columns, storage_options
+        )
         remote_parts = cls.build_partition(partition_ids, column_widths)
         if len(partition_ids) > 0:
             row_lengths = [part.length() for part in remote_parts.T[0]]
@@ -383,11 +414,7 @@ def _read(cls, path, engine, columns, **kwargs):
                     **kwargs,
                 )
         # url_to_fs returns the fs and path formatted for the specific fs
-        fs, fs_path = (
-            url_to_fs(path, **(kwargs.get("storage_options") or {}))
-            if is_fsspec_url(path)
-            else (None, path)
-        )
+        fs, fs_path = cls._get_fs_and_fs_path(path, kwargs.get("storage_options") or {})
         dataset = ParquetDataset(fs_path, filesystem=fs, use_legacy_dataset=False)
         index_columns = (
             dataset.schema.pandas_metadata.get("index_columns", [])


Test patch: