Repository: modin-project/modin

Issue:
`read_csv` does not honor `storage_options`
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Linux Ubuntu 20.04
- **Modin version** (`modin.__version__`): 0.15.2
- **Python version**: 3.9.13
- **Code we can use to reproduce**:

```python
import modin.pandas as pd

    df = pd.read_csv(
        CSV_URL,
        storage_options={
            'key': AWS_ACCESS_KEY_ID,
            'secret': AWS_SECRET_ACCESS_KEY,
            'client_kwargs': {
                'endpoint_url': 'https://minio.minio',
                'verify': False,
            },
            'config_kwargs': {
                'region_name': 'us-west-1',
                'signature_version': 's3v4',
            },
        },
    )
```

### Describe the problem
The code above to read a CSV file from a MinIO bucket works with Pandas and Modin (with `MODIN_ENGINE=python`). Does not work with Modin with Ray engine. 

The error is 
```
  File "/usr/local/lib/python3.9/site-packages/modin/logging/logger_function.py", line 65, in run_and_log
    return f(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/modin/pandas/io.py", line 140, in read_csv
    return _read(**kwargs)
  File "/usr/local/lib/python3.9/site-packages/modin/pandas/io.py", line 61, in _read
    pd_obj = FactoryDispatcher.read_csv(**kwargs)
  File "/usr/local/lib/python3.9/site-packages/modin/core/execution/dispatching/factories/dispatcher.py", line 185, in read_csv
    return cls.__factory._read_csv(**kwargs)
  File "/usr/local/lib/python3.9/site-packages/modin/core/execution/dispatching/factories/factories.py", line 217, in _read_csv
    return cls.io_cls.read_csv(**kwargs)
  File "/usr/local/lib/python3.9/site-packages/modin/logging/logger_function.py", line 65, in run_and_log
    return f(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/modin/core/io/file_dispatcher.py", line 153, in read
    query_compiler = cls._read(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/modin/logging/logger_function.py", line 65, in run_and_log
    return f(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/modin/core/io/text/text_file_dispatcher.py", line 1009, in _read
    use_modin_impl = cls.check_parameters_support(
  File "/usr/local/lib/python3.9/site-packages/modin/logging/logger_function.py", line 65, in run_and_log
    return f(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/modin/core/io/text/text_file_dispatcher.py", line 656, in check_parameters_support
    if not cls.file_exists(filepath_or_buffer):
  File "/usr/local/lib/python3.9/site-packages/modin/logging/logger_function.py", line 65, in run_and_log
    return f(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/modin/core/io/file_dispatcher.py", line 271, in file_exists
    exists = s3fs.exists(file_path) or exists
  File "/usr/local/lib/python3.9/site-packages/fsspec/asyn.py", line 86, in wrapper
    return sync(self.loop, func, *args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/fsspec/asyn.py", line 66, in sync
    raise return_result
  File "/usr/local/lib/python3.9/site-packages/fsspec/asyn.py", line 26, in _runner
    result[0] = await coro
  File "/usr/local/lib/python3.9/site-packages/s3fs/core.py", line 888, in _exists
    await self._info(path, bucket, key, version_id=version_id)
  File "/usr/local/lib/python3.9/site-packages/s3fs/core.py", line 1140, in _info
    out = await self._call_s3(
  File "/usr/local/lib/python3.9/site-packages/s3fs/core.py", line 325, in _call_s3
    await self.set_session()
  File "/usr/local/lib/python3.9/site-packages/s3fs/core.py", line 473, in set_session
    self._s3 = await s3creator.__aenter__()
  File "/usr/local/lib/python3.9/site-packages/aiobotocore/session.py", line 22, in __aenter__
    self._client = await self._coro
  File "/usr/local/lib/python3.9/site-packages/aiobotocore/session.py", line 102, in _create_client
    credentials = await self.get_credentials()
  File "/usr/local/lib/python3.9/site-packages/aiobotocore/session.py", line 133, in get_credentials
    self._credentials = await (self._components.get_component(
  File "/usr/local/lib/python3.9/site-packages/aiobotocore/credentials.py", line 814, in load_credentials
    creds = await provider.load()
  File "/usr/local/lib/python3.9/site-packages/aiobotocore/credentials.py", line 486, in load
    metadata = await fetcher.retrieve_iam_role_credentials()
  File "/usr/local/lib/python3.9/site-packages/aiobotocore/utils.py", line 175, in retrieve_iam_role_credentials
    token = await self._fetch_metadata_token()
  File "/usr/local/lib/python3.9/site-packages/aiobotocore/utils.py", line 88, in _fetch_metadata_token
    response = await session.send(request.prepare())
  File "/usr/local/lib/python3.9/site-packages/aiobotocore/httpsession.py", line 210, in send
    raise ConnectTimeoutError(endpoint_url=request.url, error=e)
botocore.exceptions.ConnectTimeoutError: Connect timeout on endpoint URL: "http://169.254.169.254/latest/api/token"
```

Looks like AWS region, key, and secret are not specified correctly on the Ray node. When I _manually_ set them on the Ray node in environment variables `AWS_DEFAULT_REGION`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, then I have another error:

```
  File "/usr/local/lib/python3.9/site-packages/modin/logging/logger_function.py", line 65, in run_and_log
    return f(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/modin/pandas/io.py", line 140, in read_csv
    return _read(**kwargs)
  File "/usr/local/lib/python3.9/site-packages/modin/pandas/io.py", line 61, in _read
    pd_obj = FactoryDispatcher.read_csv(**kwargs)
  File "/usr/local/lib/python3.9/site-packages/modin/core/execution/dispatching/factories/dispatcher.py", line 185, in read_csv
    return cls.__factory._read_csv(**kwargs)
  File "/usr/local/lib/python3.9/site-packages/modin/core/execution/dispatching/factories/factories.py", line 217, in _read_csv
    return cls.io_cls.read_csv(**kwargs)
  File "/usr/local/lib/python3.9/site-packages/modin/logging/logger_function.py", line 65, in run_and_log
    return f(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/modin/core/io/file_dispatcher.py", line 153, in read
    query_compiler = cls._read(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/modin/logging/logger_function.py", line 65, in run_and_log
    return f(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/modin/core/io/text/text_file_dispatcher.py", line 1009, in _read
    use_modin_impl = cls.check_parameters_support(
  File "/usr/local/lib/python3.9/site-packages/modin/logging/logger_function.py", line 65, in run_and_log
    return f(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/modin/core/io/text/text_file_dispatcher.py", line 656, in check_parameters_support
    if not cls.file_exists(filepath_or_buffer):
  File "/usr/local/lib/python3.9/site-packages/modin/logging/logger_function.py", line 65, in run_and_log
    return f(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/modin/core/io/file_dispatcher.py", line 271, in file_exists
    exists = s3fs.exists(file_path) or exists
  File "/usr/local/lib/python3.9/site-packages/fsspec/asyn.py", line 86, in wrapper
    return sync(self.loop, func, *args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/fsspec/asyn.py", line 66, in sync
    raise return_result
  File "/usr/local/lib/python3.9/site-packages/fsspec/asyn.py", line 26, in _runner
    result[0] = await coro
  File "/usr/local/lib/python3.9/site-packages/s3fs/core.py", line 888, in _exists
    await self._info(path, bucket, key, version_id=version_id)
  File "/usr/local/lib/python3.9/site-packages/s3fs/core.py", line 1140, in _info
    out = await self._call_s3(
  File "/usr/local/lib/python3.9/site-packages/s3fs/core.py", line 332, in _call_s3
    return await _error_wrapper(
  File "/usr/local/lib/python3.9/site-packages/s3fs/core.py", line 137, in _error_wrapper
    raise err
PermissionError: Forbidden
```

In this case, `endpoint_url` from `storage_options` is not honored on the Ray node and S3 client tries to use AWS S3 endpoint instead of the specified one.


Discussion:

Resolving patch:
diff --git a/modin/core/io/file_dispatcher.py b/modin/core/io/file_dispatcher.py
--- a/modin/core/io/file_dispatcher.py
+++ b/modin/core/io/file_dispatcher.py
@@ -236,7 +236,7 @@ def file_size(cls, f):
         return size
 
     @classmethod
-    def file_exists(cls, file_path):
+    def file_exists(cls, file_path, storage_options=None):
         """
         Check if `file_path` exists.
 
@@ -245,6 +245,8 @@ def file_exists(cls, file_path):
         file_path : str
             String that represents the path to the file (paths to S3 buckets
             are also acceptable).
+        storage_options : dict, optional
+            Keyword from `read_*` functions.
 
         Returns
         -------
@@ -261,13 +263,19 @@ def file_exists(cls, file_path):
                 )
                 from botocore.exceptions import NoCredentialsError
 
-                s3fs = S3FS.S3FileSystem(anon=False)
+                if storage_options is not None:
+                    new_storage_options = dict(storage_options)
+                    new_storage_options.pop("anon", None)
+                else:
+                    new_storage_options = {}
+
+                s3fs = S3FS.S3FileSystem(anon=False, **new_storage_options)
                 exists = False
                 try:
                     exists = s3fs.exists(file_path) or exists
-                except NoCredentialsError:
+                except (NoCredentialsError, PermissionError):
                     pass
-                s3fs = S3FS.S3FileSystem(anon=True)
+                s3fs = S3FS.S3FileSystem(anon=True, **new_storage_options)
                 return exists or s3fs.exists(file_path)
         return os.path.exists(file_path)
 
diff --git a/modin/core/io/text/csv_glob_dispatcher.py b/modin/core/io/text/csv_glob_dispatcher.py
--- a/modin/core/io/text/csv_glob_dispatcher.py
+++ b/modin/core/io/text/csv_glob_dispatcher.py
@@ -63,7 +63,7 @@ def _read(cls, filepath_or_buffer, **kwargs):
                     "Shell-style wildcard '*' must be in the filename pattern in order to read multiple "
                     + f"files at once. Did you forget it? Passed filename: '{filepath_or_buffer}'"
                 )
-            if not cls.file_exists(filepath_or_buffer):
+            if not cls.file_exists(filepath_or_buffer, kwargs.get("storage_options")):
                 return cls.single_worker_read(filepath_or_buffer, **kwargs)
             filepath_or_buffer = cls.get_path(filepath_or_buffer)
         elif not cls.pathlib_or_pypath(filepath_or_buffer):
@@ -160,7 +160,14 @@ def _read(cls, filepath_or_buffer, **kwargs):
 
         with ExitStack() as stack:
             files = [
-                stack.enter_context(OpenFile(fname, "rb", compression_type))
+                stack.enter_context(
+                    OpenFile(
+                        fname,
+                        "rb",
+                        compression_type,
+                        **(kwargs.get("storage_options", None) or {}),
+                    )
+                )
                 for fname in glob_filepaths
             ]
 
@@ -283,7 +290,7 @@ def _read(cls, filepath_or_buffer, **kwargs):
         return new_query_compiler
 
     @classmethod
-    def file_exists(cls, file_path: str) -> bool:
+    def file_exists(cls, file_path: str, storage_options=None) -> bool:
         """
         Check if the `file_path` is valid.
 
@@ -291,6 +298,8 @@ def file_exists(cls, file_path: str) -> bool:
         ----------
         file_path : str
             String representing a path.
+        storage_options : dict, optional
+            Keyword from `read_*` functions.
 
         Returns
         -------
@@ -307,13 +316,19 @@ def file_exists(cls, file_path: str) -> bool:
                 )
                 from botocore.exceptions import NoCredentialsError
 
-                s3fs = S3FS.S3FileSystem(anon=False)
+                if storage_options is not None:
+                    new_storage_options = dict(storage_options)
+                    new_storage_options.pop("anon", None)
+                else:
+                    new_storage_options = {}
+
+                s3fs = S3FS.S3FileSystem(anon=False, **new_storage_options)
                 exists = False
                 try:
                     exists = len(s3fs.glob(file_path)) > 0 or exists
-                except NoCredentialsError:
+                except (NoCredentialsError, PermissionError):
                     pass
-                s3fs = S3FS.S3FileSystem(anon=True)
+                s3fs = S3FS.S3FileSystem(anon=True, **new_storage_options)
                 return exists or len(s3fs.glob(file_path)) > 0
         return len(glob.glob(file_path)) > 0
 
diff --git a/modin/core/io/text/text_file_dispatcher.py b/modin/core/io/text/text_file_dispatcher.py
--- a/modin/core/io/text/text_file_dispatcher.py
+++ b/modin/core/io/text/text_file_dispatcher.py
@@ -638,7 +638,9 @@ def check_parameters_support(
         """
         skiprows = read_kwargs.get("skiprows")
         if isinstance(filepath_or_buffer, str):
-            if not cls.file_exists(filepath_or_buffer):
+            if not cls.file_exists(
+                filepath_or_buffer, read_kwargs.get("storage_options")
+            ):
                 return False
         elif not cls.pathlib_or_pypath(filepath_or_buffer):
             return False
@@ -1020,8 +1022,12 @@ def _read(cls, filepath_or_buffer, **kwargs):
             nrows=None,
             compression=compression_infered,
         )
-
-        with OpenFile(filepath_or_buffer_md, "rb", compression_infered) as f:
+        with OpenFile(
+            filepath_or_buffer_md,
+            "rb",
+            compression_infered,
+            **(kwargs.get("storage_options", None) or {}),
+        ) as f:
             old_pos = f.tell()
             fio = io.TextIOWrapper(f, encoding=encoding, newline="")
             newline, quotechar = cls.compute_newline(


Test patch: