Repository: modin-project/modin

Issue:
Add support of `storage_options` parameter for `read_csv_glob`
Experimental function `read_csv_glob` should support `storage_options` parameter to correctly work with buckets.

For now, if the `storage_options` parameter is specified, an error occurs:

```python
import modin.experimental.pandas as pd

df = pd.read_csv_glob("s3://noaa-ghcn-pds/csv/1788.csv", storage_options={"anon": True})
```

Log:
```bash
ValueError: storage_options passed with file object or non-fsspec file path
2021-12-10 12:18:35,728 ERROR worker.py:78 -- Unhandled error (suppress with RAY_IGNORE_UNHANDLED_ERRORS=1): ray::deploy_ray_func() (pid=1231690, ip=10.125.130.30)
  File "python/ray/_raylet.pyx", line 501, in ray._raylet.execute_task
  File "/localdisk/aprutsko/miniconda3/envs/modin/lib/python3.8/site-packages/ray/util/tracing/tracing_helper.py", line 330, in _function_with_tracing
    return function(*args, **kwargs)
  File "/localdisk/aprutsko/modin/modin/core/execution/ray/common/task_wrapper.py", line 40, in deploy_ray_func
    return func(**args)
  File "/localdisk/aprutsko/modin/modin/core/storage_formats/pandas/parsers.py", line 322, in parse
    pandas_dfs.append(pandas.read_csv(BytesIO(to_read), **kwargs))
  File "/localdisk/aprutsko/miniconda3/envs/modin/lib/python3.8/site-packages/pandas/util/_decorators.py", line 311, in wrapper
    return func(*args, **kwargs)
  File "/localdisk/aprutsko/miniconda3/envs/modin/lib/python3.8/site-packages/pandas/io/parsers/readers.py", line 586, in read_csv
    return _read(filepath_or_buffer, kwds)
  File "/localdisk/aprutsko/miniconda3/envs/modin/lib/python3.8/site-packages/pandas/io/parsers/readers.py", line 482, in _read
    parser = TextFileReader(filepath_or_buffer, **kwds)
  File "/localdisk/aprutsko/miniconda3/envs/modin/lib/python3.8/site-packages/pandas/io/parsers/readers.py", line 811, in __init__
    self._engine = self._make_engine(self.engine)
  File "/localdisk/aprutsko/miniconda3/envs/modin/lib/python3.8/site-packages/pandas/io/parsers/readers.py", line 1040, in _make_engine
    return mapping[engine](self.f, **self.options)  # type: ignore[call-arg]
  File "/localdisk/aprutsko/miniconda3/envs/modin/lib/python3.8/site-packages/pandas/io/parsers/c_parser_wrapper.py", line 51, in __init__
    self._open_handles(src, kwds)
  File "/localdisk/aprutsko/miniconda3/envs/modin/lib/python3.8/site-packages/pandas/io/parsers/base_parser.py", line 222, in _open_handles
    self.handles = get_handle(
  File "/localdisk/aprutsko/miniconda3/envs/modin/lib/python3.8/site-packages/pandas/io/common.py", line 609, in get_handle
    ioargs = _get_filepath_or_buffer(
  File "/localdisk/aprutsko/miniconda3/envs/modin/lib/python3.8/site-packages/pandas/io/common.py", line 381, in _get_filepath_or_buffer
    raise ValueError(
ValueError: storage_options passed with file object or non-fsspec file path
```

Mentioned here https://github.com/modin-project/modin/issues/3766#issuecomment-984754356


Discussion:

Resolving patch:
diff --git a/modin/core/storage_formats/pandas/parsers.py b/modin/core/storage_formats/pandas/parsers.py
--- a/modin/core/storage_formats/pandas/parsers.py
+++ b/modin/core/storage_formats/pandas/parsers.py
@@ -308,11 +308,13 @@ def parse(chunks, **kwargs):
         num_splits = kwargs.pop("num_splits", None)
         index_col = kwargs.get("index_col", None)
 
+        # pop `compression` from kwargs because `bio` below is uncompressed
+        compression = kwargs.pop("compression", "infer")
+        storage_options = kwargs.pop("storage_options", None) or {}
         pandas_dfs = []
         for fname, start, end in chunks:
             if start is not None and end is not None:
-                # pop "compression" from kwargs because bio is uncompressed
-                with OpenFile(fname, "rb", kwargs.pop("compression", "infer")) as bio:
+                with OpenFile(fname, "rb", compression, **storage_options) as bio:
                     if kwargs.get("encoding", None) is not None:
                         header = b"" + bio.readline()
                     else:
@@ -322,7 +324,12 @@ def parse(chunks, **kwargs):
                 pandas_dfs.append(pandas.read_csv(BytesIO(to_read), **kwargs))
             else:
                 # This only happens when we are reading with only one worker (Default)
-                return pandas.read_csv(fname, **kwargs)
+                return pandas.read_csv(
+                    fname,
+                    compression=compression,
+                    storage_options=storage_options,
+                    **kwargs,
+                )
 
         # Combine read in data.
         if len(pandas_dfs) > 1:


Test patch:
diff --git a/modin/experimental/pandas/test/test_io_exp.py b/modin/experimental/pandas/test/test_io_exp.py
--- a/modin/experimental/pandas/test/test_io_exp.py
+++ b/modin/experimental/pandas/test/test_io_exp.py
@@ -16,6 +16,7 @@
 import pytest
 import modin.experimental.pandas as pd
 from modin.config import Engine
+from modin.utils import get_current_execution
 from modin.pandas.test.utils import df_equals, teardown_test_files, test_data
 
 
@@ -136,6 +137,31 @@ def test_read_multiple_csv_s3():
     df_equals(modin_df, pandas_df)
 
 
+@pytest.mark.skipif(
+    get_current_execution() != "ExperimentalPandasOnRay",
+    reason=f"Execution {get_current_execution()} isn't supported.",
+)
+@pytest.mark.parametrize(
+    "storage_options",
+    [{"anon": False}, {"anon": True}, {"key": "123", "secret": "123"}, None],
+)
+def test_read_multiple_csv_s3_storage_opts(storage_options):
+    path = "s3://modin-datasets/testing/multiple_csv/"
+    # Test the fact of handling of `storage_options`
+    modin_df = pd.read_csv_glob(path, storage_options=storage_options)
+    pandas_df = pd.concat(
+        [
+            pandas.read_csv(
+                f"{path}test_data{i}.csv",
+                storage_options=storage_options,
+            )
+            for i in range(2)
+        ],
+    ).reset_index(drop=True)
+
+    df_equals(modin_df, pandas_df)
+
+
 @pytest.mark.skipif(
     not Engine.get() == "Ray",
     reason=f"{Engine.get()} does not have experimental API",