Repository: modin-project/modin

Issue:
FEAT: Add support for Distributed loading on GCS buckets
## Context
The company I work for, heavily uses pandas and Google Cloud Storage (GCS) buckets. We were interested in using Modin to improve performance of loading large datasets and improve data processing tasks. 

## Problem
However, it seems as though using GCS bucket urls for the `read_csv()` function of modin, is quite slow. The GCS bucket file patterns are `gs://`. Loading from local memory or S3 buckets works quite well, but the lack of support for GCS buckets results in Modin to default to pandas, which ends up being slower than using the vanilla pandas library!

## Feature Request
We are requesting support for GCS buckets for distributed loading so that we aren't limited to S3 or local file storage only especially since the company I work for, uses Google Suite products quite a lot and does not use Amazon offerings.



Discussion:
Connected with #4742 
@Anando304 thanks for opening the issue! This is a good point and we haven't properly addressed it yet (for reasons I don't know exactly). I'll work on a PR to add support for `gs://`. 

Resolving patch:
diff --git a/modin/core/io/file_dispatcher.py b/modin/core/io/file_dispatcher.py
--- a/modin/core/io/file_dispatcher.py
+++ b/modin/core/io/file_dispatcher.py
@@ -23,9 +23,9 @@
 from modin.config import StorageFormat
 from modin.logging import ClassLogger
 import numpy as np
+from pandas.io.common import is_url, is_fsspec_url
 
 NOT_IMPLEMENTED_MESSAGE = "Implement in children classes!"
-_SUPPORTED_PROTOCOLS = {"s3", "S3", "http", "https"}
 
 
 class OpenFile:
@@ -208,12 +208,10 @@ def get_path(cls, file_path):
 
         Notes
         -----
-        if `file_path` is an S3 bucket, parameter will be returned as is, otherwise
+        if `file_path` is a URL, parameter will be returned as is, otherwise
         absolute path will be returned.
         """
-        if isinstance(file_path, str) and (
-            fsspec.core.split_protocol(file_path)[0] in _SUPPORTED_PROTOCOLS
-        ):
+        if is_fsspec_url(file_path) or is_url(file_path):
             return file_path
         else:
             return os.path.abspath(file_path)
@@ -257,15 +255,9 @@ def file_exists(cls, file_path, storage_options=None):
         bool
             Whether file exists or not.
         """
-        if (
-            not isinstance(file_path, str)
-            or fsspec.core.split_protocol(file_path)[0] not in _SUPPORTED_PROTOCOLS
-        ):
+        if not is_fsspec_url(file_path) and not is_url(file_path):
             return os.path.exists(file_path)
 
-        # `file_path` may start with a capital letter, which isn't supported by `fsspec.core.url_to_fs` used below.
-        file_path = file_path[0].lower() + file_path[1:]
-
         from botocore.exceptions import (
             NoCredentialsError,
             EndpointConnectionError,
diff --git a/modin/core/io/text/csv_glob_dispatcher.py b/modin/core/io/text/csv_glob_dispatcher.py
--- a/modin/core/io/text/csv_glob_dispatcher.py
+++ b/modin/core/io/text/csv_glob_dispatcher.py
@@ -24,14 +24,12 @@
 
 import pandas
 import pandas._libs.lib as lib
-from pandas.io.common import is_url
+from pandas.io.common import is_url, is_fsspec_url, stringify_path
 
 from modin.config import NPartitions
 from modin.core.io.file_dispatcher import OpenFile
 from modin.core.io.text.csv_dispatcher import CSVDispatcher
 
-_SUPPORTED_PROTOCOLS = {"s3", "S3"}
-
 
 class CSVGlobDispatcher(CSVDispatcher):
     """Class contains utils for reading multiple `.csv` files simultaneously."""
@@ -54,7 +52,7 @@ def _read(cls, filepath_or_buffer, **kwargs):
             Query compiler with imported data for further processing.
         """
         # Ensures that the file is a string file path. Otherwise, default to pandas.
-        filepath_or_buffer = cls.get_path_or_buffer(filepath_or_buffer)
+        filepath_or_buffer = cls.get_path_or_buffer(stringify_path(filepath_or_buffer))
         if isinstance(filepath_or_buffer, str):
             # os.altsep == None on Linux
             is_folder = any(
@@ -331,18 +329,12 @@ def file_exists(cls, file_path: str, storage_options=None) -> bool:
         bool
             True if the path is valid.
         """
-        if isinstance(file_path, str) and is_url(file_path):
-            raise NotImplementedError("`read_csv_glob` supports only s3-like paths.")
+        if is_url(file_path):
+            raise NotImplementedError("`read_csv_glob` does not support urllib paths.")
 
-        if (
-            not isinstance(file_path, str)
-            or fsspec.core.split_protocol(file_path)[0] not in _SUPPORTED_PROTOCOLS
-        ):
+        if not is_fsspec_url(file_path):
             return len(glob.glob(file_path)) > 0
 
-        # `file_path` may start with a capital letter, which isn't supported by `fsspec.core.url_to_fs` used below.
-        file_path = file_path[0].lower() + file_path[1:]
-
         from botocore.exceptions import (
             NoCredentialsError,
             EndpointConnectionError,
@@ -384,14 +376,11 @@ def get_path(cls, file_path: str) -> list:
         list
             List of strings of absolute file paths.
         """
-        if fsspec.core.split_protocol(file_path)[0] not in _SUPPORTED_PROTOCOLS:
+        if not is_fsspec_url(file_path) and not is_url(file_path):
             relative_paths = glob.glob(file_path)
             abs_paths = [os.path.abspath(path) for path in relative_paths]
             return abs_paths
 
-        # `file_path` may start with a capital letter, which isn't supported by `fsspec.core.url_to_fs` used below.
-        file_path = file_path[0].lower() + file_path[1:]
-
         from botocore.exceptions import (
             NoCredentialsError,
             EndpointConnectionError,
@@ -402,8 +391,8 @@ def get_file_path(fs_handle) -> List[str]:
             file_paths = fs_handle.glob(file_path)
             if len(file_paths) == 0 and not fs_handle.exists(file_path):
                 raise FileNotFoundError(f"Path <{file_path}> isn't available.")
-            s3_addresses = [f"s3://{path}" for path in file_paths]
-            return s3_addresses
+            fs_addresses = [fs_handle.unstrip_protocol(path) for path in file_paths]
+            return fs_addresses
 
         fs, _ = fsspec.core.url_to_fs(file_path)
         try:


Test patch:
diff --git a/modin/experimental/pandas/test/test_io_exp.py b/modin/experimental/pandas/test/test_io_exp.py
--- a/modin/experimental/pandas/test/test_io_exp.py
+++ b/modin/experimental/pandas/test/test_io_exp.py
@@ -164,12 +164,17 @@ def test_read_single_csv_with_parse_dates(self, parse_dates):
             df_equals(modin_df, pandas_df)
 
 
+@pytest.mark.parametrize(
+    "path",
+    [
+        "s3://modin-datasets/testing/multiple_csv/test_data*.csv",
+        "gs://modin-testing/testing/multiple_csv/test_data*.csv",
+    ],
+)
 @pytest.mark.skipif(
     Engine.get() != "Ray", reason="Currently only support Ray engine for glob paths."
 )
-def test_read_multiple_csv_s3():
-    path = "S3://modin-datasets/testing/multiple_csv/test_data*.csv"
-
+def test_read_multiple_csv_cloud_store(path):
     def _pandas_read_csv_glob(path, storage_options):
         pandas_dfs = [
             pandas.read_csv(
diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py
--- a/modin/pandas/test/test_io.py
+++ b/modin/pandas/test/test_io.py
@@ -915,6 +915,13 @@ def test_read_csv_categories(self):
             dtype={"one": "int64", "two": "category"},
         )
 
+    def test_read_csv_google_cloud_storage(self):
+        eval_io(
+            fn_name="read_csv",
+            # read_csv kwargs
+            filepath_or_buffer="gs://modin-testing/testing/multiple_csv/test_data0.csv",
+        )
+
     @pytest.mark.parametrize("encoding", [None, "utf-8"])
     @pytest.mark.parametrize("encoding_errors", ["strict", "ignore"])
     @pytest.mark.parametrize(