Repository: modin-project/modin

Issue:
Can't load file from s3 bucket 
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: MAC 18.7.0
- **Modin version** (`modin.__version__`): '0.7.4'
- **Python version**: 3.7
- **Code we can use to reproduce**:

```
import modin.pandas as mpd
import padnas as pd 

path = "s3://bucket_name/data/dataframe.snappy.parquet"
df = pd.read_parquet(path) # works 
df2 = mpd.read_parquet(path) 
```

### Describe the problem
I can't load snappy.parquet files from s3. Pandas works fine. Does modin support snappy.parquet files?

### Source code / logs
Error log: 

```
---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
<ipython-input-14-814dc08ef229> in <module>
----> 1 df2 = mpd.read_parquet(path)

/opt/anaconda3/envs/myenv/lib/python3.7/site-packages/modin/pandas/io.py in read_parquet(path, engine, columns, **kwargs)
     40     return DataFrame(
     41         query_compiler=EngineDispatcher.read_parquet(
---> 42             path=path, columns=columns, engine=engine, **kwargs
     43         )
     44     )

/opt/anaconda3/envs/myenv/lib/python3.7/site-packages/modin/data_management/dispatcher.py in read_parquet(cls, **kwargs)
    105     @classmethod
    106     def read_parquet(cls, **kwargs):
--> 107         return cls.__engine._read_parquet(**kwargs)
    108 
    109     @classmethod

/opt/anaconda3/envs/myenv/lib/python3.7/site-packages/modin/data_management/factories.py in _read_parquet(cls, **kwargs)
     46     @classmethod
     47     def _read_parquet(cls, **kwargs):
---> 48         return cls.io_cls.read_parquet(**kwargs)
     49 
     50     @classmethod

/opt/anaconda3/envs/myenv/lib/python3.7/site-packages/modin/engines/base/io/file_reader.py in read(cls, *args, **kwargs)
     27     @classmethod
     28     def read(cls, *args, **kwargs):
---> 29         query_compiler = cls._read(*args, **kwargs)
     30         # TODO (devin-petersohn): Make this section more general for non-pandas kernel
     31         # implementations.

/opt/anaconda3/envs/myenv/lib/python3.7/site-packages/modin/engines/base/io/column_stores/parquet_reader.py in _read(cls, path, engine, columns, **kwargs)
     68                 column_names = pd.schema.names
     69             else:
---> 70                 meta = ParquetFile(path).metadata
     71                 column_names = meta.schema.names
     72             if meta is not None:

/opt/anaconda3/envs/myenv/lib/python3.7/site-packages/pyarrow/parquet.py in __init__(self, source, metadata, common_metadata, read_dictionary, memory_map, buffer_size)
    135         self.reader.open(source, use_memory_map=memory_map,
    136                          buffer_size=buffer_size,
--> 137                          read_dictionary=read_dictionary, metadata=metadata)
    138         self.common_metadata = common_metadata
    139         self._nested_paths_by_prefix = self._build_nested_paths()

/opt/anaconda3/envs/myenv/lib/python3.7/site-packages/pyarrow/_parquet.pyx in pyarrow._parquet.ParquetReader.open()

/opt/anaconda3/envs/myenv/lib/python3.7/site-packages/pyarrow/io.pxi in pyarrow.lib.get_reader()

/opt/anaconda3/envs/myenv/lib/python3.7/site-packages/pyarrow/io.pxi in pyarrow.lib._get_native_file()

/opt/anaconda3/envs/myenv/lib/python3.7/site-packages/pyarrow/io.pxi in pyarrow.lib.OSFile.__cinit__()

/opt/anaconda3/envs/myenv/lib/python3.7/site-packages/pyarrow/io.pxi in pyarrow.lib.OSFile._open_readable()

/opt/anaconda3/envs/myenv/lib/python3.7/site-packages/pyarrow/error.pxi in pyarrow.lib.pyarrow_internal_check_status()

/opt/anaconda3/envs/myenv/lib/python3.7/site-packages/pyarrow/error.pxi in pyarrow.lib.check_status()

FileNotFoundError: [Errno 2] Failed to open local file s3://bucket_name/data/dataframe.snappy.parquet'. Detail: [errno 2] No such file or directory
```



Discussion:
Hi @DenisVorotyntsev, thanks for posting!

This error is coming from pyarrow, and it looks like we need to provide it some more hints about how to open these types of S3 files. We use Pyarrow to read the parquet files and it should work on S3, but it looks like in this particular case it is not working. 

We will get this resolved, thanks!
@devin-petersohn, thank you for fast response! I hope to see it in future releases! 
@DenisVorotyntsev I am going to reopen this so we don't lose track of it 😄 
I wonder if it is related to https://github.com/modin-project/modin/issues/2099, seems so
As I checked `read_parquet` function can read files from local filesystem and hdfs (after #2120 merged) only. This issue is similar to #1144 and could be fixed by the same way, I think.
We don't use our internal `open_file` utility for parquet because we are using Pyarrow to read the files and the compatibility has not been tested. If we can test that it will work to use `open_file`, that will resolve this issue.
I'll try to use `open_file` for parquet files.

Resolving patch:
diff --git a/modin/backends/pandas/parsers.py b/modin/backends/pandas/parsers.py
--- a/modin/backends/pandas/parsers.py
+++ b/modin/backends/pandas/parsers.py
@@ -355,6 +355,17 @@ class PandasParquetParser(PandasParser):
     def parse(fname, **kwargs):
         num_splits = kwargs.pop("num_splits", None)
         columns = kwargs.get("columns", None)
+        if fname.startswith("s3://"):
+            from botocore.exceptions import NoCredentialsError
+            import s3fs
+
+            try:
+                fs = s3fs.S3FileSystem()
+                fname = fs.open(fname)
+            except NoCredentialsError:
+                fs = s3fs.S3FileSystem(anon=True)
+                fname = fs.open(fname)
+
         if num_splits is None:
             return pandas.read_parquet(fname, **kwargs)
         kwargs["use_pandas_metadata"] = True
diff --git a/modin/engines/base/frame/data.py b/modin/engines/base/frame/data.py
--- a/modin/engines/base/frame/data.py
+++ b/modin/engines/base/frame/data.py
@@ -285,14 +285,14 @@ def _filter_empties(self):
                 [
                     self._partitions[i][j]
                     for j in range(len(self._partitions[i]))
-                    if j < len(self._column_widths) and self._column_widths[j] > 0
+                    if j < len(self._column_widths) and self._column_widths[j] != 0
                 ]
                 for i in range(len(self._partitions))
-                if i < len(self._row_lengths) and self._row_lengths[i] > 0
+                if i < len(self._row_lengths) and self._row_lengths[i] != 0
             ]
         )
-        self._column_widths_cache = [w for w in self._column_widths if w > 0]
-        self._row_lengths_cache = [r for r in self._row_lengths if r > 0]
+        self._column_widths_cache = [w for w in self._column_widths if w != 0]
+        self._row_lengths_cache = [r for r in self._row_lengths if r != 0]
 
     def _validate_axis_equality(self, axis: int, force: bool = False):
         """
diff --git a/modin/engines/base/io/column_stores/parquet_reader.py b/modin/engines/base/io/column_stores/parquet_reader.py
--- a/modin/engines/base/io/column_stores/parquet_reader.py
+++ b/modin/engines/base/io/column_stores/parquet_reader.py
@@ -12,6 +12,7 @@
 # governing permissions and limitations under the License.
 
 import os
+import s3fs
 
 from modin.engines.base.io.column_stores.column_store_reader import ColumnStoreReader
 from modin.error_message import ErrorMessage
@@ -48,7 +49,7 @@ def _read(cls, path, engine, columns, **kwargs):
         from pyarrow.parquet import ParquetFile, ParquetDataset
         from modin.pandas.io import PQ_INDEX_REGEX
 
-        if os.path.isdir(path):
+        if isinstance(path, str) and os.path.isdir(path):
             partitioned_columns = set()
             directory = True
             # We do a tree walk of the path directory because partitioned
@@ -84,6 +85,22 @@ def _read(cls, path, engine, columns, **kwargs):
                 pd = ParquetDataset(path, filesystem=fs)
                 meta = pd.metadata
                 column_names = pd.schema.names
+            elif isinstance(path, s3fs.S3File) or (
+                isinstance(path, str) and path.startswith("s3://")
+            ):
+                from botocore.exceptions import NoCredentialsError
+
+                if isinstance(path, s3fs.S3File):
+                    bucket_path = path.url().split(".s3.amazonaws.com")
+                    path = "s3://" + bucket_path[0].split("://")[1] + bucket_path[1]
+                try:
+                    fs = s3fs.S3FileSystem()
+                    pd = ParquetDataset(path, filesystem=fs)
+                except NoCredentialsError:
+                    fs = s3fs.S3FileSystem(anon=True)
+                    pd = ParquetDataset(path, filesystem=fs)
+                meta = pd.metadata
+                column_names = pd.schema.names
             else:
                 meta = ParquetFile(path).metadata
                 column_names = meta.schema.names


Test patch:
diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py
--- a/modin/pandas/test/test_io.py
+++ b/modin/pandas/test/test_io.py
@@ -1052,6 +1052,26 @@ def test_from_csv_s3(make_csv_file):
     df_equals(modin_df, pandas_df)
 
 
+@pytest.mark.skipif(
+    Engine.get() == "Python",
+    reason="S3-like path doesn't support in pandas with anonymous credentials. See issue #2301.",
+)
+def test_read_parquet_s3():
+    import s3fs
+
+    # Pandas currently supports only default credentials for boto therefore
+    # we use S3FileSystem with `anon=True` for  to make testing possible.
+    dataset_url = "s3://aws-roda-hcls-datalake/chembl_27/chembl_27_public_tissue_dictionary/part-00000-66508102-96fa-4fd9-a0fd-5bc072a74293-c000.snappy.parquet"
+    fs = s3fs.S3FileSystem(anon=True)
+    pandas_df = pandas.read_parquet(fs.open(dataset_url, "rb"))
+    modin_df_s3fs = pd.read_parquet(fs.open(dataset_url, "rb"))
+    df_equals(pandas_df, modin_df_s3fs)
+
+    # Modin supports default and anonymous credentials and resolves this internally.
+    modin_df_s3 = pd.read_parquet(dataset_url)
+    df_equals(pandas_df, modin_df_s3)
+
+
 def test_from_csv_default(make_csv_file):
     # We haven't implemented read_csv from https, but if it's implemented, then this needs to change
     dataset_url = "https://raw.githubusercontent.com/modin-project/modin/master/modin/pandas/test/data/blah.csv"