Repository: modin-project/modin

Issue:
Unable to read entire parquet directory
<!--
General questions should be asked on the mailing list modin-dev@googlegroups.com.

Before submitting an issue, please fill out the following form.
-->

### System information
- Ubuntu 19.04
- pipenv install modin:
- **Modin version**: 0.5.0
- **Python version**: 3.7.3
- **Exact command to reproduce**: pd.read_parquet('parquet-out/foo/')

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->
It seems that read_parquet cannot read an entire directory of parquet files i.e. partitioned parquet directory. 

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->

~/.local/share/virtualenvs/foo-cFmMv4Vf/lib/python3.7/site-packages/modin/pandas/io.py in read_parquet(path, engine, columns, **kwargs)                                                                         
     27     return DataFrame(                                                                                                                                                                                      
     28         query_compiler=BaseFactory.read_parquet(                                                                                                                                                           
---> 29             path=path, columns=columns, engine=engine, **kwargs                                                                                                                                            
     30         )                                                                                                                                                                                                  
     31     )                                                                                                                                                                                                      
                                                                                                                                                                                                                   
~/.local/share/virtualenvs/foo-cFmMv4Vf/lib/python3.7/site-packages/modin/data_management/factories.py in read_parquet(cls, **kwargs)                                                                           
     49     @classmethod                                                                                                                                                                                           
     50     def read_parquet(cls, **kwargs):                                                                                                                                                                       
---> 51         return cls._determine_engine()._read_parquet(**kwargs)                                                                                                                                             
     52                                                                                                                                                                                                            
     53     @classmethod                                                                                                                                                                                           
                                                                                                                                                                                                                   
~/.local/share/virtualenvs/foo-cFmMv4Vf/lib/python3.7/site-packages/modin/data_management/factories.py in _read_parquet(cls, **kwargs)                                                                          
     53     @classmethod                                                                                                                                                                                           
     54     def _read_parquet(cls, **kwargs):                                                                                                                                                                      
---> 55         return cls.io_cls.read_parquet(**kwargs)                                                                                                                                                           
     56                                                                                                                                                                                                            
     57     @classmethod                                                                                                                                                                                           
                                                                                                                                                                                                                   
~/.local/share/virtualenvs/foo-cFmMv4Vf/lib/python3.7/site-packages/modin/engines/ray/generic/io.py in read_parquet(cls, path, engine, columns, **kwargs)                                                       
    149                                                                                                                                                                                                            
    150         if not columns:                                                                                                                                                                                   
--> 151             pf = ParquetFile(path, memory_map=False)                                                                                                                                                      
    152             columns = [                                                                                                                                                                                   
    153                 name                                                                                                                                              
                                                                                                                                                                                                                  
~/.local/share/virtualenvs/foo-cFmMv4Vf/lib/python3.7/site-packages/ray/pyarrow_files/pyarrow/parquet.py in __init__(self, source, metadata, common_metadata, memory_map)
    127                  memory_map=True):                                                                                                                                                                        
    128         self.reader = ParquetReader()                                                                                                                                                                     
--> 129         self.reader.open(source, use_memory_map=memory_map, metadata=metadata)                                                                                                                            
    130         self.common_metadata = common_metadata                                                                                                                                                             
    131         self._nested_paths_by_prefix = self._build_nested_paths()

~/.local/share/virtualenvs/foo-cFmMv4Vf/lib/python3.7/site-packages/ray/pyarrow_files/pyarrow/_parquet.pyx in pyarrow._parquet.ParquetReader.open()

~/.local/share/virtualenvs/foo-cFmMv4Vf/lib/python3.7/site-packages/ray/pyarrow_files/pyarrow/error.pxi in pyarrow.lib.check_status()
ArrowIOError: Arrow error: IOError: Error reading bytes from file: Is a directory



Discussion:
Hi @ddutt, thanks for reporting!

How was this directory created? Is this directory working to read into pandas?

From my understanding, this wouldn't work in pandas, so it could be a good addition to our `experimental` API. The goal of that API is to make clear what are "augmentations" on top of the pandas API where the behavior diverges.
Hi,

Appreciate the prompt response.

> Hi @ddutt, thanks for reporting!
> 
> How was this directory created? Is this directory working to read into pandas?

 Yes, the directory can be read in Pandas. 

> 
> From my understanding, this wouldn't work in pandas, so it could be a good addition to our `experimental` API. The goal of that API is to make clear what are "augmentations" on top of the pandas API where the behavior diverges.

Nope, it does work in pandas correctly. Showing the ipython snippet below:

In [5]: df = pd.read_parquet('parquet-out/foo')

In [6]: df.info()
<class 'pandas.core.frame.DataFrame'>
Int64Index: 454 entries, 1 to 4
Data columns (total 20 columns):
...

And an ls to show that there is more dir under parquet-out/foo:

$ ls -l parquet-out/foo/
total 56
drwxrwxr-x 13 ddutt ddutt 4096 May  5 09:39 a1
drwxrwxr-x 24 ddutt ddutt 4096 May  5 09:39 a2
drwxrwxr-x 24 ddutt ddutt 4096 May  5 09:39 a3
drwxrwxr-x  7 ddutt ddutt 4096 May  5 09:39 b1
drwxrwxr-x 30 ddutt ddutt 4096 May  5 09:39 b2
drwxrwxr-x 30 ddutt ddutt 4096 May  5 09:39 b3
drwxrwxr-x 30 ddutt ddutt 4096 May  5 09:39 b4
drwxrwxr-x 30 ddutt ddutt 4096 May  5 09:39 b5
drwxrwxr-x  9 ddutt ddutt 4096 May  5 09:39 c1
drwxrwxr-x  9 ddutt ddutt 4096 May  5 09:39 d1
drwxrwxr-x  9 ddutt ddutt 4096 May  5 09:39 d2
drwxrwxr-x  9 ddutt ddutt 4096 May  5 09:39 d3
drwxrwxr-x 14 ddutt ddutt 4096 May  5 09:39 e1
drwxrwxr-x 14 ddutt ddutt 4096 May  5 09:39 e2

Please let me know if you need further info from me. I can have many parquet files and so I'd very much like the ability to read in parallel an entire directory. Alternately, is there any other way to read the files in parallel?

Thanks for providing this software in open source. 

Dinesh
Meanwhile, I found the workaround as: pq.ParquetDataSet(folder).read().to_pandas().

where I have done import pyarrow.parquet as pq

Dinesh
Thanks @ddutt, sorry I am traveling today and have limited WiFi until this evening.

We are using parquet in the partitions, so it should be a quick fix. Link to relevant code here: https://github.com/modin-project/modin/blob/master/modin/engines/ray/pandas_on_ray/io.py#L54

Resolving patch:
diff --git a/modin/engines/ray/generic/io.py b/modin/engines/ray/generic/io.py
--- a/modin/engines/ray/generic/io.py
+++ b/modin/engines/ray/generic/io.py
@@ -142,18 +142,54 @@ def read_parquet(cls, path, engine, columns, **kwargs):
             https://arrow.apache.org/docs/python/parquet.html
         """
 
-        from pyarrow.parquet import ParquetFile
+        from pyarrow.parquet import ParquetFile, ParquetDataset
 
         if cls.read_parquet_remote_task is None:
             return super(RayIO, cls).read_parquet(path, engine, columns, **kwargs)
 
+        if os.path.isdir(path):
+            directory = True
+            partitioned_columns = set()
+            # We do a tree walk of the path directory because partitioned
+            # parquet directories have a unique column at each directory level.
+            # Thus, we can use os.walk(), which does a dfs search, to walk
+            # through the different columns that the data is partitioned on
+            for (root, dir_names, files) in os.walk(path):
+                if dir_names:
+                    partitioned_columns.add(dir_names[0].split("=")[0])
+                if files:
+                    file_path = os.path.join(root, files[0])
+                    break
+            partitioned_columns = list(partitioned_columns)
+        else:
+            directory = False
+
         if not columns:
-            pf = ParquetFile(path)
-            columns = [
-                name
-                for name in pf.metadata.schema.names
-                if not PQ_INDEX_REGEX.match(name)
-            ]
+            if directory:
+                # Path of the sample file that we will read to get the remaining
+                # columns.
+                pd = ParquetDataset(file_path)
+                column_names = pd.schema.names
+            else:
+                pf = ParquetFile(path)
+                column_names = pf.metadata.schema.names
+            columns = [name for name in column_names if not PQ_INDEX_REGEX.match(name)]
+
+        # Cannot read in parquet file by only reading in the partitioned column.
+        # Thus, we have to remove the partition columns from the columns to
+        # ensure that when we do the math for the blocks, the partition column
+        # will be read in along with a non partition column.
+        if columns and directory and any(col in partitioned_columns for col in columns):
+            # partitioned_columns = [col for col in columns if col in partitioned_columns]
+            columns = [col for col in columns if col not in partitioned_columns]
+            # If all of the columns wanted are partition columns, return an
+            # empty dataframe with the desired columns.
+            if len(columns) == 0:
+                return cls.query_compiler_cls.from_pandas(
+                    pandas.DataFrame(columns=partitioned_columns),
+                    block_partitions_cls=cls.frame_mgr_cls,
+                )
+
         num_partitions = cls.frame_mgr_cls._compute_num_partitions()
         num_splits = min(len(columns), num_partitions)
         # Each item in this list will be a list of column names of the original df
@@ -169,9 +205,16 @@ def read_parquet(cls, path, engine, columns, **kwargs):
         # Each item in this list will be a list of columns of original df
         # partitioned to smaller pieces along rows.
         # We need to transpose the oids array to fit our schema.
+        # TODO (williamma12): This part can be parallelized even more if we
+        # separate the partitioned parquet file code path from the default one.
         blk_partitions = np.array(
             [
                 cls.read_parquet_remote_task._remote(
+                    args=(path, cols + partitioned_columns, num_splits, kwargs),
+                    num_return_vals=num_splits + 1,
+                )
+                if directory and cols == col_partitions[len(col_partitions) - 1]
+                else cls.read_parquet_remote_task._remote(
                     args=(path, cols, num_splits, kwargs),
                     num_return_vals=num_splits + 1,
                 )
@@ -186,6 +229,8 @@ def read_parquet(cls, path, engine, columns, **kwargs):
         )
         index_len = ray.get(blk_partitions[-1][0])
         index = pandas.RangeIndex(index_len)
+        if directory:
+            columns += partitioned_columns
         new_query_compiler = cls.query_compiler_cls(
             cls.frame_mgr_cls(remote_partitions), index, columns
         )


Test patch:
diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py
--- a/modin/pandas/test/test_io.py
+++ b/modin/pandas/test/test_io.py
@@ -10,6 +10,7 @@
 from pathlib import Path
 import pyarrow as pa
 import os
+import shutil
 import sys
 
 from .utils import df_equals
@@ -53,13 +54,52 @@ def modin_df_equals_pandas(modin_df, pandas_df):
     return df1.equals(df2)
 
 
-def setup_parquet_file(row_size, force=False):
-    if os.path.exists(TEST_PARQUET_FILENAME) and not force:
-        pass
-    else:
-        pandas.DataFrame(
+@pytest.fixture
+def make_parquet_file():
+    """Pytest fixture factory that makes a parquet file/dir for testing.
+
+    Yields:
+        Function that generates a parquet file/dir
+    """
+
+    def _make_parquet_file(
+        row_size=SMALL_ROW_SIZE, force=False, directory=False, partitioned_columns=[]
+    ):
+        """Helper function to generate parquet files/directories.
+
+        Args:
+            row_size: Number of rows for the dataframe.
+            force: Create a new file/directory even if one already exists.
+            directory: Create a partitioned directory using pyarrow.
+            partitioned_columns: Create a partitioned directory using pandas.
+            Will be ignored if directory=True.
+        """
+        df = pandas.DataFrame(
             {"col1": np.arange(row_size), "col2": np.arange(row_size)}
-        ).to_parquet(TEST_PARQUET_FILENAME)
+        )
+        if os.path.exists(TEST_PARQUET_FILENAME) and not force:
+            pass
+        elif directory:
+            if os.path.exists(TEST_PARQUET_FILENAME):
+                shutil.rmtree(TEST_PARQUET_FILENAME)
+            else:
+                os.mkdir(TEST_PARQUET_FILENAME)
+            table = pa.Table.from_pandas(df)
+            pa.parquet.write_to_dataset(table, root_path=TEST_PARQUET_FILENAME)
+        elif len(partitioned_columns) > 0:
+            df.to_parquet(TEST_PARQUET_FILENAME, partition_cols=partitioned_columns)
+        else:
+            df.to_parquet(TEST_PARQUET_FILENAME)
+
+    # Return function that generates csv files
+    yield _make_parquet_file
+
+    # Delete parquet file that was created
+    if os.path.exists(TEST_PARQUET_FILENAME):
+        if os.path.isdir(TEST_PARQUET_FILENAME):
+            shutil.rmtree(TEST_PARQUET_FILENAME)
+        else:
+            os.remove(TEST_PARQUET_FILENAME)
 
 
 def create_test_ray_dataframe():
@@ -335,24 +375,52 @@ def teardown_fwf_file():
         os.remove(TEST_FWF_FILENAME)
 
 
-def test_from_parquet():
-    setup_parquet_file(SMALL_ROW_SIZE)
+def test_from_parquet(make_parquet_file):
+    make_parquet_file(SMALL_ROW_SIZE)
 
     pandas_df = pandas.read_parquet(TEST_PARQUET_FILENAME)
     modin_df = pd.read_parquet(TEST_PARQUET_FILENAME)
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
 
-    teardown_parquet_file()
 
+def test_from_parquet_with_columns(make_parquet_file):
+    make_parquet_file(SMALL_ROW_SIZE)
 
-def test_from_parquet_with_columns():
-    setup_parquet_file(SMALL_ROW_SIZE)
+    pandas_df = pandas.read_parquet(TEST_PARQUET_FILENAME, columns=["col1"])
+    modin_df = pd.read_parquet(TEST_PARQUET_FILENAME, columns=["col1"])
+    df_equals(modin_df, pandas_df)
+
+
+def test_from_parquet_partition(make_parquet_file):
+    make_parquet_file(SMALL_ROW_SIZE, directory=True)
+
+    pandas_df = pandas.read_parquet(TEST_PARQUET_FILENAME)
+    modin_df = pd.read_parquet(TEST_PARQUET_FILENAME)
+    df_equals(modin_df, pandas_df)
+
+
+def test_from_parquet_partition_with_columns(make_parquet_file):
+    make_parquet_file(SMALL_ROW_SIZE, directory=True)
 
     pandas_df = pandas.read_parquet(TEST_PARQUET_FILENAME, columns=["col1"])
     modin_df = pd.read_parquet(TEST_PARQUET_FILENAME, columns=["col1"])
-    assert modin_df_equals_pandas(modin_df, pandas_df)
+    df_equals(modin_df, pandas_df)
+
 
-    teardown_parquet_file()
+def test_from_parquet_partitioned_columns(make_parquet_file):
+    make_parquet_file(SMALL_ROW_SIZE, partitioned_columns=["col1"])
+
+    pandas_df = pandas.read_parquet(TEST_PARQUET_FILENAME)
+    modin_df = pd.read_parquet(TEST_PARQUET_FILENAME)
+    df_equals(modin_df, pandas_df)
+
+
+def test_from_parquet_partitioned_columns_with_columns(make_parquet_file):
+    make_parquet_file(SMALL_ROW_SIZE, partitioned_columns=["col1"])
+
+    pandas_df = pandas.read_parquet(TEST_PARQUET_FILENAME, columns=["col1"])
+    modin_df = pd.read_parquet(TEST_PARQUET_FILENAME, columns=["col1"])
+    df_equals(modin_df, pandas_df)
 
 
 def test_from_json():