Repository: modin-project/modin

Issue:
Running query over a dataframe read by read_parquet errors out with Modin 0.5.1
<!--
General questions should be asked on the mailing list modin-dev@googlegroups.com.

Before submitting an issue, please fill out the following form.
-->

### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Ubuntu 19.04
- **Modin installed from (source or binary)**: pip install modin
- **Modin version**: 0.5.1
- **Python version**: 3.7.3
- **Exact command to reproduce**:
final_df = pd.read_parquet(folder, columns=fields, filters=filters or None)
                .query(query_str)
Where the query_str was of the form ("col1 == ['bar']").
<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->
Executing the pandas code snippet errors out with the traceback shown below. Needless to say, Pandas has no error. This issue is caused with Modin 0.5.1 and ray 0.7.

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->
```
File “/home/ddutt/work/a/engines/modin/engine.py”, line 91, in get_table_df
.query(query_str)
File “/home/ddutt/.local/share/virtualenvs/a-cFmMv4Vf/lib/python3.7/site-packages/modin/pandas/dataframe.py”, line 1260, in query
new_query_compiler = self._query_compiler.query(expr, **kwargs)
File “/home/ddutt/.local/share/virtualenvs/a-cFmMv4Vf/lib/python3.7/site-packages/modin/backends/pandas/query_compiler.py”, line 1669, in query
new_index = self.compute_index(0, new_data, True)
File “/home/ddutt/.local/share/virtualenvs/a-cFmMv4Vf/lib/python3.7/site-packages/modin/backends/pandas/query_compiler.py”, line 93, in compute_index
old_blocks=old_blocks,
File “/home/ddutt/.local/share/virtualenvs/a-cFmMv4Vf/lib/python3.7/site-packages/modin/engines/base/frame/partition_manager.py”, line 535, in get_indices
if len(self._partitions_cache.T)
File “/home/ddutt/.local/share/virtualenvs/a-cFmMv4Vf/lib/python3.7/site-packages/modin/engines/base/frame/partition_manager.py”, line 534, in
[idx.apply(func).get() for idx in self._partitions_cache.T[0]]
File “/home/ddutt/.local/share/virtualenvs/a-cFmMv4Vf/lib/python3.7/site-packages/modin/engines/ray/pandas_on_ray/frame/partition.py”, line 34, in get
handle_ray_task_error(e)
File “/home/ddutt/.local/share/virtualenvs/a-cFmMv4Vf/lib/python3.7/site-packages/modin/engines/ray/utils.py”, line 13, in handle_ray_task_error
raise getattr(builtins, s.split(":")[0])("".join(s.split(":")[1:]))
ValueError: Length mismatch Expected axis has 28 elements, new values have 10 elements
```


Discussion:
Thanks @ddutt, for the report!

We're looking into the issue, others have reported running into a related issue as well. We will get this fixed as soon as possible.

Resolving patch:
diff --git a/modin/engines/ray/generic/io.py b/modin/engines/ray/generic/io.py
--- a/modin/engines/ray/generic/io.py
+++ b/modin/engines/ray/generic/io.py
@@ -147,6 +147,7 @@ def read_parquet(cls, path, engine, columns, **kwargs):
         if cls.read_parquet_remote_task is None:
             return super(RayIO, cls).read_parquet(path, engine, columns, **kwargs)
 
+        file_path = path
         if os.path.isdir(path):
             directory = True
             partitioned_columns = set()
@@ -158,6 +159,9 @@ def read_parquet(cls, path, engine, columns, **kwargs):
                 if dir_names:
                     partitioned_columns.add(dir_names[0].split("=")[0])
                 if files:
+                    # Metadata files, git files, .DSStore
+                    if files[0][0] == ".":
+                        continue
                     file_path = os.path.join(root, files[0])
                     break
             partitioned_columns = list(partitioned_columns)
@@ -168,7 +172,12 @@ def read_parquet(cls, path, engine, columns, **kwargs):
             if directory:
                 # Path of the sample file that we will read to get the remaining
                 # columns.
-                pd = ParquetDataset(file_path)
+                from pyarrow import ArrowIOError
+
+                try:
+                    pd = ParquetDataset(file_path)
+                except ArrowIOError:
+                    pd = ParquetDataset(path)
                 column_names = pd.schema.names
             else:
                 pf = ParquetFile(path)
@@ -180,7 +189,6 @@ def read_parquet(cls, path, engine, columns, **kwargs):
         # ensure that when we do the math for the blocks, the partition column
         # will be read in along with a non partition column.
         if columns and directory and any(col in partitioned_columns for col in columns):
-            # partitioned_columns = [col for col in columns if col in partitioned_columns]
             columns = [col for col in columns if col not in partitioned_columns]
             # If all of the columns wanted are partition columns, return an
             # empty dataframe with the desired columns.
diff --git a/modin/engines/ray/pandas_on_ray/io.py b/modin/engines/ray/pandas_on_ray/io.py
--- a/modin/engines/ray/pandas_on_ray/io.py
+++ b/modin/engines/ray/pandas_on_ray/io.py
@@ -51,8 +51,9 @@ def _read_parquet_columns(path, columns, num_splits, kwargs):  # pragma: no cove
     """
     import pyarrow.parquet as pq
 
-    df = pq.ParquetDataset(path, **kwargs).read(columns=columns).to_pandas()
-
+    kwargs["use_pandas_metadata"] = True
+    df = pq.read_table(path, columns=columns, **kwargs).to_pandas()
+    df = df[columns]
     # Append the length of the index here to build it externally
     return _split_result_for_readers(0, num_splits, df) + [len(df.index)]
 
diff --git a/modin/experimental/engines/pandas_on_ray/io_exp.py b/modin/experimental/engines/pandas_on_ray/io_exp.py
--- a/modin/experimental/engines/pandas_on_ray/io_exp.py
+++ b/modin/experimental/engines/pandas_on_ray/io_exp.py
@@ -7,7 +7,39 @@
 from modin.engines.ray.pandas_on_ray.frame.partition import PandasOnRayFramePartition
 
 
+@ray.remote
+def _read_parquet_columns(path, columns, num_splits, kwargs):  # pragma: no cover
+    """Use a Ray task to read columns from Parquet into a Pandas DataFrame.
+
+    Note: Ray functions are not detected by codecov (thus pragma: no cover)
+
+    Args:
+        path: The path of the Parquet file.
+        columns: The list of column names to read.
+        num_splits: The number of partitions to split the column into.
+
+    Returns:
+         A list containing the split Pandas DataFrames and the Index as the last
+            element. If there is not `index_col` set, then we just return the length.
+            This is used to determine the total length of the DataFrame to build a
+            default Index.
+    """
+    import pyarrow.parquet as pq
+
+    df = (
+        pq.ParquetDataset(path, **kwargs)
+        .read(columns=columns, use_pandas_metadata=True)
+        .to_pandas()
+    )
+    df = df[columns]
+    # Append the length of the index here to build it externally
+    return _split_result_for_readers(0, num_splits, df) + [len(df.index)]
+
+
 class ExperimentalPandasOnRayIO(PandasOnRayIO):
+
+    read_parquet_remote_task = _read_parquet_columns
+
     @classmethod
     def read_sql(
         cls,


Test patch: