Repository: modin-project/modin

Issue:
read parquet returns ValueError: Length mismatch: Expected axis has 40 elements, new values have 41 elements
### System information
- **OS Platform and Distribution** : elementary OS 5.1.4 Hera
- **Modin version** : 0.7.3
- **Python version**: 3.7.6
- **Code we can use to reproduce**: 
import modin.pandas as pd
df = pd.read_parquet('sales')

### Describe the problem
Reading parquet folder returns with value mismatch error

### Source code / logs
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/bharat/anaconda3/envs/dev_vizb/lib/python3.7/site-packages/modin/pandas/io.py", line 42, in read_parquet
    path=path, columns=columns, engine=engine, **kwargs
  File "/home/bharat/anaconda3/envs/dev_vizb/lib/python3.7/site-packages/modin/data_management/factories.py", line 57, in read_parquet
    return cls._determine_engine()._read_parquet(**kwargs)
  File "/home/bharat/anaconda3/envs/dev_vizb/lib/python3.7/site-packages/modin/data_management/factories.py", line 61, in _read_parquet
    return cls.io_cls.read_parquet(**kwargs)
  File "/home/bharat/anaconda3/envs/dev_vizb/lib/python3.7/site-packages/modin/engines/base/io/column_stores/parquet_reader.py", line 83, in read
    return cls.build_query_compiler(path, columns, **kwargs)
  File "/home/bharat/anaconda3/envs/dev_vizb/lib/python3.7/site-packages/modin/engines/base/io/column_stores/column_store_reader.py", line 113, in build_query_compiler
    dtypes = cls.build_dtypes(partition_ids[-1], columns)
  File "/home/bharat/anaconda3/envs/dev_vizb/lib/python3.7/site-packages/modin/engines/base/io/column_stores/column_store_reader.py", line 104, in build_dtypes
    dtypes.index = columns
  File "/home/bharat/anaconda3/envs/dev_vizb/lib/python3.7/site-packages/pandas/core/generic.py", line 5287, in __setattr__
    return object.__setattr__(self, name, value)
  File "pandas/_libs/properties.pyx", line 67, in pandas._libs.properties.AxisProperty.__set__
  File "/home/bharat/anaconda3/envs/dev_vizb/lib/python3.7/site-packages/pandas/core/series.py", line 401, in _set_axis
    self._data.set_axis(axis, labels)
  File "/home/bharat/anaconda3/envs/dev_vizb/lib/python3.7/site-packages/pandas/core/internals/managers.py", line 178, in set_axis
    f"Length mismatch: Expected axis has {old_len} elements, new "
ValueError: Length mismatch: Expected axis has 40 elements, new values have 41 elements



Discussion:
Hi @bharatk101 thanks for the report!

Is it possible to share the parquet file that caused this? I tried to reproduce with our test files and some other random data but haven't been able to reproduce.
Yes, you can find it here
https://drive.google.com/open?id=1vZ0eSN6Fr_ilt38LuqjWT7QlPxG2Qy4W

Resolving patch:
diff --git a/modin/engines/base/io/column_stores/parquet_reader.py b/modin/engines/base/io/column_stores/parquet_reader.py
--- a/modin/engines/base/io/column_stores/parquet_reader.py
+++ b/modin/engines/base/io/column_stores/parquet_reader.py
@@ -64,10 +64,12 @@ def read(cls, path, engine, columns, **kwargs):
             if directory:
                 # Path of the sample file that we will read to get the remaining columns
                 pd = ParquetDataset(path)
+                meta = pd.metadata
                 column_names = pd.schema.names
             else:
                 meta = ParquetFile(path).metadata
                 column_names = meta.schema.names
+            if meta is not None:
                 # This is how we convert the metadata from pyarrow to a python
                 # dictionary, from which we then get the index columns.
                 # We use these to filter out from the columns in the metadata since


Test patch:
diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py
--- a/modin/pandas/test/test_io.py
+++ b/modin/pandas/test/test_io.py
@@ -436,12 +436,31 @@ def test_from_parquet_pandas_index():
             "C": ["c"] * 2000,
         }
     )
-    pandas_df.set_index("idx").to_parquet("tmp.parquet")
+    filepath = "tmp.parquet"
+    pandas_df.set_index("idx").to_parquet(filepath)
     # read the same parquet using modin.pandas
-    df_equals(pd.read_parquet("tmp.parquet"), pandas.read_parquet("tmp.parquet"))
+    df_equals(pd.read_parquet(filepath), pandas.read_parquet(filepath))
 
-    pandas_df.set_index(["idx", "A"]).to_parquet("tmp.parquet")
-    df_equals(pd.read_parquet("tmp.parquet"), pandas.read_parquet("tmp.parquet"))
+    pandas_df.set_index(["idx", "A"]).to_parquet(filepath)
+    df_equals(pd.read_parquet(filepath), pandas.read_parquet(filepath))
+    os.remove(filepath)
+
+
+def test_from_parquet_pandas_index_partitioned():
+    # Ensure modin can read parquet files written by pandas with a non-RangeIndex object
+    pandas_df = pandas.DataFrame(
+        {
+            "idx": np.random.randint(0, 100_000, size=2000),
+            "A": np.random.randint(0, 10, size=2000),
+            "B": ["a", "b"] * 1000,
+            "C": ["c"] * 2000,
+        }
+    )
+    filepath = "tmp_folder.parquet"
+    pandas_df.set_index("idx").to_parquet(filepath, partition_cols=["A"])
+    # read the same parquet using modin.pandas
+    df_equals(pd.read_parquet(filepath), pandas.read_parquet(filepath))
+    shutil.rmtree(filepath)
 
 
 def test_from_parquet_hdfs():