Repository: pandas-dev/pandas

Issue:
we should bump our min of pyarrow to 0.7.0
_Originally posted by @jreback in https://github.com/pandas-dev/pandas/pull/23321_


Discussion:

Resolving patch:
diff --git a/pandas/io/parquet.py b/pandas/io/parquet.py
--- a/pandas/io/parquet.py
+++ b/pandas/io/parquet.py
@@ -5,7 +5,7 @@
 
 from pandas.compat import string_types
 
-from pandas import DataFrame, Int64Index, RangeIndex, get_option
+from pandas import DataFrame, get_option
 import pandas.core.common as com
 
 from pandas.io.common import get_filepath_or_buffer, is_s3_url
@@ -89,29 +89,20 @@ def __init__(self):
                 "\nor via pip\n"
                 "pip install -U pyarrow\n"
             )
-        if LooseVersion(pyarrow.__version__) < '0.4.1':
+        if LooseVersion(pyarrow.__version__) < '0.7.0':
             raise ImportError(
-                "pyarrow >= 0.4.1 is required for parquet support\n\n"
+                "pyarrow >= 0.7.0 is required for parquet support\n\n"
                 "you can install via conda\n"
                 "conda install pyarrow -c conda-forge\n"
                 "\nor via pip\n"
                 "pip install -U pyarrow\n"
             )
 
-        self._pyarrow_lt_060 = (
-            LooseVersion(pyarrow.__version__) < LooseVersion('0.6.0'))
-        self._pyarrow_lt_070 = (
-            LooseVersion(pyarrow.__version__) < LooseVersion('0.7.0'))
-
         self.api = pyarrow
 
     def write(self, df, path, compression='snappy',
               coerce_timestamps='ms', index=None, **kwargs):
         self.validate_dataframe(df)
-
-        # Only validate the index if we're writing it.
-        if self._pyarrow_lt_070 and index is not False:
-            self._validate_write_lt_070(df)
         path, _, _, _ = get_filepath_or_buffer(path, mode='wb')
 
         if index is None:
@@ -119,27 +110,17 @@ def write(self, df, path, compression='snappy',
         else:
             from_pandas_kwargs = {'preserve_index': index}
 
-        if self._pyarrow_lt_060:
-            table = self.api.Table.from_pandas(df, timestamps_to_ms=True,
-                                               **from_pandas_kwargs)
-            self.api.parquet.write_table(
-                table, path, compression=compression, **kwargs)
-
-        else:
-            table = self.api.Table.from_pandas(df, **from_pandas_kwargs)
-            self.api.parquet.write_table(
-                table, path, compression=compression,
-                coerce_timestamps=coerce_timestamps, **kwargs)
+        table = self.api.Table.from_pandas(df, **from_pandas_kwargs)
+        self.api.parquet.write_table(
+            table, path, compression=compression,
+            coerce_timestamps=coerce_timestamps, **kwargs)
 
     def read(self, path, columns=None, **kwargs):
         path, _, _, should_close = get_filepath_or_buffer(path)
-        if self._pyarrow_lt_070:
-            result = self.api.parquet.read_pandas(path, columns=columns,
-                                                  **kwargs).to_pandas()
-        else:
-            kwargs['use_pandas_metadata'] = True
-            result = self.api.parquet.read_table(path, columns=columns,
-                                                 **kwargs).to_pandas()
+
+        kwargs['use_pandas_metadata'] = True
+        result = self.api.parquet.read_table(path, columns=columns,
+                                             **kwargs).to_pandas()
         if should_close:
             try:
                 path.close()
@@ -148,39 +129,6 @@ def read(self, path, columns=None, **kwargs):
 
         return result
 
-    def _validate_write_lt_070(self, df):
-        # Compatibility shim for pyarrow < 0.7.0
-        # TODO: Remove in pandas 0.23.0
-        from pandas.core.indexes.multi import MultiIndex
-        if isinstance(df.index, MultiIndex):
-            msg = (
-                "Multi-index DataFrames are only supported "
-                "with pyarrow >= 0.7.0"
-            )
-            raise ValueError(msg)
-        # Validate index
-        if not isinstance(df.index, Int64Index):
-            msg = (
-                "pyarrow < 0.7.0 does not support serializing {} for the "
-                "index; you can .reset_index() to make the index into "
-                "column(s), or install the latest version of pyarrow or "
-                "fastparquet."
-            )
-            raise ValueError(msg.format(type(df.index)))
-        if not df.index.equals(RangeIndex(len(df))):
-            raise ValueError(
-                "pyarrow < 0.7.0 does not support serializing a non-default "
-                "index; you can .reset_index() to make the index into "
-                "column(s), or install the latest version of pyarrow or "
-                "fastparquet."
-            )
-        if df.index.name is not None:
-            raise ValueError(
-                "pyarrow < 0.7.0 does not serialize indexes with a name; you "
-                "can set the index.name to None or install the latest version "
-                "of pyarrow or fastparquet."
-            )
-
 
 class FastParquetImpl(BaseImpl):
 
@@ -197,9 +145,9 @@ def __init__(self):
                 "\nor via pip\n"
                 "pip install -U fastparquet"
             )
-        if LooseVersion(fastparquet.__version__) < '0.1.0':
+        if LooseVersion(fastparquet.__version__) < '0.1.2':
             raise ImportError(
-                "fastparquet >= 0.1.0 is required for parquet "
+                "fastparquet >= 0.1.2 is required for parquet "
                 "support\n\n"
                 "you can install via conda\n"
                 "conda install fastparquet -c conda-forge\n"


Test patch:
diff --git a/pandas/tests/io/test_parquet.py b/pandas/tests/io/test_parquet.py
--- a/pandas/tests/io/test_parquet.py
+++ b/pandas/tests/io/test_parquet.py
@@ -41,22 +41,6 @@ def engine(request):
 
 @pytest.fixture
 def pa():
-    if not _HAVE_PYARROW:
-        pytest.skip("pyarrow is not installed")
-    return 'pyarrow'
-
-
-@pytest.fixture
-def pa_lt_070():
-    if not _HAVE_PYARROW:
-        pytest.skip("pyarrow is not installed")
-    if LooseVersion(pyarrow.__version__) >= LooseVersion('0.7.0'):
-        pytest.skip("pyarrow is >= 0.7.0")
-    return 'pyarrow'
-
-
-@pytest.fixture
-def pa_ge_070():
     if not _HAVE_PYARROW:
         pytest.skip("pyarrow is not installed")
     if LooseVersion(pyarrow.__version__) < LooseVersion('0.7.0'):
@@ -337,9 +321,9 @@ def test_write_index(self, engine):
         df.index.name = 'foo'
         check_round_trip(df, engine)
 
-    def test_write_multiindex(self, pa_ge_070):
+    def test_write_multiindex(self, pa):
         # Not suppoprted in fastparquet as of 0.1.3 or older pyarrow version
-        engine = pa_ge_070
+        engine = pa
 
         df = pd.DataFrame({'A': [1, 2, 3]})
         index = pd.MultiIndex.from_tuples([('a', 1), ('a', 2), ('b', 1)])
@@ -352,8 +336,8 @@ def test_write_column_multiindex(self, engine):
         df = pd.DataFrame(np.random.randn(4, 3), columns=mi_columns)
         self.check_error_on_write(df, engine, ValueError)
 
-    def test_multiindex_with_columns(self, pa_ge_070):
-        engine = pa_ge_070
+    def test_multiindex_with_columns(self, pa):
+        engine = pa
         dates = pd.date_range('01-Jan-2018', '01-Dec-2018', freq='MS')
         df = pd.DataFrame(np.random.randn(2 * len(dates), 3),
                           columns=list('ABC'))
@@ -456,8 +440,7 @@ def test_unsupported(self, pa):
         # older pyarrows raise ArrowInvalid
         self.check_error_on_write(df, pa, Exception)
 
-    def test_categorical(self, pa_ge_070):
-        pa = pa_ge_070
+    def test_categorical(self, pa):
 
         # supported in >= 0.7.0
         df = pd.DataFrame({'a': pd.Categorical(list('abc'))})
@@ -466,13 +449,6 @@ def test_categorical(self, pa_ge_070):
         expected = df.assign(a=df.a.astype(object))
         check_round_trip(df, pa, expected=expected)
 
-    def test_categorical_unsupported(self, pa_lt_070):
-        pa = pa_lt_070
-
-        # supported in >= 0.7.0
-        df = pd.DataFrame({'a': pd.Categorical(list('abc'))})
-        self.check_error_on_write(df, pa, NotImplementedError)
-
     def test_s3_roundtrip(self, df_compat, s3_resource, pa):
         # GH #19134
         check_round_trip(df_compat, pa,