Repository: vaexio/vaex

Issue:
CI: fix the latest CI issues
This PR tries to figure out the reason behind the latest failures in the CI. 

I suspect this is due to the recent new release of pandas, and it's releases on conda-forge. 
Here we try to verify that by not allowing the latest verion of pandas.


Discussion:

Resolving patch:
diff --git a/packages/vaex-core/vaex/dataframe.py b/packages/vaex-core/vaex/dataframe.py
--- a/packages/vaex-core/vaex/dataframe.py
+++ b/packages/vaex-core/vaex/dataframe.py
@@ -3017,7 +3017,7 @@ def copy_metadata(self, other):
         self.description = other.description
 
     @docsubst
-    def to_pandas_df(self, column_names=None, selection=None, strings=True, virtual=True, index_name=None, parallel=True, chunk_size=None):
+    def to_pandas_df(self, column_names=None, selection=None, strings=True, virtual=True, index_name=None, parallel=True, chunk_size=None, array_type=None):
         """Return a pandas DataFrame containing the ndarray corresponding to the evaluated data
 
          If index is given, that column is used for the index of the dataframe.
@@ -3034,6 +3034,7 @@ def to_pandas_df(self, column_names=None, selection=None, strings=True, virtual=
         :param index_column: if this column is given it is used for the index of the DataFrame
         :param parallel: {evaluate_parallel}
         :param chunk_size: {chunk_size}
+        :param array_type: {array_type}
         :return: pandas.DataFrame object or iterator of
         """
         import pandas as pd
@@ -3053,11 +3054,11 @@ def create_pdf(data):
             return df
         if chunk_size is not None:
             def iterator():
-                for i1, i2, chunks in self.evaluate_iterator(column_names, selection=selection, parallel=parallel, chunk_size=chunk_size):
+                for i1, i2, chunks in self.evaluate_iterator(column_names, selection=selection, parallel=parallel, chunk_size=chunk_size, array_type=array_type):
                     yield i1, i2, create_pdf(dict(zip(column_names, chunks)))
             return iterator()
         else:
-            return create_pdf(self.to_dict(column_names=column_names, selection=selection, parallel=parallel))
+            return create_pdf(self.to_dict(column_names=column_names, selection=selection, parallel=parallel, array_type=array_type))
 
     @docsubst
     def to_arrow_table(self, column_names=None, selection=None, strings=True, virtual=True, parallel=True, chunk_size=None, reduce_large=False):
diff --git a/packages/vaex-core/vaex/functions.py b/packages/vaex-core/vaex/functions.py
--- a/packages/vaex-core/vaex/functions.py
+++ b/packages/vaex-core/vaex/functions.py
@@ -2412,7 +2412,6 @@ def format(x, format):
 for name in dir(scopes['str']):
     if name.startswith('__'):
         continue
-    force_string = ['get']
     def pandas_wrapper(name=name):
         def wrapper(*args, **kwargs):
             import pandas
@@ -2423,14 +2422,10 @@ def fix_arg(x):
             args = list(map(fix_arg, args))
             x = args[0]
             args = args[1:]
-            series = pandas.Series(x)
+            series = pandas.Series(x, dtype='string')
             method = getattr(series.str, name)
             value = method(*args, **kwargs)
-            if name in force_string:
-                value = _to_string_column(value.values, force=True)
-                return value
-            else:
-                return value.values
+            return pa.array(value)
         return wrapper
     wrapper = pandas_wrapper()
     wrapper.__doc__ = "Wrapper around pandas.Series.%s" % name


Test patch:
diff --git a/packages/vaex-core/vaex/test/dataset.py b/packages/vaex-core/vaex/test/dataset.py
--- a/packages/vaex-core/vaex/test/dataset.py
+++ b/packages/vaex-core/vaex/test/dataset.py
@@ -329,15 +329,15 @@ def test_equal(ds1, ds2, units=True, ucds=True, description=True, descriptions=T
 		test_equal(self.dataset, ds2, ucds=False, units=False, description=False, descriptions=False)
 
 		# as pandas
-		ds2 = vx.from_pandas(self.dataset.to_pandas_df())
+		ds2 = vx.from_pandas(self.dataset.to_pandas_df(array_type='numpy'))
 		# skip masked arrays, pandas doesn't understand that, converts it to nan, so we can't compare
 		test_equal(self.dataset, ds2, ucds=False, units=False, description=False, descriptions=False, skip=['m', 'mi'])
 
-		df = self.dataset.to_pandas_df(index_name="name")
+		df = self.dataset.to_pandas_df(index_name="name", array_type='numpy')
 		ds2 = vx.from_pandas(df, index_name="name", copy_index=True)
 		test_equal(self.dataset, ds2, ucds=False, units=False, description=False, descriptions=False, skip=['m', 'mi'])
 
-		ds2 = vx.from_pandas(self.dataset.to_pandas_df(index_name="name"))
+		ds2 = vx.from_pandas(self.dataset.to_pandas_df(index_name="name", array_type='numpy'))
 		assert "name" not in ds2.get_column_names()
 
 		# as astropy table
diff --git a/tests/countna_test.py b/tests/countna_test.py
--- a/tests/countna_test.py
+++ b/tests/countna_test.py
@@ -8,7 +8,7 @@ def test_countna(df_factory):
     y_mask = np.array([0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1])
     y = np.ma.MaskedArray(data=y_data, mask=y_mask)
     df = df_factory(x=x, y=y)
-    pandas_df = df.to_pandas_df()
+    pandas_df = df.to_pandas_df(array_type='numpy')
 
     assert df.x.countna() == pandas_df.x.isna().sum()
     assert df.y.countna() == pandas_df.y.isna().sum()
diff --git a/tests/from_json_test.py b/tests/from_json_test.py
--- a/tests/from_json_test.py
+++ b/tests/from_json_test.py
@@ -6,7 +6,7 @@ def test_from_json(ds_local):
     df = ds_local
 
     # Create temporary json files
-    pandas_df = df.to_pandas_df(virtual=True)
+    pandas_df = df.to_pandas_df(virtual=True, array_type='numpy')
     tmp = tempfile.mktemp('.json')
     with open(tmp, 'w') as f:
         f.write(pandas_df.to_json())
diff --git a/tests/groupby_test.py b/tests/groupby_test.py
--- a/tests/groupby_test.py
+++ b/tests/groupby_test.py
@@ -295,7 +295,7 @@ def test_groupby_std():
     groupby = df.groupby('s')
     dfg = groupby.agg({'g': 'std'})
     assert dfg.s.tolist() == ['0', '1', '2']
-    pandas_g = df.to_pandas_df().groupby('s').std(ddof=0).g.tolist()
+    pandas_g = df.to_pandas_df(array_type='numpy').groupby('s').std(ddof=0).g.tolist()
     np.testing.assert_array_almost_equal(dfg.g.tolist(), pandas_g)
 
 
diff --git a/tests/ml/ml_test.py b/tests/ml/ml_test.py
--- a/tests/ml/ml_test.py
+++ b/tests/ml/ml_test.py
@@ -337,7 +337,7 @@ def test_maxabs_scaler(df_factory):
     w = np.zeros_like(x)
 
     ds = df_factory(x=x, y=y, z=z, w=w)
-    df = ds.to_pandas_df()
+    df = ds.to_pandas_df(array_type='numpy')
 
     features = ['x', 'y', 'w']
 
@@ -364,7 +364,7 @@ def test_robust_scaler(df_factory):
     w = np.zeros_like(x)
 
     ds = df_factory(x=x, y=y, z=z, w=w)
-    df = ds.to_pandas_df()
+    df = ds.to_pandas_df(array_type='numpy')
 
     features = ['x', 'y']
 
diff --git a/tests/rolling_test.py b/tests/rolling_test.py
--- a/tests/rolling_test.py
+++ b/tests/rolling_test.py
@@ -4,7 +4,7 @@
 def test_rolling_sum(df_factory):
     x = [0, 1, 2, 3, 4.0]
     df = df_factory(x=x)
-    dfp = df.to_pandas_df()
+    dfp = df.to_pandas_df(array_type='numpy')
     df = df.rolling(2, fill_value=np.nan).sum()
     dfp = dfp.rolling(2).sum()
     result = df['x'].tolist()
diff --git a/tests/shift_test.py b/tests/shift_test.py
--- a/tests/shift_test.py
+++ b/tests/shift_test.py
@@ -358,7 +358,7 @@ def test_shift_dataset(chunk_size=2):
 def test_diff(df_factory, periods):
     x = [0, 1, 2, 3, 4.0]
     df = df_factory(x=x)
-    dfp = df.to_pandas_df()
+    dfp = df.to_pandas_df(array_type='numpy')
     df = df.diff(periods, fill_value=np.nan)
     dfp = dfp.diff(periods)
     result = df['x'].to_numpy()
diff --git a/tests/timedelta_test.py b/tests/timedelta_test.py
--- a/tests/timedelta_test.py
+++ b/tests/timedelta_test.py
@@ -4,7 +4,7 @@
 
 
 def test_timedelta_methods():
-    delta = np.array([17658720110, 11047049384039, 40712636304958, -18161254954], dtype='timedelta64[s]')
+    delta = np.array([187201, 1449339, 11264958, -181614], dtype='timedelta64[s]')
     df = vaex.from_arrays(delta=delta)
     pdf = pd.DataFrame({'delta': pd.Series(delta, dtype=delta.dtype)})
 
diff --git a/tests/to_test.py b/tests/to_test.py
--- a/tests/to_test.py
+++ b/tests/to_test.py
@@ -46,7 +46,7 @@ def test_to_arrow_table(df_local):
 
 def test_to_pandas_df(df_local):
     df = df_local
-    pdf = df.to_pandas_df(['x', 'y'], index_name='x')
+    pdf = df.to_pandas_df(['x', 'y'], index_name='x', array_type='numpy')
     assert pdf.index.name == 'x'
     assert pdf.columns == ['y']
     assert pdf.index.values.tolist() == df.x.tolist()
@@ -54,7 +54,7 @@ def test_to_pandas_df(df_local):
     x = df.x.values
     y = df.y.values
 
-    for i1, i2, pdf in df.to_pandas_df(['y'], index_name='x', chunk_size=3):
+    for i1, i2, pdf in df.to_pandas_df(['y'], index_name='x', chunk_size=3, array_type='numpy'):
         assert pdf.index.name == 'x'
         assert pdf.columns == ['y']
         assert pdf.index.values.tolist() == x[i1:i2].tolist()