Repository: vaexio/vaex
Pull request #226 (2019-04-23 13:55:48)

fix: pandas has issues with read only buffers, this is a workaround
Bug is fixed in pandas, we have to live with it for a while
https://github.com/pandas-dev/pandas/pull/21688

maartenbreddels: Failure is unrelated, merging this. If we find out how to not copy for pandas, we can always fix that later.

Patch:
diff --git a/packages/vaex-core/vaex/dataframe.py b/packages/vaex-core/vaex/dataframe.py
index 49cdf9a2..fc30a028 100644
--- a/packages/vaex-core/vaex/dataframe.py
+++ b/packages/vaex-core/vaex/dataframe.py
@@ -3762,7 +3762,7 @@ class DataFrame(object):
         weights_values = None
         if weights is not None:
             weights_values = self.evaluate(weights)
-            weights_values /= self.sum(weights)
+            weights_values = weights_values / self.sum(weights)
         indices = random_state.choice(len(self), n, replace=replace, p=weights_values)
         return self.take(indices)
 
@@ -4417,6 +4417,15 @@ class DataFrameLocal(DataFrame):
         self.mask = None
         self.columns = collections.OrderedDict()
 
+    def _readonly(self, inplace=False):
+        # make arrays read only if possib;e
+        df = self if inplace else self.copy()
+        for key, ar in self.columns.items():
+            if isinstance(ar, np.ndarray):
+                df.columns[key] = ar = ar.view() # make new object so we don't modify others
+                ar.flags['WRITEABLE'] = False
+        return df
+
     def categorize(self, column, labels=None, check=True):
         """Mark column as categorical, with given labels, assuming zero indexing"""
         column = _ensure_string_from_expression(column)
diff --git a/packages/vaex-core/vaex/functions.py b/packages/vaex-core/vaex/functions.py
index 1c76f448..5f7d262c 100644
--- a/packages/vaex-core/vaex/functions.py
+++ b/packages/vaex-core/vaex/functions.py
@@ -146,6 +146,16 @@ def fillna(ar, value, fill_nan=True, fill_masked=True):
 
 ########## datetime operations ##########
 
+
+def _pandas_dt_fix(x):
+    # see https://github.com/pandas-dev/pandas/issues/23276
+    import pandas as pd
+    # not sure which version this is fixed in
+    if not x.flags['WRITEABLE']:
+        x = x.copy()
+    return x
+
+
 @register_function(scope='dt', as_property=True)
 def dt_dayofweek(x):
     """Obtain the day of the week with Monday=0 and Sunday=6
@@ -318,7 +328,7 @@ def dt_month_name(x):
     2  November
     """
     import pandas as pd
-    return pd.Series(x).dt.month_name().values.astype(str)
+    return pd.Series(_pandas_dt_fix(x)).dt.month_name().values.astype(str)
 
 @register_function(scope='dt', as_property=True)
 def dt_day(x):
@@ -376,7 +386,7 @@ def dt_day_name(x):
     2  Thursday
     """
     import pandas as pd
-    return pd.Series(x).dt.day_name().values.astype(str)
+    return pd.Series(_pandas_dt_fix(x)).dt.day_name().values.astype(str)
 
 @register_function(scope='dt', as_property=True)
 def dt_weekofyear(x):


Test patch:
diff --git a/tests/common.py b/tests/common.py
index 25eb7b61..ff07bd98 100644
--- a/tests/common.py
+++ b/tests/common.py
@@ -131,6 +131,4 @@ def create_base_ds():
     booleans[[4, 6, 8, 14, 16, 19]] = False
     dataset.add_column("bool", booleans)
 
-    return dataset
-
-# dsf = create_filtered()
\ No newline at end of file
+    return dataset._readonly()
diff --git a/tests/datetime_test.py b/tests/datetime_test.py
index f440c119..65ee50c1 100644
--- a/tests/datetime_test.py
+++ b/tests/datetime_test.py
@@ -3,14 +3,14 @@ import numpy as np
 
 
 def test_datetime_operations():
-    date = [np.datetime64('2009-10-12T03:31:00'),
-            np.datetime64('2016-02-11T10:17:34'),
-            np.datetime64('2015-11-12T11:34:22'),
-            np.datetime64('2003-03-03T00:33:15'),
-            np.datetime64('2014-07-23T15:08:05'),
-            np.datetime64('2011-01-01T07:02:01')]
+    date = np.array([np.datetime64('2009-10-12T03:31:00'),
+        np.datetime64('2016-02-11T10:17:34'),
+        np.datetime64('2015-11-12T11:34:22'),
+        np.datetime64('2003-03-03T00:33:15'),
+        np.datetime64('2014-07-23T15:08:05'),
+        np.datetime64('2011-01-01T07:02:01')], dtype='<M8[ns]')
 
-    df = vaex.from_arrays(date=date)
+    df = vaex.from_arrays(date=date)._readonly()
     pandas_df = df.to_pandas_df()
 
     assert df.date.dt.hour.values.tolist() == pandas_df.date.dt.hour.values.tolist()