Repository: modin-project/modin

Issue:
pd.to_datetime() - ValueError: buffer source array is read-only
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Amazon Linux 2018.03
- **Modin version** (`modin.__version__`): 0.8.3 (with ray==1.2.0)
- **Python version**: 3.6.13
- **Code we can use to reproduce**:

```
import modin.pandas as pd
df = pd.DataFrame({'date': [1618193360,  
                            1618193360, 
                            1618193360],
                   'value': [2, 3, 4]})
pd.to_datetime(df.date, unit='s')
```
<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->

I have a column in a dataframe that are linux timestamps, I want to convert them to datetime format, but I'm geting the error below using modin. It works fine with pandas. 

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->

```
---------------------------------------------------------------------------
RayTaskError(ValueError)                  Traceback (most recent call last)
<ipython-input-78-5e2507587681> in <module>
----> 1 pd.to_datetime(data_interactions['TIMESTAMP'], unit='s')

~/anaconda3/envs/python3/lib/python3.6/site-packages/modin/pandas/general.py in to_datetime(arg, errors, dayfirst, yearfirst, utc, format, exact, unit, infer_datetime_format, origin, cache)
    539         infer_datetime_format=infer_datetime_format,
    540         origin=origin,
--> 541         cache=cache,
    542     )
    543 

~/anaconda3/envs/python3/lib/python3.6/site-packages/modin/pandas/series.py in _to_datetime(self, **kwargs)
   1514         """
   1515         return self.__constructor__(
-> 1516             query_compiler=self._query_compiler.to_datetime(**kwargs)
   1517         )
   1518 

~/anaconda3/envs/python3/lib/python3.6/site-packages/modin/data_management/functions/reductionfunction.py in caller(query_compiler, *args, **kwargs)
     25                     cls.validate_axis(axis),
     26                     lambda x: reduction_function(x, *args, **kwargs),
---> 27                     preserve_index=preserve_index,
     28                 )
     29             )

~/anaconda3/envs/python3/lib/python3.6/site-packages/modin/engines/base/frame/data.py in _fold_reduce(self, axis, func, preserve_index)
   1157         )
   1158         return self._compute_map_reduce_metadata(
-> 1159             axis, new_parts, preserve_index=preserve_index
   1160         )
   1161 

~/anaconda3/envs/python3/lib/python3.6/site-packages/modin/engines/base/frame/data.py in _compute_map_reduce_metadata(self, axis, new_parts, preserve_index)
   1131             *new_axes_lengths,
   1132             new_dtypes,
-> 1133             validate_axes=("reduced" if preserve_index else "reduced+other"),
   1134         )
   1135 

~/anaconda3/envs/python3/lib/python3.6/site-packages/modin/engines/base/frame/data.py in __init__(self, partitions, index, columns, row_lengths, column_widths, dtypes, validate_axes)
     85         self._dtypes = dtypes
     86         if validate_axes is not False:
---> 87             self._validate_internal_indices(mode=validate_axes)
     88         self._filter_empties()
     89 

~/anaconda3/envs/python3/lib/python3.6/site-packages/modin/engines/base/frame/data.py in _validate_internal_indices(self, mode, **kwargs)
    406             self._validate_axis_equality(axis=0, force=force_getter(0))
    407         if args.get("validate_columns", True):
--> 408             self._validate_axis_equality(axis=1, force=force_getter(1))
    409 
    410     def _apply_index_objs(self, axis=None):

~/anaconda3/envs/python3/lib/python3.6/site-packages/modin/engines/base/frame/data.py in _validate_axis_equality(self, axis, force)
    325                 do not match or raise an exception in that case.
    326         """
--> 327         internal_axis = self._compute_axis_labels(axis)
    328         self_axis = self.axes[axis]
    329         is_equals = self_axis.equals(internal_axis)

~/anaconda3/envs/python3/lib/python3.6/site-packages/modin/engines/base/frame/data.py in _compute_axis_labels(self, axis, partitions)
    288             partitions = self._partitions
    289         return self._frame_mgr_cls.get_indices(
--> 290             axis, partitions, lambda df: df.axes[axis]
    291         )
    292 

~/anaconda3/envs/python3/lib/python3.6/site-packages/modin/engines/ray/pandas_on_ray/frame/partition_manager.py in get_indices(cls, axis, partitions, index_func)
     97                 else []
     98             )
---> 99         new_idx = ray.get(new_idx)
    100         return new_idx[0].append(new_idx[1:]) if len(new_idx) else new_idx
    101 

~/anaconda3/envs/python3/lib/python3.6/site-packages/ray/_private/client_mode_hook.py in wrapper(*args, **kwargs)
     45         if client_mode_enabled and _client_hook_enabled:
     46             return getattr(ray, func.__name__)(*args, **kwargs)
---> 47         return func(*args, **kwargs)
     48 
     49     return wrapper

~/anaconda3/envs/python3/lib/python3.6/site-packages/ray/worker.py in get(object_refs, timeout)
   1454                     worker.core_worker.dump_object_store_memory_usage()
   1455                 if isinstance(value, RayTaskError):
-> 1456                     raise value.as_instanceof_cause()
   1457                 else:
   1458                     raise value

RayTaskError(ValueError): ray::deploy_ray_func() (pid=3159, ip=172.16.60.41)
  File "python/ray/_raylet.pyx", line 442, in ray._raylet.execute_task
  File "python/ray/_raylet.pyx", line 464, in ray._raylet.execute_task
ray.exceptions.RayTaskError: ray::deploy_ray_func() (pid=3153, ip=172.16.60.41)
  File "python/ray/_raylet.pyx", line 480, in ray._raylet.execute_task
  File "/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/modin/engines/ray/pandas_on_ray/frame/axis_partition.py", line 105, in deploy_ray_func
    result = func(*args)
  File "/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/modin/engines/base/frame/axis_partition.py", line 225, in deploy_axis_func
    result = func(dataframe, **kwargs)
  File "/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/modin/engines/base/frame/data.py", line 1078, in _map_reduce_func
    series_result = func(df, *args, **kwargs)
  File "/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/modin/data_management/functions/reductionfunction.py", line 26, in <lambda>
    lambda x: reduction_function(x, *args, **kwargs),
  File "/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/modin/backends/pandas/query_compiler.py", line 777, in <lambda>
    df.squeeze(axis=1), *args, **kwargs
  File "/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/pandas/core/tools/datetimes.py", line 803, in to_datetime
    values = convert_listlike(arg._values, format)
  File "/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/pandas/core/tools/datetimes.py", line 340, in _convert_listlike_datetimes
    arg, unit, errors=errors
  File "pandas/_libs/tslib.pyx", line 236, in pandas._libs.tslib.array_with_unit_to_datetime
ValueError: buffer source array is read-only
```



Discussion:
Hi @laphang, thanks for posting! Thanks for the concise reproducible example as well!

I can reproduce this on the latest commit.

As a side note, we dropped support for Python 3.6 when pandas did, current minimum supported Python version is 3.7.1. The latest release is 0.9.1 (with the next release coming soon). This issue is still present, and should be fixed by the next release, but it will not solve the issue for Python < 3.7.1.
After some investigation, this is an issue with using Arrow serialization.
thanks @devin-petersohn, that comment was helpful. I was confused as to why I couldn't install the latest modin version - I can now in python 3.7.10. 

Looking forward to the fix. 


Resolving patch:
diff --git a/modin/backends/pandas/query_compiler.py b/modin/backends/pandas/query_compiler.py
--- a/modin/backends/pandas/query_compiler.py
+++ b/modin/backends/pandas/query_compiler.py
@@ -788,12 +788,19 @@ def value_counts(df):
     prod_min_count = ReductionFunction.register(pandas.DataFrame.prod)
     quantile_for_single_value = ReductionFunction.register(pandas.DataFrame.quantile)
     mad = ReductionFunction.register(pandas.DataFrame.mad)
-    to_datetime = ReductionFunction.register(
-        lambda df, *args, **kwargs: pandas.to_datetime(
-            df.squeeze(axis=1), *args, **kwargs
-        ),
-        axis=1,
-    )
+
+    def to_datetime(self, *args, **kwargs):
+        if len(self.columns) == 1:
+            return MapFunction.register(
+                # to_datetime has inplace side effects, see GH#3063
+                lambda df, *args, **kwargs: pandas.to_datetime(
+                    df.squeeze(axis=1), *args, **kwargs
+                ).to_frame()
+            )(self, *args, **kwargs)
+        else:
+            return ReductionFunction.register(pandas.to_datetime, axis=1)(
+                self, *args, **kwargs
+            )
 
     # END Reduction operations
 


Test patch:
diff --git a/modin/pandas/test/test_general.py b/modin/pandas/test/test_general.py
--- a/modin/pandas/test/test_general.py
+++ b/modin/pandas/test/test_general.py
@@ -644,6 +644,18 @@ def test_to_datetime():
     )
 
 
+def test_to_datetime_inplace_side_effect():
+    # See GH#3063
+    times = list(range(1617993360, 1618193360))
+    values = list(range(215441, 415441))
+    modin_df = pd.DataFrame({"time": times, "value": values})
+    pandas_df = pandas.DataFrame({"time": times, "value": values})
+    df_equals(
+        pd.to_datetime(modin_df["time"], unit="s"),
+        pandas.to_datetime(pandas_df["time"], unit="s"),
+    )
+
+
 @pytest.mark.parametrize(
     "data, errors, downcast",
     [