Repository: ray-project/ray

Issue:
Worker dies when passed pandas DataFrame.
### System information
- **Ray version**: 0.3.0
- **Python version**: 3.6.0
- **Exact command to reproduce**:

```python
import pandas as pd
import ray

pd.__version__  # '0.19.2'

ray.init()

df = pd.DataFrame(data={'col1': [1, 2, 3, 4], 'col2': [3, 4, 5, 6]})

@ray.remote
def f(x):
    pass

f.remote(df)
```

The last line causes the following error to be printed in the background.

```
A worker died or was killed while executing a task.
```

cc @devin-petersohn


Discussion:
The original application that failed was

```python
import pandas as pd
import ray

ray.init()

data = pd.DataFrame(data={'col1': [1, 2, 3, 4], 'col2': [3, 4, 5, 6]})

@ray.remote
def test_fn(df):
    return df.assign(result=df.col1 * df.col2)

remote_data = ray.put(data)
x = test_fn.remote(remote_data)
```
Note that as a workaround for now you can do

```python
ray.register_custom_serializer(type(data), use_pickle=True)
```
So while ray.put and ray.get works in the same process, it doesn't work if the DataFrame is put in one process and then gotten in a different one. Interesting.
Backtrace:
```
* thread #1: tid = 0xf9367, 0x0000000000000000, queue = 'com.apple.main-thread', stop reason = EXC_BAD_ACCESS (code=1, address=0x0)
  * frame #0: 0x0000000000000000
    frame #1: 0x0000000103cb0ff2 libarrow_python.0.dylib`arrow::py::GetValue(context=0x000000010de149f8, parent=0x000000010db944f8, arr=0x000000010db88488, index=0, type=0, base=<unavailable>, blobs=<unavailable>, result=<unavailable>) + 2562 at arrow_to_python.cc:171 [opt]
    frame #2: 0x0000000103cb0459 libarrow_python.0.dylib`arrow::py::DeserializeList(context=0x000000010de149f8, array=<unavailable>, start_idx=<unavailable>, stop_idx=2, base=0x000000010ddf3fa8, blobs=<unavailable>, out=<unavailable>) + 265 at arrow_to_python.cc:206 [opt]
    frame #3: 0x0000000103cb0133 libarrow_python.0.dylib`arrow::py::DeserializeDict(context=0x000000010de149f8, array=0x000000010db94318, start_idx=0, stop_idx=2, base=0x000000010ddf3fa8, blobs=<unavailable>, out=<unavailable>) + 227 at arrow_to_python.cc:72 [opt]
    frame #4: 0x0000000103cb0ef7 libarrow_python.0.dylib`arrow::py::GetValue(context=0x000000010de149f8, parent=0x00000001007dceb8, arr=0x000000010db883e8, index=<unavailable>, type=0, base=<unavailable>, blobs=<unavailable>, result=<unavailable>) + 2311 at arrow_to_python.cc:156 [opt]
    frame #5: 0x0000000103cb0459 libarrow_python.0.dylib`arrow::py::DeserializeList(context=0x000000010de149f8, array=<unavailable>, start_idx=<unavailable>, stop_idx=1, base=0x000000010ddf3fa8, blobs=<unavailable>, out=<unavailable>) + 265 at arrow_to_python.cc:206 [opt]
    frame #6: 0x0000000103cb187f libarrow_python.0.dylib`arrow::py::DeserializeObject(context=0x000000010de149f8, obj=0x000000010ddf2c90, base=0x000000010ddf3fa8, out=0x00007fff5fbfe058) + 111 at arrow_to_python.cc:285 [opt]
    frame #7: 0x00000001038057de lib.cpython-35m-darwin.so`__pyx_pf_7pyarrow_3lib_18SerializedPyObject_2deserialize(__pyx_v_self=0x000000010ddf2c78, __pyx_v_context=0x000000010de149f8) + 462 at lib.cxx:73367
    frame #8: 0x0000000103805014 lib.cpython-35m-darwin.so`__pyx_pw_7pyarrow_3lib_18SerializedPyObject_3deserialize(__pyx_v_self=0x000000010ddf2c78, __pyx_args=0x000000010de2b2b0, __pyx_kwds=0x0000000000000000) + 772 at lib.cxx:73290
    frame #9: 0x00000001000561fc libpython3.5m.dylib`PyCFunction_Call + 60```
@robertnishihara created a PR in Arrow: https://github.com/apache/arrow/pull/1463 that fixes this. Once that gets pulled in we will need to update the serialization.

Resolving patch:
diff --git a/python/ray/dataframe/__init__.py b/python/ray/dataframe/__init__.py
--- a/python/ray/dataframe/__init__.py
+++ b/python/ray/dataframe/__init__.py
@@ -6,10 +6,5 @@
 from .dataframe import from_pandas
 from .dataframe import to_pandas
 from .series import Series
-import ray
-import pandas as pd
 
 __all__ = ["DataFrame", "from_pandas", "to_pandas", "Series"]
-
-ray.register_custom_serializer(pd.DataFrame, use_pickle=True)
-ray.register_custom_serializer(pd.core.indexes.base.Index, use_pickle=True)


Test patch:
diff --git a/src/plasma/test/client_tests.cc b/src/plasma/test/client_tests.cc
--- a/src/plasma/test/client_tests.cc
+++ b/src/plasma/test/client_tests.cc
@@ -32,7 +32,7 @@ TEST plasma_status_tests(void) {
   int64_t data_size = 100;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  uint8_t *data;
+  std::shared_ptr<MutableBuffer> data;
   ARROW_CHECK_OK(
       client1.Create(oid1, data_size, metadata, metadata_size, &data));
   ARROW_CHECK_OK(client1.Seal(oid1));
@@ -73,7 +73,7 @@ TEST plasma_fetch_tests(void) {
   int64_t data_size = 100;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  uint8_t *data;
+  std::shared_ptr<MutableBuffer> data;
   ARROW_CHECK_OK(
       client1.Create(oid1, data_size, metadata, metadata_size, &data));
   ARROW_CHECK_OK(client1.Seal(oid1));
@@ -116,7 +116,9 @@ void init_data_123(uint8_t *data, uint64_t size, uint8_t base) {
   }
 }
 
-bool is_equal_data_123(uint8_t *data1, uint8_t *data2, uint64_t size) {
+bool is_equal_data_123(const uint8_t *data1,
+                       const uint8_t *data2,
+                       uint64_t size) {
   for (size_t i = 0; i < size; i++) {
     if (data1[i] != data2[i]) {
       return false;
@@ -142,14 +144,15 @@ TEST plasma_nonblocking_get_tests(void) {
   int64_t data_size = 4;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  uint8_t *data;
+  std::shared_ptr<MutableBuffer> data;
   ARROW_CHECK_OK(client.Create(oid, data_size, metadata, metadata_size, &data));
-  init_data_123(data, data_size, 0);
+  init_data_123(data->mutable_data(), data_size, 0);
   ARROW_CHECK_OK(client.Seal(oid));
 
   sleep(1);
   ARROW_CHECK_OK(client.Get(oid_array, 1, 0, &obj_buffer));
-  ASSERT(is_equal_data_123(data, obj_buffer.data, data_size) == true);
+  ASSERT(is_equal_data_123(data->data(), obj_buffer.data->data(), data_size) ==
+         true);
 
   sleep(1);
   ARROW_CHECK_OK(client.Disconnect());
@@ -191,7 +194,7 @@ TEST plasma_wait_for_objects_tests(void) {
   int64_t data_size = 4;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  uint8_t *data;
+  std::shared_ptr<MutableBuffer> data;
   ARROW_CHECK_OK(
       client1.Create(oid1, data_size, metadata, metadata_size, &data));
   ARROW_CHECK_OK(client1.Seal(oid1));
@@ -245,23 +248,23 @@ TEST plasma_get_tests(void) {
   int64_t data_size = 4;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  uint8_t *data;
+  std::shared_ptr<MutableBuffer> data;
   ARROW_CHECK_OK(
       client1.Create(oid1, data_size, metadata, metadata_size, &data));
-  init_data_123(data, data_size, 1);
+  init_data_123(data->mutable_data(), data_size, 1);
   ARROW_CHECK_OK(client1.Seal(oid1));
 
   ARROW_CHECK_OK(client1.Get(oid_array1, 1, -1, &obj_buffer));
-  ASSERT(data[0] == obj_buffer.data[0]);
+  ASSERT(data->data()[0] == obj_buffer.data->data()[0]);
 
   ARROW_CHECK_OK(
       client2.Create(oid2, data_size, metadata, metadata_size, &data));
-  init_data_123(data, data_size, 2);
+  init_data_123(data->mutable_data(), data_size, 2);
   ARROW_CHECK_OK(client2.Seal(oid2));
 
   ARROW_CHECK_OK(client1.Fetch(1, oid_array2));
   ARROW_CHECK_OK(client1.Get(oid_array2, 1, -1, &obj_buffer));
-  ASSERT(data[0] == obj_buffer.data[0]);
+  ASSERT(data->data()[0] == obj_buffer.data->data()[0]);
 
   sleep(1);
   ARROW_CHECK_OK(client1.Disconnect());
@@ -288,25 +291,25 @@ TEST plasma_get_multiple_tests(void) {
   int64_t data_size = 4;
   uint8_t metadata[] = {5};
   int64_t metadata_size = sizeof(metadata);
-  uint8_t *data;
+  std::shared_ptr<MutableBuffer> data;
   ARROW_CHECK_OK(
       client1.Create(oid1, data_size, metadata, metadata_size, &data));
-  init_data_123(data, data_size, obj1_first);
+  init_data_123(data->mutable_data(), data_size, obj1_first);
   ARROW_CHECK_OK(client1.Seal(oid1));
 
   /* This only waits for oid1. */
   ARROW_CHECK_OK(client1.Get(obj_ids, 1, -1, obj_buffer));
-  ASSERT(data[0] == obj_buffer[0].data[0]);
+  ASSERT(data->data()[0] == obj_buffer[0].data->data()[0]);
 
   ARROW_CHECK_OK(
       client2.Create(oid2, data_size, metadata, metadata_size, &data));
-  init_data_123(data, data_size, obj2_first);
+  init_data_123(data->mutable_data(), data_size, obj2_first);
   ARROW_CHECK_OK(client2.Seal(oid2));
 
   ARROW_CHECK_OK(client1.Fetch(2, obj_ids));
   ARROW_CHECK_OK(client1.Get(obj_ids, 2, -1, obj_buffer));
-  ASSERT(obj1_first == obj_buffer[0].data[0]);
-  ASSERT(obj2_first == obj_buffer[1].data[0]);
+  ASSERT(obj1_first == obj_buffer[0].data->data()[0]);
+  ASSERT(obj2_first == obj_buffer[1].data->data()[0]);
 
   sleep(1);
   ARROW_CHECK_OK(client1.Disconnect());