Repository: mars-project/mars

Issue:
Support serialize pandas DataFrame in dataserializer, protobuf and json serializer
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Is your feature request related to a problem? Please describe.**

Currently dataserializer support numpy ndarray, SparseNDArray etc, we should let it support serialize pandas DataFrame. Also, we should add a field type in protobuf and json serializer which indicates the DataFrame type.

**Describe the solution you'd like**

pyarrow should support pandas dataframe so that we might need to do nothing for dataserializer. 



Discussion:

Resolving patch:
diff --git a/mars/serialize/dataserializer.py b/mars/serialize/dataserializer.py
--- a/mars/serialize/dataserializer.py
+++ b/mars/serialize/dataserializer.py
@@ -252,10 +252,6 @@ def peek_serialized_size(file):
         file.seek(0)
 
 
-class DataTuple(tuple):
-    pass
-
-
 def _serialize_numpy_array_list(obj):
     if obj.dtype.str != '|O':
         # Make the array c_contiguous if necessary so that we can call change
@@ -303,37 +299,6 @@ def _deserialize_sparse_csr_list(data):
     return SparseNDArray(target_csr)
 
 
-def _serialize_data_tuple(obj):
-    data_meta = []
-    outputs = [None]
-    for item in obj:
-        if isinstance(item, np.ndarray):
-            serialized_items = _serialize_numpy_array_list(item)
-            outputs.extend(serialized_items)
-            data_meta.append(DATA_FLAG_DENSE)
-            data_meta.append(len(serialized_items))
-        else:
-            serialized_items = _serialize_sparse_csr_list(item)
-            outputs.extend(serialized_items)
-            data_meta.append(DATA_FLAG_CSR)
-            data_meta.append(len(serialized_items))
-    outputs[0] = struct.pack('<' + 'B' * len(data_meta), *data_meta)
-    return tuple(outputs)
-
-
-def _deserialize_data_tuple(data):
-    data_meta = struct.unpack('<' + 'B' * len(data[0]), data[0])
-    pos = 1
-    data_parts = []
-    for flag, size in zip(data_meta[0::2], data_meta[1::2]):
-        if flag == DATA_FLAG_DENSE:
-            data_parts.append(_deserialize_numpy_array_list(data[pos:pos + size]))
-        elif flag == DATA_FLAG_CSR:
-            data_parts.append(_deserialize_sparse_csr_list(data[pos:pos + size]))
-        pos += size
-    return DataTuple(data_parts)
-
-
 _serialize_context = None
 
 
@@ -344,8 +309,5 @@ def mars_serialize_context():
         ctx.register_type(SparseNDArray, 'mars.SparseNDArray',
                           custom_serializer=_serialize_sparse_csr_list,
                           custom_deserializer=_deserialize_sparse_csr_list)
-        ctx.register_type(DataTuple, 'mars.DataTuple',
-                          custom_serializer=_serialize_data_tuple,
-                          custom_deserializer=_deserialize_data_tuple)
         _serialize_context = ctx
     return _serialize_context
diff --git a/mars/serialize/protos/value_pb2.py b/mars/serialize/protos/value_pb2.py
--- a/mars/serialize/protos/value_pb2.py
+++ b/mars/serialize/protos/value_pb2.py
@@ -19,7 +19,7 @@
   package='',
   syntax='proto3',
   serialized_options=None,
-  serialized_pb=_b('\n!mars/serialize/protos/value.proto\"\xa6\x04\n\x05Value\x12\x11\n\x07is_null\x18\x01 \x01(\x08H\x00\x12\x0b\n\x01\x62\x18\x02 \x01(\x08H\x00\x12\x0b\n\x01i\x18\x03 \x01(\x03H\x00\x12\x0b\n\x01\x66\x18\x04 \x01(\x02H\x00\x12\x0b\n\x01s\x18\x05 \x01(\x0cH\x00\x12\x0b\n\x01u\x18\x06 \x01(\tH\x00\x12\x1b\n\x04list\x18\x07 \x01(\x0b\x32\x0b.Value.ListH\x00\x12\x1b\n\x04\x64ict\x18\x08 \x01(\x0b\x32\x0b.Value.DictH\x00\x12\x1d\n\x05slice\x18\t \x01(\x0b\x32\x0c.Value.SliceH\x00\x12\r\n\x03\x61rr\x18\n \x01(\x0cH\x00\x12\x0f\n\x05\x64type\x18\x0b \x01(\x0cH\x00\x12\x19\n\x03key\x18\x0c \x01(\x0b\x32\n.Value.KeyH\x00\x12\x14\n\ndatetime64\x18\r \x01(\x0cH\x00\x12\x15\n\x0btimedelta64\x18\x0e \x01(\x0cH\x00\x1a/\n\x04List\x12\x10\n\x08is_tuple\x18\x01 \x01(\x08\x12\x15\n\x05value\x18\x02 \x03(\x0b\x32\x06.Value\x1an\n\x05Slice\x12\x0f\n\x07is_null\x18\x04 \x01(\x08\x12\x13\n\tstart_val\x18\x01 \x01(\x03H\x00\x12\x12\n\x08stop_val\x18\x02 \x01(\x03H\x01\x12\x12\n\x08step_val\x18\x03 \x01(\x03H\x02\x42\x07\n\x05startB\x06\n\x04stopB\x06\n\x04step\x1a>\n\x04\x44ict\x12\x19\n\x04keys\x18\x01 \x01(\x0b\x32\x0b.Value.List\x12\x1b\n\x06values\x18\x02 \x01(\x0b\x32\x0b.Value.List\x1a\x1e\n\x03Key\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\tB\x07\n\x05valueb\x06proto3')
+  serialized_pb=_b('\n!mars/serialize/protos/value.proto\"\xde\x04\n\x05Value\x12\x11\n\x07is_null\x18\x01 \x01(\x08H\x00\x12\x0b\n\x01\x62\x18\x02 \x01(\x08H\x00\x12\x0b\n\x01i\x18\x03 \x01(\x03H\x00\x12\x0b\n\x01\x66\x18\x04 \x01(\x02H\x00\x12\x0b\n\x01s\x18\x05 \x01(\x0cH\x00\x12\x0b\n\x01u\x18\x06 \x01(\tH\x00\x12\x1b\n\x04list\x18\x07 \x01(\x0b\x32\x0b.Value.ListH\x00\x12\x1b\n\x04\x64ict\x18\x08 \x01(\x0b\x32\x0b.Value.DictH\x00\x12\x1d\n\x05slice\x18\t \x01(\x0b\x32\x0c.Value.SliceH\x00\x12\r\n\x03\x61rr\x18\n \x01(\x0cH\x00\x12\x0f\n\x05\x64type\x18\x0b \x01(\x0cH\x00\x12\x19\n\x03key\x18\x0c \x01(\x0b\x32\n.Value.KeyH\x00\x12\x14\n\ndatetime64\x18\r \x01(\x0cH\x00\x12\x15\n\x0btimedelta64\x18\x0e \x01(\x0cH\x00\x12\x0f\n\x05index\x18\x0f \x01(\x0cH\x00\x12\x10\n\x06series\x18\x10 \x01(\x0cH\x00\x12\x13\n\tdataframe\x18\x11 \x01(\x0cH\x00\x1a/\n\x04List\x12\x10\n\x08is_tuple\x18\x01 \x01(\x08\x12\x15\n\x05value\x18\x02 \x03(\x0b\x32\x06.Value\x1an\n\x05Slice\x12\x0f\n\x07is_null\x18\x04 \x01(\x08\x12\x13\n\tstart_val\x18\x01 \x01(\x03H\x00\x12\x12\n\x08stop_val\x18\x02 \x01(\x03H\x01\x12\x12\n\x08step_val\x18\x03 \x01(\x03H\x02\x42\x07\n\x05startB\x06\n\x04stopB\x06\n\x04step\x1a>\n\x04\x44ict\x12\x19\n\x04keys\x18\x01 \x01(\x0b\x32\x0b.Value.List\x12\x1b\n\x06values\x18\x02 \x01(\x0b\x32\x0b.Value.List\x1a\x1e\n\x03Key\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\tB\x07\n\x05valueb\x06proto3')
 )
 
 
@@ -58,8 +58,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=324,
-  serialized_end=371,
+  serialized_start=380,
+  serialized_end=427,
 )
 
 _VALUE_SLICE = _descriptor.Descriptor(
@@ -118,8 +118,8 @@
       name='step', full_name='Value.Slice.step',
       index=2, containing_type=None, fields=[]),
   ],
-  serialized_start=373,
-  serialized_end=483,
+  serialized_start=429,
+  serialized_end=539,
 )
 
 _VALUE_DICT = _descriptor.Descriptor(
@@ -155,8 +155,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=485,
-  serialized_end=547,
+  serialized_start=541,
+  serialized_end=603,
 )
 
 _VALUE_KEY = _descriptor.Descriptor(
@@ -192,8 +192,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=549,
-  serialized_end=579,
+  serialized_start=605,
+  serialized_end=635,
 )
 
 _VALUE = _descriptor.Descriptor(
@@ -301,6 +301,27 @@
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='index', full_name='Value.index', index=14,
+      number=15, type=12, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b(""),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='series', full_name='Value.series', index=15,
+      number=16, type=12, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b(""),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='dataframe', full_name='Value.dataframe', index=16,
+      number=17, type=12, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b(""),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
   ],
   extensions=[
   ],
@@ -317,7 +338,7 @@
       index=0, containing_type=None, fields=[]),
   ],
   serialized_start=38,
-  serialized_end=588,
+  serialized_end=644,
 )
 
 _VALUE_LIST.fields_by_name['value'].message_type = _VALUE
@@ -382,6 +403,15 @@
 _VALUE.oneofs_by_name['value'].fields.append(
   _VALUE.fields_by_name['timedelta64'])
 _VALUE.fields_by_name['timedelta64'].containing_oneof = _VALUE.oneofs_by_name['value']
+_VALUE.oneofs_by_name['value'].fields.append(
+  _VALUE.fields_by_name['index'])
+_VALUE.fields_by_name['index'].containing_oneof = _VALUE.oneofs_by_name['value']
+_VALUE.oneofs_by_name['value'].fields.append(
+  _VALUE.fields_by_name['series'])
+_VALUE.fields_by_name['series'].containing_oneof = _VALUE.oneofs_by_name['value']
+_VALUE.oneofs_by_name['value'].fields.append(
+  _VALUE.fields_by_name['dataframe'])
+_VALUE.fields_by_name['dataframe'].containing_oneof = _VALUE.oneofs_by_name['value']
 DESCRIPTOR.message_types_by_name['Value'] = _VALUE
 _sym_db.RegisterFileDescriptor(DESCRIPTOR)
 
diff --git a/mars/worker/chunkstore.py b/mars/worker/chunkstore.py
--- a/mars/worker/chunkstore.py
+++ b/mars/worker/chunkstore.py
@@ -150,11 +150,8 @@ def put(self, session_id, chunk_key, value):
         """
         import pyarrow
         from pyarrow.lib import PlasmaStoreFull, PlasmaObjectExists
-        from ..serialize.dataserializer import DataTuple
 
         data_size = calc_data_size(value)
-        if isinstance(value, tuple):
-            value = DataTuple(*value)
 
         obj_id = self._calc_object_id(session_id, chunk_key)
 


Test patch:
diff --git a/mars/serialize/tests/test_serialize.py b/mars/serialize/tests/test_serialize.py
--- a/mars/serialize/tests/test_serialize.py
+++ b/mars/serialize/tests/test_serialize.py
@@ -21,11 +21,14 @@
 import unittest
 
 import numpy as np
-
 try:
     import pyarrow
 except ImportError:
     pyarrow = None
+try:
+    import pandas as pd
+except ImportError:
+    pd = None
 
 from mars.compat import six, OrderedDict, BytesIO
 from mars.lib import sparse
@@ -34,7 +37,7 @@
     UInt32Field, UInt64Field, Float16Field, Float32Field, Float64Field, BoolField, \
     Datetime64Field, Timedelta64Field, DataTypeField, KeyField, ReferenceField, OneOfField, \
     ListField, NDArrayField, DictField, TupleField, ValueType, serializes, deserializes, \
-    ProviderType, AttributeAsDict
+    IndexField, SeriesField, DataFrameField, ProviderType, AttributeAsDict
 from mars.serialize import dataserializer
 from mars.serialize.pbserializer import ProtobufSerializeProvider
 from mars.serialize.jsonserializer import JsonSerializeProvider
@@ -138,6 +141,11 @@ class Node4(AttributeAsDict):
     j = ReferenceField('k', Node5)
     k = ListField('l', ValueType.reference('Node5'))
     l = OneOfField('m', n5=Node5, n6=Node6)
+    m = IndexField('n')
+    mm = IndexField('mn')
+    n = SeriesField('o')
+    o = DataFrameField('p')
+    p = ListField('q')
 
     @classmethod
     def cls(cls, provider):
@@ -248,6 +256,15 @@ def testJSONSerialize(self):
         self.assertNotIsInstance(node3.value.i[0], Node8)
 
     def testAttributeAsDict(self):
+        other_data = {}
+        if pd:
+            df = pd.DataFrame({'a': [1, 2, 3], 'b': [to_text('测试'), to_binary('属性'), 'c']},
+                              index=[[0, 0, 1], ['测试', '属性', '测试']])
+            other_data['m'] = df.columns
+            other_data['mm'] = df.index
+            other_data['n'] = df['b']
+            other_data['o'] = df
+            other_data['p'] = [df.columns, df.index, df['a'], df]
         node4 = Node4(a=to_binary('中文'),
                       b=np.random.randint(4, size=(3, 4)),
                       c=np.datetime64(datetime.datetime.now()),
@@ -259,7 +276,7 @@ def testAttributeAsDict(self):
                       i=(slice(10), slice(0, 2), None, slice(2, 0, -1)),
                       j=Node5(a='aa'),
                       k=[Node5(a='bb'), None],
-                      l=Node6(b=3, nid=1))
+                      l=Node6(b=3, nid=1), **other_data)
 
         pbs = ProtobufSerializeProvider()
 
@@ -267,7 +284,7 @@ def testAttributeAsDict(self):
         d_node4 = Node4.deserialize(pbs, serial)
 
         self.assertEqual(node4.a, d_node4.a)
-        self.assertTrue(np.array_equal(node4.b, d_node4.b))
+        np.testing.assert_array_equal(node4.b, d_node4.b)
         self.assertEqual(node4.c, d_node4.c)
         self.assertEqual(node4.d, d_node4.d)
         self.assertEqual(node4.e, d_node4.e)
@@ -280,6 +297,15 @@ def testAttributeAsDict(self):
         self.assertIsNone(d_node4.k[1])
         self.assertIsInstance(d_node4.l, Node7)
         self.assertEqual(d_node4.l.b, 3)
+        if pd:
+            pd.testing.assert_index_equal(node4.m, d_node4.m)
+            pd.testing.assert_index_equal(node4.mm, d_node4.mm)
+            pd.testing.assert_series_equal(node4.n, d_node4.n)
+            pd.testing.assert_frame_equal(node4.o, d_node4.o)
+            pd.testing.assert_index_equal(node4.p[0], d_node4.p[0])
+            pd.testing.assert_index_equal(node4.p[1], d_node4.p[1])
+            pd.testing.assert_series_equal(node4.p[2], d_node4.p[2])
+            pd.testing.assert_frame_equal(node4.p[3], d_node4.p[3])
 
         jss = JsonSerializeProvider()
 
@@ -288,7 +314,7 @@ def testAttributeAsDict(self):
         d_node4 = Node4.deserialize(jss, serial)
 
         self.assertEqual(node4.a, d_node4.a)
-        self.assertTrue(np.array_equal(node4.b, d_node4.b))
+        np.testing.assert_array_equal(node4.b, d_node4.b)
         self.assertEqual(node4.c, d_node4.c)
         self.assertEqual(node4.d, d_node4.d)
         self.assertEqual(node4.e, d_node4.e)
@@ -301,6 +327,15 @@ def testAttributeAsDict(self):
         self.assertIsNone(d_node4.k[1])
         self.assertIsInstance(d_node4.l, Node7)
         self.assertEqual(d_node4.l.b, 3)
+        if pd:
+            pd.testing.assert_index_equal(node4.m, d_node4.m)
+            pd.testing.assert_index_equal(node4.mm, d_node4.mm)
+            pd.testing.assert_series_equal(node4.n, d_node4.n)
+            pd.testing.assert_frame_equal(node4.o, d_node4.o)
+            pd.testing.assert_index_equal(node4.p[0], d_node4.p[0])
+            pd.testing.assert_index_equal(node4.p[1], d_node4.p[1])
+            pd.testing.assert_series_equal(node4.p[2], d_node4.p[2])
+            pd.testing.assert_frame_equal(node4.p[3], d_node4.p[3])
 
     def testException(self):
         node1 = Node1(h=[object()])
@@ -386,7 +421,7 @@ def testArrowSerialize(self):
         except ImportError:
             sps = None
 
-        from mars.serialize.dataserializer import DataTuple, mars_serialize_context
+        from mars.serialize.dataserializer import mars_serialize_context
         context = mars_serialize_context()
 
         if np:
@@ -401,7 +436,7 @@ def testArrowSerialize(self):
         if np and sps:
             array = np.random.rand(1000, 100)
             mat = sparse.SparseMatrix(sps.random(100, 100, 0.1, format='csr'))
-            tp = DataTuple((array, mat))
+            tp = (array, mat)
             des_tp = pyarrow.deserialize(pyarrow.serialize(tp, context).to_buffer(), context)
             assert_array_equal(tp[0], des_tp[0])
             self.assertTrue((tp[1].spmatrix != des_tp[1].spmatrix).nnz == 0)