Repository: mars-project/mars

Issue:
[BUG] Serialize tiled tensor failed
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**

Currently, if I want to serialize a tiled tensor without inputs, I may add the tensor to a graph, and serialize the graph. The serialization works, but deserialize failed.

**To Reproduce**

```python
In [1]: import mars.tensor as mt                                                

In [2]: from  mars.graph import DAG                                             

In [3]: a = mt.ones((10, 20), chunk_size=5)                                     

In [4]: a.tiles()                                                               
Out[4]: <mars.tensor.core.Tensor at 0x1399bbc78>

In [5]: g = DAG()                                                               

In [6]: g.add_node(a)                                                           

In [7]: j = g.to_json()                                                         

In [8]: DAG.from_json(j)                                                        
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-8-1e09b7847f5b> in <module>
----> 1 DAG.from_json(j)

~/Workspace/mars/mars/graph.pyx in mars.graph.DirectedGraph.from_json()
    426     @classmethod
    427     def from_json(cls, json_obj):
--> 428         return cls.deserialize(SerializableGraph.from_json(json_obj))
    429 
    430     def compose(self, list keys=None):

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Serializable.from_json()
    576 
    577         provider = JsonSerializeProvider()
--> 578         return cls.deserialize(provider, obj)
    579 
    580 

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Serializable.deserialize()
    547         callbacks = [] if callbacks is None else callbacks
    548         key_to_instance = {} if key_to_instance is None else key_to_instance
--> 549         obj = provider.deserialize_model(cls, obj, callbacks, key_to_instance)
    550         if call_cb:
    551             [cb(key_to_instance) for cb in callbacks]

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Provider.deserialize_model()
    678 
    679         for name, field in model_instance._FIELDS.items():
--> 680             field.deserialize(self, model_instance, obj, callbacks, key_to_instance)
    681 
    682         if hasattr(model_instance, 'key') and hasattr(model_instance, 'id'):

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Field.deserialize()
    143 
    144     cpdef deserialize(self, Provider provider, model_instance, obj, list callbacks, dict key_to_instance):
--> 145         return provider.deserialize_field(self, model_instance, obj, callbacks, key_to_instance)
    146 
    147     @property

~/Workspace/mars/mars/serialize/jsonserializer.pyx in mars.serialize.jsonserializer.JsonSerializeProvider.deserialize_field()
    525             setattr(model_instance, field.attr,
    526                     self._on_deserial(
--> 527                         field, [field.type.type.model.deserialize(self, it_obj, callbacks, key_to_instance)
    528                                 if it_obj is not None else None
    529                                 for it_obj in obj[tag]]))

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Serializable.deserialize()
    547         callbacks = [] if callbacks is None else callbacks
    548         key_to_instance = {} if key_to_instance is None else key_to_instance
--> 549         obj = provider.deserialize_model(cls, obj, callbacks, key_to_instance)
    550         if call_cb:
    551             [cb(key_to_instance) for cb in callbacks]

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Provider.deserialize_model()
    678 
    679         for name, field in model_instance._FIELDS.items():
--> 680             field.deserialize(self, model_instance, obj, callbacks, key_to_instance)
    681 
    682         if hasattr(model_instance, 'key') and hasattr(model_instance, 'id'):

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Field.deserialize()
    143 
    144     cpdef deserialize(self, Provider provider, model_instance, obj, list callbacks, dict key_to_instance):
--> 145         return provider.deserialize_field(self, model_instance, obj, callbacks, key_to_instance)
    146 
    147     @property

~/Workspace/mars/mars/serialize/jsonserializer.pyx in mars.serialize.jsonserializer.JsonSerializeProvider.deserialize_field()
    518 
    519                     setattr(model_instance, f.attr,
--> 520                             self._on_deserial(field, f.type.model.deserialize(self, obj[tag], callbacks, key_to_instance)))
    521         elif isinstance(field, ListField) and type(field.type.type) == Reference:
    522             tag = field.tag_name(self)

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Serializable.deserialize()
    547         callbacks = [] if callbacks is None else callbacks
    548         key_to_instance = {} if key_to_instance is None else key_to_instance
--> 549         obj = provider.deserialize_model(cls, obj, callbacks, key_to_instance)
    550         if call_cb:
    551             [cb(key_to_instance) for cb in callbacks]

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Provider.deserialize_model()
    678 
    679         for name, field in model_instance._FIELDS.items():
--> 680             field.deserialize(self, model_instance, obj, callbacks, key_to_instance)
    681 
    682         if hasattr(model_instance, 'key') and hasattr(model_instance, 'id'):

~/Workspace/mars/mars/serialize/core.pyx in mars.serialize.core.Field.deserialize()
    143 
    144     cpdef deserialize(self, Provider provider, model_instance, obj, list callbacks, dict key_to_instance):
--> 145         return provider.deserialize_field(self, model_instance, obj, callbacks, key_to_instance)
    146 
    147     @property

~/Workspace/mars/mars/serialize/jsonserializer.pyx in mars.serialize.jsonserializer.JsonSerializeProvider.deserialize_field()
    525             setattr(model_instance, field.attr,
    526                     self._on_deserial(
--> 527                         field, [field.type.type.model.deserialize(self, it_obj, callbacks, key_to_instance)
    528                                 if it_obj is not None else None
    529                                 for it_obj in obj[tag]]))

AttributeError: type object 'Chunk' has no attribute 'deserialize'
```




Discussion:

Resolving patch:
diff --git a/mars/core.py b/mars/core.py
--- a/mars/core.py
+++ b/mars/core.py
@@ -23,7 +23,7 @@
 from .compat import six, izip, builtins, reduce
 from .utils import tokenize, AttributeDict, on_serialize_shape, on_deserialize_shape
 from .serialize import ValueType, ProviderType, Serializable, AttributeAsDict, \
-    ListField, TupleField, DictField, KeyField, BoolField, StringField
+    TupleField, DictField, KeyField, BoolField, StringField
 from .tiles import Tilesable, handler
 from .graph import DAG
 
@@ -275,7 +275,6 @@ class TilesableData(SerializableWithKey, Tilesable):
     # optional fields
     # `nsplits` means the sizes of chunks for each dimension
     _nsplits = TupleField('nsplits', ValueType.tuple(ValueType.uint64))
-    _chunks = ListField('chunks', ValueType.reference(Chunk))
     _params = DictField('params', key_type=ValueType.string, on_deserialize=AttributeDict)
 
     def __init__(self, *args, **kwargs):
diff --git a/mars/dataframe/core.py b/mars/dataframe/core.py
--- a/mars/dataframe/core.py
+++ b/mars/dataframe/core.py
@@ -15,7 +15,7 @@
 # limitations under the License.
 
 from ..core import ChunkData, Chunk, Entity, TilesableData
-from ..serialize import Serializable, ProviderType, DataTypeField, AnyField, SeriesField, \
+from ..serialize import Serializable, ValueType, ProviderType, DataTypeField, AnyField, SeriesField, \
     BoolField, Int64Field, Int32Field, ListField, SliceField, OneOfField, ReferenceField
 
 
@@ -136,6 +136,9 @@ class IndexData(TilesableData):
     # optional field
     _dtype = DataTypeField('dtype')
     _index_value = ReferenceField('index_value', IndexValue)
+    _chunks = ListField('chunks', ValueType.reference(IndexChunkData),
+                        on_serialize=lambda x: [it.data for it in x] if x is not None else x,
+                        on_deserialize=lambda x: [IndexChunk(it) for it in x] if x is not None else x)
 
     @classmethod
     def cls(cls, provider):
@@ -188,6 +191,9 @@ class SeriesData(TilesableData):
     _dtype = DataTypeField('dtype')
     _name = AnyField('name')
     _index_value = ReferenceField('index_value', IndexValue)
+    _chunks = ListField('chunks', ValueType.reference(SeriesChunkData),
+                        on_serialize=lambda x: [it.data for it in x] if x is not None else x,
+                        on_deserialize=lambda x: [SeriesChunk(it) for it in x] if x is not None else x)
 
     @property
     def dtype(self):
@@ -238,6 +244,9 @@ class DataFrameData(TilesableData):
     # optional field
     _dtypes = SeriesField('dtypes')
     _index_value = ReferenceField('index_value', IndexValue)
+    _chunks = ListField('chunks', ValueType.reference(DataFrameChunkData),
+                        on_serialize=lambda x: [it.data for it in x] if x is not None else x,
+                        on_deserialize=lambda x: [DataFrameChunk(it) for it in x] if x is not None else x)
 
     @property
     def dtypes(self):
diff --git a/mars/tensor/core.py b/mars/tensor/core.py
--- a/mars/tensor/core.py
+++ b/mars/tensor/core.py
@@ -50,11 +50,19 @@ def rough_shape(self):
         return calc_rough_shape(self)
 
 
+class TensorChunk(Chunk):
+    __slots__ = ()
+    _allow_data_type_ = (TensorChunkData,)
+
+
 class TensorData(TilesableData):
     __slots__ = ()
 
     # required fields
     _dtype = DataTypeField('dtype')
+    _chunks = ListField('chunks', ValueType.reference(TensorChunkData),
+                        on_serialize=lambda x: [it.data for it in x] if x is not None else x,
+                        on_deserialize=lambda x: [TensorChunk(it) for it in x] if x is not None else x)
 
     @classmethod
     def cls(cls, provider):
@@ -270,11 +278,6 @@ def execute(self, session=None, **kw):
         return session.run(*self, **kw)
 
 
-class TensorChunk(Chunk):
-    __slots__ = ()
-    _allow_data_type_ = (TensorChunkData,)
-
-
 class Tensor(Entity):
     __slots__ = ()
     _allow_data_type_ = (TensorData,)


Test patch:
diff --git a/mars/dataframe/expressions/tests/test_datasource.py b/mars/dataframe/expressions/tests/test_datasource.py
--- a/mars/dataframe/expressions/tests/test_datasource.py
+++ b/mars/dataframe/expressions/tests/test_datasource.py
@@ -23,7 +23,7 @@
 
 from mars import opcodes as OperandDef
 from mars.graph import DAG
-from mars.dataframe.core import IndexValue
+from mars.dataframe.core import IndexValue, DataFrameChunk
 from mars.dataframe.expressions.datasource.dataframe import from_pandas
 from mars.tests.core import TestBase
 
@@ -86,6 +86,29 @@ def testDataFrameGraphSerialize(self):
         self.assertEqual(t.shape, t2.shape)
         self.assertEqual(sorted(i.key for i in t.inputs), sorted(i.key for i in t2.inputs))
 
+        # test graph with tiled DataFrame
+        t2 = from_pandas(pd.DataFrame(np.random.rand(10, 10)), chunk_size=(5, 4)).tiles()
+        graph = DAG()
+        graph.add_node(t2)
+
+        pb = graph.to_pb()
+        graph2 = DAG.from_pb(pb)
+        self.assertEqual(len(graph), len(graph2))
+        chunks = next(iter(graph2)).chunks
+        self.assertEqual(len(chunks), 6)
+        self.assertIsInstance(chunks[0], DataFrameChunk)
+        self.assertEqual(chunks[0].index, t2.chunks[0].index)
+        self.assertBaseEqual(chunks[0].op, t2.chunks[0].op)
+
+        jsn = graph.to_json()
+        graph2 = DAG.from_json(jsn)
+        self.assertEqual(len(graph), len(graph2))
+        chunks = next(iter(graph2)).chunks
+        self.assertEqual(len(chunks), 6)
+        self.assertIsInstance(chunks[0], DataFrameChunk)
+        self.assertEqual(chunks[0].index, t2.chunks[0].index)
+        self.assertBaseEqual(chunks[0].op, t2.chunks[0].op)
+
     def testFromPandas(self):
         data = pd.DataFrame(np.random.rand(10, 10))
         df = from_pandas(data, chunk_size=4)
diff --git a/mars/tensor/expressions/tests/test_core.py b/mars/tensor/expressions/tests/test_core.py
--- a/mars/tensor/expressions/tests/test_core.py
+++ b/mars/tensor/expressions/tests/test_core.py
@@ -28,7 +28,7 @@
 from mars.tensor.expressions.datasource.array import CSRMatrixDataSource
 from mars.tensor.expressions.datasource.ones import TensorOnesLike
 from mars.tensor.expressions.fuse.core import TensorFuseChunk
-from mars.tensor.core import Tensor, SparseTensor, build_mode
+from mars.tensor.core import Tensor, SparseTensor, TensorChunk, build_mode
 from mars.graph import DAG
 from mars.serialize.protos.operand_pb2 import OperandDef
 from mars.tests.core import TestBase, calc_shape
@@ -271,6 +271,29 @@ def testTensorGraphSerialize(self):
         self.assertEqual(t.shape, t2.shape)
         self.assertEqual(sorted(i.key for i in t.inputs), sorted(i.key for i in t2.inputs))
 
+        # test graph with tiled tensor
+        t2 = ones((10, 10), chunk_size=(5, 4)).tiles()
+        graph = DAG()
+        graph.add_node(t2)
+
+        pb = graph.to_pb()
+        graph2 = DAG.from_pb(pb)
+        self.assertEqual(len(graph), len(graph2))
+        chunks = next(iter(graph2)).chunks
+        self.assertEqual(len(chunks), 6)
+        self.assertIsInstance(chunks[0], TensorChunk)
+        self.assertEqual(chunks[0].index, t2.chunks[0].index)
+        self.assertBaseEqual(chunks[0].op, t2.chunks[0].op)
+
+        jsn = graph.to_json()
+        graph2 = DAG.from_json(jsn)
+        self.assertEqual(len(graph), len(graph2))
+        chunks = next(iter(graph2)).chunks
+        self.assertEqual(len(chunks), 6)
+        self.assertIsInstance(chunks[0], TensorChunk)
+        self.assertEqual(chunks[0].index, t2.chunks[0].index)
+        self.assertBaseEqual(chunks[0].op, t2.chunks[0].op)
+
     def testTensorGraphTiledSerialize(self):
         t = ones((10, 3), chunk_size=(5, 2)) + tensor(np.random.random((10, 3)), chunk_size=(5, 2))
         graph = t.build_graph(tiled=True)