Repository: mars-project/mars

Issue:
Add test for multiple workers
As the bug mentioned in #129 , we should add a test in which multiple workers are created.


Discussion:

Resolving patch:
diff --git a/mars/scheduler/graph.py b/mars/scheduler/graph.py
--- a/mars/scheduler/graph.py
+++ b/mars/scheduler/graph.py
@@ -275,7 +275,7 @@ def stop_graph(self):
         for chunk in chunk_graph:
             if chunk.op.key not in self._operand_infos:
                 continue
-            if self._operand_infos[chunk.op.key]['state'] in \
+            if self._operand_infos[chunk.op.key].get('state') in \
                     (OperandState.READY, OperandState.RUNNING, OperandState.FINISHED):
                 # we only need to stop on ready, running and finished operands
                 op_uid = OperandActor.gen_uid(self._session_id, chunk.op.key)
@@ -608,7 +608,7 @@ def get_executable_operand_dag(self, op_key, serialize=True):
         else:
             return graph
 
-    def create_operand_actors(self, _clean_io_meta=True):
+    def create_operand_actors(self, _clean_io_meta=True, _start=True):
         """
         Create operand actors for all operands
         """
@@ -675,10 +675,11 @@ def create_operand_actors(self, _clean_io_meta=True):
 
         self.state = GraphState.RUNNING
 
-        op_refs = dict((k, v.result()) for k, v in op_refs.items())
-        start_futures = [op_refs[op_key].start_operand(_tell=True, _wait=False)
-                         for op_key in initial_keys]
-        [future.result() for future in start_futures]
+        if _start:
+            op_refs = dict((k, v.result()) for k, v in op_refs.items())
+            start_futures = [op_refs[op_key].start_operand(_tell=True, _wait=False)
+                             for op_key in initial_keys]
+            [future.result() for future in start_futures]
 
     def mark_terminal_finished(self, op_key, final_state=None):
         """
diff --git a/mars/scheduler/operand.py b/mars/scheduler/operand.py
--- a/mars/scheduler/operand.py
+++ b/mars/scheduler/operand.py
@@ -108,6 +108,9 @@ def post_create(self):
         if not self.ctx.has_actor(self._kv_store_ref):
             self._kv_store_ref = None
 
+    def get_state(self):
+        return self._state
+
     @property
     def state(self):
         return self._state
@@ -176,6 +179,12 @@ def add_finished_predecessor(self, op_key):
         self.update_demand_depths(self._info.get('optimize', {}).get('depth', 0))
         return False
 
+    def remove_finished_predecessor(self, op_key):
+        try:
+            self._finish_preds.remove(op_key)
+        except KeyError:
+            pass
+
     def add_finished_successor(self, op_key):
         self._finish_succs.add(op_key)
         if not self._is_terminal and all(k in self._finish_succs for k in self._succ_keys):
@@ -267,12 +276,6 @@ def propose_descendant_workers(self, input_key, worker_scores, depth=1):
                          self._op_key, self._op_name, max_worker, self._worker_scores)
             return self._input_chunks, max_worker
 
-    def get_op_info(self):
-        info = dict()
-        info['name'] = self._op_name
-        info['state'] = self.state
-        return info
-
     def start_operand(self, state=None):
         """
         Start handling operand given self.state


Test patch:
diff --git a/mars/actors/pool/tests/test_gevent_pool.py b/mars/actors/pool/tests/test_gevent_pool.py
--- a/mars/actors/pool/tests/test_gevent_pool.py
+++ b/mars/actors/pool/tests/test_gevent_pool.py
@@ -454,6 +454,11 @@ def testProcessTell(self):
             self.assertIsNone(ref1.send(('tell_async', ref3, 'add', 3)))
             self.assertEqual(ref3.send(('get',)), 8)
 
+            ref3.tell(('add', 3), wait=False, delay=0.3)
+            self.assertEqual(ref3.send(('get',)), 8)
+            pool.sleep(0.5)
+            self.assertEqual(ref3.send(('get',)), 11)
+
             ref1.send(('tell_delay', ref2, 'add', 4, 0.3))  # delay 0.5 secs
             self.assertEqual(ref2.send(('get',)), 5)
             pool.sleep(0.5)
@@ -470,8 +475,8 @@ def testProcessDestroyHas(self):
 
             ref1 = pool.create_actor(DummyActor, 1, uid='admin-1')
             ref2 = ref1.send(('create', (DummyActor, 2)))
-
-            self.assertTrue(pool.has_actor(ref2))
+            future = pool.has_actor(ref2, wait=False)
+            self.assertTrue(future.result())
 
             ref1.send(('delete', ref2))
             self.assertFalse(ref1.send(('has', ref2)))
@@ -1054,6 +1059,11 @@ def testRemoteTellProcessPool(self):
             self.assertIsNone(ref1.send(('tell_async', ref3, 'add', 3)))
             self.assertEqual(ref3.send(('get',)), 8)
 
+            ref3.tell(('add', 3), wait=False, delay=0.3)
+            self.assertEqual(ref3.send(('get',)), 8)
+            client.sleep(.5)
+            self.assertEqual(ref3.send(('get',)), 11)
+
             ref1.send(('tell_delay', ref2, 'add', 4, 0.3))  # delay 0.3 secs
             self.assertEqual(ref2.send(('get',)), 5)
             client.sleep(.5)
@@ -1248,6 +1258,11 @@ def testRemoteDestroyHasProcessPool(self):
             future.result()
             self.assertFalse(pool.has_actor(ref3))
 
+            ref3 = client.create_actor(DummyActor, 1, address=addr)
+            future = pool.destroy_actor(ref3, wait=False)
+            future.result()
+            self.assertFalse(pool.has_actor(ref3))
+
             ref3 = client.create_actor(DummyActor, 1, address=addr)
             ref4 = ref3.send(('create', (DummyActor, 2)))
             ref3.send(('delete_async', ref4))
diff --git a/mars/scheduler/tests/test_graph.py b/mars/scheduler/tests/test_graph.py
--- a/mars/scheduler/tests/test_graph.py
+++ b/mars/scheduler/tests/test_graph.py
@@ -12,18 +12,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import contextlib
 import uuid
 import unittest
 
 import mars.tensor as mt
 from mars.cluster_info import ClusterInfoActor
-from mars.scheduler import GraphActor, ResourceActor, ChunkMetaActor, AssignerActor
+from mars.scheduler import GraphActor, GraphMetaActor, ResourceActor, ChunkMetaActor, \
+    AssignerActor, GraphState
 from mars.utils import serialize_graph, get_next_port
 from mars.actors import create_actor_pool
+from mars.tests.core import patch_method
 
 
 class Test(unittest.TestCase):
-    def run_expr_suite(self, expr, compose=False):
+    @contextlib.contextmanager
+    def prepare_graph_in_pool(self, expr, clean_io_meta=True, compose=False):
         session_id = str(uuid.uuid4())
         graph_key = str(uuid.uuid4())
 
@@ -68,62 +72,67 @@ def run_expr_suite(self, expr, compose=False):
                 target_worker = op_infos[n.op.key]['target_worker']
                 self.assertIsNotNone(target_worker)
 
-            graph_ref.create_operand_actors(_clean_io_meta=False)
+            graph_ref.create_operand_actors(_clean_io_meta=clean_io_meta)
             op_infos = graph_ref.get_operand_info()
 
-            orig_metas = dict()
-            for n in fetched_graph:
-                try:
-                    meta = orig_metas[n.op.key]
-                except KeyError:
-                    meta = orig_metas[n.op.key] = dict(
-                        predecessors=set(), successors=set(), input_chunks=set(), chunks=set()
-                    )
-                meta['predecessors'].update([pn.op.key for pn in fetched_graph.iter_predecessors(n)])
-                meta['successors'].update([sn.op.key for sn in fetched_graph.iter_successors(n)])
-                meta['input_chunks'].update([pn.key for pn in fetched_graph.iter_predecessors(n)])
-                meta['chunks'].update([c.key for c in n.op.outputs])
+            if not clean_io_meta:
+                orig_metas = dict()
+                for n in fetched_graph:
+                    try:
+                        meta = orig_metas[n.op.key]
+                    except KeyError:
+                        meta = orig_metas[n.op.key] = dict(
+                            predecessors=set(), successors=set(), input_chunks=set(), chunks=set()
+                        )
+                    meta['predecessors'].update([pn.op.key for pn in fetched_graph.iter_predecessors(n)])
+                    meta['successors'].update([sn.op.key for sn in fetched_graph.iter_successors(n)])
+                    meta['input_chunks'].update([pn.key for pn in fetched_graph.iter_predecessors(n)])
+                    meta['chunks'].update([c.key for c in n.op.outputs])
 
-            for n in fetched_graph:
-                self.assertEqual(op_infos[n.op.key]['op_name'], type(n.op).__name__)
+                for n in fetched_graph:
+                    self.assertEqual(op_infos[n.op.key]['op_name'], type(n.op).__name__)
 
-                io_meta = op_infos[n.op.key]['io_meta']
-                orig_io_meta = orig_metas[n.op.key]
+                    io_meta = op_infos[n.op.key]['io_meta']
+                    orig_io_meta = orig_metas[n.op.key]
 
-                self.assertSetEqual(set(io_meta['predecessors']), set(orig_io_meta['predecessors']))
-                self.assertSetEqual(set(io_meta['successors']), set(orig_io_meta['successors']))
-                self.assertSetEqual(set(io_meta['input_chunks']), set(orig_io_meta['input_chunks']))
-                self.assertSetEqual(set(io_meta['chunks']), set(orig_io_meta['chunks']))
+                    self.assertSetEqual(set(io_meta['predecessors']), set(orig_io_meta['predecessors']))
+                    self.assertSetEqual(set(io_meta['successors']), set(orig_io_meta['successors']))
+                    self.assertSetEqual(set(io_meta['input_chunks']), set(orig_io_meta['input_chunks']))
+                    self.assertSetEqual(set(io_meta['chunks']), set(orig_io_meta['chunks']))
 
-                self.assertEqual(op_infos[n.op.key]['output_size'], sum(ch.nbytes for ch in n.op.outputs))
+                    self.assertEqual(op_infos[n.op.key]['output_size'], sum(ch.nbytes for ch in n.op.outputs))
 
-        return fetched_graph
+            yield pool, graph_ref
 
-    def testGraphActor(self):
+    def testSimpleGraphPreparation(self, *_):
         arr = mt.random.randint(10, size=(10, 8), chunk_size=4)
         arr_add = mt.random.randint(10, size=(10, 8), chunk_size=4)
         arr2 = arr + arr_add
-        self.run_expr_suite(arr2)
+        with self.prepare_graph_in_pool(arr2, clean_io_meta=False):
+            pass
 
-    def testGraphWithSplit(self):
+    def testSplitPreparation(self, *_):
         arr = mt.ones(12, chunk_size=4)
         arr_split = mt.split(arr, 2)
         arr_sum = arr_split[0] + arr_split[1]
-        self.run_expr_suite(arr_sum)
+        with self.prepare_graph_in_pool(arr_sum, clean_io_meta=False):
+            pass
 
-    def testSameKey(self):
+    def testSameKeyPreparation(self, *_):
         arr = mt.ones((5, 5), chunk_size=3)
         arr2 = mt.concatenate((arr, arr))
-        self.run_expr_suite(arr2)
+        with self.prepare_graph_in_pool(arr2, clean_io_meta=False):
+            pass
 
-    def testFuseExist(self):
+    def testFusePreparation(self, *_):
         from mars.tensor.expressions.fuse.core import TensorFuseChunk
         arr = mt.ones((5, 5), chunk_size=3)
         arr2 = (arr + 5) * 2
-        out_graph = self.run_expr_suite(arr2, compose=True)
-        self.assertTrue(all(isinstance(v.op, TensorFuseChunk) for v in out_graph))
+        with self.prepare_graph_in_pool(arr2, compose=True) as (pool, graph_ref):
+            out_graph = graph_ref.get_chunk_graph()
+            self.assertTrue(all(isinstance(v.op, TensorFuseChunk) for v in out_graph))
 
-    def testMultipleAdd(self):
+    def testMultipleAddPreparation(self, *_):
         import numpy as np
         import operator
         from mars.compat import reduce
@@ -131,4 +140,89 @@ def testMultipleAdd(self):
         base_arr = np.random.random((100, 100))
         a = mt.array(base_arr)
         sumv = reduce(operator.add, [a[:10, :10] for _ in range(10)])
-        self.run_expr_suite(sumv)
+        with self.prepare_graph_in_pool(sumv):
+            pass
+
+    def testGraphTermination(self, *_):
+        from mars.tensor.expressions.arithmetic.add import TensorAddConstant
+        arr = mt.random.random((8, 2), chunk_size=2)
+        arr2 = arr + 1
+        with self.prepare_graph_in_pool(arr2) as (pool, graph_ref):
+            out_graph = graph_ref.get_chunk_graph()
+            for c in out_graph:
+                if not isinstance(c.op, TensorAddConstant):
+                    continue
+                self.assertNotEqual(graph_ref.get_state(), GraphState.SUCCEEDED)
+                graph_ref.mark_terminal_finished(c.op.key)
+
+            self.assertEqual(graph_ref.get_state(), GraphState.SUCCEEDED)
+
+        arr = mt.random.random((8, 2), chunk_size=2)
+        arr2 = arr + 1
+        with self.prepare_graph_in_pool(arr2) as (pool, graph_ref):
+            out_graph = graph_ref.get_chunk_graph()
+            for c in out_graph:
+                if not isinstance(c.op, TensorAddConstant):
+                    continue
+                self.assertNotEqual(graph_ref.get_state(), GraphState.FAILED)
+                graph_ref.mark_terminal_finished(c.op.key, GraphState.FAILED)
+
+            self.assertEqual(graph_ref.get_state(), GraphState.FAILED)
+
+    def testErrorOnPrepare(self, *_):
+        session_id = str(uuid.uuid4())
+
+        addr = '127.0.0.1:%d' % get_next_port()
+        with create_actor_pool(n_process=1, backend='gevent', address=addr) as pool:
+            pool.create_actor(ClusterInfoActor, [pool.cluster_info.address],
+                              uid=ClusterInfoActor.default_name())
+            resource_ref = pool.create_actor(ResourceActor, uid=ResourceActor.default_name())
+            pool.create_actor(ChunkMetaActor, uid=ChunkMetaActor.default_name())
+            pool.create_actor(AssignerActor, uid=AssignerActor.default_name())
+
+            resource_ref.set_worker_meta('localhost:12345', dict(hardware=dict(cpu_total=4)))
+            resource_ref.set_worker_meta('localhost:23456', dict(hardware=dict(cpu_total=4)))
+
+            # error occurred in create_operand_actors
+            graph_key = str(uuid.uuid4())
+            expr = mt.random.random((8, 2), chunk_size=2) + 1
+            graph = expr.build_graph(compose=False)
+            serialized_graph = serialize_graph(graph)
+
+            graph_ref = pool.create_actor(GraphActor, session_id, graph_key, serialized_graph,
+                                          uid=GraphActor.gen_name(session_id, graph_key))
+
+            def _mock_raises(*_):
+                raise RuntimeError
+
+            with patch_method(GraphActor.create_operand_actors, new=_mock_raises):
+                with self.assertRaises(RuntimeError):
+                    graph_ref.execute_graph()
+            self.assertEqual(graph_ref.get_state(), GraphState.FAILED)
+            graph_ref.destroy()
+
+            # interrupted during create_operand_actors
+            graph_key = str(uuid.uuid4())
+            graph_ref = pool.create_actor(GraphActor, session_id, graph_key, serialized_graph,
+                                          uid=GraphActor.gen_name(session_id, graph_key))
+
+            def _mock_cancels(*_):
+                graph_meta_ref = pool.actor_ref(GraphMetaActor.gen_name(session_id, graph_key))
+                graph_meta_ref.set_state(GraphState.CANCELLING)
+
+            with patch_method(GraphActor.create_operand_actors, new=_mock_cancels):
+                graph_ref.execute_graph()
+            self.assertEqual(graph_ref.get_state(), GraphState.CANCELLED)
+
+            # interrupted during previous steps
+            graph_key = str(uuid.uuid4())
+            graph_ref = pool.create_actor(GraphActor, session_id, graph_key, serialized_graph,
+                                          uid=GraphActor.gen_name(session_id, graph_key))
+
+            def _mock_cancels(*_):
+                graph_meta_ref = pool.actor_ref(GraphMetaActor.gen_name(session_id, graph_key))
+                graph_meta_ref.set_state(GraphState.CANCELLING)
+
+            with patch_method(GraphActor.place_initial_chunks, new=_mock_cancels):
+                graph_ref.execute_graph()
+            self.assertEqual(graph_ref.get_state(), GraphState.CANCELLED)
diff --git a/mars/scheduler/tests/test_main.py b/mars/scheduler/tests/test_main.py
--- a/mars/scheduler/tests/test_main.py
+++ b/mars/scheduler/tests/test_main.py
@@ -55,13 +55,20 @@ def tearDownClass(cls):
 
     def setUp(self):
         scheduler_port = str(get_next_port())
-        proc_worker = subprocess.Popen([sys.executable, '-m', 'mars.worker',
-                                        '-a', '127.0.0.1',
-                                        '--cpu-procs', '2',
-                                        '--level', 'debug',
-                                        '--cache-mem', '16m',
-                                        '--schedulers', '127.0.0.1:' + scheduler_port,
-                                        '--ignore-avail-mem'])
+        proc_worker1 = subprocess.Popen([sys.executable, '-m', 'mars.worker',
+                                         '-a', '127.0.0.1',
+                                         '--cpu-procs', '2',
+                                         '--level', 'debug',
+                                         '--cache-mem', '16m',
+                                         '--schedulers', '127.0.0.1:' + scheduler_port,
+                                         '--ignore-avail-mem'])
+        proc_worker2 = subprocess.Popen([sys.executable, '-m', 'mars.worker',
+                                         '-a', '127.0.0.1',
+                                         '--cpu-procs', '2',
+                                         '--level', 'debug',
+                                         '--cache-mem', '16m',
+                                         '--schedulers', '127.0.0.1:' + scheduler_port,
+                                         '--ignore-avail-mem'])
         proc_scheduler = subprocess.Popen([sys.executable, '-m', 'mars.scheduler',
                                            '-H', '127.0.0.1',
                                            '--level', 'debug',
@@ -69,7 +76,7 @@ def setUp(self):
                                            '--format', '%(asctime)-15s %(message)s'])
 
         self.scheduler_port = scheduler_port
-        self.proc_worker = proc_worker
+        self.proc_workers = [proc_worker1, proc_worker2]
         self.proc_scheduler = proc_scheduler
 
         time.sleep(2)
@@ -89,7 +96,7 @@ def setUp(self):
 
         check_time = time.time()
         while True:
-            if not resource_ref.get_worker_count():
+            if resource_ref.get_worker_count() < 2:
                 time.sleep(0.5)
                 self.check_process_statuses()
                 if time.time() - check_time > 20:
@@ -101,7 +108,7 @@ def setUp(self):
         gevent.hub.Hub.NOT_ERROR = (Exception,)
 
     def tearDown(self):
-        procs = (self.proc_worker, self.proc_scheduler)
+        procs = (self.proc_scheduler,) + tuple(self.proc_workers)
         for p in procs:
             p.send_signal(signal.SIGINT)
 
@@ -120,8 +127,9 @@ def tearDown(self):
     def check_process_statuses(self):
         if self.proc_scheduler.poll() is not None:
             raise SystemError('Scheduler not started. exit code %s' % self.proc_scheduler.poll())
-        if self.proc_worker.poll() is not None:
-            raise SystemError('Worker not started. exit code %s' % self.proc_worker.poll())
+        for worker_proc in self.proc_workers:
+            if worker_proc.poll() is not None:
+                raise SystemError('Worker not started. exit code %s' % worker_proc.poll())
 
     def wait_for_termination(self, session_ref, graph_key):
         check_time = time.time()
diff --git a/mars/serialize/tests/test_serialize.py b/mars/serialize/tests/test_serialize.py
--- a/mars/serialize/tests/test_serialize.py
+++ b/mars/serialize/tests/test_serialize.py
@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import datetime
 import json
 import os
 import tempfile
@@ -28,9 +29,12 @@
 
 from mars.compat import six, OrderedDict, BytesIO
 from mars.lib import sparse
-from mars.serialize.core import Serializable, IdentityField, StringField, Int32Field, BytesField, \
-    KeyField, ReferenceField, OneOfField, ListField, NDArrayField, DictField, TupleField, \
-    ValueType, serializes, deserializes, ProviderType, AttributeAsDict
+from mars.serialize.core import Serializable, IdentityField, StringField, UnicodeField, \
+    BytesField, Int8Field, Int16Field, Int32Field, Int64Field, UInt8Field, UInt16Field, \
+    UInt32Field, UInt64Field, Float16Field, Float32Field, Float64Field, BoolField, \
+    Datetime64Field, Timedelta64Field, DataTypeField, KeyField, ReferenceField, OneOfField, \
+    ListField, NDArrayField, DictField, TupleField, ValueType, serializes, deserializes, \
+    ProviderType, AttributeAsDict
 from mars.serialize import dataserializer
 from mars.serialize.pbserializer import ProtobufSerializeProvider
 from mars.serialize.jsonserializer import JsonSerializeProvider
@@ -40,11 +44,22 @@
 
 class Node1(Serializable):
     a = IdentityField('a', ValueType.string)
-    b = Int32Field('b')
-    c = KeyField('c')
-    d = ReferenceField('d', 'Node2')
-    e = ListField('e')
-    f = ListField('f', ValueType.reference('self'))
+    b1 = Int8Field('b1')
+    b2 = Int16Field('b2')
+    b3 = Int32Field('b3')
+    b4 = Int64Field('b4')
+    c1 = UInt8Field('c1')
+    c2 = UInt16Field('c2')
+    c3 = UInt32Field('c3')
+    c4 = UInt64Field('c4')
+    d1 = Float16Field('d1')
+    d2 = Float32Field('d2')
+    d3 = Float64Field('d3')
+    e = BoolField('e')
+    f = KeyField('f')
+    g = ReferenceField('g', 'Node2')
+    h = ListField('h')
+    i = ListField('i', ValueType.reference('self'))
 
     def __new__(cls, *args, **kwargs):
         if 'a' in kwargs and kwargs['a'] == 'test1':
@@ -67,6 +82,7 @@ class Node2(Serializable, BaseWithKey):
     a = ListField('a', ValueType.list(ValueType.string))
     _key = StringField('key')
     _id = StringField('id')
+    _name = UnicodeField('name')
     data = ListField('data', ValueType.int32)
 
     @classmethod
@@ -111,14 +127,17 @@ class Node4(AttributeAsDict):
 
     a = BytesField('b')
     b = NDArrayField('c')
-    c = DictField('d', ValueType.string, ValueType.list(ValueType.bool))
-    d = DictField('e')
-    e = TupleField('f', ValueType.int64, ValueType.unicode, ValueType.string, ValueType.float32,
+    c = Datetime64Field('d')
+    d = Timedelta64Field('e')
+    e = DataTypeField('f')
+    f = DictField('g', ValueType.string, ValueType.list(ValueType.bool))
+    g = DictField('h')
+    h = TupleField('i', ValueType.int64, ValueType.unicode, ValueType.string, ValueType.float32,
                    ValueType.datetime64, ValueType.timedelta64, ValueType.dtype)
-    f = TupleField('g', ValueType.slice)
-    g = ReferenceField('h', Node5)
-    h = ListField('i', ValueType.reference('Node5'))
-    i = OneOfField('j', n5=Node5, n6=Node6)
+    i = TupleField('j', ValueType.slice)
+    j = ReferenceField('k', Node5)
+    k = ListField('l', ValueType.reference('Node5'))
+    l = OneOfField('m', n5=Node5, n6=Node6)
 
     @classmethod
     def cls(cls, provider):
@@ -133,10 +152,15 @@ def testPBSerialize(self):
         provider = ProtobufSerializeProvider()
 
         node2 = Node2(a=[['ss'], ['dd']], data=[3, 7, 212])
-        node1 = Node1(a='test1', b=2, d=Node2(a=[['1', '2'], ['3', '4']]),
-                      c=node2,
-                      e=[[2, 3], node2, True, {1: node2}, np.datetime64('1066-10-13'), np.timedelta64(1, 'D')],
-                      f=[Node1(b=111), Node1(b=222)])
+        node1 = Node1(a='test1',
+                      b1=-2, b2=2000, b3=-5000, b4=500000,
+                      c1=2, c2=2000, c3=5000, c4=500000,
+                      d1=2.5, d2=7.37, d3=5.976321,
+                      e=False,
+                      f=node2,
+                      g=Node2(a=[['1', '2'], ['3', '4']]),
+                      h=[[2, 3], node2, True, {1: node2}, np.datetime64('1066-10-13'), np.timedelta64(1, 'D')],
+                      i=[Node1(b1=111), Node1(b1=222)])
         node3 = Node3(value=node1)
 
         serials = serializes(provider, [node2, node3])
@@ -150,26 +174,42 @@ def testPBSerialize(self):
         self.assertIsInstance(d_node3.value, Node8)
         self.assertIsNot(node3.value, d_node3.value)
         self.assertEqual(node3.value.a, d_node3.value.a)
-        self.assertEqual(node3.value.b, d_node3.value.b)
-        self.assertIsNot(node3.value.c, d_node3.value.c)
-        self.assertEqual(node3.value.c.a, d_node3.value.c.a)
-        self.assertIsNot(node3.value.d, d_node3.value.d)
-        self.assertEqual(node3.value.d.a, d_node3.value.d.a)
-        self.assertEqual(node3.value.e[0], d_node3.value.e[0])
-        self.assertNotIsInstance(d_node3.value.e[1], six.string_types)
-        self.assertIs(d_node3.value.e[1], d_node3.value.c)
-        self.assertEqual(node3.value.e[2], True)
-        self.assertEqual([n.b for n in node3.value.f], [n.b for n in d_node3.value.f])
-        self.assertNotIsInstance(node3.value.f[0], Node8)
+        self.assertEqual(node3.value.b1, d_node3.value.b1)
+        self.assertEqual(node3.value.b2, d_node3.value.b2)
+        self.assertEqual(node3.value.b3, d_node3.value.b3)
+        self.assertEqual(node3.value.b4, d_node3.value.b4)
+        self.assertEqual(node3.value.c1, d_node3.value.c1)
+        self.assertEqual(node3.value.c2, d_node3.value.c2)
+        self.assertEqual(node3.value.c3, d_node3.value.c3)
+        self.assertEqual(node3.value.c4, d_node3.value.c4)
+        self.assertAlmostEqual(node3.value.d1, d_node3.value.d1, places=2)
+        self.assertAlmostEqual(node3.value.d2, d_node3.value.d2, places=4)
+        self.assertAlmostEqual(node3.value.d3, d_node3.value.d3)
+        self.assertEqual(node3.value.e, d_node3.value.e)
+        self.assertIsNot(node3.value.f, d_node3.value.f)
+        self.assertEqual(node3.value.f.a, d_node3.value.f.a)
+        self.assertIsNot(node3.value.g, d_node3.value.g)
+        self.assertEqual(node3.value.g.a, d_node3.value.g.a)
+        self.assertEqual(node3.value.h[0], d_node3.value.h[0])
+        self.assertNotIsInstance(d_node3.value.h[1], six.string_types)
+        self.assertIs(d_node3.value.h[1], d_node3.value.f)
+        self.assertEqual(node3.value.h[2], True)
+        self.assertEqual([n.b1 for n in node3.value.i], [n.b1 for n in d_node3.value.i])
+        self.assertNotIsInstance(node3.value.i[0], Node8)
 
     def testJSONSerialize(self):
         provider = JsonSerializeProvider()
 
         node2 = Node2(a=[['ss'], ['dd']], data=[3, 7, 212])
-        node1 = Node1(a='test1', b=2, d=Node2(a=[['1', '2'], ['3', '4']]),
-                      c=node2,
-                      e=[[2, 3], node2, True, {1: node2}, np.datetime64('1066-10-13'), np.timedelta64(1, 'D')],
-                      f=[Node1(b=111), Node1(b=222)])
+        node1 = Node1(a='test1',
+                      b1=2, b2=2000, b3=5000, b4=500000,
+                      c1=2, c2=2000, c3=5000, c4=500000,
+                      d1=2.5, d2=7.37, d3=5.976321,
+                      e=False,
+                      f=node2,
+                      g=Node2(a=[['1', '2'], ['3', '4']]),
+                      h=[[2, 3], node2, True, {1: node2}, np.datetime64('1066-10-13'), np.timedelta64(1, 'D')],
+                      i=[Node1(b1=111), Node1(b1=222)])
         node3 = Node3(value=node1)
 
         serials = serializes(provider, [node2, node3])
@@ -184,27 +224,42 @@ def testJSONSerialize(self):
         self.assertIsInstance(d_node3.value, Node8)
         self.assertIsNot(node3.value, d_node3.value)
         self.assertEqual(node3.value.a, d_node3.value.a)
-        self.assertEqual(node3.value.b, d_node3.value.b)
-        self.assertIsNot(node3.value.c, d_node3.value.c)
-        self.assertEqual(node3.value.c.a, d_node3.value.c.a)
-        self.assertIsNot(node3.value.d, d_node3.value.d)
-        self.assertEqual(node3.value.d.a, d_node3.value.d.a)
-        self.assertEqual(node3.value.e[0], d_node3.value.e[0])
-        self.assertNotIsInstance(d_node3.value.e[1], six.string_types)
-        self.assertIs(d_node3.value.e[1], d_node3.value.c)
-        self.assertEqual(node3.value.e[2], True)
-        self.assertEqual([n.b for n in node3.value.f], [n.b for n in d_node3.value.f])
-        self.assertNotIsInstance(node3.value.f[0], Node8)
+        self.assertEqual(node3.value.b1, d_node3.value.b1)
+        self.assertEqual(node3.value.b2, d_node3.value.b2)
+        self.assertEqual(node3.value.b3, d_node3.value.b3)
+        self.assertEqual(node3.value.b4, d_node3.value.b4)
+        self.assertEqual(node3.value.c1, d_node3.value.c1)
+        self.assertEqual(node3.value.c2, d_node3.value.c2)
+        self.assertEqual(node3.value.c3, d_node3.value.c3)
+        self.assertEqual(node3.value.c4, d_node3.value.c4)
+        self.assertAlmostEqual(node3.value.d1, d_node3.value.d1, places=2)
+        self.assertAlmostEqual(node3.value.d2, d_node3.value.d2, places=4)
+        self.assertAlmostEqual(node3.value.d3, d_node3.value.d3)
+        self.assertEqual(node3.value.e, d_node3.value.e)
+        self.assertIsNot(node3.value.f, d_node3.value.f)
+        self.assertEqual(node3.value.f.a, d_node3.value.f.a)
+        self.assertIsNot(node3.value.g, d_node3.value.g)
+        self.assertEqual(node3.value.g.a, d_node3.value.g.a)
+        self.assertEqual(node3.value.h[0], d_node3.value.h[0])
+        self.assertNotIsInstance(d_node3.value.h[1], six.string_types)
+        self.assertIs(d_node3.value.h[1], d_node3.value.f)
+        self.assertEqual(node3.value.h[2], True)
+        self.assertEqual([n.b1 for n in node3.value.i], [n.b1 for n in d_node3.value.i])
+        self.assertNotIsInstance(node3.value.i[0], Node8)
 
     def testAttributeAsDict(self):
-        node4 = Node4(a=to_binary('中文'), b=np.random.randint(4, size=(3, 4)),
-                      c={'a': [True, False, False], 'd': [False, None]},
-                      e=(1234, to_text('测试'), '属性', None, np.datetime64('1066-10-13'),
+        node4 = Node4(a=to_binary('中文'),
+                      b=np.random.randint(4, size=(3, 4)),
+                      c=np.datetime64(datetime.datetime.now()),
+                      d=np.timedelta64(datetime.timedelta(seconds=1234)),
+                      e=np.dtype('int'),
+                      f={'a': [True, False, False], 'd': [False, None]},
+                      h=(1234, to_text('测试'), '属性', None, np.datetime64('1066-10-13'),
                          np.timedelta64(1, 'D'), np.dtype([('x', 'i4'), ('y', 'f4')])),
-                      f=(slice(10), slice(0, 2), None, slice(2, 0, -1)),
-                      g=Node5(a='aa'),
-                      h=[Node5(a='bb'), None],
-                      i=Node6(b=3, nid=1))
+                      i=(slice(10), slice(0, 2), None, slice(2, 0, -1)),
+                      j=Node5(a='aa'),
+                      k=[Node5(a='bb'), None],
+                      l=Node6(b=3, nid=1))
 
         pbs = ProtobufSerializeProvider()
 
@@ -214,14 +269,17 @@ def testAttributeAsDict(self):
         self.assertEqual(node4.a, d_node4.a)
         self.assertTrue(np.array_equal(node4.b, d_node4.b))
         self.assertEqual(node4.c, d_node4.c)
-        self.assertFalse(hasattr(d_node4, 'd'))
+        self.assertEqual(node4.d, d_node4.d)
         self.assertEqual(node4.e, d_node4.e)
         self.assertEqual(node4.f, d_node4.f)
-        self.assertEqual(node4.g.a, d_node4.g.a)
-        self.assertEqual(node4.h[0].a, d_node4.h[0].a)
-        self.assertIsNone(d_node4.h[1])
-        self.assertIsInstance(d_node4.i, Node7)
-        self.assertEqual(d_node4.i.b, 3)
+        self.assertFalse(hasattr(d_node4, 'g'))
+        self.assertEqual(node4.h, d_node4.h)
+        self.assertEqual(node4.i, d_node4.i)
+        self.assertEqual(node4.j.a, d_node4.j.a)
+        self.assertEqual(node4.k[0].a, d_node4.k[0].a)
+        self.assertIsNone(d_node4.k[1])
+        self.assertIsInstance(d_node4.l, Node7)
+        self.assertEqual(d_node4.l.b, 3)
 
         jss = JsonSerializeProvider()
 
@@ -232,17 +290,20 @@ def testAttributeAsDict(self):
         self.assertEqual(node4.a, d_node4.a)
         self.assertTrue(np.array_equal(node4.b, d_node4.b))
         self.assertEqual(node4.c, d_node4.c)
-        self.assertFalse(hasattr(d_node4, 'd'))
+        self.assertEqual(node4.d, d_node4.d)
         self.assertEqual(node4.e, d_node4.e)
         self.assertEqual(node4.f, d_node4.f)
-        self.assertEqual(node4.g.a, d_node4.g.a)
-        self.assertEqual(node4.h[0].a, d_node4.h[0].a)
-        self.assertIsNone(d_node4.h[1])
-        self.assertIsInstance(d_node4.i, Node7)
-        self.assertEqual(d_node4.i.b, 3)
+        self.assertFalse(hasattr(d_node4, 'g'))
+        self.assertEqual(node4.h, d_node4.h)
+        self.assertEqual(node4.i, d_node4.i)
+        self.assertEqual(node4.j.a, d_node4.j.a)
+        self.assertEqual(node4.k[0].a, d_node4.k[0].a)
+        self.assertIsNone(d_node4.k[1])
+        self.assertIsInstance(d_node4.l, Node7)
+        self.assertEqual(d_node4.l.b, 3)
 
     def testException(self):
-        node1 = Node1(e=[object()])
+        node1 = Node1(h=[object()])
 
         pbs = ProtobufSerializeProvider()
 
diff --git a/mars/serialize/tests/testser.proto b/mars/serialize/tests/testser.proto
--- a/mars/serialize/tests/testser.proto
+++ b/mars/serialize/tests/testser.proto
@@ -5,11 +5,22 @@ import "mars/serialize/protos/value.proto";
 
 message Node1Def {
     string a = 1;
-    int32 b = 2;
-    Value c = 3;
-    Node2Def d = 4;
-    repeated Value e = 5;
-    repeated Node1Def f = 6;
+    int32 b1 = 2;
+    int32 b2 = 3;
+    int32 b3 = 4;
+    int64 b4 = 5;
+    uint32 c1 = 6;
+    uint32 c2 = 7;
+    uint32 c3 = 8;
+    uint64 c4 = 9;
+    float d1 = 10;
+    float d2 = 11;
+    double d3 = 12;
+    bool e = 13;
+    Value f = 14;
+    Node2Def g = 15;
+    repeated Value h = 16;
+    repeated Node1Def i = 17;
 }
 
 
@@ -18,6 +29,7 @@ message Node2Def {
     string key = 2;
     repeated int32 data = 3;
     string id = 4;
+    string name = 5;
 }
 
 
diff --git a/mars/serialize/tests/testser_pb2.py b/mars/serialize/tests/testser_pb2.py
--- a/mars/serialize/tests/testser_pb2.py
+++ b/mars/serialize/tests/testser_pb2.py
@@ -20,7 +20,7 @@
   package='',
   syntax='proto3',
   serialized_options=None,
-  serialized_pb=_b('\n\"mars/serialize/tests/testser.proto\x1a!mars/serialize/protos/value.proto\"r\n\x08Node1Def\x12\t\n\x01\x61\x18\x01 \x01(\t\x12\t\n\x01\x62\x18\x02 \x01(\x05\x12\x11\n\x01\x63\x18\x03 \x01(\x0b\x32\x06.Value\x12\x14\n\x01\x64\x18\x04 \x01(\x0b\x32\t.Node2Def\x12\x11\n\x01\x65\x18\x05 \x03(\x0b\x32\x06.Value\x12\x14\n\x01\x66\x18\x06 \x03(\x0b\x32\t.Node1Def\"D\n\x08Node2Def\x12\x11\n\x01\x61\x18\x01 \x01(\x0b\x32\x06.Value\x12\x0b\n\x03key\x18\x02 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x03 \x03(\x05\x12\n\n\x02id\x18\x04 \x01(\t\"E\n\x08Node3Def\x12\x17\n\x02n1\x18\x01 \x01(\x0b\x32\t.Node1DefH\x00\x12\x17\n\x02n2\x18\x02 \x01(\x0b\x32\t.Node2DefH\x00\x42\x07\n\x05value\"b\n\x08Node4Def\x12!\n\x04\x61ttr\x18\x01 \x03(\x0b\x32\x13.Node4Def.AttrEntry\x1a\x33\n\tAttrEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x15\n\x05value\x18\x02 \x01(\x0b\x32\x06.Value:\x02\x38\x01\x62\x06proto3')
+  serialized_pb=_b('\n\"mars/serialize/tests/testser.proto\x1a!mars/serialize/protos/value.proto\"\xf6\x01\n\x08Node1Def\x12\t\n\x01\x61\x18\x01 \x01(\t\x12\n\n\x02\x62\x31\x18\x02 \x01(\x05\x12\n\n\x02\x62\x32\x18\x03 \x01(\x05\x12\n\n\x02\x62\x33\x18\x04 \x01(\x05\x12\n\n\x02\x62\x34\x18\x05 \x01(\x03\x12\n\n\x02\x63\x31\x18\x06 \x01(\r\x12\n\n\x02\x63\x32\x18\x07 \x01(\r\x12\n\n\x02\x63\x33\x18\x08 \x01(\r\x12\n\n\x02\x63\x34\x18\t \x01(\x04\x12\n\n\x02\x64\x31\x18\n \x01(\x02\x12\n\n\x02\x64\x32\x18\x0b \x01(\x02\x12\n\n\x02\x64\x33\x18\x0c \x01(\x01\x12\t\n\x01\x65\x18\r \x01(\x08\x12\x11\n\x01\x66\x18\x0e \x01(\x0b\x32\x06.Value\x12\x14\n\x01g\x18\x0f \x01(\x0b\x32\t.Node2Def\x12\x11\n\x01h\x18\x10 \x03(\x0b\x32\x06.Value\x12\x14\n\x01i\x18\x11 \x03(\x0b\x32\t.Node1Def\"R\n\x08Node2Def\x12\x11\n\x01\x61\x18\x01 \x01(\x0b\x32\x06.Value\x12\x0b\n\x03key\x18\x02 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x03 \x03(\x05\x12\n\n\x02id\x18\x04 \x01(\t\x12\x0c\n\x04name\x18\x05 \x01(\t\"E\n\x08Node3Def\x12\x17\n\x02n1\x18\x01 \x01(\x0b\x32\t.Node1DefH\x00\x12\x17\n\x02n2\x18\x02 \x01(\x0b\x32\t.Node2DefH\x00\x42\x07\n\x05value\"b\n\x08Node4Def\x12!\n\x04\x61ttr\x18\x01 \x03(\x0b\x32\x13.Node4Def.AttrEntry\x1a\x33\n\tAttrEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x15\n\x05value\x18\x02 \x01(\x0b\x32\x06.Value:\x02\x38\x01\x62\x06proto3')
   ,
   dependencies=[mars_dot_serialize_dot_protos_dot_value__pb2.DESCRIPTOR,])
 
@@ -42,36 +42,113 @@
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
-      name='b', full_name='Node1Def.b', index=1,
+      name='b1', full_name='Node1Def.b1', index=1,
       number=2, type=5, cpp_type=1, label=1,
       has_default_value=False, default_value=0,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
-      name='c', full_name='Node1Def.c', index=2,
-      number=3, type=11, cpp_type=10, label=1,
+      name='b2', full_name='Node1Def.b2', index=2,
+      number=3, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='b3', full_name='Node1Def.b3', index=3,
+      number=4, type=5, cpp_type=1, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='b4', full_name='Node1Def.b4', index=4,
+      number=5, type=3, cpp_type=2, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='c1', full_name='Node1Def.c1', index=5,
+      number=6, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='c2', full_name='Node1Def.c2', index=6,
+      number=7, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='c3', full_name='Node1Def.c3', index=7,
+      number=8, type=13, cpp_type=3, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='c4', full_name='Node1Def.c4', index=8,
+      number=9, type=4, cpp_type=4, label=1,
+      has_default_value=False, default_value=0,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='d1', full_name='Node1Def.d1', index=9,
+      number=10, type=2, cpp_type=6, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='d2', full_name='Node1Def.d2', index=10,
+      number=11, type=2, cpp_type=6, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='d3', full_name='Node1Def.d3', index=11,
+      number=12, type=1, cpp_type=5, label=1,
+      has_default_value=False, default_value=float(0),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='e', full_name='Node1Def.e', index=12,
+      number=13, type=8, cpp_type=7, label=1,
+      has_default_value=False, default_value=False,
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='f', full_name='Node1Def.f', index=13,
+      number=14, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
-      name='d', full_name='Node1Def.d', index=3,
-      number=4, type=11, cpp_type=10, label=1,
+      name='g', full_name='Node1Def.g', index=14,
+      number=15, type=11, cpp_type=10, label=1,
       has_default_value=False, default_value=None,
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
-      name='e', full_name='Node1Def.e', index=4,
-      number=5, type=11, cpp_type=10, label=3,
+      name='h', full_name='Node1Def.h', index=15,
+      number=16, type=11, cpp_type=10, label=3,
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR),
     _descriptor.FieldDescriptor(
-      name='f', full_name='Node1Def.f', index=5,
-      number=6, type=11, cpp_type=10, label=3,
+      name='i', full_name='Node1Def.i', index=16,
+      number=17, type=11, cpp_type=10, label=3,
       has_default_value=False, default_value=[],
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
@@ -88,8 +165,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=73,
-  serialized_end=187,
+  serialized_start=74,
+  serialized_end=320,
 )
 
 
@@ -128,6 +205,13 @@
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       serialized_options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='name', full_name='Node2Def.name', index=4,
+      number=5, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      serialized_options=None, file=DESCRIPTOR),
   ],
   extensions=[
   ],
@@ -140,8 +224,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=189,
-  serialized_end=257,
+  serialized_start=322,
+  serialized_end=404,
 )
 
 
@@ -181,8 +265,8 @@
       name='value', full_name='Node3Def.value',
       index=0, containing_type=None, fields=[]),
   ],
-  serialized_start=259,
-  serialized_end=328,
+  serialized_start=406,
+  serialized_end=475,
 )
 
 
@@ -219,8 +303,8 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=377,
-  serialized_end=428,
+  serialized_start=524,
+  serialized_end=575,
 )
 
 _NODE4DEF = _descriptor.Descriptor(
@@ -249,14 +333,14 @@
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=330,
-  serialized_end=428,
+  serialized_start=477,
+  serialized_end=575,
 )
 
-_NODE1DEF.fields_by_name['c'].message_type = mars_dot_serialize_dot_protos_dot_value__pb2._VALUE
-_NODE1DEF.fields_by_name['d'].message_type = _NODE2DEF
-_NODE1DEF.fields_by_name['e'].message_type = mars_dot_serialize_dot_protos_dot_value__pb2._VALUE
-_NODE1DEF.fields_by_name['f'].message_type = _NODE1DEF
+_NODE1DEF.fields_by_name['f'].message_type = mars_dot_serialize_dot_protos_dot_value__pb2._VALUE
+_NODE1DEF.fields_by_name['g'].message_type = _NODE2DEF
+_NODE1DEF.fields_by_name['h'].message_type = mars_dot_serialize_dot_protos_dot_value__pb2._VALUE
+_NODE1DEF.fields_by_name['i'].message_type = _NODE1DEF
 _NODE2DEF.fields_by_name['a'].message_type = mars_dot_serialize_dot_protos_dot_value__pb2._VALUE
 _NODE3DEF.fields_by_name['n1'].message_type = _NODE1DEF
 _NODE3DEF.fields_by_name['n2'].message_type = _NODE2DEF