Repository: mars-project/mars

Issue:
[BUG] Remote function can not return None for distributed
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**

Remote function can not return None for distributed.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version
2. The version of Mars you use
3. Versions of crucial packages, such as numpy, scipy and protobuf
4. Full stack of the error.
5. Minimized code to reproduce the error.

```
In [1]: import mars.remote as mr                                                

In [2]: from mars.deploy.local import new_cluster                               

In [3]: c = new_cluster()                                                       
WARNING: Logging before InitGoogleLogging() is written to STDERR
I0604 14:30:01.529353 132840896 store.cc:1149] Allowing the Plasma store to use up to 3.43597GB of memory.
I0604 14:30:01.534931 132840896 store.cc:1176] Starting object store with directory /tmp and huge page support disabled

In [4]: def f(x): 
   ...:     return 
   ...:                                                                         

In [5]: mr.spawn(f, 3).execute()                                                
Unhandled exception in promise
Traceback (most recent call last):
  File "/Users/qinxuye/Workspace/mars/mars/worker/storage/client.py", line 299, in copy_to
    return promise.finished(*build_exc_info(KeyError, err_msg), _accept=False)
KeyError: 'Data key (6f8edc26-0267-44a0-a64d-e939bac89fbe, d9f9097c4ccb8ca4716366cf6680fff2) not exist, proc_id=9'
Unhandled exception in promise
Traceback (most recent call last):
  File "/Users/qinxuye/Workspace/mars/mars/worker/storage/client.py", line 299, in copy_to
    return promise.finished(*build_exc_info(KeyError, err_msg), _accept=False)
KeyError: 'Data key (6f8edc26-0267-44a0-a64d-e939bac89fbe, d9f9097c4ccb8ca4716366cf6680fff2) not exist, proc_id=9'
Unexpected error occurred in executing graph e0879bcab1971118787b182578bff091
Traceback (most recent call last):
  File "/Users/qinxuye/Workspace/mars/mars/worker/storage/client.py", line 299, in copy_to
    return promise.finished(*build_exc_info(KeyError, err_msg), _accept=False)
KeyError: 'Data key (6f8edc26-0267-44a0-a64d-e939bac89fbe, d9f9097c4ccb8ca4716366cf6680fff2) not exist, proc_id=9'
Attempt 1: Unexpected error KeyError occurred in executing operand e0879bcab1971118787b182578bff091 in 0.0.0.0:37928
Traceback (most recent call last):
  File "/Users/qinxuye/Workspace/mars/mars/worker/storage/client.py", line 299, in copy_to
    return promise.finished(*build_exc_info(KeyError, err_msg), _accept=False)
KeyError: 'Data key (6f8edc26-0267-44a0-a64d-e939bac89fbe, d9f9097c4ccb8ca4716366cf6680fff2) not exist, proc_id=9'
Unhandled exception in promise
Traceback (most recent call last):
  File "/Users/qinxuye/Workspace/mars/mars/worker/storage/client.py", line 299, in copy_to
    return promise.finished(*build_exc_info(KeyError, err_msg), _accept=False)
KeyError: 'Data key (6f8edc26-0267-44a0-a64d-e939bac89fbe, d9f9097c4ccb8ca4716366cf6680fff2) not exist, proc_id=9'
Unhandled exception in promise
Traceback (most recent call last):
  File "/Users/qinxuye/Workspace/mars/mars/worker/storage/client.py", line 299, in copy_to
    return promise.finished(*build_exc_info(KeyError, err_msg), _accept=False)
KeyError: 'Data key (6f8edc26-0267-44a0-a64d-e939bac89fbe, d9f9097c4ccb8ca4716366cf6680fff2) not exist, proc_id=9'
Unexpected error occurred in executing graph e0879bcab1971118787b182578bff091
Traceback (most recent call last):
  File "/Users/qinxuye/Workspace/mars/mars/worker/storage/client.py", line 299, in copy_to
    return promise.finished(*build_exc_info(KeyError, err_msg), _accept=False)
KeyError: 'Data key (6f8edc26-0267-44a0-a64d-e939bac89fbe, d9f9097c4ccb8ca4716366cf6680fff2) not exist, proc_id=9'
Attempt 2: Unexpected error KeyError occurred in executing operand e0879bcab1971118787b182578bff091 in 0.0.0.0:37928
Traceback (most recent call last):
  File "/Users/qinxuye/Workspace/mars/mars/worker/storage/client.py", line 299, in copy_to
    return promise.finished(*build_exc_info(KeyError, err_msg), _accept=False)
KeyError: 'Data key (6f8edc26-0267-44a0-a64d-e939bac89fbe, d9f9097c4ccb8ca4716366cf6680fff2) not exist, proc_id=9'

```


Discussion:

Resolving patch:
diff --git a/mars/web/server.py b/mars/web/server.py
--- a/mars/web/server.py
+++ b/mars/web/server.py
@@ -244,7 +244,10 @@ def start(self, event=None, block=False):
             if event:
                 event.set()
 
-            self._server.io_loop.start()
+            try:
+                self._server.io_loop.start()
+            except KeyboardInterrupt:
+                pass
 
     def stop(self):
         if self._server is not None:
diff --git a/mars/worker/service.py b/mars/worker/service.py
--- a/mars/worker/service.py
+++ b/mars/worker/service.py
@@ -300,26 +300,28 @@ def handle_process_down(self, pool, proc_indices):
 
     def stop(self):
         try:
+            destroy_futures = []
             for actor in (self._cpu_calc_actors + self._sender_actors + self._inproc_holder_actors
                           + self._inproc_io_runner_actors + self._cuda_calc_actors
                           + self._cuda_holder_actors + self._receiver_actors + self._spill_actors
                           + self._process_helper_actors):
                 if actor and actor.ctx:
-                    actor.destroy(wait=False)
+                    destroy_futures.append(actor.destroy(wait=False))
 
             if self._result_sender_ref:
-                self._result_sender_ref.destroy(wait=False)
+                destroy_futures.append(self._result_sender_ref.destroy(wait=False))
             if self._status_ref:
-                self._status_ref.destroy(wait=False)
+                destroy_futures.append(self._status_ref.destroy(wait=False))
             if self._shared_holder_ref:
-                self._shared_holder_ref.destroy(wait=False)
+                destroy_futures.append(self._shared_holder_ref.destroy(wait=False))
             if self._storage_manager_ref:
-                self._storage_manager_ref.destroy(wait=False)
+                destroy_futures.append(self._storage_manager_ref.destroy(wait=False))
             if self._events_ref:
-                self._events_ref.destroy(wait=False)
+                destroy_futures.append(self._events_ref.destroy(wait=False))
             if self._dispatch_ref:
-                self._dispatch_ref.destroy(wait=False)
+                destroy_futures.append(self._dispatch_ref.destroy(wait=False))
             if self._execution_ref:
-                self._execution_ref.destroy(wait=False)
+                destroy_futures.append(self._execution_ref.destroy(wait=False))
+            [f.result(5) for f in destroy_futures]
         finally:
             self._plasma_store.__exit__(None, None, None)
diff --git a/mars/worker/storage/client.py b/mars/worker/storage/client.py
--- a/mars/worker/storage/client.py
+++ b/mars/worker/storage/client.py
@@ -294,7 +294,7 @@ def copy_to(self, session_id, data_keys, device_order, ensure=True, pin_token=No
         device_total_size = defaultdict(lambda: 0)
         lift_reqs = defaultdict(list)
         for k, devices, size in zip(data_keys, existing_devs, data_sizes):
-            if not devices or not size:
+            if not devices or size is None:
                 err_msg = 'Data key (%s, %s) not exist, proc_id=%s' % (session_id, k, self.proc_id)
                 return promise.finished(*build_exc_info(KeyError, err_msg), _accept=False)
 


Test patch:
diff --git a/mars/scheduler/tests/integrated/test_normal_execution.py b/mars/scheduler/tests/integrated/test_normal_execution.py
--- a/mars/scheduler/tests/integrated/test_normal_execution.py
+++ b/mars/scheduler/tests/integrated/test_normal_execution.py
@@ -30,6 +30,7 @@
 from mars.scheduler.resource import ResourceActor
 from mars.scheduler.tests.integrated.base import SchedulerIntegratedTest
 from mars.scheduler.tests.integrated.no_prepare_op import NoPrepareOperand
+from mars.remote import spawn
 from mars.utils import build_tileable_graph
 from mars.serialize.dataserializer import loads
 from mars.tests.core import EtcdProcessHelper, require_cupy, require_cudf
@@ -413,3 +414,55 @@ def testOperandsWithoutPrepareInputs(self):
 
         state = self.wait_for_termination(actor_client, session_ref, graph_key)
         self.assertEqual(state, GraphState.SUCCEEDED)
+
+    def testRemoteWithoutEtcd(self):
+        self.start_processes(etcd=False, modules=['mars.scheduler.tests.integrated.no_prepare_op'])
+
+        session_id = uuid.uuid1()
+        actor_client = new_client()
+
+        session_ref = actor_client.actor_ref(self.session_manager_ref.create_session(session_id))
+
+        rs = np.random.RandomState(0)
+        raw1 = rs.rand(10, 10)
+        raw2 = rs.rand(10, 10)
+
+        def f_none(_x):
+            return None
+
+        r_none = spawn(f_none, raw1)
+
+        graph = r_none.build_graph()
+        targets = [r_none.key]
+        graph_key = uuid.uuid1()
+        session_ref.submit_tileable_graph(json.dumps(graph.to_json()),
+                                          graph_key, target_tileables=targets)
+
+        state = self.wait_for_termination(actor_client, session_ref, graph_key)
+        self.assertEqual(state, GraphState.SUCCEEDED)
+
+        result = session_ref.fetch_result(graph_key, r_none.key)
+        self.assertIsNone(loads(result))
+
+        def f1(x):
+            return x + 1
+
+        def f2(x, y, z=None):
+            return x * y * (z[0] + z[1])
+
+        r1 = spawn(f1, raw1)
+        r2 = spawn(f1, raw2)
+        r3 = spawn(f2, (r1, r2), {'z': [r1, r2]})
+
+        graph = r3.build_graph()
+        targets = [r3.key]
+        graph_key = uuid.uuid1()
+        session_ref.submit_tileable_graph(json.dumps(graph.to_json()),
+                                          graph_key, target_tileables=targets)
+
+        state = self.wait_for_termination(actor_client, session_ref, graph_key)
+        self.assertEqual(state, GraphState.SUCCEEDED)
+
+        result = session_ref.fetch_result(graph_key, r3.key)
+        expected = (raw1 + 1) * (raw2 + 1) * (raw1 + 1 + raw2 + 1)
+        assert_allclose(loads(result), expected)