Repository: mars-project/mars

Issue:
[BUG] Fail to start local cluster within IPython
When starting Mars local cluster within IPython with code below

```python
from mars.deploy.local import new_cluster                                                              
cl = new_cluster(web=True)
```

Error occurred:

```
Traceback (most recent call last):
  File "/Users/wenjun.swj/miniconda3/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
    self.run()
  File "/Users/wenjun.swj/miniconda3/lib/python3.7/multiprocessing/process.py", line 99, in run
    self._target(*self._args, **self._kwargs)
  File "mars/lib/gipc.pyx", line 419, in mars.lib.gipc._child
    target(*args, **kwargs)
  File "/Users/wenjun.swj/Code/mars/mars/deploy/local/core.py", line 211, in _start_web
    web.start(event=event, block=True)
  File "/Users/wenjun.swj/Code/mars/mars/web/server.py", line 234, in start
    self._try_start_web_server()
  File "/Users/wenjun.swj/Code/mars/mars/web/server.py", line 219, in _try_start_web_server
    http_server_kwargs={'max_buffer_size': 2 ** 32},
  File "/Users/wenjun.swj/miniconda3/lib/python3.7/site-packages/bokeh/server/server.py", line 400, in __init__
    http_server.add_sockets(sockets)
  File "/Users/wenjun.swj/miniconda3/lib/python3.7/site-packages/tornado/tcpserver.py", line 166, in add_sockets
    sock, self._handle_connection
  File "/Users/wenjun.swj/miniconda3/lib/python3.7/site-packages/tornado/netutil.py", line 279, in add_accept_handler
    io_loop.add_handler(sock, accept_handler, IOLoop.READ)
  File "/Users/wenjun.swj/miniconda3/lib/python3.7/site-packages/tornado/platform/asyncio.py", line 100, in add_handler
    self.asyncio_loop.add_reader(fd, self._handle_events, fd, IOLoop.READ)
  File "/Users/wenjun.swj/miniconda3/lib/python3.7/asyncio/selector_events.py", line 329, in add_reader
    return self._add_reader(fd, callback, *args)
  File "/Users/wenjun.swj/miniconda3/lib/python3.7/asyncio/selector_events.py", line 259, in _add_reader
    (handle, None))
  File "/Users/wenjun.swj/miniconda3/lib/python3.7/selectors.py", line 520, in register
    kev = select.kevent(key.fd, select.KQ_FILTER_READ,
AttributeError: module 'select' has no attribute 'kevent'
```


Discussion:

Resolving patch:
diff --git a/mars/deploy/local/core.py b/mars/deploy/local/core.py
--- a/mars/deploy/local/core.py
+++ b/mars/deploy/local/core.py
@@ -24,7 +24,6 @@
 from ...actors import create_actor_pool
 from ...cluster_info import StaticSchedulerDiscoverer
 from ...config import options
-from ...lib import gipc
 from ...resource import cpu_count
 from ...scheduler.service import SchedulerService
 from ...session import new_session
@@ -32,6 +31,8 @@
 from ...worker.service import WorkerService
 from .distributor import gen_distributor
 
+_mp_spawn_context = multiprocessing.get_context('spawn')
+
 _local_cluster_clients = dict()
 atexit.register(lambda: [v.stop() for v in list(_local_cluster_clients.values())])
 
@@ -169,6 +170,10 @@ def gen_endpoint(address):
 
 
 def _start_cluster(endpoint, event, n_process=None, shared_memory=None, **kw):
+    modules = kw.pop('modules', None) or []
+    for m in modules:
+        __import__(m, globals(), locals(), [])
+
     cluster = LocalDistributedCluster(endpoint, n_process=n_process,
                                       shared_memory=shared_memory, **kw)
     cluster.start_service()
@@ -180,12 +185,14 @@ def _start_cluster(endpoint, event, n_process=None, shared_memory=None, **kw):
 
 
 def _start_cluster_process(endpoint, n_process, shared_memory, **kw):
-    event = multiprocessing.Event()
+    event = _mp_spawn_context.Event()
 
     kw = kw.copy()
     kw['n_process'] = n_process
     kw['shared_memory'] = shared_memory or '20%'
-    process = gipc.start_process(_start_cluster, args=(endpoint, event), kwargs=kw)
+    process = _mp_spawn_context.Process(
+        target=_start_cluster, args=(endpoint, event), kwargs=kw)
+    process.start()
 
     while True:
         event.wait(5)
@@ -214,10 +221,12 @@ def _start_web(scheduler_address, ui_port, event):
 
 
 def _start_web_process(scheduler_endpoint, web_endpoint):
-    web_event = multiprocessing.Event()
     ui_port = int(web_endpoint.rsplit(':', 1)[1])
-    web_process = gipc.start_process(
-        _start_web, args=(scheduler_endpoint, ui_port, web_event), daemon=True)
+
+    web_event = _mp_spawn_context.Event()
+    web_process = _mp_spawn_context.Process(
+        target=_start_web, args=(scheduler_endpoint, ui_port, web_event), daemon=True)
+    web_process.start()
 
     while True:
         web_event.wait(5)
diff --git a/mars/utils.py b/mars/utils.py
--- a/mars/utils.py
+++ b/mars/utils.py
@@ -277,8 +277,10 @@ def deserialize_graph(ser_graph, graph_cls=None):
         g.ParseFromString(ser_graph_bin)
         return graph_cls.from_pb(g)
     except (zlib.error, DecodeError):
-        json_obj = json.loads(to_str(ser_graph))
-        return graph_cls.from_json(json_obj)
+        pass
+
+    json_obj = json.loads(to_str(ser_graph))
+    return graph_cls.from_json(json_obj)
 
 
 def calc_data_size(dt):
diff --git a/mars/web/__main__.py b/mars/web/__main__.py
--- a/mars/web/__main__.py
+++ b/mars/web/__main__.py
@@ -15,7 +15,7 @@
 # limitations under the License.
 
 import gevent.monkey
-gevent.monkey.patch_all(thread=False)
+gevent.monkey.patch_all(thread=False, select=False)
 
 import argparse  # noqa: E402
 import logging   # noqa: E402


Test patch:
diff --git a/mars/deploy/local/tests/test_cluster.py b/mars/deploy/local/tests/test_cluster.py
--- a/mars/deploy/local/tests/test_cluster.py
+++ b/mars/deploy/local/tests/test_cluster.py
@@ -66,6 +66,34 @@ def __init__(self, f=None, **kw):
         super().__init__(_f=f, **kw)
 
 
+class FakeOp(TensorAbs):
+    _op_type_ = 9870102948
+
+    _multiplier = Int64Field('multiplier')
+
+    @classmethod
+    def tile(cls, op):
+        context = get_context()
+
+        if context.running_mode != RunningMode.local_cluster:
+            raise AssertionError
+
+        inp_chunk = op.inputs[0].chunks[0]
+        inp_size = context.get_chunk_metas([inp_chunk.key])[0].chunk_size
+        chunk_op = op.copy().reset_key()
+        chunk_op._multiplier = inp_size
+        chunk = chunk_op.new_chunk([inp_chunk], shape=inp_chunk.shape)
+
+        new_op = op.copy()
+        return new_op.new_tensors(op.inputs, shape=op.outputs[0].shape,
+                                  order=op.outputs[0].order, nsplits=op.inputs[0].nsplits,
+                                  chunks=[chunk])
+
+    @classmethod
+    def execute(cls, ctx, op):
+        ctx[op.outputs[0].key] = ctx[op.inputs[0].key] * op._multiplier
+
+
 @unittest.skipIf(sys.platform == 'win32', 'does not run in windows')
 @mock.patch('webbrowser.open_new_tab', new=lambda *_, **__: True)
 class Test(unittest.TestCase):
@@ -879,34 +907,8 @@ def testCudaCluster(self, *_):
             pd.testing.assert_frame_equal(pdf, result)
 
     def testTileContextInLocalCluster(self):
-        class FakeOp(TensorAbs):
-            _op_type_ = 9870102948
-
-            _multiplier = Int64Field('multiplier')
-
-            @classmethod
-            def tile(cls, op):
-                context = get_context()
-
-                self.assertEqual(context.running_mode, RunningMode.local_cluster)
-
-                inp_chunk = op.inputs[0].chunks[0]
-                inp_size = context.get_chunk_metas([inp_chunk.key])[0].chunk_size
-                chunk_op = op.copy().reset_key()
-                chunk_op._multiplier = inp_size
-                chunk = chunk_op.new_chunk([inp_chunk], shape=inp_chunk.shape)
-
-                new_op = op.copy()
-                return new_op.new_tensors(op.inputs, shape=op.outputs[0].shape,
-                                          order=op.outputs[0].order, nsplits=op.inputs[0].nsplits,
-                                          chunks=[chunk])
-
-            @classmethod
-            def execute(cls, ctx, op):
-                ctx[op.outputs[0].key] = ctx[op.inputs[0].key] * op._multiplier
-
         with new_cluster(scheduler_n_process=2, worker_n_process=2,
-                         shared_memory='20M', web=True) as cluster:
+                         shared_memory='20M', modules=[__name__], web=True) as cluster:
             session = cluster.session
 
             raw = np.random.rand(10, 20)