Repository: mars-project/mars

Issue:
[BUG] Can't create session when there's an event loop running
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
When there is en event loop running, create session by `mars.core.session.new_session` will fail.

**To Reproduce**
1. Python version: py3.8
2. Mars version: master
3. numpy 1.20.1  pandas 1.2.3 scipy 1.6.1
4. Full stack of the error:
```
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-4-801ae027078e> in <module>
     29     assert get_default_session() is None
     30 
---> 31 test_sync_execute()

<ipython-input-4-801ae027078e> in test_sync_execute()
      5     start_method = os.environ.get('POOL_START_METHOD', 'forkserver') \
      6         if sys.platform != 'win32' else None
----> 7     session = new_session(n_cpu=2, subprocess_start_method=start_method,
      8                           default=True)
      9 

~/ant/Development/DevProjects/python/mars/mars/core/session.py in new_session(address, session_id, backend, default, **kwargs)
    394         if 'init_local' not in kwargs:
    395             kwargs['init_local'] = True
--> 396     session = _loop.run_until_complete(
    397         _new_session(address, session_id=session_id,
    398                      backend=backend, default=default, **kwargs))

~/.pyenv/versions/3.8.1/lib/python3.8/asyncio/base_events.py in run_until_complete(self, future)
    597         future.add_done_callback(_run_until_complete_cb)
    598         try:
--> 599             self.run_forever()
    600         except:
    601             if new_task and future.done() and not future.cancelled():

~/.pyenv/versions/3.8.1/lib/python3.8/asyncio/base_events.py in run_forever(self)
    554             raise RuntimeError('This event loop is already running')
    555         if events._get_running_loop() is not None:
--> 556             raise RuntimeError(
    557                 'Cannot run the event loop while another loop is running')
    558         self._set_coroutine_origin_tracking(self._debug)

RuntimeError: Cannot run the event loop while another loop is running
```
5. Minimized code to reproduce the error.
```
import sys, os
from mars.core.session import get_default_session, \
    new_session, execute, fetch, stop_server
def test_sync_execute():
    start_method = os.environ.get('POOL_START_METHOD', 'forkserver') \
        if sys.platform != 'win32' else None
    session = new_session(n_cpu=2, subprocess_start_method=start_method,
                          default=True)

    with session:
        raw = np.random.RandomState(0).rand(10, 5)
        a = mt.tensor(raw, chunk_size=5).sum(axis=1)
        b = a.execute(show_progress=False)
        assert b is a
        result = a.fetch()
        np.testing.assert_array_equal(result, raw.sum(axis=1))

        c = b + 1
        c.execute(show_progress=False)
        result = c.fetch()
        np.testing.assert_array_equal(result, raw.sum(axis=1) + 1)

        c = mt.tensor(raw, chunk_size=5).sum()
        d = session.execute(c)
        assert d is c
        assert abs(session.fetch(d) - raw.sum()) < 0.001

    session.stop_server()
    assert get_default_session() is None

test_sync_execute()
```

**Expected behavior**
Whether there is an event loop running or not, creating session should always secceed.

**Additional context**
Running pandas in notebook is very important for engineers, mars should support too.



Discussion:
An potential solution would be using `nest_asyncio` library.
After running 
```python
import nest_asyncio
nest_asyncio.apply()
```
The `test_sync_execute` will succeed in jupyter notebook.
The `nest_asyncio` library also mentioned this:

![image](https://user-images.githubusercontent.com/12445254/116022415-bb180080-a67c-11eb-9b80-ef347db6f101.png)


Resolving patch:
diff --git a/mars/__init__.py b/mars/__init__.py
--- a/mars/__init__.py
+++ b/mars/__init__.py
@@ -16,3 +16,8 @@
 
 from ._version import __version__
 from .config import options
+
+from .core.session import new_session, execute, fetch, stop_server
+# register
+from .deploy.oscar.session import Session
+del Session
diff --git a/mars/core/session.py b/mars/core/session.py
--- a/mars/core/session.py
+++ b/mars/core/session.py
@@ -13,11 +13,12 @@
 # limitations under the License.
 
 import asyncio
+import concurrent.futures
 import threading
 import uuid
 import warnings
 from abc import ABC, abstractmethod
-from typing import Dict, List, Type, Tuple, Union
+from typing import Callable, Dict, List, Type, Tuple, Union
 
 from ..config import options
 from ..core import TileableGraph, enter_mode
@@ -27,6 +28,37 @@
 
 _loop = asyncio.new_event_loop()
 _get_session_lock = asyncio.Lock(loop=_loop)
+_pool = concurrent.futures.ThreadPoolExecutor(1)
+
+
+def _wrap_in_thread(func: Callable):
+    """
+    Function is those wrapping async function,
+    when there is a running event loop,
+    error will raise (see GH#2108),
+    so we wrap them to run in a thread.
+    """
+
+    def sync_default_session(sess: "AbstractSession"):
+        if sess:
+            sess.as_default()
+        else:
+            AbstractSession.reset_default()
+
+    def inner(*args, **kwargs):
+        default_session = get_default_session()
+
+        def run_in_thread():
+            # set default session in this thread
+            sync_default_session(default_session)
+            return func(*args, **kwargs), get_default_session()
+
+        fut = _pool.submit(run_in_thread)
+        result, default_session_in_thread = fut.result()
+        sync_default_session(default_session_in_thread)
+
+        return result
+    return inner
 
 
 class ExecutionInfo(ABC):
@@ -181,6 +213,17 @@ async def fetch(self, *tileables, **kwargs) -> list:
         data
         """
 
+    def decref(self, *tileables):
+        """
+        Decref tileables.
+
+        Parameters
+        ----------
+        tileables
+            Tileables.
+        """
+        # TODO(qinxuye): implement this function when lifecycle service ready.
+
     async def stop_server(self):
         """
         Stop server.
@@ -318,6 +361,7 @@ def _attach_session(fut: asyncio.Future):
         return execution_info
 
 
+@_wrap_in_thread
 def execute(tileable: TileableType,
             *tileables: Tuple[TileableType],
             session: AbstractSession = None,
@@ -349,6 +393,7 @@ async def _fetch(tileable: TileableType,
     return data[0] if len(tileables) == 0 else data
 
 
+@_wrap_in_thread
 def fetch(*tileables, session: AbstractSession = None):
     return _loop.run_until_complete(
         _fetch(*tileables, session=session))
@@ -367,10 +412,12 @@ def execute(self,
     def fetch(self, *tileables):
         return fetch(*tileables, session=self._session)
 
+    @_wrap_in_thread
     def destroy(self):
         return _loop.run_until_complete(
             self._session.destroy())
 
+    @_wrap_in_thread
     def stop_server(self):
         return _loop.run_until_complete(
             self._session.stop_server())
@@ -384,6 +431,7 @@ def __exit__(self, *_):
             AbstractSession.reset_default()
 
 
+@_wrap_in_thread
 def new_session(address: str = None,
                 session_id: str = None,
                 backend: str = 'oscar',
diff --git a/mars/storage/__init__.py b/mars/storage/__init__.py
--- a/mars/storage/__init__.py
+++ b/mars/storage/__init__.py
@@ -16,7 +16,15 @@
 
 from .base import StorageLevel, get_storage_backend
 from .filesystem import FileSystemStorage
-from .plasma import PlasmaStorage
 from .ray import RayStorage
 from .shared_memory import SharedMemoryStorage
-from .vineyard import VineyardStorage
+try:
+    # require vineyard, pyarrow
+    from .vineyard import VineyardStorage
+except ImportError:
+    pass
+try:
+    # require pyarrow
+    from .plasma import PlasmaStorage
+except ImportError:
+    pass


Test patch: