Repository: smarie/mkdocs-gallery

Issue:
asyncio functionality is mostly unusable after async handling was applied
Reporting on my own failures here. In #90 we added automatic `async` handling and did so by using `asyncio.run`:

https://github.com/smarie/mkdocs-gallery/blob/8a16972a3d1dd9728a5406bb83f4fe631b9ad4c6/src/mkdocs_gallery/gen_single.py#L782-L783

The problem is, and I didn't know this before, that `asyncio.run` is meant to be only called once. This is what the [documentation](https://docs.python.org/3/library/asyncio-runner.html#asyncio.run) says about this:

> This function should be used as a main entry point for asyncio programs, and should ideally only be called once.

However, we call it for every code block that needs `async` handling. That in itself is not an issue as the gallery here builds just fine. Unfortunately, it breaks down when other libraries that are part of the documentation building also need to use some `async` functionality.

For example, [`panel`](https://panel.holoviz.org/) creates an `asyncio.Lock` on import. When building some `async` galleries before the import, we effectively have the following situation

```python
import asyncio
import asyncio.events

async def foo():
    pass

asyncio.run(foo())

loop = asyncio.events.get_event_loop()
```

And the `get_event_loop` fails with

```
RuntimeError: There is no current event loop in thread 'MainThread'.
```

because we have previously used `asyncio.run`.

My absolute crude workaround for now is the following snippet that has to be placed before any further `asyncio` functionality is used:

```python
import asyncio

asyncio.get_event_loop_policy()._local._set_called = False
```

Please note that this works for my case, but I have no idea if this screws up other cases as we are messing with low-level private attributes here.

I'll look on how to restructure the async handling to avoid using `asyncio.run` to avoid this mess all together.

Discussion:

Patch:
diff --git a/docs/changelog.md b/docs/changelog.md
index dfc6317..9b0b636 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -1,5 +1,9 @@
 # Changelog
 
+### 0.10.3 - Bugfixes
+
+-  Don't use `asyncio.run` for async handling. Fixes [#93](https://github.com/smarie/mkdocs-gallery/issues/93).
+
 ### 0.10.2 - Bugfixes
 
 - **SECURITY** removed insecure polyfill extra javascript from example. Fixes [#99](https://github.com/smarie/mkdocs-gallery/issues/99).
diff --git a/docs/examples/plot_12_async.py b/examples/plot_12_async.py
similarity index 87%
rename from docs/examples/plot_12_async.py
rename to examples/plot_12_async.py
index 4aa6f3e..1ff79f3 100644
--- a/docs/examples/plot_12_async.py
+++ b/examples/plot_12_async.py
@@ -31,16 +31,9 @@ f"I waited for {await afn():.1f} seconds!"
 
 # %%
 # Without any handling, the snippet above would trigger a `SyntaxError`, since we are using `await` outside of an
-# asynchronous context. With the handling, it works just fine.
+# asynchronous context. With the background handling, it works just fine.
 #
-# The background handling will only be applied if it is actually needed. Meaning, you can still run your asynchronous
-# code manually if required.
-
-asyncio.run(afn())
-
-
-# %%
-# Apart from `await` all other asynchronous syntax is supported as well.
+# Apart from `await` that we used above, all other asynchronous syntax is supported as well.
 #
 # ## Asynchronous Generators
 
diff --git a/src/mkdocs_gallery/gen_single.py b/src/mkdocs_gallery/gen_single.py
index 416b867..3e5a03d 100644
--- a/src/mkdocs_gallery/gen_single.py
+++ b/src/mkdocs_gallery/gen_single.py
@@ -41,7 +41,7 @@ from .gen_data_model import GalleryBase, GalleryScript, GalleryScriptResults
 from .notebook import jupyter_notebook, save_notebook
 from .py_source_parser import remove_config_comments, split_code_and_text_blocks
 from .scrapers import ImageNotFoundError, _find_image_ext, clean_modules, save_figures
-from .utils import _new_file, _replace_by_new_if_needed, optipng, rescale_image
+from .utils import _new_file, _replace_by_new_if_needed, optipng, rescale_image, run_async
 
 logger = mkdocs_compatibility.getLogger("mkdocs-gallery")
 
@@ -780,8 +780,7 @@ def _apply_async_handling(code_ast, *, compiler_flags):
                 async def __async_wrapper__():
                     # original AST goes here
                     return locals()
-                import asyncio as __asyncio__
-                __async_wrapper_locals__ = __asyncio__.run(__async_wrapper__())
+                __async_wrapper_locals__ = __run_async__(__async_wrapper__())
                 __async_wrapper_result__ = __async_wrapper_locals__.pop("__async_wrapper_result__", None)
                 globals().update(__async_wrapper_locals__)
                 __async_wrapper_result__
@@ -950,6 +949,7 @@ def parse_and_execute(script: GalleryScript, script_blocks):
             # Don't ever support __file__: Issues #166 #212
             # Don't let them use input()
             "input": _check_input,
+            "__run_async__": run_async,
         }
     )
     script.run_vars.example_globals = example_globals
@@ -960,7 +960,7 @@ def parse_and_execute(script: GalleryScript, script_blocks):
     # Remember the original argv so that we can put them back after run
     argv_orig = sys.argv[:]
 
-    # Remember the original sys.path so that we can reset it after run
+    # Remember the original sys.path so that we can reset it after run
     sys_path_orig = deepcopy(sys.path)
 
     # Python file is the original one (not the copy for download)
diff --git a/src/mkdocs_gallery/utils.py b/src/mkdocs_gallery/utils.py
index 58d835f..d5b3496 100644
--- a/src/mkdocs_gallery/utils.py
+++ b/src/mkdocs_gallery/utils.py
@@ -12,6 +12,7 @@ Miscellaneous utilities.
 
 from __future__ import absolute_import, division, print_function
 
+import asyncio
 import hashlib
 import os
 import re
@@ -376,3 +377,15 @@ def is_relative_to(parentpath: Path, subpath: Path) -> bool:
 
     except ValueError:
         return False
+
+
+def run_async(coro):
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError:
+        loop = asyncio.new_event_loop()
+
+    try:
+        return loop.run_until_complete(coro)
+    finally:
+        loop.close()


Test patch:
diff --git a/tests/test_gen_single.py b/tests/test_gen_single.py
index 2702bf8..23dda72 100644
--- a/tests/test_gen_single.py
+++ b/tests/test_gen_single.py
@@ -1,4 +1,5 @@
 import ast
+import asyncio
 import codeop
 import sys
 from textwrap import dedent
@@ -6,10 +7,11 @@ from textwrap import dedent
 import pytest
 
 from mkdocs_gallery.gen_single import _needs_async_handling, _parse_code
+from mkdocs_gallery.utils import run_async
 
 SRC_FILE = __file__
 COMPILER = codeop.Compile()
-COMPILER_FLAGS = codeop.Compile().flags
+COMPILER_FLAGS = COMPILER.flags
 
 
 needs_ast_unparse = pytest.mark.skipif(
@@ -17,29 +19,29 @@ needs_ast_unparse = pytest.mark.skipif(
 )
 
 
+def make_globals():
+    return {"__run_async__": run_async}
+
+
 def test_non_async_syntax_error():
     with pytest.raises(SyntaxError, match="unexpected indent"):
         _parse_code("foo = None\n bar = None", src_file=SRC_FILE, compiler_flags=COMPILER_FLAGS)
 
 
+@needs_ast_unparse
+def test_no_async_roundtrip():
+    code = "None"
+    assert not _needs_async_handling(code, src_file=SRC_FILE, compiler_flags=COMPILER_FLAGS)
+
+    code_unparsed = ast.unparse(_parse_code(code, src_file=SRC_FILE, compiler_flags=COMPILER_FLAGS))
+
+    assert code_unparsed == code
+
+
 @needs_ast_unparse
 @pytest.mark.parametrize(
-    ("code", "needs"),
+    "code",
     [
-        pytest.param("None", False, id="no_async"),
-        pytest.param(
-            dedent(
-                """
-                async def afn():
-                    return True
-
-                import asyncio
-                assert asyncio.run(afn())
-                """
-            ),
-            False,
-            id="asyncio_run",
-        ),
         pytest.param(
             dedent(
                 """
@@ -49,7 +51,6 @@ def test_non_async_syntax_error():
                 assert await afn()
                 """
             ),
-            True,
             id="await",
         ),
         pytest.param(
@@ -62,7 +63,6 @@ def test_non_async_syntax_error():
                     assert item
                 """
             ),
-            True,
             id="async_for",
         ),
         pytest.param(
@@ -74,7 +74,6 @@ def test_non_async_syntax_error():
                 assert [item async for item in agen()] == [True]
                 """
             ),
-            True,
             id="async_comprehension",
         ),
         pytest.param(
@@ -90,25 +89,23 @@ def test_non_async_syntax_error():
                     assert ctx
                 """
             ),
-            True,
             id="async_context_manager",
         ),
     ],
 )
-def test_async_handling(code, needs):
-    assert _needs_async_handling(code, src_file=SRC_FILE, compiler_flags=COMPILER_FLAGS) is needs
+def test_async_handling(code):
+    assert _needs_async_handling(code, src_file=SRC_FILE, compiler_flags=COMPILER_FLAGS)
 
     # Since AST objects are quite involved to compare, we unparse again and check that nothing has changed. Note that
     # since we are dealing with AST and not CST here, all whitespace is eliminated in the process and this needs to be
     # reflected in the input as well.
     code_stripped = "\n".join(line for line in code.splitlines() if line)
     code_unparsed = ast.unparse(_parse_code(code, src_file=SRC_FILE, compiler_flags=COMPILER_FLAGS))
-    assert (code_unparsed == code_stripped) ^ needs
+    assert code_unparsed != code_stripped
 
-    if needs:
-        assert not _needs_async_handling(code_unparsed, src_file=SRC_FILE, compiler_flags=COMPILER_FLAGS)
+    assert not _needs_async_handling(code_unparsed, src_file=SRC_FILE, compiler_flags=COMPILER_FLAGS)
 
-    exec(COMPILER(code_unparsed, SRC_FILE, "exec"), {})
+    exec(COMPILER(code_unparsed, SRC_FILE, "exec"), make_globals())
 
 
 @needs_ast_unparse
@@ -128,10 +125,10 @@ def test_async_handling_locals():
     )
     code_unparsed = ast.unparse(_parse_code(code, src_file=SRC_FILE, compiler_flags=COMPILER_FLAGS))
 
-    locals = {}
-    exec(COMPILER(code_unparsed, SRC_FILE, "exec"), locals)
+    globals_ = make_globals()
+    exec(COMPILER(code_unparsed, SRC_FILE, "exec"), globals_)
 
-    assert "sentinel" in locals and locals["sentinel"] == sentinel
+    assert "sentinel" in globals_ and globals_["sentinel"] == sentinel
 
 
 @needs_ast_unparse
@@ -153,6 +150,24 @@ def test_async_handling_last_expression():
     last = code_unparsed_ast.body[-1]
     assert isinstance(last, ast.Expr)
 
-    locals = {}
-    exec(COMPILER(code_unparsed, SRC_FILE, "exec"), locals)
-    assert eval(ast.unparse(last.value), locals)
+    globals_ = make_globals()
+    exec(COMPILER(code_unparsed, SRC_FILE, "exec"), globals_)
+    assert eval(ast.unparse(last.value), globals_)
+
+
+@needs_ast_unparse
+def test_get_event_loop_after_async_handling():
+    # Non-regression test for https://github.com/smarie/mkdocs-gallery/issues/93
+    code = dedent(
+        """
+        async def afn():
+            return True
+        
+        assert await afn()
+        """
+    )
+
+    code_unparsed = ast.unparse(_parse_code(code, src_file=SRC_FILE, compiler_flags=COMPILER_FLAGS))
+    exec(COMPILER(code_unparsed, SRC_FILE, "exec"), make_globals())
+
+    asyncio.events.get_event_loop()