Repository: tornadoweb/tornado

Issue:
Issue #2229: allow GCing of suspended coroutines
A suspended coroutine should be GCed if the underlying loop is closed and no other outside reference exists to it.  However, a suspended coroutine with a refcycle would be kept alive by the `_futures_to_runners` mapping. Instead use a private attribute on the future.
IOLoop.clear_instance() and IOLoop.clear_current() broken
This is on git master:
```
>>> from tornado.ioloop import IOLoop
>>> IOLoop.clear_current()
Traceback (most recent call last):
  File "<ipython-input-4-59b136f0bd53>", line 1, in <module>
    IOLoop.clear_current()
  File "/home/antoine/tornado/tornado/ioloop.py", line 322, in clear_current
    old = IOLoop._current.instance
AttributeError: '_thread._local' object has no attribute 'instance'

>>> IOLoop.clear_instance()
Traceback (most recent call last):
  File "<ipython-input-5-cf608ef320e5>", line 1, in <module>
    IOLoop.clear_instance()
  File "/home/antoine/tornado/tornado/ioloop.py", line 263, in clear_instance
    IOLoop.clear_current()
  File "/home/antoine/tornado/tornado/ioloop.py", line 322, in clear_current
    old = IOLoop._current.instance
AttributeError: '_thread._local' object has no attribute 'instance'

```

Compare with v4.5.2:
```
>>> from tornado.ioloop import IOLoop
>>> IOLoop.clear_current()
>>> IOLoop.clear_instance()
```


Discussion:
Do you think it's just bad luck (travis-ci server load) that every build configurations of https://travis-ci.org/tornadoweb/tornado/builds/323627058 took more than twice as long as usual to run?
@ploxiln I just noticed that. It seems Travis-CI is [saturated](https://github.com/travis-ci/travis-ci/issues/8991) these days. When I run the test suite locally, I don't get any runtime difference on the test suite between this PR and git master (17 seconds on both).

Even `pip install Twisted` seems to take a full minute:
https://travis-ci.org/tornadoweb/tornado/jobs/323627063#L461

Perhaps @bdarnell can restart the build at some point to see if runtimes vary. I don't have the access rights to do so.
Now we're even getting "import timed out" errors:
https://travis-ci.org/tornadoweb/tornado/jobs/323627065#L1133
OK yeah it's just travis, nevermind me :)
FTR, builds have also become awfully long on CPython.


Resolving patch:
diff --git a/tornado/gen.py b/tornado/gen.py
--- a/tornado/gen.py
+++ b/tornado/gen.py
@@ -82,7 +82,6 @@ def get(self):
 import os
 import sys
 import types
-import weakref
 
 from tornado.concurrent import (Future, is_future, chain_future, future_set_exc_info,
                                 future_add_done_callback, future_set_result_unless_cancelled)
@@ -257,26 +256,6 @@ def coroutine(func, replace_callback=True):
     return _make_coroutine_wrapper(func, replace_callback=True)
 
 
-# Ties lifetime of runners to their result futures. Github Issue #1769
-# Generators, like any object in Python, must be strong referenced
-# in order to not be cleaned up by the garbage collector. When using
-# coroutines, the Runner object is what strong-refs the inner
-# generator. However, the only item that strong-reffed the Runner
-# was the last Future that the inner generator yielded (via the
-# Future's internal done_callback list). Usually this is enough, but
-# it is also possible for this Future to not have any strong references
-# other than other objects referenced by the Runner object (usually
-# when using other callback patterns and/or weakrefs). In this
-# situation, if a garbage collection ran, a cycle would be detected and
-# Runner objects could be destroyed along with their inner generators
-# and everything in their local scope.
-# This map provides strong references to Runner objects as long as
-# their result future objects also have strong references (typically
-# from the parent coroutine's Runner). This keeps the coroutine's
-# Runner alive.
-_futures_to_runners = weakref.WeakKeyDictionary()
-
-
 def _make_coroutine_wrapper(func, replace_callback):
     """The inner workings of ``@gen.coroutine`` and ``@gen.engine``.
 
@@ -331,7 +310,16 @@ def wrapper(*args, **kwargs):
                 except Exception:
                     future_set_exc_info(future, sys.exc_info())
                 else:
-                    _futures_to_runners[future] = Runner(result, future, yielded)
+                    # Provide strong references to Runner objects as long
+                    # as their result future objects also have strong
+                    # references (typically from the parent coroutine's
+                    # Runner). This keeps the coroutine's Runner alive.
+                    # We do this by exploiting the public API
+                    # add_done_callback() instead of putting a private
+                    # attribute on the Future.
+                    # (Github issues #1769, #2229).
+                    runner = Runner(result, future, yielded)
+                    future.add_done_callback(lambda _: runner)
                 yielded = None
                 try:
                     return future
diff --git a/tornado/ioloop.py b/tornado/ioloop.py
--- a/tornado/ioloop.py
+++ b/tornado/ioloop.py
@@ -310,6 +310,9 @@ def make_current(self):
         .. versionchanged:: 5.0
            This method also sets the current `asyncio` event loop.
         """
+        old = getattr(IOLoop._current, "instance", None)
+        if old is not None:
+            old.clear_current()
         IOLoop._current.instance = self
 
     @staticmethod
@@ -321,7 +324,7 @@ def clear_current():
         .. versionchanged:: 5.0
            This method also clears the current `asyncio` event loop.
         """
-        old = IOLoop._current.instance
+        old = getattr(IOLoop._current, "instance", None)
         if old is not None:
             old._clear_current_hook()
         IOLoop._current.instance = None
@@ -924,8 +927,9 @@ def start(self):
         if self._stopped:
             self._stopped = False
             return
-        old_current = getattr(IOLoop._current, "instance", None)
-        IOLoop._current.instance = self
+        old_current = IOLoop.current(instance=False)
+        if old_current is not self:
+            self.make_current()
         self._thread_ident = thread.get_ident()
         self._running = True
 
@@ -1068,7 +1072,10 @@ def start(self):
             self._stopped = False
             if self._blocking_signal_threshold is not None:
                 signal.setitimer(signal.ITIMER_REAL, 0, 0)
-            IOLoop._current.instance = old_current
+            if old_current is None:
+                IOLoop.clear_current()
+            elif old_current is not self:
+                old_current.make_current()
             if old_wakeup_fd is not None:
                 signal.set_wakeup_fd(old_wakeup_fd)
 
diff --git a/tornado/platform/asyncio.py b/tornado/platform/asyncio.py
--- a/tornado/platform/asyncio.py
+++ b/tornado/platform/asyncio.py
@@ -30,9 +30,8 @@
 
 
 class BaseAsyncIOLoop(IOLoop):
-    def initialize(self, asyncio_loop, close_loop=False, **kwargs):
+    def initialize(self, asyncio_loop, **kwargs):
         self.asyncio_loop = asyncio_loop
-        self.close_loop = close_loop
         # Maps fd to (fileobj, handler function) pair (as in IOLoop.add_handler)
         self.handlers = {}
         # Set of fds listening for reads/writes
@@ -48,8 +47,7 @@ def close(self, all_fds=False):
             self.remove_handler(fd)
             if all_fds:
                 self.close_fd(fileobj)
-        if self.close_loop:
-            self.asyncio_loop.close()
+        self.asyncio_loop.close()
 
     def add_handler(self, fd, handler, events):
         fd, fileobj = self.split_fd(fd)
@@ -159,10 +157,13 @@ class AsyncIOMainLoop(BaseAsyncIOLoop):
 
        Now used automatically when appropriate; it is no longer necessary
        to refer to this class directly.
+
+    .. versionchanged:: 5.0
+
+       Closing an `AsyncIOMainLoop` now closes the underlying asyncio loop.
     """
     def initialize(self, **kwargs):
-        super(AsyncIOMainLoop, self).initialize(asyncio.get_event_loop(),
-                                                close_loop=False, **kwargs)
+        super(AsyncIOMainLoop, self).initialize(asyncio.get_event_loop(), **kwargs)
 
 
 class AsyncIOLoop(BaseAsyncIOLoop):
@@ -185,25 +186,35 @@ class AsyncIOLoop(BaseAsyncIOLoop):
        to refer to this class directly.
     """
     def initialize(self, **kwargs):
+        self.is_current = False
         loop = asyncio.new_event_loop()
         try:
-            super(AsyncIOLoop, self).initialize(loop, close_loop=True, **kwargs)
+            super(AsyncIOLoop, self).initialize(loop, **kwargs)
         except Exception:
             # If initialize() does not succeed (taking ownership of the loop),
             # we have to close it.
             loop.close()
             raise
 
+    def close(self, all_fds=False):
+        if self.is_current:
+            self.clear_current()
+        super(AsyncIOLoop, self).close(all_fds=all_fds)
+
     def make_current(self):
-        super(AsyncIOLoop, self).make_current()
-        try:
-            self.old_asyncio = asyncio.get_event_loop()
-        except RuntimeError:
-            self.old_asyncio = None
+        if not self.is_current:
+            super(AsyncIOLoop, self).make_current()
+            try:
+                self.old_asyncio = asyncio.get_event_loop()
+            except RuntimeError:
+                self.old_asyncio = None
+            self.is_current = True
         asyncio.set_event_loop(self.asyncio_loop)
 
     def _clear_current_hook(self):
-        asyncio.set_event_loop(self.old_asyncio)
+        if self.is_current:
+            asyncio.set_event_loop(self.old_asyncio)
+            self.is_current = False
 
 
 def to_tornado_future(asyncio_future):


Test patch:
diff --git a/tornado/test/concurrent_test.py b/tornado/test/concurrent_test.py
--- a/tornado/test/concurrent_test.py
+++ b/tornado/test/concurrent_test.py
@@ -213,6 +213,9 @@ def f():
                        ".*ZeroDivisionError"):
             yield gen.moment
             yield gen.moment
+            # For some reason, TwistedIOLoop and pypy3 need a third iteration
+            # in order to drain references to the future
+            yield gen.moment
             del g
             gc.collect()  # for PyPy
 
@@ -425,3 +428,7 @@ def f(self):
         o = Object()
         answer = yield o.f()
         self.assertEqual(answer, 42)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/tornado/test/gen_test.py b/tornado/test/gen_test.py
--- a/tornado/test/gen_test.py
+++ b/tornado/test/gen_test.py
@@ -4,6 +4,7 @@
 import contextlib
 import datetime
 import functools
+import platform
 import sys
 import textwrap
 import time
@@ -1478,12 +1479,14 @@ def test_no_ref(self):
 
 
 class RunnerGCTest(AsyncTestCase):
-    """Github issue 1769: Runner objects can get GCed unexpectedly"""
-    @skipOnTravis
+    def is_pypy3(self):
+        return (platform.python_implementation() == 'PyPy' and
+                sys.version_info > (3,))
+
     @gen_test
     def test_gc(self):
-        """Runners shouldn't GC if future is alive"""
-        # Create the weakref
+        # Github issue 1769: Runner objects can get GCed unexpectedly
+        # while their future is alive.
         weakref_scope = [None]
 
         def callback():
@@ -1502,6 +1505,85 @@ def tester():
             tester()
         )
 
+    def test_gc_infinite_coro(self):
+        # Github issue 2229: suspended coroutines should be GCed when
+        # their loop is closed, even if they're involved in a reference
+        # cycle.
+        if IOLoop.configured_class().__name__.endswith('TwistedIOLoop'):
+            raise unittest.SkipTest("Test may fail on TwistedIOLoop")
+
+        loop = self.get_new_ioloop()
+        result = []
+        wfut = []
+
+        @gen.coroutine
+        def infinite_coro():
+            try:
+                while True:
+                    yield gen.sleep(1e-3)
+                    result.append(True)
+            finally:
+                # coroutine finalizer
+                result.append(None)
+
+        @gen.coroutine
+        def do_something():
+            fut = infinite_coro()
+            fut._refcycle = fut
+            wfut.append(weakref.ref(fut))
+            yield gen.sleep(0.2)
+
+        loop.run_sync(do_something)
+        loop.close()
+        gc.collect()
+        # Future was collected
+        self.assertIs(wfut[0](), None)
+        # At least one wakeup
+        self.assertGreaterEqual(len(result), 2)
+        if not self.is_pypy3():
+            # coroutine finalizer was called (not on PyPy3 apparently)
+            self.assertIs(result[-1], None)
+
+    @skipBefore35
+    def test_gc_infinite_async_await(self):
+        # Same as test_gc_infinite_coro, but with a `async def` function
+        import asyncio
+
+        namespace = exec_test(globals(), locals(), """
+        async def infinite_coro(result):
+            try:
+                while True:
+                    await gen.sleep(1e-3)
+                    result.append(True)
+            finally:
+                # coroutine finalizer
+                result.append(None)
+        """)
+
+        infinite_coro = namespace['infinite_coro']
+        loop = self.get_new_ioloop()
+        result = []
+        wfut = []
+
+        @gen.coroutine
+        def do_something():
+            fut = asyncio.get_event_loop().create_task(infinite_coro(result))
+            fut._refcycle = fut
+            wfut.append(weakref.ref(fut))
+            yield gen.sleep(0.2)
+
+        loop.run_sync(do_something)
+        with ExpectLog('asyncio', "Task was destroyed but it is pending"):
+            loop.close()
+            gc.collect()
+        # Future was collected
+        self.assertIs(wfut[0](), None)
+        # At least one wakeup and one finally
+        self.assertGreaterEqual(len(result), 2)
+        if not self.is_pypy3():
+            # coroutine finalizer was called (not on PyPy3 apparently)
+            self.assertIs(result[-1], None)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tornado/test/ioloop_test.py b/tornado/test/ioloop_test.py
--- a/tornado/test/ioloop_test.py
+++ b/tornado/test/ioloop_test.py
@@ -1,5 +1,6 @@
 from __future__ import absolute_import, division, print_function
 
+from concurrent.futures import ThreadPoolExecutor
 import contextlib
 import datetime
 import functools
@@ -495,6 +496,16 @@ def test_force_current(self):
         self.assertIs(self.io_loop, IOLoop.current())
 
 
+class TestIOLoopCurrentAsync(AsyncTestCase):
+    @gen_test
+    def test_clear_without_current(self):
+        # If there is no current IOLoop, clear_current is a no-op (but
+        # should not fail). Use a thread so we see the threading.Local
+        # in a pristine state.
+        with ThreadPoolExecutor(1) as e:
+            yield e.submit(IOLoop.clear_current)
+
+
 class TestIOLoopAddCallback(AsyncTestCase):
     def setUp(self):
         super(TestIOLoopAddCallback, self).setUp()