Repository: tornadoweb/tornado

Issue:
ValueError with ContextVar reset within decorated coroutine
I was getting the following error while updating code for Python 3.7 and Tornado 6:

```
ValueError: <Token var=<ContextVar ...> at 0x1059e4fc0> was created in a different Context
```

I was able to reproduce this issue with the sample script below. The error only occurs when using decorate coroutines, whereas native coroutines work as expected.

``` python
import asyncio
import contextvars
from tornado import gen, ioloop

cv = contextvars.ContextVar("cv", default=False)

async def foo():
    await asyncio.sleep(1)

async def main_native():
    token = cv.set(True)
    print(f"main_native before foo: Context={contextvars.Context()}")
    await foo()
    print(f"main_native after foo: Context={contextvars.Context()}")
    cv.reset(token)
    print("main_native: OK")

@gen.coroutine
def main_decorated():
    print(f"main_decorated before foo: Context={contextvars.Context()}")
    token = cv.set(True)
    yield foo()
    print(f"main_decorated after foo: Context={contextvars.Context()}")
    cv.reset(token)
    print("main_decorated OK")

if __name__ == "__main__":
    ioloop.IOLoop.current().run_sync(main_native)
    ioloop.IOLoop.current().run_sync(main_decorated)
```

The output from Python 3.7 and Tornado 6.0.3 is:

```
main_native before foo: Context=<Context object at 0x10ee2b948>
main_native after foo: Context=<Context object at 0x10ee2b948>
main_native: OK
main_decorated before foo: Context=<Context object at 0x10ee2b7e0>
main_decorated after foo: Context=<Context object at 0x10ee2bcf0>
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/var/folders/9_/k78_39fn08991mz6r_ws9fc00000gn/T/babel-ZbUBrS/python-QPUJC9", line 29, in <module>
    ioloop.IOLoop.current().run_sync(main_decorated)
  File "/Users/tahir.butt/.virtualenvs/py37-tornado6/lib/python3.7/site-packages/tornado/ioloop.py", line 532, in run_sync
    return future_cell[0].result()
  File "/Users/tahir.butt/.virtualenvs/py37-tornado6/lib/python3.7/site-packages/tornado/gen.py", line 748, in run
    yielded = self.gen.send(value)
  File "/var/folders/9_/k78_39fn08991mz6r_ws9fc00000gn/T/babel-ZbUBrS/python-QPUJC9", line 24, in main_decorated
    cv.reset(token)
ValueError: <Token var=<ContextVar name='cv' default=False at 0x10ee1c570> at 0x10ee2bbd0> was created in a different Context
```

Whereas the value of `contextvars.Context()` is not changed within the native coroutine `main_native` when calling the `foo` coroutine, this is not true for within the decorated courtine  `main_decorated`.

We see here where this error originates in the cPython implementation for `ContextVar.reset()`:

https://github.com/python/cpython/blob/aacc77fbd77640a8f03638216fa09372cc21673d/Python/context.c#L295-L300

This issue reflects the fundamental problem addressed in #2716. Using that PR branch resolves the problem raised in this issue.


Discussion:
Sorry for the silence here; there's been some discussion of `contextvars` issues on #2716. I have reproduced the problem here and am working on a fix. 

Resolving patch:
diff --git a/tornado/gen.py b/tornado/gen.py
--- a/tornado/gen.py
+++ b/tornado/gen.py
@@ -90,6 +90,11 @@ def get(self):
 from tornado.log import app_log
 from tornado.util import TimeoutError
 
+try:
+    import contextvars
+except ImportError:
+    contextvars = None  # type: ignore
+
 import typing
 from typing import Union, Any, Callable, List, Type, Tuple, Awaitable, Dict, overload
 
@@ -153,6 +158,10 @@ def _create_future() -> Future:
     return future
 
 
+def _fake_ctx_run(f: Callable[..., _T], *args: Any, **kw: Any) -> _T:
+    return f(*args, **kw)
+
+
 @overload
 def coroutine(
     func: Callable[..., "Generator[Any, Any, _T]"]
@@ -199,8 +208,12 @@ def wrapper(*args, **kwargs):
         # This function is type-annotated with a comment to work around
         # https://bitbucket.org/pypy/pypy/issues/2868/segfault-with-args-type-annotation-in
         future = _create_future()
+        if contextvars is not None:
+            ctx_run = contextvars.copy_context().run  # type: Callable
+        else:
+            ctx_run = _fake_ctx_run
         try:
-            result = func(*args, **kwargs)
+            result = ctx_run(func, *args, **kwargs)
         except (Return, StopIteration) as e:
             result = _value_from_stopiteration(e)
         except Exception:
@@ -218,7 +231,7 @@ def wrapper(*args, **kwargs):
                 # use "optional" coroutines in critical path code without
                 # performance penalty for the synchronous case.
                 try:
-                    yielded = next(result)
+                    yielded = ctx_run(next, result)
                 except (StopIteration, Return) as e:
                     future_set_result_unless_cancelled(
                         future, _value_from_stopiteration(e)
@@ -234,7 +247,7 @@ def wrapper(*args, **kwargs):
                     # add_done_callback() instead of putting a private
                     # attribute on the Future.
                     # (GitHub issues #1769, #2229).
-                    runner = Runner(result, future, yielded)
+                    runner = Runner(ctx_run, result, future, yielded)
                     future.add_done_callback(lambda _: runner)
                 yielded = None
                 try:
@@ -711,10 +724,12 @@ class Runner(object):
 
     def __init__(
         self,
+        ctx_run: Callable,
         gen: "Generator[_Yieldable, Any, _T]",
         result_future: "Future[_T]",
         first_yielded: _Yieldable,
     ) -> None:
+        self.ctx_run = ctx_run
         self.gen = gen
         self.result_future = result_future
         self.future = _null_future  # type: Union[None, Future]
@@ -723,7 +738,7 @@ def __init__(
         self.io_loop = IOLoop.current()
         if self.handle_yield(first_yielded):
             gen = result_future = first_yielded = None  # type: ignore
-            self.run()
+            self.ctx_run(self.run)
 
     def run(self) -> None:
         """Starts or resumes the generator, running until it reaches a
@@ -787,7 +802,7 @@ def handle_yield(self, yielded: _Yieldable) -> bool:
             future_set_exc_info(self.future, sys.exc_info())
 
         if self.future is moment:
-            self.io_loop.add_callback(self.run)
+            self.io_loop.add_callback(self.ctx_run, self.run)
             return False
         elif self.future is None:
             raise Exception("no pending future")
@@ -796,7 +811,7 @@ def handle_yield(self, yielded: _Yieldable) -> bool:
             def inner(f: Any) -> None:
                 # Break a reference cycle to speed GC.
                 f = None  # noqa: F841
-                self.run()
+                self.ctx_run(self.run)
 
             self.io_loop.add_future(self.future, inner)
             return False
@@ -808,7 +823,7 @@ def handle_exception(
         if not self.running and not self.finished:
             self.future = Future()
             future_set_exc_info(self.future, (typ, value, tb))
-            self.run()
+            self.ctx_run(self.run)
             return True
         else:
             return False


Test patch:
diff --git a/tornado/test/gen_test.py b/tornado/test/gen_test.py
--- a/tornado/test/gen_test.py
+++ b/tornado/test/gen_test.py
@@ -1106,6 +1106,14 @@ def test_propagate(self):
             self.gen_root(4),
         ]
 
+    @gen_test
+    def test_reset(self):
+        token = ctx_var.set(1)
+        yield
+        # reset asserts that we are still at the same level of the context tree,
+        # so we must make sure that we maintain that property across yield.
+        ctx_var.reset(token)
+
 
 if __name__ == "__main__":
     unittest.main()