Repository: tornadoweb/tornado

Issue:
NotImplementedError when calling HTTPServer.listen in Python 3.8.1 and Tornado 6.0.x
A few of our devs upgraded to 3.8.1 and now get an error when calling listen on an HTTPServer object.

Here is the callstack:

```
  File "C:\Temp\tornado_bugp\service.py", line 565, in _make_private_app
    server = app.listen(
  File "C:\Users\dev\.virtualenvs\tornado-web-app-LhID9pTb\lib\site-packages\tornado\web.py", line 2112, in listen
    server.listen(port, address)
  File "C:\Users\dev\.virtualenvs\tornado-web-app-LhID9pTb\lib\site-packages\tornado\tcpserver.py", line 152, in listen
    self.add_sockets(sockets)
  File "C:\Users\dev\.virtualenvs\tornado-web-app-LhID9pTb\lib\site-packages\tornado\tcpserver.py", line 165, in add_sockets
    self._handlers[sock.fileno()] = add_accept_handler(
  File "C:\Users\dev\.virtualenvs\tornado-web-app-LhID9pTb\lib\site-packages\tornado\netutil.py", line 279, in add_accept_handler
    io_loop.add_handler(sock, accept_handler, IOLoop.READ)
  File "C:\Users\dev\.virtualenvs\tornado-web-app-LhID9pTb\lib\site-packages\tornado\platform\asyncio.py", line 99, in add_handler
    self.asyncio_loop.add_reader(fd, self._handle_events, fd, IOLoop.READ)
  File "C:\Users\dev\AppData\Local\Programs\Python\Python38-32\Lib\asyncio\events.py", line 501, in add_reader
    raise NotImplementedError
NotImplementedError
```

Here is our call to listen:

```
handlers = [
        tornado.web.URLSpec(
            r'/config',
            _ConfigHandler
        ),
        tornado.web.URLSpec(
            r'/health',
            _HealthHandler,
            {
                'service_state': service_state
            }
        ),
        tornado.web.URLSpec(
            r'/shutdown',
            _ShutdownHandler,
            {
                'service_state': service_state
            }
        ),
        tornado.web.URLSpec(
            r'/status',
            relayserver.StatusHandler,
            {
                'relay_server': relay_server,
                'stats': stats
            }
        )
    ]

    app = tornado.web.Application(
        handlers
    )
server = app.listen(
        tornado.options.options.private_port,
        tornado.options.options.private_address,
        xheaders=True,
        max_body_size=tornado.options.options.max_body_size,
        idle_connection_timeout=tornado.options.options.idle_timeout
    )
```

I double checked to make sure all the handlers actually exist and they do.
When changing to 3.7 either system wide or in a virtual environment this works without issue. 


Discussion:
It appears you are using Windows. Python 3.8 changed the default event loop on Windows. Working around this is documented here: https://www.tornadoweb.org/en/stable/index.html#installation
Excellent. 
Thanks so much.

Resolving patch:
diff --git a/tornado/platform/asyncio.py b/tornado/platform/asyncio.py
--- a/tornado/platform/asyncio.py
+++ b/tornado/platform/asyncio.py
@@ -19,30 +19,74 @@
    Windows. Use the `~asyncio.SelectorEventLoop` instead.
 """
 
+import asyncio
+import atexit
 import concurrent.futures
+import errno
 import functools
+import select
+import socket
 import sys
-
-from threading import get_ident
+import threading
+import typing
 from tornado.gen import convert_yielded
 from tornado.ioloop import IOLoop, _Selectable
 
-import asyncio
-
-import typing
-from typing import Any, TypeVar, Awaitable, Callable, Union, Optional
+from typing import Any, TypeVar, Awaitable, Callable, Union, Optional, List, Tuple, Dict
 
 if typing.TYPE_CHECKING:
-    from typing import Set, Dict, Tuple  # noqa: F401
+    from typing import Set  # noqa: F401
+    from typing_extensions import Protocol
+
+    class _HasFileno(Protocol):
+        def fileno(self) -> int:
+            pass
+
+    _FileDescriptorLike = Union[int, _HasFileno]
 
 _T = TypeVar("_T")
 
 
+# Collection of sockets to write to at shutdown to wake up any selector threads.
+_waker_sockets = set()  # type: Set[socket.socket]
+
+
+def _atexit_callback() -> None:
+    for fd in _waker_sockets:
+        try:
+            fd.send(b"a")
+        except BlockingIOError:
+            pass
+
+
+# atexit callbacks are run in LIFO order. Our callback must run before
+# ThreadPoolExecutor's or it will deadlock (the pool's threads can't
+# finish their work items until we write to their waker sockets). In
+# recent versions of Python the thread pool atexit callback is
+# registered in a getattr hook the first time TPE is *referenced*
+# (instead of older versions of python where it was registered when
+# concurrent.futures was imported).
+concurrent.futures.ThreadPoolExecutor
+atexit.register(_atexit_callback)
+
+
 class BaseAsyncIOLoop(IOLoop):
     def initialize(  # type: ignore
         self, asyncio_loop: asyncio.AbstractEventLoop, **kwargs: Any
     ) -> None:
+        # asyncio_loop is always the real underlying IOLoop. This is used in
+        # ioloop.py to maintain the asyncio-to-ioloop mappings.
         self.asyncio_loop = asyncio_loop
+        # selector_loop is an event loop that implements the add_reader family of
+        # methods. Usually the same as asyncio_loop but differs on platforms such
+        # as windows where the default event loop does not implement these methods.
+        self.selector_loop = asyncio_loop
+        if hasattr(asyncio, "ProactorEventLoop") and isinstance(
+            asyncio_loop, asyncio.ProactorEventLoop  # type: ignore
+        ):
+            # Ignore this line for mypy because the abstract method checker
+            # doesn't understand dynamic proxies.
+            self.selector_loop = AddThreadSelectorEventLoop(asyncio_loop)  # type: ignore
         # Maps fd to (fileobj, handler function) pair (as in IOLoop.add_handler)
         self.handlers = {}  # type: Dict[int, Tuple[Union[int, _Selectable], Callable]]
         # Set of fds listening for reads/writes
@@ -70,7 +114,7 @@ def initialize(  # type: ignore
         super(BaseAsyncIOLoop, self).initialize(**kwargs)
 
         def assign_thread_identity() -> None:
-            self._thread_identity = get_ident()
+            self._thread_identity = threading.get_ident()
 
         self.add_callback(assign_thread_identity)
 
@@ -87,6 +131,8 @@ def close(self, all_fds: bool = False) -> None:
         # assume it was closed from the asyncio side, and do this
         # cleanup for us, leading to a KeyError.
         del IOLoop._ioloop_for_asyncio[self.asyncio_loop]
+        if self.selector_loop is not self.asyncio_loop:
+            self.selector_loop.close()
         self.asyncio_loop.close()
 
     def add_handler(
@@ -97,29 +143,29 @@ def add_handler(
             raise ValueError("fd %s added twice" % fd)
         self.handlers[fd] = (fileobj, handler)
         if events & IOLoop.READ:
-            self.asyncio_loop.add_reader(fd, self._handle_events, fd, IOLoop.READ)
+            self.selector_loop.add_reader(fd, self._handle_events, fd, IOLoop.READ)
             self.readers.add(fd)
         if events & IOLoop.WRITE:
-            self.asyncio_loop.add_writer(fd, self._handle_events, fd, IOLoop.WRITE)
+            self.selector_loop.add_writer(fd, self._handle_events, fd, IOLoop.WRITE)
             self.writers.add(fd)
 
     def update_handler(self, fd: Union[int, _Selectable], events: int) -> None:
         fd, fileobj = self.split_fd(fd)
         if events & IOLoop.READ:
             if fd not in self.readers:
-                self.asyncio_loop.add_reader(fd, self._handle_events, fd, IOLoop.READ)
+                self.selector_loop.add_reader(fd, self._handle_events, fd, IOLoop.READ)
                 self.readers.add(fd)
         else:
             if fd in self.readers:
-                self.asyncio_loop.remove_reader(fd)
+                self.selector_loop.remove_reader(fd)
                 self.readers.remove(fd)
         if events & IOLoop.WRITE:
             if fd not in self.writers:
-                self.asyncio_loop.add_writer(fd, self._handle_events, fd, IOLoop.WRITE)
+                self.selector_loop.add_writer(fd, self._handle_events, fd, IOLoop.WRITE)
                 self.writers.add(fd)
         else:
             if fd in self.writers:
-                self.asyncio_loop.remove_writer(fd)
+                self.selector_loop.remove_writer(fd)
                 self.writers.remove(fd)
 
     def remove_handler(self, fd: Union[int, _Selectable]) -> None:
@@ -127,10 +173,10 @@ def remove_handler(self, fd: Union[int, _Selectable]) -> None:
         if fd not in self.handlers:
             return
         if fd in self.readers:
-            self.asyncio_loop.remove_reader(fd)
+            self.selector_loop.remove_reader(fd)
             self.readers.remove(fd)
         if fd in self.writers:
-            self.asyncio_loop.remove_writer(fd)
+            self.selector_loop.remove_writer(fd)
             self.writers.remove(fd)
         del self.handlers[fd]
 
@@ -169,7 +215,7 @@ def remove_timeout(self, timeout: object) -> None:
         timeout.cancel()  # type: ignore
 
     def add_callback(self, callback: Callable, *args: Any, **kwargs: Any) -> None:
-        if get_ident() == self._thread_identity:
+        if threading.get_ident() == self._thread_identity:
             call_soon = self.asyncio_loop.call_soon
         else:
             call_soon = self.asyncio_loop.call_soon_threadsafe
@@ -182,6 +228,11 @@ def add_callback(self, callback: Callable, *args: Any, **kwargs: Any) -> None:
             # add_callback that completes without error will
             # eventually execute).
             pass
+        except AttributeError:
+            # ProactorEventLoop may raise this instead of RuntimeError
+            # if call_soon_threadsafe races with a call to close().
+            # Swallow it too for consistency.
+            pass
 
     def add_callback_from_signal(
         self, callback: Callable, *args: Any, **kwargs: Any
@@ -344,3 +395,187 @@ def get_event_loop(self) -> asyncio.AbstractEventLoop:
             loop = self.new_event_loop()
             self.set_event_loop(loop)
             return loop
+
+
+class AddThreadSelectorEventLoop(asyncio.AbstractEventLoop):
+    """Wrap an event loop to add implementations of the ``add_reader`` method family.
+
+    Instances of this class start a second thread to run a selector.
+    This thread is completely hidden from the user; all callbacks are
+    run on the wrapped event loop's thread.
+
+    This class is used automatically by Tornado; applications should not need
+    to refer to it directly.
+
+    It is safe to wrap any event loop with this class, although it only makes sense
+    for event loops that do not implement the ``add_reader`` family of methods
+    themselves (i.e. ``WindowsProactorEventLoop``)
+
+    Closing the ``AddThreadSelectorEventLoop`` also closes the wrapped event loop.
+
+    """
+
+    # This class is a __getattribute__-based proxy. All attributes other than those
+    # in this set are proxied through to the underlying loop.
+    MY_ATTRIBUTES = {
+        "_consume_waker",
+        "_executor",
+        "_handle_event",
+        "_readers",
+        "_real_loop",
+        "_start_select",
+        "_run_select",
+        "_handle_select",
+        "_wake_selector",
+        "_waker_r",
+        "_waker_w",
+        "_writers",
+        "add_reader",
+        "add_writer",
+        "close",
+        "remove_reader",
+        "remove_writer",
+    }
+
+    def __getattribute__(self, name: str) -> Any:
+        if name in AddThreadSelectorEventLoop.MY_ATTRIBUTES:
+            return super().__getattribute__(name)
+        return getattr(self._real_loop, name)
+
+    def __init__(self, real_loop: asyncio.AbstractEventLoop) -> None:
+        self._real_loop = real_loop
+        # Create our own executor to ensure we always have a thread
+        # available (we'll keep it 100% busy) instead of contending
+        # with the application for a thread in the default executor.
+        self._executor = concurrent.futures.ThreadPoolExecutor(1)
+        # Start the select loop once the loop is started.
+        self._real_loop.call_soon(self._start_select)
+
+        self._readers = {}  # type: Dict[_FileDescriptorLike, Callable]
+        self._writers = {}  # type: Dict[_FileDescriptorLike, Callable]
+
+        # Writing to _waker_w will wake up the selector thread, which
+        # watches for _waker_r to be readable.
+        self._waker_r, self._waker_w = socket.socketpair()
+        self._waker_r.setblocking(False)
+        self._waker_w.setblocking(False)
+        _waker_sockets.add(self._waker_w)
+        self.add_reader(self._waker_r, self._consume_waker)
+
+    def __del__(self) -> None:
+        # If the top-level application code uses asyncio interfaces to
+        # start and stop the event loop, no objects created in Tornado
+        # can get a clean shutdown notification. If we're just left to
+        # be GC'd, we must explicitly close our sockets to avoid
+        # logging warnings.
+        _waker_sockets.discard(self._waker_w)
+        self._waker_r.close()
+        self._waker_w.close()
+
+    def close(self) -> None:
+        self._wake_selector()
+        self._executor.shutdown()
+        _waker_sockets.discard(self._waker_w)
+        self._waker_r.close()
+        self._waker_w.close()
+        self._real_loop.close()
+
+    def _wake_selector(self) -> None:
+        try:
+            self._waker_w.send(b"a")
+        except BlockingIOError:
+            pass
+
+    def _consume_waker(self) -> None:
+        try:
+            self._waker_r.recv(1024)
+        except BlockingIOError:
+            pass
+
+    def _start_select(self) -> None:
+        # Capture reader and writer sets here in the event loop
+        # thread to avoid any problems with concurrent
+        # modification while the select loop uses them.
+        f = self.run_in_executor(
+            self._executor,
+            self._run_select,
+            list(self._readers.keys()),
+            list(self._writers.keys()),
+        )
+        asyncio.ensure_future(f).add_done_callback(self._handle_select)
+
+    def _run_select(
+        self, to_read: List[int], to_write: List[int]
+    ) -> Tuple[List[int], List[int]]:
+        # We use the simpler interface of the select module instead of
+        # the more stateful interface in the selectors module because
+        # this class is only intended for use on windows, where
+        # select.select is the only option. The selector interface
+        # does not have well-documented thread-safety semantics that
+        # we can rely on so ensuring proper synchronization would be
+        # tricky.
+        try:
+            # On windows, selecting on a socket for write will not
+            # return the socket when there is an error (but selecting
+            # for reads works). Also select for errors when selecting
+            # for writes, and merge the results.
+            #
+            # This pattern is also used in
+            # https://github.com/python/cpython/blob/v3.8.0/Lib/selectors.py#L312-L317
+            rs, ws, xs = select.select(to_read, to_write, to_write)
+            ws = ws + xs
+        except OSError as e:
+            # After remove_reader or remove_writer is called, the file
+            # descriptor may subsequently be closed on the event loop
+            # thread. It's possible that this select thread hasn't
+            # gotten into the select system call by the time that
+            # happens in which case (at least on macOS), select may
+            # raise a "bad file descriptor" error. If we get that
+            # error, check and see if we're also being woken up by
+            # polling the waker alone. If we are, just return to the
+            # event loop and we'll get the updated set of file
+            # descriptors on the next iteration. Otherwise, raise the
+            # original error.
+            if e.errno == getattr(errno, "WSAENOTSOCK", errno.EBADF):
+                rs, _, _ = select.select([self._waker_r.fileno()], [], [], 0)
+                if rs:
+                    return rs, []
+            raise
+        return rs, ws
+
+    def _handle_select(self, f: "asyncio.Future[Tuple[List[int], List[int]]]") -> None:
+        rs, ws = f.result()
+        for r in rs:
+            self._handle_event(r, self._readers)
+        for w in ws:
+            self._handle_event(w, self._writers)
+        self._start_select()
+
+    def _handle_event(
+        self, fd: "_FileDescriptorLike", cb_map: Dict["_FileDescriptorLike", Callable],
+    ) -> None:
+        try:
+            callback = cb_map[fd]
+        except KeyError:
+            return
+        callback()
+
+    def add_reader(
+        self, fd: "_FileDescriptorLike", callback: Callable[..., None], *args: Any
+    ) -> None:
+        self._readers[fd] = functools.partial(callback, *args)
+        self._wake_selector()
+
+    def add_writer(
+        self, fd: "_FileDescriptorLike", callback: Callable[..., None], *args: Any
+    ) -> None:
+        self._writers[fd] = functools.partial(callback, *args)
+        self._wake_selector()
+
+    def remove_reader(self, fd: "_FileDescriptorLike") -> None:
+        del self._readers[fd]
+        self._wake_selector()
+
+    def remove_writer(self, fd: "_FileDescriptorLike") -> None:
+        del self._writers[fd]
+        self._wake_selector()


Test patch:
diff --git a/tornado/test/__init__.py b/tornado/test/__init__.py
deleted file mode 100644
--- a/tornado/test/__init__.py
+++ /dev/null
@@ -1,8 +0,0 @@
-import asyncio
-import sys
-
-# Use the selector event loop on windows. Do this in tornado/test/__init__.py
-# instead of runtests.py so it happens no matter how the test is run (such as
-# through editor integrations).
-if sys.platform == "win32" and hasattr(asyncio, "WindowsSelectorEventLoopPolicy"):
-    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())  # type: ignore
diff --git a/tornado/test/httpclient_test.py b/tornado/test/httpclient_test.py
--- a/tornado/test/httpclient_test.py
+++ b/tornado/test/httpclient_test.py
@@ -500,10 +500,12 @@ def accept_callback(conn, address):
                 stream.close()
 
             netutil.add_accept_handler(sock, accept_callback)  # type: ignore
-            resp = self.fetch("http://127.0.0.1:%d/" % port)
-            resp.rethrow()
-            self.assertEqual(resp.headers["X-XSS-Protection"], "1; mode=block")
-            self.io_loop.remove_handler(sock.fileno())
+            try:
+                resp = self.fetch("http://127.0.0.1:%d/" % port)
+                resp.rethrow()
+                self.assertEqual(resp.headers["X-XSS-Protection"], "1; mode=block")
+            finally:
+                self.io_loop.remove_handler(sock.fileno())
 
     def test_304_with_content_length(self):
         # According to the spec 304 responses SHOULD NOT include
diff --git a/tornado/test/ioloop_test.py b/tornado/test/ioloop_test.py
--- a/tornado/test/ioloop_test.py
+++ b/tornado/test/ioloop_test.py
@@ -400,9 +400,14 @@ def handle_read(fd, events):
             client.close()
             server.close()
 
+    @skipIfNonUnix
     @gen_test
     def test_init_close_race(self):
         # Regression test for #2367
+        #
+        # Skipped on windows because of what looks like a bug in the
+        # proactor event loop when started and stopped on non-main
+        # threads.
         def f():
             for i in range(10):
                 loop = IOLoop()
diff --git a/tornado/test/iostream_test.py b/tornado/test/iostream_test.py
--- a/tornado/test/iostream_test.py
+++ b/tornado/test/iostream_test.py
@@ -1,6 +1,7 @@
 from tornado.concurrent import Future
 from tornado import gen
 from tornado import netutil
+from tornado.ioloop import IOLoop
 from tornado.iostream import (
     IOStream,
     SSLIOStream,
@@ -12,6 +13,7 @@
 from tornado.locks import Condition, Event
 from tornado.log import gen_log
 from tornado.netutil import ssl_wrap_socket
+from tornado.platform.asyncio import AddThreadSelectorEventLoop
 from tornado.tcpserver import TCPServer
 from tornado.testing import (
     AsyncHTTPTestCase,
@@ -246,11 +248,11 @@ def test_read_until_close_after_close(self: typing.Any):
         rs, ws = yield self.make_iostream_pair()
         try:
             ws.write(b"1234")
-            ws.close()
             # Read one byte to make sure the client has received the data.
             # It won't run the close callback as long as there is more buffered
             # data that could satisfy a later read.
             data = yield rs.read_bytes(1)
+            ws.close()
             self.assertEqual(data, b"1")
             data = yield rs.read_until_close()
             self.assertEqual(data, b"234")
@@ -298,6 +300,10 @@ async def test_read_until_with_close_after_second_packet(self):
         # in size
         async with self.iostream_pair() as (rs, ws):
             rf = asyncio.ensure_future(rs.read_until(b"done"))
+            # We need to wait for the read_until to actually start. On
+            # windows that's tricky because the selector runs in
+            # another thread; sleeping is the simplest way.
+            await asyncio.sleep(0.1)
             await ws.write(b"x" * 2048)
             ws.write(b"done")
             ws.close()
@@ -796,6 +802,17 @@ def test_inline_read_error(self: typing.Any):
         # on socket FDs, but we can't close the socket object normally
         # because we won't get the error we want if the socket knows
         # it's closed.
+        #
+        # This test is also disabled when the
+        # AddThreadSelectorEventLoop is used, because a race between
+        # this thread closing the socket and the selector thread
+        # calling the select system call can make this test flaky.
+        # This event loop implementation is normally only used on
+        # windows, making this check redundant with skipIfNonUnix, but
+        # we sometimes enable it on other platforms for testing.
+        io_loop = IOLoop.current()
+        if isinstance(io_loop.selector_loop, AddThreadSelectorEventLoop):
+            self.skipTest("AddThreadSelectorEventLoop not supported")
         server, client = yield self.make_iostream_pair()
         try:
             os.close(server.socket.fileno())
@@ -943,6 +960,7 @@ def tearDown(self):
             self.server_stream.close()
         if self.client_stream is not None:
             self.client_stream.close()
+        self.io_loop.remove_handler(self.listener.fileno())
         self.listener.close()
         super(TestIOStreamStartTLS, self).tearDown()
 
diff --git a/tornado/test/runtests.py b/tornado/test/runtests.py
--- a/tornado/test/runtests.py
+++ b/tornado/test/runtests.py
@@ -12,7 +12,7 @@
 from tornado.httpclient import AsyncHTTPClient
 from tornado.httpserver import HTTPServer
 from tornado.netutil import Resolver
-from tornado.options import define, add_parse_callback
+from tornado.options import define, add_parse_callback, options
 
 
 TEST_MODULES = [
@@ -182,6 +182,11 @@ def main():
             reduce(operator.or_, (getattr(gc, v) for v in values))
         ),
     )
+    define(
+        "fail-if-logs",
+        default=True,
+        help="If true, fail the tests if any log output is produced (unless captured by ExpectLog)",
+    )
 
     def set_locale(x):
         locale.setlocale(locale.LC_ALL, x)
@@ -228,7 +233,8 @@ def set_locale(x):
                 log_counter.error_count,
                 counting_stderr.byte_count,
             )
-            sys.exit(1)
+            if options.fail_if_logs:
+                sys.exit(1)
 
 
 if __name__ == "__main__":
diff --git a/tornado/test/simple_httpclient_test.py b/tornado/test/simple_httpclient_test.py
--- a/tornado/test/simple_httpclient_test.py
+++ b/tornado/test/simple_httpclient_test.py
@@ -752,7 +752,7 @@ def test_small_headers(self):
         self.assertEqual(response.body, b"ok")
 
     def test_large_headers(self):
-        with ExpectLog(gen_log, "Unsatisfiable read"):
+        with ExpectLog(gen_log, "Unsatisfiable read", level=logging.INFO):
             with self.assertRaises(UnsatisfiableReadError):
                 self.fetch("/large", raise_error=True)
 
diff --git a/tornado/test/twisted_test.py b/tornado/test/twisted_test.py
--- a/tornado/test/twisted_test.py
+++ b/tornado/test/twisted_test.py
@@ -75,6 +75,13 @@ def restore_signal_handlers(saved):
 class CompatibilityTests(unittest.TestCase):
     def setUp(self):
         self.saved_signals = save_signal_handlers()
+        self.saved_policy = asyncio.get_event_loop_policy()
+        if hasattr(asyncio, "WindowsSelectorEventLoopPolicy"):
+            # Twisted requires a selector event loop, even if Tornado is
+            # doing its own tricks in AsyncIOLoop to support proactors.
+            # Setting an AddThreadSelectorEventLoop exposes various edge
+            # cases so just use a regular selector.
+            asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())  # type: ignore
         self.io_loop = IOLoop()
         self.io_loop.make_current()
         self.reactor = AsyncioSelectorReactor()
@@ -83,6 +90,7 @@ def tearDown(self):
         self.reactor.disconnectAll()
         self.io_loop.clear_current()
         self.io_loop.close(all_fds=True)
+        asyncio.set_event_loop_policy(self.saved_policy)
         restore_signal_handlers(self.saved_signals)
 
     def start_twisted_server(self):