Repository: tornadoweb/tornado

Issue:
  http_server.start multiple instance fails in python3.5
the very basic multi instances code fails on python 3.5 env. It works with python2.7 and 3.6.4. 

Python 3.5.2
```
tornado                5.0.1
Flask                  0.12.2
Flask-Cors             3.0.3
gevent                 1.2.2
greenlet               0.4.13
gunicorn               19.7.1
lxml                   4.2.0
```

```
from flask import Flask, request, jsonify
from flask_cors import CORS, cross_origin
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop

app = Flask(__name__)
PROD = True
CORS(app)


@app.route('/api/check/alive', methods=['GET'])
def handle_api_alive():
    resp = jsonify(dict(status="OK"))

    return resp


if __name__ == '__main__':
    # start HTTP server
    http_server = HTTPServer(WSGIContainer(app))
    http_server.bind(9999)
    http_server.start(num_processes=2)
    IOLoop.instance().start()
```


Discussion:
part of error log:

```
During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "test_server.py", line 32, in <module>
    http_server.start(num_processes=2)
  File "/home/jin/pyenv/py3DL/lib/python3.5/site-packages/tornado/tcpserver.py", line 223, in start
    self.add_sockets(sockets)
  File "/home/jin/pyenv/py3DL/lib/python3.5/site-packages/tornado/tcpserver.py", line 159, in add_sockets
    sock, self._handle_connection)
  File "/home/jin/pyenv/py3DL/lib/python3.5/site-packages/tornado/netutil.py", line 258, in add_accept_handler
    io_loop.add_handler(sock, accept_handler, IOLoop.READ)
  File "/home/jin/pyenv/py3DL/lib/python3.5/site-packages/tornado/platform/asyncio.py", line 60, in add_handler
    fd, self._handle_events, fd, IOLoop.READ)
  File "/usr/lib/python3.5/asyncio/selector_events.py", line 237, in add_reader
    (handle, None))
  File "/usr/lib/python3.5/selectors.py", line 411, in register
    self._epoll.register(key.fd, epoll_events)
FileExistsError: [Errno 17] File exists
child 1 (pid 23316) exited with status 1, restarting
Traceback (most recent call last):
  File "/usr/lib/python3.5/asyncio/selector_events.py", line 234, in add_reader
    key = self._selector.get_key(fd)
  File "/usr/lib/python3.5/selectors.py", line 191, in get_key
    raise KeyError("{!r} is not registered".format(fileobj)) from None
KeyError: '6 is not registered'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "test_server.py", line 32, in <module>
    http_server.start(num_processes=2)
  File "/home/jin/pyenv/py3DL/lib/python3.5/site-packages/tornado/tcpserver.py", line 223, in start
    self.add_sockets(sockets)
  File "/home/jin/pyenv/py3DL/lib/python3.5/site-packages/tornado/tcpserver.py", line 159, in add_sockets
    sock, self._handle_connection)
  File "/home/jin/pyenv/py3DL/lib/python3.5/site-packages/tornado/netutil.py", line 258, in add_accept_handler
    io_loop.add_handler(sock, accept_handler, IOLoop.READ)
  File "/home/jin/pyenv/py3DL/lib/python3.5/site-packages/tornado/platform/asyncio.py", line 60, in add_handler
    fd, self._handle_events, fd, IOLoop.READ)
  File "/usr/lib/python3.5/asyncio/selector_events.py", line 237, in add_reader
    (handle, None))
  File "/usr/lib/python3.5/selectors.py", line 411, in register
    self._epoll.register(key.fd, epoll_events)
FileExistsError: [Errno 17] File exists
child 1 (pid 23317) exited with status 1, restarting
Traceback (most recent call last):
  File "/usr/lib/python3.5/asyncio/selector_events.py", line 234, in add_reader
    key = self._selector.get_key(fd)
  File "/usr/lib/python3.5/selectors.py", line 191, in get_key
    raise KeyError("{!r} is not registered".format(fileobj)) from None
KeyError: '6 is not registered'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "test_server.py", line 32, in <module>
    http_server.start(num_processes=2)
  File "/home/jin/pyenv/py3DL/lib/python3.5/site-packages/tornado/tcpserver.py", line 223, in start
    self.add_sockets(sockets)
  File "/home/jin/pyenv/py3DL/lib/python3.5/site-packages/tornado/tcpserver.py", line 159, in add_sockets
    sock, self._handle_connection)
  File "/home/jin/pyenv/py3DL/lib/python3.5/site-packages/tornado/netutil.py", line 258, in add_accept_handler
    io_loop.add_handler(sock, accept_handler, IOLoop.READ)
  File "/home/jin/pyenv/py3DL/lib/python3.5/site-packages/tornado/platform/asyncio.py", line 60, in add_handler
    fd, self._handle_events, fd, IOLoop.READ)
  File "/usr/lib/python3.5/asyncio/selector_events.py", line 237, in add_reader
    (handle, None))
  File "/usr/lib/python3.5/selectors.py", line 411, in register
    self._epoll.register(key.fd, epoll_events)
FileExistsError: [Errno 17] File exists
child 1 (pid 23318) exited with status 1, restarting
Traceback (most recent call last):
  File "/usr/lib/python3.5/asyncio/selector_events.py", line 234, in add_reader
    key = self._selector.get_key(fd)
  File "/usr/lib/python3.5/selectors.py", line 191, in get_key
    raise KeyError("{!r} is not registered".format(fileobj)) from None
KeyError: '6 is not registered'

During handling of the above exception, another exception occurred:

```
more specifically, the example code works under the following env: 
```
flask==0.12.2
flask-cors==3.0.3
tornado==4.5.3
```
Same problem here, using Tornado’s web framework and HTTP server, `tornado==5.0.2` and `Python 3.6.4`
```python
def start_application(app, loop, port):
    """Start a tornado application."""
    parse_command_line()
    _app = app()
    _server = HTTPServer(_app)
    _server.bind(port)
    _server.start(2)

    logging.info(f"Running on port {port}")

    tornado.ioloop.IOLoop.current().start()
```
```
[I 180620 15:40:08 process:128] Starting 2 processes
Traceback (most recent call last):
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/selector_events.py", line 264, in _add_reader
Traceback (most recent call last):
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/selector_events.py", line 264, in _add_reader
    key = self._selector.get_key(fd)
    key = self._selector.get_key(fd)
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/selectors.py", line 191, in get_key
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/selectors.py", line 191, in get_key
    raise KeyError("{!r} is not registered".format(fileobj)) from None
    raise KeyError("{!r} is not registered".format(fileobj)) from None
KeyError: '8 is not registered'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/mio/Work/h3c/face_service/web/app.py", line 35, in <module>
KeyError: '8 is not registered'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/mio/Work/h3c/face_service/web/app.py", line 35, in <module>
    run()
  File "/Users/mio/Work/h3c/face_service/web/app.py", line 31, in run
    run()
  File "/Users/mio/Work/h3c/face_service/web/app.py", line 31, in run
    start_application(Configurer, loop, port=APP_PORT)
  File "/Users/mio/Work/h3c/face_service/web/utils/helpers.py", line 41, in start_application
    start_application(Configurer, loop, port=APP_PORT)
  File "/Users/mio/Work/h3c/face_service/web/utils/helpers.py", line 41, in start_application
    _server.start(2)
  File "/Users/mio/Work/h3c/face_service/env/lib/python3.6/site-packages/tornado/tcpserver.py", line 223, in start
    _server.start(2)
  File "/Users/mio/Work/h3c/face_service/env/lib/python3.6/site-packages/tornado/tcpserver.py", line 223, in start
    self.add_sockets(sockets)
    self.add_sockets(sockets)
  File "/Users/mio/Work/h3c/face_service/env/lib/python3.6/site-packages/tornado/tcpserver.py", line 159, in add_sockets
  File "/Users/mio/Work/h3c/face_service/env/lib/python3.6/site-packages/tornado/tcpserver.py", line 159, in add_sockets
    sock, self._handle_connection)
  File "/Users/mio/Work/h3c/face_service/env/lib/python3.6/site-packages/tornado/netutil.py", line 258, in add_accept_handler
    sock, self._handle_connection)
  File "/Users/mio/Work/h3c/face_service/env/lib/python3.6/site-packages/tornado/netutil.py", line 258, in add_accept_handler
    io_loop.add_handler(sock, accept_handler, IOLoop.READ)
  File "/Users/mio/Work/h3c/face_service/env/lib/python3.6/site-packages/tornado/platform/asyncio.py", line 75, in add_handler
    io_loop.add_handler(sock, accept_handler, IOLoop.READ)
  File "/Users/mio/Work/h3c/face_service/env/lib/python3.6/site-packages/tornado/platform/asyncio.py", line 75, in add_handler
    fd, self._handle_events, fd, IOLoop.READ)
    fd, self._handle_events, fd, IOLoop.READ)
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/selector_events.py", line 337, in add_reader
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/selector_events.py", line 337, in add_reader
    return self._add_reader(fd, callback, *args)
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/selector_events.py", line 267, in _add_reader
    return self._add_reader(fd, callback, *args)
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/selector_events.py", line 267, in _add_reader
    (handle, None))
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/selectors.py", line 541, in register
    (handle, None))
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/selectors.py", line 541, in register
    self._kqueue.control([kev], 0, 0)
OSError: [Errno 9] Bad file descriptor
    self._kqueue.control([kev], 0, 0)
OSError: [Errno 9] Bad file descriptor
[W 180620 15:40:09 process:163] child 1 (pid 3255) exited with status 1, restarting
```
I think the issue in these line of code:
```
http_server.bind(9999)
http_server.start(num_processes=2)
```
Both the process you create are binding on the same port.

```
http_server.listen(9999)
http_server.start(num_processes=2)
```
@ajayuranakar http://www.tornadoweb.org/en/stable/guide/running.html#processes-and-ports
```
def main():
    app = make_app()
    server = tornado.httpserver.HTTPServer(app)
    server.bind(8888)
    server.start(0)  # forks one process per cpu
    IOLoop.current().start()
```
BTW, if use `listen`:

```
[I 180626 13:38:54 helpers:45] Running on port 8888
    IOLoop.current().start()
  File "/Users/mio/Work/h3c/face_service/env/lib/python3.6/site-packages/tornado/platform/asyncio.py", line 127, in start
    self.asyncio_loop.run_forever()
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/base_events.py", line 421, in run_forever
    self._run_once()
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/base_events.py", line 1395, in _run_once
    event_list = self._selector.select(timeout)
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/selectors.py", line 577, in select
    kev_list = self._kqueue.control(None, max_ev, timeout)
OSError: [Errno 9] Bad file descriptor
Traceback (most recent call last):
  File "app.py", line 35, in <module>
    run()
  File "app.py", line 31, in run
    start_application(Configurer, loop, port=APP_PORT)
  File "/Users/mio/Work/h3c/face_service/web/utils/helpers.py", line 52, in start_application
    IOLoop.current().start()
  File "/Users/mio/Work/h3c/face_service/env/lib/python3.6/site-packages/tornado/platform/asyncio.py", line 127, in start
    self.asyncio_loop.run_forever()
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/base_events.py", line 421, in run_forever
    self._run_once()
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/base_events.py", line 1395, in _run_once
    event_list = self._selector.select(timeout)
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/selectors.py", line 577, in select
    kev_list = self._kqueue.control(None, max_ev, timeout)
OSError: [Errno 9] Bad file descriptor
[W 180626 13:38:54 process:163] child 1 (pid 5396) exited with status 1, restarting
[W 180626 13:38:54 process:163] child 0 (pid 5395) exited with status 1, restarting
```
@ajayuranakar on Linux, `listen` works, i tried on my centos7 server and debian docker container on macOS ; on macOS(10.13.5), `listen` don't  work.

Thank you for your help.
The original `bind()` way is pretty much as is suggested in the docs. I think this is a tricky low-level macOS kqueue related bug.
actually I tried above in ubuntu 16.04, not mac OS
@chrjxj yeah, the problem is that the `bind()` way which is suggested in the docs does not work on Linux or macOS; the `listen` way works on Linux, does not work on macOS.

Resolving patch:
diff --git a/tornado/wsgi.py b/tornado/wsgi.py
--- a/tornado/wsgi.py
+++ b/tornado/wsgi.py
@@ -85,8 +85,10 @@ def __call__(self, environ, start_response):
 
 # WSGI has no facilities for flow control, so just return an already-done
 # Future when the interface requires it.
-_dummy_future = Future()
-_dummy_future.set_result(None)
+def _dummy_future():
+    f = Future()
+    f.set_result(None)
+    return f
 
 
 class _WSGIConnection(httputil.HTTPConnection):
@@ -118,7 +120,7 @@ def write_headers(self, start_line, headers, chunk=None, callback=None):
             self.write(chunk, callback)
         elif callback is not None:
             callback()
-        return _dummy_future
+        return _dummy_future()
 
     def write(self, chunk, callback=None):
         if self._expected_content_remaining is not None:
@@ -130,7 +132,7 @@ def write(self, chunk, callback=None):
         self._write_buffer.append(chunk)
         if callback is not None:
             callback()
-        return _dummy_future
+        return _dummy_future()
 
     def finish(self):
         if (self._expected_content_remaining is not None and


Test patch:
diff --git a/tornado/test/import_test.py b/tornado/test/import_test.py
--- a/tornado/test/import_test.py
+++ b/tornado/test/import_test.py
@@ -1,50 +1,68 @@
 # flake8: noqa
 from __future__ import absolute_import, division, print_function
+
+import subprocess
+import sys
+
 from tornado.test.util import unittest
 
+_import_everything = b"""
+# The event loop is not fork-safe, and it's easy to initialize an asyncio.Future
+# at startup, which in turn creates the default event loop and prevents forking.
+# Explicitly disallow the default event loop so that an error will be raised
+# if something tries to touch it.
+try:
+    import asyncio
+except ImportError:
+    pass
+else:
+    asyncio.set_event_loop(None)
+
+import tornado.auth
+import tornado.autoreload
+import tornado.concurrent
+import tornado.escape
+import tornado.gen
+import tornado.http1connection
+import tornado.httpclient
+import tornado.httpserver
+import tornado.httputil
+import tornado.ioloop
+import tornado.iostream
+import tornado.locale
+import tornado.log
+import tornado.netutil
+import tornado.options
+import tornado.process
+import tornado.simple_httpclient
+import tornado.stack_context
+import tornado.tcpserver
+import tornado.tcpclient
+import tornado.template
+import tornado.testing
+import tornado.util
+import tornado.web
+import tornado.websocket
+import tornado.wsgi
+
+try:
+    import pycurl
+except ImportError:
+    pass
+else:
+    import tornado.curl_httpclient
+"""
+
 
 class ImportTest(unittest.TestCase):
     def test_import_everything(self):
-        # Some of our modules are not otherwise tested.  Import them
-        # all (unless they have external dependencies) here to at
-        # least ensure that there are no syntax errors.
-        import tornado.auth
-        import tornado.autoreload
-        import tornado.concurrent
-        import tornado.escape
-        import tornado.gen
-        import tornado.http1connection
-        import tornado.httpclient
-        import tornado.httpserver
-        import tornado.httputil
-        import tornado.ioloop
-        import tornado.iostream
-        import tornado.locale
-        import tornado.log
-        import tornado.netutil
-        import tornado.options
-        import tornado.process
-        import tornado.simple_httpclient
-        import tornado.stack_context
-        import tornado.tcpserver
-        import tornado.tcpclient
-        import tornado.template
-        import tornado.testing
-        import tornado.util
-        import tornado.web
-        import tornado.websocket
-        import tornado.wsgi
-
-    # for modules with dependencies, if those dependencies can be loaded,
-    # load them too.
-
-    def test_import_pycurl(self):
-        try:
-            import pycurl  # type: ignore
-        except ImportError:
-            pass
-        else:
-            import tornado.curl_httpclient
+        # Test that all Tornado modules can be imported without side effects,
+        # specifically without initializing the default asyncio event loop.
+        # Since we can't tell which modules may have already beein imported
+        # in our process, do it in a subprocess for a clean slate.
+        proc = subprocess.Popen([sys.executable], stdin=subprocess.PIPE)
+        proc.communicate(_import_everything)
+        self.assertEqual(proc.returncode, 0)
 
     def test_import_aliases(self):
         # Ensure we don't delete formerly-documented aliases accidentally.