Repository: tornadoweb/tornado

Issue:
Inconsistency in add_callback() between PollIOLoop and AsyncIOLoop
When the loop is closed, `add_callback` returns quietly in `PollIOLoop` but raises RuntimeError in `AsyncIOLoop`. I don't have an opinion on what the best behavior is (I do find the RuntimeError mildly irritating, especially as you can't easily inspect the loop to know whether it is closed already -- perhaps add a `is_closed()` method? :-)), but IMHO it would be better if the two were consistent with each other.


Discussion:
Good point. There's even a comment in AsyncIOLoop.add_callback pointing out that this is racy and we can't guarantee that we'll correctly raise the exception (which is why I changed PollIOLoop to accept the callback without running it). AsyncIOLoop.add_callback should be updated to do the same (and while we're there, get rid of AsyncIOLoop.closing in favor of `AsyncIOLoop.asyncio_loop.is_closing` (which wasn't there when this class was first written)

Resolving patch:
diff --git a/tornado/httpclient.py b/tornado/httpclient.py
--- a/tornado/httpclient.py
+++ b/tornado/httpclient.py
@@ -657,6 +657,8 @@ def main():
     define("print_body", type=bool, default=True)
     define("follow_redirects", type=bool, default=True)
     define("validate_cert", type=bool, default=True)
+    define("proxy_host", type=str)
+    define("proxy_port", type=int)
     args = parse_command_line()
     client = HTTPClient()
     for arg in args:
@@ -664,6 +666,8 @@ def main():
             response = client.fetch(arg,
                                     follow_redirects=options.follow_redirects,
                                     validate_cert=options.validate_cert,
+                                    proxy_host=options.proxy_host,
+                                    proxy_port=options.proxy_port,
                                     )
         except HTTPError as e:
             if e.response is not None:
diff --git a/tornado/options.py b/tornado/options.py
--- a/tornado/options.py
+++ b/tornado/options.py
@@ -16,9 +16,19 @@
 
 """A command line parsing module that lets modules define their own options.
 
-Each module defines its own options which are added to the global
-option namespace, e.g.::
-
+This module is inspired by Google's `gflags
+<https://github.com/google/python-gflags>`_. The primary difference
+with libraries such as `argparse` is that a global registry is used so
+that options may be defined in any module (it also enables
+`tornado.log` by default). The rest of Tornado does not depend on this
+module, so feel free to use `argparse` or other configuration
+libraries if you prefer them.
+
+Options must be defined with `tornado.options.define` before use,
+generally at the top level of a module. The options are then
+accessible as attributes of `tornado.options.options`::
+
+    # myapp/db.py
     from tornado.options import define, options
 
     define("mysql_host", default="127.0.0.1:3306", help="Main user DB")
@@ -29,35 +39,36 @@ def connect():
         db = database.Connection(options.mysql_host)
         ...
 
+    # myapp/server.py
+    from tornado.options import define, options
+
+    define("port", default=8080, help="port to listen on")
+
+    def start_server():
+        app = make_app()
+        app.listen(options.port)
+
 The ``main()`` method of your application does not need to be aware of all of
 the options used throughout your program; they are all automatically loaded
 when the modules are loaded.  However, all modules that define options
 must have been imported before the command line is parsed.
 
 Your ``main()`` method can parse the command line or parse a config file with
-either::
+either `parse_command_line` or `parse_config_file`::
 
-    tornado.options.parse_command_line()
-    # or
-    tornado.options.parse_config_file("/etc/server.conf")
+    import myapp.db, myapp.server
+    import tornado.options
 
-.. note:
+    if __name__ == '__main__':
+        tornado.options.parse_command_line()
+        # or
+        tornado.options.parse_config_file("/etc/server.conf")
 
-   When using tornado.options.parse_command_line or
-   tornado.options.parse_config_file, the only options that are set are
-   ones that were previously defined with tornado.options.define.
-
-Command line formats are what you would expect (``--myoption=myvalue``
-or ``--myflag`` in the case of boolean flags). Config files are just Python
-files. Global names become options, e.g.::
-
-    myoption = "myvalue"
-    myotheroption = "myothervalue"
+.. note::
 
-We support `datetimes <datetime.datetime>`, `timedeltas
-<datetime.timedelta>`, bools, ints, and floats (just pass a ``type`` kwarg to
-`define`). We also accept multi-value options. See the documentation for
-`define()` below.
+   When using multiple ``parse_*`` functions, pass ``final=False`` to all
+   but the last one, or side effects may occur twice (in particular,
+   this can result in log messages being doubled).
 
 `tornado.options.options` is a singleton instance of `OptionParser`, and
 the top-level functions in this module (`define`, `parse_command_line`, etc)
@@ -81,6 +92,7 @@ def connect():
    options can be defined, set, and read with any mix of the two.
    Dashes are typical for command-line usage while config files require
    underscores.
+
 """
 
 from __future__ import absolute_import, division, print_function
@@ -191,13 +203,13 @@ def define(self, name, default=None, type=None, help=None, metavar=None,
                multiple=False, group=None, callback=None):
         """Defines a new command line option.
 
-        If ``type`` is given (one of str, float, int, bool, datetime, or timedelta)
-        or can be inferred from the ``default``, we parse the command line
-        arguments based on the given type. If ``multiple`` is True, we accept
-        comma-separated values, and the option value is always a list.
+        ``type`` can be any of `str`, `int`, `float`, `bool`,
+        `~datetime.datetime`, or `~datetime.timedelta`. If no ``type``
+        is given but a ``default`` is, ``type`` is the type of
+        ``default``. Otherwise, ``type`` defaults to `str`.
 
-        For multi-value integers, we also accept the syntax ``x:y``, which
-        turns into ``range(x, y)`` - very useful for long integer ranges.
+        If ``multiple`` is True, the option value is a list of ``type``
+        instead of an instance of ``type``.
 
         ``help`` and ``metavar`` are used to construct the
         automatically generated command line help string. The help
@@ -209,9 +221,7 @@ def define(self, name, default=None, type=None, help=None, metavar=None,
         groups. By default, command line options are grouped by the
         file in which they are defined.
 
-        Command line option names must be unique globally. They can be parsed
-        from the command line with `parse_command_line` or parsed from a
-        config file with `parse_config_file`.
+        Command line option names must be unique globally.
 
         If a ``callback`` is given, it will be run with the new value whenever
         the option is changed.  This can be used to combine command-line
@@ -223,6 +233,7 @@ def define(self, name, default=None, type=None, help=None, metavar=None,
         With this definition, options in the file specified by ``--config`` will
         override options set earlier on the command line, but can be overridden
         by later flags.
+
         """
         normalized = self._normalize_name(name)
         if normalized in self._options:
@@ -260,6 +271,14 @@ def parse_command_line(self, args=None, final=True):
         """Parses all options given on the command line (defaults to
         `sys.argv`).
 
+        Options look like ``--option=value`` and are parsed according
+        to their ``type``. For boolean options, ``--option`` is
+        equivalent to ``--option=true``
+
+        If the option has ``multiple=True``, comma-separated values
+        are accepted. For multi-value integer options, the syntax
+        ``x:y`` is also accepted and equivalent to ``range(x, y)``.
+
         Note that ``args[0]`` is ignored since it is the program name
         in `sys.argv`.
 
@@ -268,6 +287,7 @@ def parse_command_line(self, args=None, final=True):
         If ``final`` is ``False``, parse callbacks will not be run.
         This is useful for applications that wish to combine configurations
         from multiple sources.
+
         """
         if args is None:
             args = sys.argv
@@ -300,12 +320,37 @@ def parse_command_line(self, args=None, final=True):
         return remaining
 
     def parse_config_file(self, path, final=True):
-        """Parses and loads the Python config file at the given path.
+        """Parses and loads the config file at the given path.
+
+        The config file contains Python code that will be executed (so
+        it is **not safe** to use untrusted config files). Anything in
+        the global namespace that matches a defined option will be
+        used to set that option's value.
+
+        Options are not parsed from strings as they would be on the
+        command line; they should be set to the correct type (this
+        means if you have ``datetime`` or ``timedelta`` options you
+        will need to import those modules in the config file.
+
+        Example (using the options defined in the top-level docs of
+        this module)::
+
+            port = 80
+            mysql_host = 'mydb.example.com:3306'
+            memcache_hosts = ['cache1.example.com:11011',
+                              'cache2.example.com:11011']
 
         If ``final`` is ``False``, parse callbacks will not be run.
         This is useful for applications that wish to combine configurations
         from multiple sources.
 
+        .. note::
+
+            `tornado.options` is primarily a command-line library.
+            Config file support is provided for applications that wish
+            to use it, but applications that prefer config files may
+            wish to look at other libraries instead.
+
         .. versionchanged:: 4.1
            Config files are now always interpreted as utf-8 instead of
            the system default encoding.
@@ -313,6 +358,7 @@ def parse_config_file(self, path, final=True):
         .. versionchanged:: 4.4
            The special variable ``__file__`` is available inside config
            files, specifying the absolute path to the config file itself.
+
         """
         config = {'__file__': os.path.abspath(path)}
         with open(path, 'rb') as f:
diff --git a/tornado/platform/asyncio.py b/tornado/platform/asyncio.py
--- a/tornado/platform/asyncio.py
+++ b/tornado/platform/asyncio.py
@@ -130,13 +130,17 @@ def remove_timeout(self, timeout):
         timeout.cancel()
 
     def add_callback(self, callback, *args, **kwargs):
-        if self.closing:
-            # TODO: this is racy; we need a lock to ensure that the
-            # loop isn't closed during call_soon_threadsafe.
-            raise RuntimeError("IOLoop is closing")
-        self.asyncio_loop.call_soon_threadsafe(
-            self._run_callback,
-            functools.partial(stack_context.wrap(callback), *args, **kwargs))
+        try:
+            self.asyncio_loop.call_soon_threadsafe(
+                self._run_callback,
+                functools.partial(stack_context.wrap(callback), *args, **kwargs))
+        except RuntimeError:
+            # "Event loop is closed". Swallow the exception for
+            # consistency with PollIOLoop (and logical consistency
+            # with the fact that we can't guarantee that an
+            # add_callback that completes without error will
+            # eventually execute).
+            pass
 
     add_callback_from_signal = add_callback
 
@@ -229,3 +233,30 @@ def to_asyncio_future(tornado_future):
        so this method is now equivalent to `tornado.gen.convert_yielded`.
     """
     return convert_yielded(tornado_future)
+
+
+class AnyThreadEventLoopPolicy(asyncio.DefaultEventLoopPolicy):
+    """Event loop policy that allows loop creation on any thread.
+
+    The default `asyncio` event loop policy only automatically creates
+    event loops in the main threads. Other threads must create event
+    loops explicitly or `asyncio.get_event_loop` (and therefore
+    `.IOLoop.current`) will fail. Installing this policy allows event
+    loops to be created automatically on any thread, matching the
+    behavior of Tornado versions prior to 5.0 (or 5.0 on Python 2).
+
+    Usage::
+
+        asyncio.set_event_loop_policy(AnyThreadEventLoopPolicy())
+
+    .. versionadded:: 5.0
+
+    """
+    def get_event_loop(self):
+        try:
+            return super().get_event_loop()
+        except RuntimeError:
+            # "There is no current event loop in thread %r"
+            loop = self.new_event_loop()
+            self.set_event_loop(loop)
+            return loop


Test patch:
diff --git a/tornado/test/asyncio_test.py b/tornado/test/asyncio_test.py
--- a/tornado/test/asyncio_test.py
+++ b/tornado/test/asyncio_test.py
@@ -12,7 +12,9 @@
 
 from __future__ import absolute_import, division, print_function
 
+from concurrent.futures import ThreadPoolExecutor
 from tornado import gen
+from tornado.ioloop import IOLoop
 from tornado.testing import AsyncTestCase, gen_test
 from tornado.test.util import unittest, skipBefore33, skipBefore35, exec_test
 
@@ -21,7 +23,7 @@
 except ImportError:
     asyncio = None
 else:
-    from tornado.platform.asyncio import AsyncIOLoop, to_asyncio_future
+    from tornado.platform.asyncio import AsyncIOLoop, to_asyncio_future, AnyThreadEventLoopPolicy
     # This is used in dynamically-evaluated code, so silence pyflakes.
     to_asyncio_future
 
@@ -117,3 +119,49 @@ async def native_coroutine_with_adapter2():
             asyncio.get_event_loop().run_until_complete(
                 native_coroutine_with_adapter2()),
             42)
+
+
+@unittest.skipIf(asyncio is None, "asyncio module not present")
+class AnyThreadEventLoopPolicyTest(unittest.TestCase):
+    def setUp(self):
+        self.orig_policy = asyncio.get_event_loop_policy()
+        self.executor = ThreadPoolExecutor(1)
+
+    def tearDown(self):
+        asyncio.set_event_loop_policy(self.orig_policy)
+        self.executor.shutdown()
+
+    def get_event_loop_on_thread(self):
+        def get_and_close_event_loop():
+            """Get the event loop. Close it if one is returned.
+
+            Returns the (closed) event loop. This is a silly thing
+            to do and leaves the thread in a broken state, but it's
+            enough for this test. Closing the loop avoids resource
+            leak warnings.
+            """
+            loop = asyncio.get_event_loop()
+            loop.close()
+            return loop
+        future = self.executor.submit(get_and_close_event_loop)
+        return future.result()
+
+    def run_policy_test(self, accessor, expected_type):
+        # With the default policy, non-main threads don't get an event
+        # loop.
+        self.assertRaises(RuntimeError,
+                          self.executor.submit(accessor).result)
+        # Set the policy and we can get a loop.
+        asyncio.set_event_loop_policy(AnyThreadEventLoopPolicy())
+        self.assertIsInstance(
+            self.executor.submit(accessor).result(),
+            expected_type)
+        # Clean up to silence leak warnings. Always use asyncio since
+        # IOLoop doesn't (currently) close the underlying loop.
+        self.executor.submit(lambda: asyncio.get_event_loop().close()).result()
+
+    def test_asyncio_accessor(self):
+        self.run_policy_test(asyncio.get_event_loop, asyncio.AbstractEventLoop)
+
+    def test_tornado_accessor(self):
+        self.run_policy_test(IOLoop.current, IOLoop)
diff --git a/tornado/test/gen_test.py b/tornado/test/gen_test.py
--- a/tornado/test/gen_test.py
+++ b/tornado/test/gen_test.py
@@ -1480,6 +1480,7 @@ def test_no_ref(self):
 
 class RunnerGCTest(AsyncTestCase):
     """Github issue 1769: Runner objects can get GCed unexpectedly"""
+    @skipOnTravis
     @gen_test
     def test_gc(self):
         """Runners shouldn't GC if future is alive"""
diff --git a/tornado/test/ioloop_test.py b/tornado/test/ioloop_test.py
--- a/tornado/test/ioloop_test.py
+++ b/tornado/test/ioloop_test.py
@@ -171,10 +171,9 @@ def test_add_callback_from_signal_other_thread(self):
         other_ioloop.close()
 
     def test_add_callback_while_closing(self):
-        # Issue #635: add_callback() should raise a clean exception
-        # if called while another thread is closing the IOLoop.
-        if IOLoop.configured_class().__name__.endswith('AsyncIOLoop'):
-            raise unittest.SkipTest("AsyncIOMainLoop shutdown not thread safe")
+        # add_callback should not fail if it races with another thread
+        # closing the IOLoop. The callbacks are dropped silently
+        # without executing.
         closing = threading.Event()
 
         def target():
@@ -187,11 +186,7 @@ def target():
         thread.start()
         closing.wait()
         for i in range(1000):
-            try:
-                other_ioloop.add_callback(lambda: None)
-            except RuntimeError as e:
-                self.assertEqual("IOLoop is closing", str(e))
-                break
+            other_ioloop.add_callback(lambda: None)
 
     def test_handle_callback_exception(self):
         # IOLoop.handle_callback_exception can be overridden to catch