Repository: encode/uvicorn

Issue:
Exception on controlled worker shutdown
Currently when running gunicorn + uvicorn ([a suggested production deployment pattern](https://www.uvicorn.org/deployment/)), I get a exception on shutdown (on SIGTERM):

```
[2020-02-04 17:40:31 -0500] [37823] [ERROR] Exception in worker process
Traceback (most recent call last):
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/gunicorn/arbiter.py", line 583, in spawn_worker
    worker.init_process()
    └ <uvicorn.workers.UvicornWorker object at 0x10b9f9110>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/uvicorn/workers.py", line 57, in init_process
    super(UvicornWorker, self).init_process()
          │              └ <uvicorn.workers.UvicornWorker object at 0x10b9f9110>
          └ <class 'uvicorn.workers.UvicornWorker'>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/gunicorn/workers/base.py", line 134, in init_process
    self.run()
    └ <uvicorn.workers.UvicornWorker object at 0x10b9f9110>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/uvicorn/workers.py", line 66, in run
    loop.run_until_complete(server.serve(sockets=self.sockets))
    │                       │                    └ <uvicorn.workers.UvicornWorker object at 0x10b9f9110>
    │                       └ <uvicorn.main.Server object at 0x10bc80d90>
    └ <_UnixSelectorEventLoop running=False closed=False debug=False>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/lib/python3.7/asyncio/base_events.py", line 579, in run_until_complete
    return future.result()
           └ <Task finished coro=<Server.serve() done, defined at /Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-pack...
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/uvicorn/main.py", line 403, in serve
    await self.shutdown(sockets=sockets)
          │                     └ [<gunicorn.sock.TCPSocket object at 0x10bc77890>]
          └ <uvicorn.main.Server object at 0x10bc80d90>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/uvicorn/main.py", line 533, in shutdown
    server.close()
    └ <Server sockets=[]>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/lib/python3.7/asyncio/base_events.py", line 306, in close
    self._loop._stop_serving(sock)
    │                        └ <gunicorn.sock.TCPSocket object at 0x10bc77890>
    └ <Server sockets=[]>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/lib/python3.7/asyncio/selector_events.py", line 562, in _stop_serving
    self._remove_reader(sock.fileno())
    │                   └ <gunicorn.sock.TCPSocket object at 0x10bc77890>
    └ <_UnixSelectorEventLoop running=False closed=False debug=False>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/gunicorn/sock.py", line 38, in __getattr__
    return getattr(self.sock, name)
                   │          └ 'fileno'
                   └ <gunicorn.sock.TCPSocket object at 0x10bc77890>
AttributeError: 'NoneType' object has no attribute 'fileno'
```
This appears to be because [sockets created by servers are explicitly closed before closing servers](https://github.com/encode/uvicorn/blob/0.11.2/uvicorn/main.py#L530-L533), however, a server close may [also attempt to operate on the socket](https://github.com/python/cpython/blob/97d2a9832494e4c85da124ceab18802eed9a4ab1/Lib/asyncio/selector_events.py#L572-L573). 

Command being run: 
```
gunicorn <...myapp...> --workers 2 --bind 0.0.0.0:4201 --worker-class uvicorn.workers.UvicornWorker --timeout 15 --graceful-timeout 10 --log-level debug
```

Full Debug log output:
```
[2020-02-04 17:40:27 -0500] [37792] [DEBUG] Current configuration:
  config: None
  bind: ['0.0.0.0:4201']
  backlog: 2048
  workers: 2
  worker_class: uvicorn.workers.UvicornWorker
  threads: 1
  worker_connections: 1000
  max_requests: 0
  max_requests_jitter: 0
  timeout: 15
  graceful_timeout: 10
  keepalive: 2
  limit_request_line: 4094
  limit_request_fields: 100
  limit_request_field_size: 8190
  reload: False
  reload_engine: auto
  reload_extra_files: []
  spew: False
  check_config: False
  preload_app: False
  sendfile: None
  reuse_port: False
  chdir: <...snip...>
  daemon: False
  raw_env: []
  pidfile: None
  worker_tmp_dir: None
  user: 501
  group: 20
  umask: 0
  initgroups: False
  tmp_upload_dir: None
  secure_scheme_headers: {'X-FORWARDED-PROTOCOL': 'ssl', 'X-FORWARDED-PROTO': 'https', 'X-FORWARDED-SSL': 'on'}
  forwarded_allow_ips: ['127.0.0.1']
  accesslog: None
  disable_redirect_access_to_syslog: False
  access_log_format: %(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"
  errorlog: -
  loglevel: debug
  capture_output: False
  logger_class: gunicorn.glogging.Logger
  logconfig: None
  logconfig_dict: {}
  syslog_addr: unix:///var/run/syslog
  syslog: False
  syslog_prefix: None
  syslog_facility: user
  enable_stdio_inheritance: False
  statsd_host: None
  statsd_prefix:
  proc_name: None
  default_proc_name: <...snip...>
  pythonpath: None
  paste: None
  on_starting: <function OnStarting.on_starting at 0x10b9a2b00>
  on_reload: <function OnReload.on_reload at 0x10b9a2c20>
  when_ready: <function WhenReady.when_ready at 0x10b9a2d40>
  pre_fork: <function Prefork.pre_fork at 0x10b9a2e60>
  post_fork: <function Postfork.post_fork at 0x10b9a2f80>
  post_worker_init: <function PostWorkerInit.post_worker_init at 0x10b9b70e0>
  worker_int: <function WorkerInt.worker_int at 0x10b9b7200>
  worker_abort: <function WorkerAbort.worker_abort at 0x10b9b7320>
  pre_exec: <function PreExec.pre_exec at 0x10b9b7440>
  pre_request: <function PreRequest.pre_request at 0x10b9b7560>
  post_request: <function PostRequest.post_request at 0x10b9b75f0>
  child_exit: <function ChildExit.child_exit at 0x10b9b7710>
  worker_exit: <function WorkerExit.worker_exit at 0x10b9b7830>
  nworkers_changed: <function NumWorkersChanged.nworkers_changed at 0x10b9b7950>
  on_exit: <function OnExit.on_exit at 0x10b9b7a70>
  proxy_protocol: False
  proxy_allow_ips: ['127.0.0.1']
  keyfile: None
  certfile: None
  ssl_version: 2
  cert_reqs: 0
  ca_certs: None
  suppress_ragged_eofs: True
  do_handshake_on_connect: False
  ciphers: TLSv1
  raw_paste_global_conf: []
[2020-02-04 17:40:27 -0500] [37792] [INFO] Starting gunicorn 19.9.0
[2020-02-04 17:40:27 -0500] [37792] [DEBUG] Arbiter booted
[2020-02-04 17:40:27 -0500] [37792] [INFO] Listening at: http://0.0.0.0:4201 (37792)
[2020-02-04 17:40:27 -0500] [37792] [INFO] Using worker: uvicorn.workers.UvicornWorker
[2020-02-04 17:40:27 -0500] [37822] [INFO] Booting worker with pid: 37822
[2020-02-04 17:40:27 -0500] [37823] [INFO] Booting worker with pid: 37823
[2020-02-04 17:40:27 -0500] [37792] [DEBUG] 2 workers
[2020-02-04 17:40:29 -0500] [37822] [INFO] Started server process [37822]
[2020-02-04 17:40:29 -0500] [37823] [INFO] Started server process [37823]
[2020-02-04 17:40:29 -0500] [37822] [INFO] Waiting for application startup.
[2020-02-04 17:40:29 -0500] [37823] [INFO] Waiting for application startup.
[2020-02-04 17:40:29 -0500] [37823] [INFO] Application startup complete.
[2020-02-04 17:40:29 -0500] [37822] [INFO] Application startup complete.
[2020-02-04 17:40:31 -0500] [37792] [INFO] Handling signal: term
[2020-02-04 17:40:31 -0500] [37823] [INFO] Shutting down
[2020-02-04 17:40:31 -0500] [37822] [INFO] Shutting down
[2020-02-04 17:40:31 -0500] [37822] [ERROR] Exception in worker process
Traceback (most recent call last):
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/gunicorn/arbiter.py", line 583, in spawn_worker
    worker.init_process()
    └ <uvicorn.workers.UvicornWorker object at 0x10babc950>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/uvicorn/workers.py", line 57, in init_process
    super(UvicornWorker, self).init_process()
          │              └ <uvicorn.workers.UvicornWorker object at 0x10babc950>
          └ <class 'uvicorn.workers.UvicornWorker'>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/gunicorn/workers/base.py", line 134, in init_process
    self.run()
    └ <uvicorn.workers.UvicornWorker object at 0x10babc950>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/uvicorn/workers.py", line 66, in run
    loop.run_until_complete(server.serve(sockets=self.sockets))
    │                       │                    └ <uvicorn.workers.UvicornWorker object at 0x10babc950>
    │                       └ <uvicorn.main.Server object at 0x10bc03b90>
    └ <_UnixSelectorEventLoop running=False closed=False debug=False>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/lib/python3.7/asyncio/base_events.py", line 579, in run_until_complete
    return future.result()
           └ <Task finished coro=<Server.serve() done, defined at /Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-pack...
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/uvicorn/main.py", line 403, in serve
    await self.shutdown(sockets=sockets)
          │                     └ [<gunicorn.sock.TCPSocket object at 0x10bc77890>]
          └ <uvicorn.main.Server object at 0x10bc03b90>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/uvicorn/main.py", line 533, in shutdown
    server.close()
    └ <Server sockets=[]>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/lib/python3.7/asyncio/base_events.py", line 306, in close
    self._loop._stop_serving(sock)
    │                        └ <gunicorn.sock.TCPSocket object at 0x10bc77890>
    └ <Server sockets=[]>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/lib/python3.7/asyncio/selector_events.py", line 562, in _stop_serving
    self._remove_reader(sock.fileno())
    │                   └ <gunicorn.sock.TCPSocket object at 0x10bc77890>
    └ <_UnixSelectorEventLoop running=False closed=False debug=False>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/gunicorn/sock.py", line 38, in __getattr__
    return getattr(self.sock, name)
                   │          └ 'fileno'
                   └ <gunicorn.sock.TCPSocket object at 0x10bc77890>
AttributeError: 'NoneType' object has no attribute 'fileno'

[2020-02-04 17:40:31 -0500] [37823] [ERROR] Exception in worker process
Traceback (most recent call last):
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/gunicorn/arbiter.py", line 583, in spawn_worker
    worker.init_process()
    └ <uvicorn.workers.UvicornWorker object at 0x10b9f9110>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/uvicorn/workers.py", line 57, in init_process
    super(UvicornWorker, self).init_process()
          │              └ <uvicorn.workers.UvicornWorker object at 0x10b9f9110>
          └ <class 'uvicorn.workers.UvicornWorker'>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/gunicorn/workers/base.py", line 134, in init_process
    self.run()
    └ <uvicorn.workers.UvicornWorker object at 0x10b9f9110>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/uvicorn/workers.py", line 66, in run
    loop.run_until_complete(server.serve(sockets=self.sockets))
    │                       │                    └ <uvicorn.workers.UvicornWorker object at 0x10b9f9110>
    │                       └ <uvicorn.main.Server object at 0x10bc80d90>
    └ <_UnixSelectorEventLoop running=False closed=False debug=False>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/lib/python3.7/asyncio/base_events.py", line 579, in run_until_complete
    return future.result()
           └ <Task finished coro=<Server.serve() done, defined at /Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-pack...
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/uvicorn/main.py", line 403, in serve
    await self.shutdown(sockets=sockets)
          │                     └ [<gunicorn.sock.TCPSocket object at 0x10bc77890>]
          └ <uvicorn.main.Server object at 0x10bc80d90>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/uvicorn/main.py", line 533, in shutdown
    server.close()
    └ <Server sockets=[]>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/lib/python3.7/asyncio/base_events.py", line 306, in close
    self._loop._stop_serving(sock)
    │                        └ <gunicorn.sock.TCPSocket object at 0x10bc77890>
    └ <Server sockets=[]>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/lib/python3.7/asyncio/selector_events.py", line 562, in _stop_serving
    self._remove_reader(sock.fileno())
    │                   └ <gunicorn.sock.TCPSocket object at 0x10bc77890>
    └ <_UnixSelectorEventLoop running=False closed=False debug=False>
  File "/Users/wagoodman/.pyenv/versions/3.7.4/envs/venv374/lib/python3.7/site-packages/gunicorn/sock.py", line 38, in __getattr__
    return getattr(self.sock, name)
                   │          └ 'fileno'
                   └ <gunicorn.sock.TCPSocket object at 0x10bc77890>
AttributeError: 'NoneType' object has no attribute 'fileno'

[2020-02-04 17:40:31 -0500] [37822] [INFO] Worker exiting (pid: 37822)
[2020-02-04 17:40:31 -0500] [37823] [INFO] Worker exiting (pid: 37823)
[2020-02-04 17:40:31 -0500] [37792] [INFO] Shutting down: Master
```


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -13,8 +13,8 @@ def get_version(package):
     """
     Return package version as listed in `__version__` in `init.py`.
     """
-    path = os.path.join(package, '__init__.py')
-    init_py = open(path, 'r', encoding='utf8').read()
+    path = os.path.join(package, "__init__.py")
+    init_py = open(path, "r", encoding="utf8").read()
     return re.search("__version__ = ['\"]([^'\"]+)['\"]", init_py).group(1)
 
 
@@ -22,16 +22,18 @@ def get_long_description():
     """
     Return the README.
     """
-    return open('README.md', 'r', encoding='utf8').read()
+    return open("README.md", "r", encoding="utf8").read()
 
 
 def get_packages(package):
     """
     Return root package and all sub-packages.
     """
-    return [dirpath
-            for dirpath, dirnames, filenames in os.walk(package)
-            if os.path.exists(os.path.join(dirpath, '__init__.py'))]
+    return [
+        dirpath
+        for dirpath, dirnames, filenames in os.walk(package)
+        if os.path.exists(os.path.join(dirpath, "__init__.py"))
+    ]
 
 
 env_marker = (
@@ -50,34 +52,34 @@ def get_packages(package):
 
 
 setup(
-    name='uvicorn',
-    version=get_version('uvicorn'),
-    url='https://github.com/encode/uvicorn',
-    license='BSD',
-    description='The lightning-fast ASGI server.',
+    name="uvicorn",
+    version=get_version("uvicorn"),
+    url="https://github.com/encode/uvicorn",
+    license="BSD",
+    description="The lightning-fast ASGI server.",
     long_description=get_long_description(),
-    long_description_content_type='text/markdown',
-    author='Tom Christie',
-    author_email='tom@tomchristie.com',
-    packages=get_packages('uvicorn'),
+    long_description_content_type="text/markdown",
+    author="Tom Christie",
+    author_email="tom@tomchristie.com",
+    packages=get_packages("uvicorn"),
     install_requires=requirements,
-    data_files = [("", ["LICENSE.md"])],
+    data_files=[("", ["LICENSE.md"])],
     classifiers=[
-        'Development Status :: 4 - Beta',
-        'Environment :: Web Environment',
-        'Intended Audience :: Developers',
-        'License :: OSI Approved :: BSD License',
-        'Operating System :: OS Independent',
-        'Topic :: Internet :: WWW/HTTP',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: Implementation :: CPython',
-        'Programming Language :: Python :: Implementation :: PyPy',
+        "Development Status :: 4 - Beta",
+        "Environment :: Web Environment",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: BSD License",
+        "Operating System :: OS Independent",
+        "Topic :: Internet :: WWW/HTTP",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: Implementation :: CPython",
+        "Programming Language :: Python :: Implementation :: PyPy",
     ],
     entry_points="""
     [console_scripts]
     uvicorn=uvicorn.main:main
-    """
+    """,
 )
diff --git a/uvicorn/main.py b/uvicorn/main.py
--- a/uvicorn/main.py
+++ b/uvicorn/main.py
@@ -527,10 +527,10 @@ async def shutdown(self, sockets=None):
         logger.info("Shutting down")
 
         # Stop accepting new connections.
-        for socket in sockets or []:
-            socket.close()
         for server in self.servers:
             server.close()
+        for socket in sockets or []:
+            socket.close()
         for server in self.servers:
             await server.wait_closed()
 


Test patch:
diff --git a/tests/test_main.py b/tests/test_main.py
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -1,5 +1,6 @@
 import threading
 import time
+import asyncio
 
 import requests
 
@@ -80,3 +81,43 @@ def install_signal_handlers(self):
     response = requests.get("http://127.0.0.1:8000")
     assert response.status_code == 204
     thread.join()
+
+
+def test_run_with_shutdown():
+    class App:
+        def __init__(self, scope):
+            if scope["type"] != "http":
+                raise Exception()
+
+        async def __call__(self, receive, send):
+            while True:
+                time.sleep(1)
+
+    class CustomServer(Server):
+        def install_signal_handlers(self):
+            pass
+
+    config = Config(app=App, loop="asyncio", workers=2, limit_max_requests=1)
+    server = CustomServer(config=config)
+    sock = config.bind_socket()
+    exc = True
+
+    def safe_run():
+        nonlocal exc, server
+        try:
+            exc = None
+            config.setup_event_loop()
+            loop = asyncio.get_event_loop()
+            loop.run_until_complete(server.serve(sockets=[sock]))
+        except Exception as e:
+            exc = e
+
+    thread = threading.Thread(target=safe_run)
+    thread.start()
+
+    while not server.started:
+        time.sleep(0.01)
+
+    server.should_exit = True
+    thread.join()
+    assert exc is None