Repository: jupyterhub/jupyterhub

Issue:
proxy.check_routes can call again by timer before the last call stop
user route add is slow,so if we lost a large number of user's routes，proxy.check_routes will still running in next timer clock.


Discussion:

Resolving patch:
diff --git a/jupyterhub/app.py b/jupyterhub/app.py
--- a/jupyterhub/app.py
+++ b/jupyterhub/app.py
@@ -1655,7 +1655,6 @@ async def start(self):
 
         await self.proxy.check_routes(self.users, self._service_map)
 
-
         if self.service_check_interval and any(s.url for s in self._service_map.values()):
             pc = PeriodicCallback(self.check_services_health, 1e3 * self.service_check_interval)
             pc.start()
diff --git a/jupyterhub/proxy.py b/jupyterhub/proxy.py
--- a/jupyterhub/proxy.py
+++ b/jupyterhub/proxy.py
@@ -15,9 +15,11 @@
 - Route paths should be normalized to always start and end with '/'
 """
 
-# Copyright (c) IPython Development Team.
+# Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
 
+import asyncio
+from functools import wraps
 import json
 import os
 from subprocess import Popen
@@ -30,17 +32,31 @@
 
 from traitlets import (
     Any, Bool, Instance, Integer, Unicode,
-    default,
+    default, observe,
 )
 from jupyterhub.traitlets import Command
 
 from traitlets.config import LoggingConfigurable
 from .objects import Server
-from .orm import Service, User
 from . import utils
 from .utils import url_path_join
 
 
+def _one_at_a_time(method):
+    """decorator to limit an async method to be called only once
+
+    If multiple concurrent calls to this method are made,
+    queue them instead of allowing them to be concurrently outstanding.
+    """
+    method._lock = asyncio.Lock()
+    @wraps(method)
+    async def locked_method(*args, **kwargs):
+        async with method._lock:
+            return await method(*args, **kwargs)
+
+    return locked_method
+
+
 class Proxy(LoggingConfigurable):
     """Base class for configurable proxies that JupyterHub can use.
 
@@ -251,7 +267,6 @@ async def add_all_services(self, service_dict):
 
         Used when loading up a new proxy.
         """
-        db = self.db
         futures = []
         for service in service_dict.values():
             if service.server:
@@ -264,7 +279,6 @@ async def add_all_users(self, user_dict):
 
         Used when loading up a new proxy.
         """
-        db = self.db
         futures = []
         for user in user_dict.values():
             for name, spawner in user.spawners.items():
@@ -273,20 +287,25 @@ async def add_all_users(self, user_dict):
         # wait after submitting them all
         await gen.multi(futures)
 
+    @_one_at_a_time
     async def check_routes(self, user_dict, service_dict, routes=None):
         """Check that all users are properly routed on the proxy."""
         if not routes:
+            self.log.debug("Fetching routes to check")
             routes = await self.get_all_routes()
+        # log info-level that we are starting the route-checking
+        # this may help diagnose performance issues,
+        # as we are about
+        self.log.info("Checking routes")
 
         user_routes = {path for path, r in routes.items() if 'user' in r['data']}
         futures = []
-        db = self.db
 
         good_routes = {'/'}
 
         hub = self.app.hub
         if '/' not in routes:
-            self.log.warning("Adding missing default route")
+            self.log.warning("Adding default route")
             futures.append(self.add_hub_route(hub))
         else:
             route = routes['/']
@@ -342,8 +361,7 @@ async def check_routes(self, user_dict, service_dict, routes=None):
                 self.log.warning("Deleting stale route %s", routespec)
                 futures.append(self.delete_route(routespec))
 
-        for f in futures:
-            await f
+        await gen.multi(futures)
 
     def add_hub_route(self, hub):
         """Add the default route for the Hub"""
@@ -367,13 +385,33 @@ class ConfigurableHTTPProxy(Proxy):
     If the proxy should not be run as a subprocess of the Hub,
     (e.g. in a separate container),
     set::
-    
+
         c.ConfigurableHTTPProxy.should_start = False
     """
 
     proxy_process = Any()
     client = Instance(AsyncHTTPClient, ())
 
+    concurrency = Integer(
+        10,
+        config=True,
+        help="""
+        The number of requests allowed to be concurrently outstanding to the proxy
+
+        Limiting this number avoids potential timeout errors
+        by sending too many requests to update the proxy at once
+        """,
+        )
+    semaphore = Any()
+
+    @default('semaphore')
+    def _default_semaphore(self):
+        return asyncio.BoundedSemaphore(self.concurrency)
+
+    @observe('concurrency')
+    def _concurrency_changed(self, change):
+        self.semaphore = asyncio.BoundedSemaphore(change.new)
+
     debug = Bool(False, help="Add debug-level logging to the Proxy.", config=True)
     auth_token = Unicode(
         help="""The Proxy auth token
@@ -453,9 +491,7 @@ def _check_process():
             if status is not None:
                 e = RuntimeError(
                     "Proxy failed to start with exit code %i" % status)
-                # py2-compatible `raise e from None`
-                e.__cause__ = None
-                raise e
+                raise e from None
 
         for server in (public_server, api_server):
             for i in range(10):
@@ -524,7 +560,7 @@ def _routespec_from_chp_path(self, chp_path):
             routespec = routespec + '/'
         return routespec
 
-    def api_request(self, path, method='GET', body=None, client=None):
+    async def api_request(self, path, method='GET', body=None, client=None):
         """Make an authenticated API request of the proxy."""
         client = client or AsyncHTTPClient()
         url = url_path_join(self.api_url, 'api/routes', path)
@@ -538,23 +574,25 @@ def api_request(self, path, method='GET', body=None, client=None):
                               self.auth_token)},
                           body=body,
                           )
+        async with self.semaphore:
+            result = await client.fetch(req)
+            return result
 
-        return client.fetch(req)
-
-    def add_route(self, routespec, target, data):
+    async def add_route(self, routespec, target, data):
         body = data or {}
         body['target'] = target
         body['jupyterhub'] = True
         path = self._routespec_to_chp_path(routespec)
-        return self.api_request(path,
-                                method='POST',
-                                body=body,
-                                )
+        await self.api_request(
+            path,
+            method='POST',
+            body=body,
+        )
 
-    def delete_route(self, routespec):
+    async def delete_route(self, routespec):
         path = self._routespec_to_chp_path(routespec)
         try:
-            return self.api_request(path, method='DELETE')
+            await self.api_request(path, method='DELETE')
         except HTTPError as e:
             if e.status_code == 404:
                 # Warn about 404s because something might be wrong


Test patch: