Repository: jupyterhub/jupyterhub

Issue:
Provice explicit Retry-After guidance when throttling users
When we throttle server starts because too many spawns are pending, we should explicitly provide users with guidance on how long to wait before retrying. This should be a uniform distribution with a min, max that is configured.

In general, if average server start time is X, we want users to try again, on average, after X. This lets us maximize throughput of users starting while minimizing user wait time.


Discussion:
I discovered this while performance testing!


Resolving patch:
diff --git a/jupyterhub/apihandlers/base.py b/jupyterhub/apihandlers/base.py
--- a/jupyterhub/apihandlers/base.py
+++ b/jupyterhub/apihandlers/base.py
@@ -72,6 +72,7 @@ def write_error(self, status_code, **kwargs):
         """Write JSON errors instead of HTML"""
         exc_info = kwargs.get('exc_info')
         message = ''
+        exception = None
         status_message = responses.get(status_code, 'Unknown Error')
         if exc_info:
             exception = exc_info[1]
@@ -85,6 +86,15 @@ def write_error(self, status_code, **kwargs):
             reason = getattr(exception, 'reason', '')
             if reason:
                 status_message = reason
+
+        self.set_header('Content-Type', 'application/json')
+        # allow setting headers from exceptions
+        # since exception handler clears headers
+        headers = getattr(exception, 'headers', None)
+        if headers:
+            for key, value in headers.items():
+                self.set_header(key, value)
+
         self.write(json.dumps({
             'status': status_code,
             'message': message or status_message,
diff --git a/jupyterhub/app.py b/jupyterhub/app.py
--- a/jupyterhub/app.py
+++ b/jupyterhub/app.py
@@ -36,7 +36,7 @@
 
 from traitlets import (
     Unicode, Integer, Dict, TraitError, List, Bool, Any,
-    Type, Set, Instance, Bytes, Float,
+    Tuple, Type, Set, Instance, Bytes, Float,
     observe, default,
 )
 from traitlets.config import Application, catch_config_error
@@ -616,6 +616,23 @@ def _authenticator_default(self):
         """
     ).tag(config=True)
 
+    spawn_throttle_retry_range = Tuple(
+        (30, 60),
+        help="""
+        (min seconds, max seconds) range to suggest a user wait before retrying.
+
+        When `concurrent_spawn_limit` is exceeded, spawning is throttled.
+        We suggest users wait random period of time within this range
+        before retrying.
+
+        A Retry-After header is set with a random value within this range.
+        Error pages will display a rounded version of this value.
+
+        The lower bound should ideally be approximately
+        the median spawn time for your deployment.
+        """
+    )
+
     active_server_limit = Integer(
         0,
         help="""
@@ -1423,6 +1440,7 @@ def init_tornado_settings(self):
             allow_named_servers=self.allow_named_servers,
             oauth_provider=self.oauth_provider,
             concurrent_spawn_limit=self.concurrent_spawn_limit,
+            spawn_throttle_retry_range=self.spawn_throttle_retry_range,
             active_server_limit=self.active_server_limit,
         )
         # allow configured settings to have priority
diff --git a/jupyterhub/handlers/base.py b/jupyterhub/handlers/base.py
--- a/jupyterhub/handlers/base.py
+++ b/jupyterhub/handlers/base.py
@@ -4,10 +4,12 @@
 # Distributed under the terms of the Modified BSD License.
 
 import copy
-import re
-import time
 from datetime import datetime, timedelta
 from http.client import responses
+import math
+import random
+import re
+import time
 from urllib.parse import urlparse, urlunparse, parse_qs, urlencode
 import uuid
 
@@ -526,17 +528,37 @@ async def spawn_single_user(self, user, server_name='', options=None):
         active_server_limit = self.active_server_limit
 
         if concurrent_spawn_limit and spawn_pending_count >= concurrent_spawn_limit:
-            self.log.info(
-                '%s pending spawns, throttling',
-                spawn_pending_count,
-            )
             SERVER_SPAWN_DURATION_SECONDS.labels(
                 status=ServerSpawnStatus.throttled
             ).observe(time.perf_counter() - spawn_start_time)
-            raise web.HTTPError(
+            # Suggest number of seconds client should wait before retrying
+            # This helps prevent thundering herd problems, where users simply
+            # immediately retry when we are overloaded.
+            retry_range = self.settings['spawn_throttle_retry_range']
+            retry_time = int(random.uniform(*retry_range))
+
+            # round suggestion to nicer human value (nearest 10 seconds or minute)
+            if retry_time <= 90:
+                # round human seconds up to nearest 10
+                human_retry_time = "%i0 seconds" % math.ceil(retry_time / 10.)
+            else:
+                # round number of minutes
+                human_retry_time = "%i minutes" % math.round(retry_time / 60.)
+
+            self.log.warning(
+                '%s pending spawns, throttling. Suggested retry in %s seconds.',
+                spawn_pending_count, retry_time,
+            )
+            err = web.HTTPError(
                 429,
-                "User startup rate limit exceeded. Try again in a few minutes.",
+                "Too many users trying to log in right now. Try again in {}.".format(human_retry_time)
             )
+            # can't call set_header directly here because it gets ignored
+            # when errors are raised
+            # we handle err.headers ourselves in Handler.write_error
+            err.headers = {'Retry-After': retry_time}
+            raise err
+
         if active_server_limit and active_count >= active_server_limit:
             self.log.info(
                 '%s servers active, no space available',
@@ -770,6 +792,13 @@ def write_error(self, status_code, **kwargs):
         )
 
         self.set_header('Content-Type', 'text/html')
+        # allow setting headers from exceptions
+        # since exception handler clears headers
+        headers = getattr(exception, 'headers', None)
+        if headers:
+            for key, value in headers.items():
+                self.set_header(key, value)
+
         # render the template
         try:
             html = self.render_template('%s.html' % status_code, **ns)


Test patch: