Repository: tornadoweb/tornado
Pull request #2330 (2018-03-27 11:07:59)

Support jitter in PeriodicCallback
A performance issue I have run into in several tornado applications is alignment of callbacks under load, causing brief periods of unresponsiveness at regular intervals, e.g. several similar PeriodicCallbacks aligning so that they fire in the same eventloop iteration. A solution that I have applied on a number of occasions is adding jitter to my periodic events, to reduce the occurrence of alignment. I thought it might be nice to support this in the default PeriodicCallback class.

`jitter` is a fraction and `callback_time` is used as the lower bound, so callbacks are scheduled on the interval

```python
[callback_time, callback_time * (1 + jitter)]
```

bdarnell: I was also just in PeriodicCallback to fix #2333 and did some refactoring to make testing easier. You'll have to rebase onto #2338 when it merges but the result will be simpler (and the tests won't be skipped on python 3). 
bdarnell: OK, #2338 is merged. It had some changes so you'll need to rebase again (and if you don't mind, please squash your commits into one when you do so). Other than that I think this is ready to merge. 

Patch:
diff --git a/tornado/ioloop.py b/tornado/ioloop.py
index 48700139..123f2ba5 100644
--- a/tornado/ioloop.py
+++ b/tornado/ioloop.py
@@ -47,6 +47,7 @@ import threading
 import time
 import traceback
 import math
+import random
 
 from tornado.concurrent import Future, is_future, chain_future, future_set_exc_info, future_add_done_callback  # noqa: E501
 from tornado.log import app_log, gen_log
@@ -1161,6 +1162,14 @@ class PeriodicCallback(object):
     Note that the timeout is given in milliseconds, while most other
     time-related functions in Tornado use seconds.
 
+    If ``jitter`` is specified, each callback time will be randomly selected
+    within a window of ``jitter * callback_time`` milliseconds.
+    Jitter can be used to reduce alignment of events with similar periods.
+    A jitter of 0.1 means allowing a 10% variation in callback time.
+    The window is centered on ``callback_time`` so the total number of calls
+    within a given interval should not be significantly affected by adding
+    jitter.
+
     If the callback runs for longer than ``callback_time`` milliseconds,
     subsequent invocations will be skipped to get back on schedule.
 
@@ -1168,12 +1177,16 @@ class PeriodicCallback(object):
 
     .. versionchanged:: 5.0
        The ``io_loop`` argument (deprecated since version 4.1) has been removed.
+
+    .. versionchanged:: 5.1
+       The ``jitter`` argument is added.
     """
-    def __init__(self, callback, callback_time):
+    def __init__(self, callback, callback_time, jitter=0):
         self.callback = callback
         if callback_time <= 0:
             raise ValueError("Periodic callback must have a positive callback_time")
         self.callback_time = callback_time
+        self.jitter = jitter
         self._running = False
         self._timeout = None
 
@@ -1218,6 +1231,9 @@ class PeriodicCallback(object):
 
     def _update_next(self, current_time):
         callback_time_sec = self.callback_time / 1000.0
+        if self.jitter:
+            # apply jitter fraction
+            callback_time_sec *= 1 + (self.jitter * (random.random() - 0.5))
         if self._next_timeout <= current_time:
             # The period should be measured from the start of one call
             # to the start of the next. If one call takes too long,


Test patch:
diff --git a/tornado/test/ioloop_test.py b/tornado/test/ioloop_test.py
index 9f7c1847..b1b22b1a 100644
--- a/tornado/test/ioloop_test.py
+++ b/tornado/test/ioloop_test.py
@@ -10,6 +10,13 @@ import sys
 import threading
 import time
 import types
+try:
+    from unittest import mock  # type: ignore
+except ImportError:
+    try:
+        import mock  # type: ignore
+    except ImportError:
+        mock = None
 
 from tornado.escape import native_str
 from tornado import gen
@@ -845,6 +852,20 @@ class TestPeriodicCallbackMath(unittest.TestCase):
         self.assertEqual(self.simulate_calls(pc, [-100, 0, 0]),
                          [1010, 1020, 1030])
 
+    @unittest.skipIf(mock is None, 'mock package not present')
+    def test_jitter(self):
+        random_times = [0.5, 1, 0, 0.75]
+        expected = [1010, 1022.5, 1030, 1041.25]
+        call_durations = [0] * len(random_times)
+        pc = PeriodicCallback(None, 10000, jitter=0.5)
+
+        def mock_random():
+            return random_times.pop(0)
+        with mock.patch('random.random', mock_random):
+            self.assertEqual(self.simulate_calls(pc, call_durations),
+                             expected)
+
+
 class TestIOLoopConfiguration(unittest.TestCase):
     def run_python(self, *statements):
         statements = [