Repository: matrix-org/synapse

Issue:
Synchrotron metric collection appears to fail if large numbers of background tasks present.
```
2020-05-28 21:35:06,821 - twisted - 192 - ERROR -  - ----------------------------------------
2020-05-28 21:35:06,837 - twisted - 192 - ERROR -  - Exception happened during processing of request from ('MONITORING-HOST', remote-port)
2020-05-28 21:35:06,877 - twisted - 192 - ERROR -  - Traceback (most recent call last):
2020-05-28 21:35:06,895 - twisted - 192 - ERROR -  -   File "/usr/local/lib/python3.7/socketserver.py", line 650, in process_request_thread
2020-05-28 21:35:06,922 - twisted - 192 - ERROR -  -     self.finish_request(request, client_address)
2020-05-28 21:35:06,948 - twisted - 192 - ERROR -  -   File "/usr/local/lib/python3.7/socketserver.py", line 360, in finish_request
2020-05-28 21:35:06,968 - twisted - 192 - ERROR -  -     self.RequestHandlerClass(request, client_address, self)
2020-05-28 21:35:06,982 - twisted - 192 - ERROR -  -   File "/usr/local/lib/python3.7/socketserver.py", line 720, in __init__
2020-05-28 21:35:07,013 - twisted - 192 - ERROR -  -     self.handle()
2020-05-28 21:35:07,041 - twisted - 192 - ERROR -  -   File "/usr/local/lib/python3.7/http/server.py", line 426, in handle
2020-05-28 21:35:07,056 - twisted - 192 - ERROR -  -     self.handle_one_request()
2020-05-28 21:35:07,080 - twisted - 192 - ERROR -  -   File "/usr/local/lib/python3.7/http/server.py", line 414, in handle_one_request
2020-05-28 21:35:07,100 - twisted - 192 - ERROR -  -     method()
2020-05-28 21:35:07,107 - twisted - 192 - ERROR -  -   File "/home/synapse/src/synapse/metrics/_exposition.py", line 205, in do_GET
2020-05-28 21:35:07,114 - twisted - 192 - ERROR -  -     output = generate_latest(registry, emit_help=emit_help)
2020-05-28 21:35:07,130 - twisted - 192 - ERROR -  -   File "/home/synapse/src/synapse/metrics/_exposition.py", line 116, in generate_latest
2020-05-28 21:35:07,139 - twisted - 192 - ERROR -  -     for metric in registry.collect():
2020-05-28 21:35:07,163 - twisted - 192 - ERROR -  -   File "/home/synapse/src/synapse/metrics/__init__.py", line 59, in collect
2020-05-28 21:35:07,175 - twisted - 192 - ERROR -  -     for metric in REGISTRY.collect():
2020-05-28 21:35:07,196 - twisted - 192 - ERROR -  -   File "/home/synapse/env-py37/lib/python3.7/site-packages/prometheus_client/registry.py", line 75, in collect
2020-05-28 21:35:07,216 - twisted - 192 - ERROR -  -     for metric in collector.collect():
2020-05-28 21:35:07,238 - twisted - 192 - ERROR -  -   File "/home/synapse/src/synapse/metrics/background_process_metrics.py", line 120, in collect
2020-05-28 21:35:07,260 - twisted - 192 - ERROR -  -     process.update_metrics()
2020-05-28 21:35:07,289 - twisted - 192 - ERROR -  -   File "/home/synapse/src/synapse/metrics/background_process_metrics.py", line 155, in update_metrics
2020-05-28 21:35:07,305 - twisted - 192 - ERROR -  -     _background_process_ru_utime.labels(self.desc).inc(diff.ru_utime)
2020-05-28 21:35:07,320 - twisted - 192 - ERROR -  -   File "/home/synapse/env-py37/lib/python3.7/site-packages/prometheus_client/metrics.py", line 243, in inc
2020-05-28 21:35:07,326 - twisted - 192 - ERROR -  -     raise ValueError('Counters can only be incremented by non-negative amounts.')
2020-05-28 21:35:07,334 - twisted - 192 - ERROR -  - ValueError: Counters can only be incremented by non-negative amounts.
2020-05-28 21:35:07,341 - twisted - 192 - ERROR -  - ----------------------------------------
```
At the time this fired, the background task count for this process was through the roof (150,000 +) 


Discussion:
This causes pages so we should do something here.

I *think* this is likely due to the replication RDATA linearizer so we could do something there to stop bg tasks stacking up, or we could try and figure out a way of fixing the more general problem.

Resolving patch:
diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py
--- a/synapse/metrics/background_process_metrics.py
+++ b/synapse/metrics/background_process_metrics.py
@@ -17,16 +17,18 @@
 import threading
 from asyncio import iscoroutine
 from functools import wraps
-from typing import Dict, Set
+from typing import TYPE_CHECKING, Dict, Optional, Set
 
-import six
-
-from prometheus_client.core import REGISTRY, Counter, GaugeMetricFamily
+from prometheus_client.core import REGISTRY, Counter, Gauge
 
 from twisted.internet import defer
 
 from synapse.logging.context import LoggingContext, PreserveLoggingContext
 
+if TYPE_CHECKING:
+    import resource
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -36,6 +38,12 @@
     ["name"],
 )
 
+_background_process_in_flight_count = Gauge(
+    "synapse_background_process_in_flight_count",
+    "Number of background processes in flight",
+    labelnames=["name"],
+)
+
 # we set registry=None in all of these to stop them getting registered with
 # the default registry. Instead we collect them all via the CustomCollector,
 # which ensures that we can update them before they are collected.
@@ -83,13 +91,17 @@
 # it's much simpler to do so than to try to combine them.)
 _background_process_counts = {}  # type: Dict[str, int]
 
-# map from description to the currently running background processes.
+# Set of all running background processes that became active active since the
+# last time metrics were scraped (i.e. background processes that performed some
+# work since the last scrape.)
 #
-# it's kept as a dict of sets rather than a big set so that we can keep track
-# of process descriptions that no longer have any active processes.
-_background_processes = {}  # type: Dict[str, Set[_BackgroundProcess]]
+# We do it like this to handle the case where we have a large number of
+# background processes stacking up behind a lock or linearizer, where we then
+# only need to iterate over and update metrics for the process that have
+# actually been active and can ignore the idle ones.
+_background_processes_active_since_last_scrape = set()  # type: Set[_BackgroundProcess]
 
-# A lock that covers the above dicts
+# A lock that covers the above set and dict
 _bg_metrics_lock = threading.Lock()
 
 
@@ -101,25 +113,16 @@ class _Collector(object):
     """
 
     def collect(self):
-        background_process_in_flight_count = GaugeMetricFamily(
-            "synapse_background_process_in_flight_count",
-            "Number of background processes in flight",
-            labels=["name"],
-        )
+        global _background_processes_active_since_last_scrape
 
-        # We copy the dict so that it doesn't change from underneath us.
-        # We also copy the process lists as that can also change
+        # We swap out the _background_processes set with an empty one so that
+        # we can safely iterate over the set without holding the lock.
         with _bg_metrics_lock:
-            _background_processes_copy = {
-                k: list(v) for k, v in six.iteritems(_background_processes)
-            }
+            _background_processes_copy = _background_processes_active_since_last_scrape
+            _background_processes_active_since_last_scrape = set()
 
-        for desc, processes in six.iteritems(_background_processes_copy):
-            background_process_in_flight_count.add_metric((desc,), len(processes))
-            for process in processes:
-                process.update_metrics()
-
-        yield background_process_in_flight_count
+        for process in _background_processes_copy:
+            process.update_metrics()
 
         # now we need to run collect() over each of the static Counters, and
         # yield each metric they return.
@@ -191,13 +194,10 @@ def run():
             _background_process_counts[desc] = count + 1
 
         _background_process_start_count.labels(desc).inc()
+        _background_process_in_flight_count.labels(desc).inc()
 
-        with LoggingContext(desc) as context:
+        with BackgroundProcessLoggingContext(desc) as context:
             context.request = "%s-%i" % (desc, count)
-            proc = _BackgroundProcess(desc, context)
-
-            with _bg_metrics_lock:
-                _background_processes.setdefault(desc, set()).add(proc)
 
             try:
                 result = func(*args, **kwargs)
@@ -214,10 +214,7 @@ def run():
             except Exception:
                 logger.exception("Background process '%s' threw an exception", desc)
             finally:
-                proc.update_metrics()
-
-                with _bg_metrics_lock:
-                    _background_processes[desc].remove(proc)
+                _background_process_in_flight_count.labels(desc).dec()
 
     with PreserveLoggingContext():
         return run()
@@ -238,3 +235,42 @@ def wrap_as_background_process_inner_2(*args, **kwargs):
         return wrap_as_background_process_inner_2
 
     return wrap_as_background_process_inner
+
+
+class BackgroundProcessLoggingContext(LoggingContext):
+    """A logging context that tracks in flight metrics for background
+    processes.
+    """
+
+    __slots__ = ["_proc"]
+
+    def __init__(self, name: str):
+        super().__init__(name)
+
+        self._proc = _BackgroundProcess(name, self)
+
+    def start(self, rusage: "Optional[resource._RUsage]"):
+        """Log context has started running (again).
+        """
+
+        super().start(rusage)
+
+        # We've become active again so we make sure we're in the list of active
+        # procs. (Note that "start" here means we've become active, as opposed
+        # to starting for the first time.)
+        with _bg_metrics_lock:
+            _background_processes_active_since_last_scrape.add(self._proc)
+
+    def __exit__(self, type, value, traceback) -> None:
+        """Log context has finished.
+        """
+
+        super().__exit__(type, value, traceback)
+
+        # The background process has finished. We explictly remove and manually
+        # update the metrics here so that if nothing is scraping metrics the set
+        # doesn't infinitely grow.
+        with _bg_metrics_lock:
+            _background_processes_active_since_last_scrape.discard(self._proc)
+
+        self._proc.update_metrics()


Test patch: