Repository: matrix-org/synapse

Issue:
Intermittent uncaught(?) errors causing sentry spam with `ValueError: Counters can only be incremented by non-negative amounts`
E.g. https://sentry.tools.element.io/organizations/element/issues/27/?project=2&query=is%3Aunresolved&statsPeriod=24h

```
2022-10-28 21:46:03,088 - twisted - 274 - ERROR - sentinel - Traceback (most recent call last):
2022-10-28 21:46:03,095 - twisted - 274 - ERROR - sentinel -   File "/usr/lib/python3.9/wsgiref/handlers.py", line 137, in run
2022-10-28 21:46:03,098 - twisted - 274 - ERROR - sentinel -     self.result = application(self.environ, self.start_response)
2022-10-28 21:46:03,100 - twisted - 274 - ERROR - sentinel -   File "/home/synapse/env-poetry/lib/python3.9/site-packages/prometheus_client/exposition.py", line 127, in prometheus_app
2022-10-28 21:46:03,101 - twisted - 274 - ERROR - sentinel -     status, headers, output = _bake_output(registry, accept_header, accept_encoding_header, params, disable_compression)
2022-10-28 21:46:03,104 - twisted - 274 - ERROR - sentinel -   File "/home/synapse/env-poetry/lib/python3.9/site-packages/prometheus_client/exposition.py", line 103, in _bake_output
2022-10-28 21:46:03,106 - twisted - 274 - ERROR - sentinel -     output = formatter(registry)
2022-10-28 21:46:03,109 - twisted - 274 - ERROR - sentinel -   File "/home/synapse/env-poetry/lib/python3.9/site-packages/prometheus_client/openmetrics/exposition.py", line 21, in generate_latest
2022-10-28 21:46:03,112 - twisted - 274 - ERROR - sentinel -     for metric in registry.collect():
2022-10-28 21:46:03,115 - twisted - 274 - ERROR - sentinel -   File "/home/synapse/src/synapse/metrics/__init__.py", line 70, in collect
2022-10-28 21:46:03,117 - twisted - 274 - ERROR - sentinel -     for metric in REGISTRY.collect():
2022-10-28 21:46:03,119 - twisted - 274 - ERROR - sentinel -   File "/home/synapse/env-poetry/lib/python3.9/site-packages/prometheus_client/registry.py", line 97, in collect
2022-10-28 21:46:03,122 - twisted - 274 - ERROR - sentinel -     yield from collector.collect()
2022-10-28 21:46:03,124 - twisted - 274 - ERROR - sentinel -   File "/home/synapse/src/synapse/metrics/background_process_metrics.py", line 145, in collect
2022-10-28 21:46:03,130 - twisted - 274 - ERROR - sentinel -     process.update_metrics()
2022-10-28 21:46:03,136 - twisted - 274 - ERROR - sentinel -   File "/home/synapse/src/synapse/metrics/background_process_metrics.py", line 177, in update_metrics
2022-10-28 21:46:03,147 - twisted - 274 - ERROR - sentinel -     _background_process_ru_utime.labels(self.desc).inc(diff.ru_utime)
2022-10-28 21:46:03,151 - twisted - 274 - ERROR - sentinel -   File "/home/synapse/env-poetry/lib/python3.9/site-packages/prometheus_client/me	trics.py", line 285, in inc
2022-10-28 21:46:03,156 - twisted - 274 - ERROR - sentinel -     raise ValueError('Counters can only be incremented by non-negative amounts.')
2022-10-28 21:46:03,160 - twisted - 274 - ERROR - sentinel - ValueError: Counters can only be incremented by non-negative amounts.
```

Presumably `diff.ru_utime` is negative depending how `self._reported_stats` and `new_stats` interact(??)

https://github.com/matrix-org/synapse/blob/50122754c8743df5c904e81b634fdfdeea64e795/synapse/metrics/background_process_metrics.py#L168-L185


Discussion:

Resolving patch:
diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py
--- a/synapse/metrics/background_process_metrics.py
+++ b/synapse/metrics/background_process_metrics.py
@@ -174,8 +174,10 @@ def update_metrics(self) -> None:
             diff = new_stats - self._reported_stats
         self._reported_stats = new_stats
 
-        _background_process_ru_utime.labels(self.desc).inc(diff.ru_utime)
-        _background_process_ru_stime.labels(self.desc).inc(diff.ru_stime)
+        # For unknown reasons, the difference in times can be negative. See comment in
+        # synapse.http.request_metrics.RequestMetrics.update_metrics.
+        _background_process_ru_utime.labels(self.desc).inc(max(diff.ru_utime, 0))
+        _background_process_ru_stime.labels(self.desc).inc(max(diff.ru_stime, 0))
         _background_process_db_txn_count.labels(self.desc).inc(diff.db_txn_count)
         _background_process_db_txn_duration.labels(self.desc).inc(
             diff.db_txn_duration_sec


Test patch:
diff --git a/tests/metrics/__init__.py b/tests/metrics/__init__.py
new file mode 100644
diff --git a/tests/metrics/test_background_process_metrics.py b/tests/metrics/test_background_process_metrics.py
new file mode 100644
--- /dev/null
+++ b/tests/metrics/test_background_process_metrics.py
@@ -0,0 +1,19 @@
+from unittest import TestCase as StdlibTestCase
+from unittest.mock import Mock
+
+from synapse.logging.context import ContextResourceUsage, LoggingContext
+from synapse.metrics.background_process_metrics import _BackgroundProcess
+
+
+class TestBackgroundProcessMetrics(StdlibTestCase):
+    def test_update_metrics_with_negative_time_diff(self) -> None:
+        """We should ignore negative reported utime and stime differences"""
+        usage = ContextResourceUsage()
+        usage.ru_stime = usage.ru_utime = -1.0
+
+        mock_logging_context = Mock(spec=LoggingContext)
+        mock_logging_context.get_resource_usage.return_value = usage
+
+        process = _BackgroundProcess("test process", mock_logging_context)
+        # Should not raise
+        process.update_metrics()
diff --git a/tests/test_metrics.py b/tests/metrics/test_metrics.py
similarity index 96%
rename from tests/test_metrics.py
rename to tests/metrics/test_metrics.py
--- a/tests/test_metrics.py
+++ b/tests/metrics/test_metrics.py
@@ -12,6 +12,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from typing_extensions import Protocol
+
 try:
     from importlib import metadata
 except ImportError:
@@ -52,7 +54,11 @@ def get_sample_labels_value(sample):
 
 class TestMauLimit(unittest.TestCase):
     def test_basic(self):
-        gauge = InFlightGauge(
+        class MetricEntry(Protocol):
+            foo: int
+            bar: int
+
+        gauge: InFlightGauge[MetricEntry] = InFlightGauge(
             "test1", "", labels=["test_label"], sub_metrics=["foo", "bar"]
         )
 
@@ -146,7 +152,7 @@ def test_cache_metric(self):
         Caches produce metrics reflecting their state when scraped.
         """
         CACHE_NAME = "cache_metrics_test_fgjkbdfg"
-        cache = DeferredCache(CACHE_NAME, max_entries=777)
+        cache: DeferredCache[str, str] = DeferredCache(CACHE_NAME, max_entries=777)
 
         items = {
             x.split(b"{")[0].decode("ascii"): x.split(b" ")[1].decode("ascii")