Repository: rucio/rucio

Issue:
duplicate time series in server-traces 
Motivation
----------
```
Internal Error
Duplicated timeseries in CollectorRegistry: {'rucio_trace_trace_total', 'rucio_trace_trace', 'rucio_trace_trace_created'}
  File "/usr/local/lib/python3.6/site-packages/rucio/web/rest/flaskapi/v1/common.py", line 54, in dispatch_request
    return super(ErrorHandlingMethodView, self).dispatch_request(*args, **kwargs)
  File "/usr/local/lib/python3.6/site-packages/flask/views.py", line 158, in dispatch_request
    return current_app.ensure_sync(meth)(*args, **kwargs)
  File "/usr/local/lib/python3.6/site-packages/rucio/web/rest/flaskapi/v1/traces.py", line 67, in post
    trace(payload=payload)
  File "/usr/local/lib/python3.6/site-packages/rucio/core/trace.py", line 301, in trace
    record_counter('trace.trace')
  File "/usr/local/lib/python3.6/site-packages/rucio/core/monitor.py", line 213, in record_counter
    COUNTERS[name] = counter = MultiCounter(statsd=name, labelnames=labels.keys() if labels else ())
  File "/usr/local/lib/python3.6/site-packages/rucio/core/monitor.py", line 153, in __init__
    self._prom = self.init_prometheus_metric(prom, self._documentation, labelnames=labelnames)
  File "/usr/local/lib/python3.6/site-packages/rucio/core/monitor.py", line 179, in init_prometheus_metric
    return Counter(name, documentation, labelnames=labelnames, registry=self._registry)
  File "/usr/local/lib/python3.6/site-packages/prometheus_client/metrics.py", line 134, in __init__
    registry.register(self)
  File "/usr/local/lib/python3.6/site-packages/prometheus_client/registry.py", line 31, in register
    duplicates))
```





Discussion:

Resolving patch:
diff --git a/lib/rucio/core/monitor.py b/lib/rucio/core/monitor.py
--- a/lib/rucio/core/monitor.py
+++ b/lib/rucio/core/monitor.py
@@ -86,6 +86,7 @@ def get(self):
 COUNTERS = {}
 GAUGES = {}
 TIMINGS = {}
+METRICS_LOCK = Lock()
 
 
 _HISTOGRAM_DEFAULT_BUCKETS = Histogram.DEFAULT_BUCKETS
@@ -221,7 +222,9 @@ def record_counter(name, delta=1, labels=None):
 
     counter = COUNTERS.get(name)
     if not counter:
-        COUNTERS[name] = counter = MultiCounter(statsd=name, labelnames=labels.keys() if labels else ())
+        with METRICS_LOCK:
+            if not COUNTERS.get(name):
+                COUNTERS[name] = counter = MultiCounter(statsd=name, labelnames=labels.keys() if labels else ())
 
     delta = abs(delta)
 
@@ -241,7 +244,9 @@ def record_gauge(name, value, labels=None):
     """
     gauge = GAUGES.get(name)
     if not gauge:
-        GAUGES[name] = gauge = MultiGauge(statsd=name, labelnames=labels.keys() if labels else ())
+        with METRICS_LOCK:
+            if not GAUGES.get(name):
+                GAUGES[name] = gauge = MultiGauge(statsd=name, labelnames=labels.keys() if labels else ())
 
     if labels:
         gauge.labels(**labels).set(value)
@@ -263,10 +268,11 @@ def record_timer(name: str,
     :param labels: labels used to parametrize the metric
     :param buckets: Optional iterable of histogram bucket separators.
     """
-    if name not in TIMINGS:
-        TIMINGS[name] = MultiTiming(statsd=name, labelnames=labels.keys() if labels else (), buckets=buckets)
-
-    histogram = TIMINGS[name]
+    histogram = TIMINGS.get(name)
+    if not histogram:
+        with METRICS_LOCK:
+            if not TIMINGS.get(name):
+                TIMINGS[name] = histogram = MultiTiming(statsd=name, labelnames=labels.keys() if labels else (), buckets=buckets)
 
     if labels:
         histogram.labels(**labels).observe(time)


Test patch: