Repository: prometheus/client_python
Pull request #301 (2018-08-31 07:40:29)

Add detailed Garbage Collection statistics.
We've found this useful in monitoring pythons Garbage Collection. If you are
interested in including this:

- Should the name default to python_gc perhaps?
- Are you happy for these all to be histograms?
- If so, how would you prefer the buckets to be set up (presumably defaulted and
  pass in to the constructor?)
- I'm not actually sure if having uncollected as a histogram is of much use. Uncollected
  should be monitored, but histogram is probably overkill.
- I'll fill out the tests?

tcolgate: @brian-brazil removed the configurable namespace (this was cargo-culted from another module in this package). I've also removed the bucket specification for the timings.

re: other packages. The gc.callbacks was added in python 3.3, the gc package is certainly in 2.7 and the docs allude to it having been in as far as 2.2.

I've tested the basic import of gc on jython (2.5.3 available on debian), and this appears to work without problems.

What is your minimal supported version?
tcolgate: @brian-brazil udpated to include a test.
pypy has it's own gc, those stats wont get picked up here. Perhaps an interested party will come along.

tcolgate: It would break if no gc module exists, but jython, python and pypy all have one, I can probably do something to work around the eventuality if you prefer.
brian-brazil: If the main implementations have one since 2.6 then we're good for now, we can wait until someone complains to see if it's an actual problem.
tcolgate: The docs for the gc module in 2.7 have some functions labeled as "new in 2.2", and I'd seen some other references to 2.2 as well.
tcolgate: @brian-brazil I've rebased this on master. Reviewed all the comments and , so far as I can tell, they are all resolved.

tcolgate: @brian-brazil updated.

Patch:
diff --git a/prometheus_client/__init__.py b/prometheus_client/__init__.py
index 8208d45..3a23935 100644
--- a/prometheus_client/__init__.py
+++ b/prometheus_client/__init__.py
@@ -4,6 +4,7 @@ from . import core
 from . import exposition
 from . import process_collector
 from . import platform_collector
+from . import gc_collector
 
 __all__ = ['Counter', 'Gauge', 'Summary', 'Histogram', 'Info', 'Enum']
 
@@ -35,6 +36,8 @@ PROCESS_COLLECTOR = process_collector.PROCESS_COLLECTOR
 PlatformCollector = platform_collector.PlatformCollector
 PLATFORM_COLLECTOR = platform_collector.PLATFORM_COLLECTOR
 
+GCCollector = gc_collector.GCCollector
+GC_COLLECTOR = gc_collector.GC_COLLECTOR
 
 if __name__ == '__main__':
     c = Counter('cc', 'A counter')
diff --git a/prometheus_client/gc_collector.py b/prometheus_client/gc_collector.py
new file mode 100644
index 0000000..34efade
--- /dev/null
+++ b/prometheus_client/gc_collector.py
@@ -0,0 +1,59 @@
+#!/usr/bin/python
+
+from __future__ import unicode_literals
+
+import gc
+import time
+
+from . import core
+
+class GCCollector(object):
+    """Collector for Garbage collection statistics."""
+    def __init__(self, registry=core.REGISTRY, gc=gc):
+        if not hasattr(gc, 'callbacks'):
+            return
+
+        collected = core.Histogram(
+           'python_gc_collected_objects',
+           'Objects collected during gc',
+           ['generation'],
+           buckets=[500, 1000, 5000, 10000, 50000],
+           registry=registry
+        )
+
+        uncollectable = core.Histogram(
+           'python_gc_uncollectable_objects',
+           'Uncollectable object found during GC',
+           ['generation'],
+           buckets=[500, 1000, 5000, 10000, 50000],
+           registry=registry
+        )
+
+        latency = core.Histogram(
+           'python_gc_duration_seconds',
+           'Time spent in garbage collection',
+           ['generation'],
+           registry=registry
+        )
+
+        times = {}
+
+        def _cb(phase, info):
+            gen = info['generation']
+
+            if phase == 'start':
+                times[gen] = time.time()
+
+            if phase == 'stop':
+                delta = time.time() - times[gen]
+                latency.labels(gen).observe(delta)
+                if 'collected' in info:
+                    collected.labels(gen).observe(info['collected'])
+                if 'uncollectable' in info:
+                    uncollectable.labels(gen).observe(info['uncollectable'])
+
+        gc.callbacks.append(_cb)
+
+
+GC_COLLECTOR = GCCollector()
+"""Default GCCollector in default Registry REGISTRY."""


Test patch:
diff --git a/tests/test_gc_collector.py b/tests/test_gc_collector.py
new file mode 100644
index 0000000..573a7da
--- /dev/null
+++ b/tests/test_gc_collector.py
@@ -0,0 +1,54 @@
+from __future__ import unicode_literals
+
+import unittest
+
+from prometheus_client import CollectorRegistry, GCCollector
+
+
+class TestGCCollector(unittest.TestCase):
+    def setUp(self):
+        self.registry = CollectorRegistry()
+        self.gc = _MockGC()
+
+    def test_working(self):
+        collector = GCCollector(registry=self.registry, gc=self.gc)
+        self.gc.start_gc({'generation': 0})
+        self.gc.stop_gc({'generation': 0, 'collected': 10, 'uncollectable': 2})
+
+        self.assertEqual(1,
+            self.registry.get_sample_value(
+                'python_gc_duration_seconds_count',
+                labels={"generation": "0"}))
+
+        self.assertEqual(1,
+            self.registry.get_sample_value(
+                'python_gc_collected_objects_count',
+                labels={"generation": "0"}))
+
+        self.assertEqual(1,
+            self.registry.get_sample_value(
+                'python_gc_uncollectable_objects_count',
+                labels={"generation": "0"}))
+
+        self.assertEqual(10,
+            self.registry.get_sample_value(
+                'python_gc_collected_objects_sum',
+                labels={"generation": "0"}))
+
+        self.assertEqual(2,
+            self.registry.get_sample_value(
+                'python_gc_uncollectable_objects_sum',
+                labels={"generation": "0"}))
+
+
+class _MockGC(object):
+    def __init__(self):
+        self.callbacks = []
+
+    def start_gc(self, info):
+        for cb in self.callbacks:
+            cb('start', info)
+
+    def stop_gc(self, info):
+        for cb in self.callbacks:
+            cb('stop', info)