Repository: prometheus/client_python
Pull request #19 (2015-04-06 11:06:57)

Standard exports
Andrea's changes with my code review comments.


Patch:
diff --git a/AUTHORS.md b/AUTHORS.md
index d51b46b..daa8ceb 100644
--- a/AUTHORS.md
+++ b/AUTHORS.md
@@ -5,4 +5,5 @@ Maintainers of this repository:
 The following individuals have contributed code to this repository
 (listed in alphabetical order):
 
+* Andrea Fagan <andreafagan28@gmail.com>
 * Brian Brazil <brian.brazil@gmail.com>
diff --git a/README.md b/README.md
index fedfd1e..ed9524c 100644
--- a/README.md
+++ b/README.md
@@ -133,6 +133,12 @@ c.labels('get', '/').inc()
 c.labels('post', '/submit').inc()
 ```
 
+### Process Collector
+
+The Python Client automatically exports metrics about process CPU usage, RAM,
+file descriptors and start time. These all have the prefix `process\_`, and
+are only currently available on Linux.
+
 ## Exporting
 
 There are several options for exporting metrics.
@@ -180,4 +186,5 @@ g.set(1)
 write_to_textfile('/configured/textfile/path/raid.prom', registry)
 ```
 
-A separate registry is used, as the default registry may contain other metrics.
+A separate registry is used, as the default registry may contain other metrics
+such as those from the Process Collector.
diff --git a/prometheus_client/__init__.py b/prometheus_client/__init__.py
index 0407577..232c057 100644
--- a/prometheus_client/__init__.py
+++ b/prometheus_client/__init__.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 
 import copy
 import re
+import resource
 import os
 import time
 import threading
@@ -419,7 +420,7 @@ def generate_latest(registry=REGISTRY):
                 labelstr = '{{{0}}}'.format(','.join(
                     ['{0}="{1}"'.format(
                      k, v.replace('\\', r'\\').replace('\n', r'\n').replace('"', r'\"'))
-                     for k, v in labels.items()]))
+                     for k, v in sorted(labels.items())]))
             else:
                 labelstr = ''
             output.append('{0}{1} {2}\n'.format(name, labelstr, _floatToGoString(value)))
@@ -446,6 +447,83 @@ def write_to_textfile(path, registry):
     os.rename(tmppath, path)
 
 
+class ProcessCollector(object):
+    """Collector for Standard Exports such as cpu and memory."""
+    def __init__(self, namespace='', pid='self', proc='/proc', registry=REGISTRY):
+        self._namespace = namespace
+        self._pid = os.path.join(proc, str(pid))
+        self._proc = proc
+        self._pagesize = resource.getpagesize()
+        if namespace:
+            self._prefix = namespace + '_process_'
+        else:
+            self._prefix = 'process_'
+        self._ticks = 100.0
+        try:
+            self._ticks = os.sysconf('SC_CLK_TCK')
+        except (ValueError, TypeError):
+            pass
+
+        # This is used to test if we can access /proc.
+        self._btime = 0
+        try:
+            self._btime = self._boot_time()
+        except IOError:
+            pass
+        if registry:
+          registry.register(self)
+
+    def _boot_time(self):
+        with open(os.path.join(self._proc, 'stat')) as stat:
+            for line in stat:
+                if line.startswith('btime '):
+                    return float(line.split()[1])
+
+    def collect(self):
+        if not self._btime:
+            return []
+
+        result = []
+        try:
+            with open(os.path.join(self._pid, 'stat')) as stat:
+                parts = (stat.read().split(')')[-1].split())
+            vmem = Metric(self._prefix + 'virtual_memory_bytes', 'Virtual memory size in bytes', 'gauge')
+            vmem.add_sample(self._prefix + 'virtual_memory_bytes', {}, float(parts[20]))
+            rss = Metric(self._prefix + 'resident_memory_bytes', 'Resident memory size in bytes', 'gauge')
+            rss.add_sample(self._prefix + 'resident_memory_bytes', {}, float(parts[21]) * self._pagesize)
+            start_time = Metric(self._prefix + 'start_time_seconds',
+                                'Start time of the process since unix epoch in seconds.', 'gauge')
+            start_time_secs = float(parts[19]) / self._ticks
+            start_time.add_sample(self._prefix + 'start_time_seconds',{} , start_time_secs + self._btime)
+            utime = float(parts[11]) / self._ticks
+            stime = float(parts[12]) / self._ticks
+            cpu = Metric(self._prefix + 'cpu_seconds_total',
+                         'Total user and system CPU time spent in seconds.', 'counter')
+            cpu.add_sample(self._prefix + 'cpu_seconds_total', {}, utime + stime)
+            result.extend([vmem, rss, start_time, cpu])
+        except IOError:
+            pass
+
+        try:
+            max_fds = Metric(self._prefix + 'max_fds', 'Maximum number of open file descriptors.', 'gauge')
+            with open(os.path.join(self._pid, 'limits')) as limits:
+                for line in limits:
+                    if line.startswith('Max open file'):
+                        max_fds.add_sample(self._prefix + 'max_fds', {}, float(line.split()[3]))
+                        break
+            open_fds = Metric(self._prefix + 'open_fds', 'Number of open file descriptors.', 'gauge')
+            open_fds.add_sample(self._prefix + 'open_fds', {}, len(os.listdir(os.path.join(self._pid, 'fd'))))
+            result.extend([open_fds, max_fds])
+        except IOError:
+            pass
+
+        return result
+
+
+PROCESS_COLLECTOR = ProcessCollector()
+"""Default ProcessCollector in default Registry REGISTRY."""
+
+
 if __name__ == '__main__':
     c = Counter('cc', 'A counter')
     c.inc()


Test patch:
diff --git a/tests/proc/26231/fd/0 b/tests/proc/26231/fd/0
new file mode 100644
index 0000000..e69de29
diff --git a/tests/proc/26231/fd/1 b/tests/proc/26231/fd/1
new file mode 100644
index 0000000..e69de29
diff --git a/tests/proc/26231/fd/2 b/tests/proc/26231/fd/2
new file mode 100644
index 0000000..e69de29
diff --git a/tests/proc/26231/fd/3 b/tests/proc/26231/fd/3
new file mode 100644
index 0000000..e69de29
diff --git a/tests/proc/26231/fd/4 b/tests/proc/26231/fd/4
new file mode 100644
index 0000000..e69de29
diff --git a/tests/proc/26231/limits b/tests/proc/26231/limits
new file mode 100644
index 0000000..d477b9b
--- /dev/null
+++ b/tests/proc/26231/limits
@@ -0,0 +1,15 @@
+Limit                     Soft Limit           Hard Limit           Units
+Max cpu time              unlimited            unlimited            seconds
+Max file size             unlimited            unlimited            bytes
+Max data size             unlimited            unlimited            bytes
+Max stack size            8388608              unlimited            bytes
+Max core file size        0                    unlimited            bytes
+Max resident set          unlimited            unlimited            bytes
+Max processes             62898                62898                processes
+Max open files            2048                 4096                 files
+Max locked memory         65536                65536                bytes
+Max address space         unlimited            unlimited            bytes
+Max file locks            unlimited            unlimited            locks
+Max pending signals       62898                62898                signals
+Max msgqueue size         819200               819200               bytes
+Max nice priority         0                    0
diff --git a/tests/proc/26231/stat b/tests/proc/26231/stat
new file mode 100644
index 0000000..438aaa9
--- /dev/null
+++ b/tests/proc/26231/stat
@@ -0,0 +1,1 @@
+26231 (vim) R 5392 7446 5392 34835 7446 4218880 32533 309516 26 82 1677 44 158 99 20 0 1 0 82375 56274944 1981 18446744073709551615 4194304 6294284 140736914091744 140736914087944 139965136429984 0 0 12288 1870679807 0 0 0 17 0 0 0 31 0 0 8391624 8481048 16420864 140736914093252 140736914093279 140736914093279 140736914096107 0
diff --git a/tests/proc/584/stat b/tests/proc/584/stat
new file mode 100644
index 0000000..af0d385
--- /dev/null
+++ b/tests/proc/584/stat
@@ -0,0 +1,1 @@
+1020 ((a b ) ( c d) ) R 28378 1020 28378 34842 1020 4218880 286 0 0 0 0 0 0 0 20 0 1 0 10839175 10395648 155 18446744073709551615 4194304 4238788 140736466511168 140736466511168 140609271124624 0 0 0 0 0 0 0 17 5 0 0 0 0 0 6336016 6337300 25579520 140736466515030 140736466515061 140736466515061 140736466518002 0
diff --git a/tests/proc/stat b/tests/proc/stat
new file mode 100644
index 0000000..32fd6d2
--- /dev/null
+++ b/tests/proc/stat
@@ -0,0 +1,16 @@
+cpu  301854 612 111922 8979004 3552 2 3944 0 0 0
+cpu0 44490 19 21045 1087069 220 1 3410 0 0 0
+cpu1 47869 23 16474 1110787 591 0 46 0 0 0
+cpu2 46504 36 15916 1112321 441 0 326 0 0 0
+cpu3 47054 102 15683 1113230 533 0 60 0 0 0
+cpu4 28413 25 10776 1140321 217 0 8 0 0 0
+cpu5 29271 101 11586 1136270 672 0 30 0 0 0
+cpu6 29152 36 10276 1139721 319 0 29 0 0 0
+cpu7 29098 268 10164 1139282 555 0 31 0 0 0
+intr 8885917 17 0 0 0 0 0 0 0 1 79281 0 0 0 0 0 0 0 231237 0 0 0 0 250586 103 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 223424 190745 13 906 1283803 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0intr 8885917 17 0 0 0 0 0 0 0 1 79281 0 0 0 0 00 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
+ctxt 38014093
+btime 1418183276
+processes 26442
+procs_running 2
+procs_blocked 0
+softirq 5057579 250191 1481983 1647 211099 186066 0 1783454 622196 12499 508444
diff --git a/tests/test_client.py b/tests/test_client.py
index a28f3f4..385998e 100644
--- a/tests/test_client.py
+++ b/tests/test_client.py
@@ -1,8 +1,10 @@
 from __future__ import unicode_literals
+import os
 import unittest
 
+
 from prometheus_client import Gauge, Counter, Summary, Histogram
-from prometheus_client import CollectorRegistry, generate_latest
+from prometheus_client import CollectorRegistry, generate_latest, ProcessCollector
 
 
 class TestCounter(unittest.TestCase):
@@ -266,5 +268,64 @@ class TestGenerateText(unittest.TestCase):
         self.assertEqual(b'# HELP cc A\\ncount\\\\er\n# TYPE cc counter\ncc{a="\\\\x\\n\\""} 1.0\n', generate_latest(self.registry))
 
 
+class TestProcessCollector(unittest.TestCase):
+    def setUp(self):
+        self.registry = CollectorRegistry()
+        self.test_proc = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'proc')
+
+    def test_working(self):
+        collector = ProcessCollector(proc=self.test_proc, pid=26231, registry=self.registry)
+        collector._pagesize = 4096
+        collector._ticks = 100
+
+        self.assertEqual(17.21, self.registry.get_sample_value('process_cpu_seconds_total'))
+        self.assertEqual(56274944.0, self.registry.get_sample_value('process_virtual_memory_bytes'))
+        self.assertEqual(8114176, self.registry.get_sample_value('process_resident_memory_bytes'))
+        self.assertEqual(1418184099.75, self.registry.get_sample_value('process_start_time_seconds'))
+        self.assertEqual(2048.0, self.registry.get_sample_value('process_max_fds'))
+        self.assertEqual(5.0, self.registry.get_sample_value('process_open_fds'))
+        self.assertEqual(None, self.registry.get_sample_value('process_fake_namespace'))
+
+    def test_namespace(self):
+        collector = ProcessCollector(proc=self.test_proc, pid=26231, registry=self.registry, namespace='n')
+        collector._pagesize = 4096
+        collector._ticks = 100
+
+        self.assertEqual(17.21, self.registry.get_sample_value('n_process_cpu_seconds_total'))
+        self.assertEqual(56274944.0, self.registry.get_sample_value('n_process_virtual_memory_bytes'))
+        self.assertEqual(8114176, self.registry.get_sample_value('n_process_resident_memory_bytes'))
+        self.assertEqual(1418184099.75, self.registry.get_sample_value('n_process_start_time_seconds'))
+        self.assertEqual(2048.0, self.registry.get_sample_value('n_process_max_fds'))
+        self.assertEqual(5.0, self.registry.get_sample_value('n_process_open_fds'))
+        self.assertEqual(None, self.registry.get_sample_value('process_cpu_seconds_total'))
+
+    def test_working_584(self):
+        collector = ProcessCollector(proc=self.test_proc, pid=584, registry=self.registry)
+        collector._pagesize = 4096
+        collector._ticks = 100
+
+        self.assertEqual(0.0, self.registry.get_sample_value('process_cpu_seconds_total'))
+        self.assertEqual(10395648.0, self.registry.get_sample_value('process_virtual_memory_bytes'))
+        self.assertEqual(634880, self.registry.get_sample_value('process_resident_memory_bytes'))
+        self.assertEqual(1418291667.75, self.registry.get_sample_value('process_start_time_seconds'))
+        self.assertEqual(None, self.registry.get_sample_value('process_max_fds'))
+        self.assertEqual(None, self.registry.get_sample_value('process_open_fds'))
+
+    def test_working_fake_pid(self):
+        collector = ProcessCollector(proc=self.test_proc, pid=123, registry=self.registry)
+        collector._pagesize = 4096
+        collector._ticks = 100
+
+        self.assertEqual(None, self.registry.get_sample_value('process_cpu_seconds_total'))
+        self.assertEqual(None, self.registry.get_sample_value('process_virtual_memory_bytes'))
+        self.assertEqual(None, self.registry.get_sample_value('process_resident_memory_bytes'))
+        self.assertEqual(None, self.registry.get_sample_value('process_start_time_seconds'))
+        self.assertEqual(None, self.registry.get_sample_value('process_max_fds'))
+        self.assertEqual(None, self.registry.get_sample_value('process_open_fds'))
+        self.assertEqual(None, self.registry.get_sample_value('process_fake_namespace'))
+
+
+
+
 if __name__ == '__main__':
     unittest.main()