Repository: DataDog/dd-agent

Issue:
[zookeeper] metrics should be reported as counts not gauges
For instance:
- `zookeeper.bytes_received` is a monotonically increasing gauge since ZK start
- `zookeeper.bytes_sent`



Discussion:
this is a compatibility breaking change, we should use new metrics instead

Yes, that's also a good candidate for the "What to do when we change metrics in a backwards incompatible way" convo.


Resolving patch:
diff --git a/checks.d/zk.py b/checks.d/zk.py
--- a/checks.d/zk.py
+++ b/checks.d/zk.py
@@ -24,9 +24,9 @@
 
 '''
 # stdlib
+from StringIO import StringIO
 import re
 import socket
-from StringIO import StringIO
 import struct
 
 # project
@@ -38,6 +38,15 @@ class ZKConnectionFailure(Exception):
     pass
 
 
+class ZKMetric(tuple):
+    """
+    A Zookeeper metric.
+    Tuple with an optional metric type (default is 'gauge').
+    """
+    def __new__(cls, name, value, m_type="gauge"):
+        return super(ZKMetric, cls).__new__(cls, [name, value, m_type])
+
+
 class ZookeeperCheck(AgentCheck):
     version_pattern = re.compile(r'Zookeeper version: ([^.]+)\.([^.]+)\.([^-]+)', flags=re.I)
 
@@ -82,8 +91,9 @@ def check(self, instance):
             metrics, new_tags, mode = self.parse_stat(stat_out)
 
             # Write the data
-            for metric, value in metrics:
-                self.gauge(metric, value, tags=tags + new_tags)
+            for metric, value, m_type in metrics:
+                submit_metric = getattr(self, m_type)
+                submit_metric(metric, value, tags=tags + new_tags)
 
             if expected_mode:
                 if mode == expected_mode:
@@ -146,7 +156,7 @@ def parse_stat(self, buf):
         has_connections_val = version_tuple >= ('3', '4', '4')
 
         # Clients:
-        buf.readline() # skip the Clients: header
+        buf.readline()  # skip the Clients: header
         connections = 0
         client_line = buf.readline().strip()
         if client_line:
@@ -159,33 +169,35 @@ def parse_stat(self, buf):
         # Latency min/avg/max: -10/0/20007
         _, value = buf.readline().split(':')
         l_min, l_avg, l_max = [int(v) for v in value.strip().split('/')]
-        metrics.append(('zookeeper.latency.min', l_min))
-        metrics.append(('zookeeper.latency.avg', l_avg))
-        metrics.append(('zookeeper.latency.max', l_max))
+        metrics.append(ZKMetric('zookeeper.latency.min', l_min))
+        metrics.append(ZKMetric('zookeeper.latency.avg', l_avg))
+        metrics.append(ZKMetric('zookeeper.latency.max', l_max))
 
         # Received: 101032173
         _, value = buf.readline().split(':')
-        metrics.append(('zookeeper.bytes_received', long(value.strip())))
+        metrics.append(ZKMetric('zookeeper.bytes_received', long(value.strip())))
+        metrics.append(ZKMetric('zookeeper.bytes_received_per_second', long(value.strip()), "rate"))
 
         # Sent: 1324
         _, value = buf.readline().split(':')
-        metrics.append(('zookeeper.bytes_sent', long(value.strip())))
+        metrics.append(ZKMetric('zookeeper.bytes_sent', long(value.strip())))
+        metrics.append(ZKMetric('zookeeper.bytes_sent_per_second', long(value.strip()), "rate"))
 
         if has_connections_val:
             # Connections: 1
             _, value = buf.readline().split(':')
-            metrics.append(('zookeeper.connections', int(value.strip())))
+            metrics.append(ZKMetric('zookeeper.connections', int(value.strip())))
         else:
             # If the zk version doesnt explicitly give the Connections val,
             # use the value we computed from the client list.
-            metrics.append(('zookeeper.connections', connections))
+            metrics.append(ZKMetric('zookeeper.connections', connections))
 
         # Outstanding: 0
         _, value = buf.readline().split(':')
         # Fixme: This metric name is wrong. It should be removed in a major version of the agent
         # See https://github.com/DataDog/dd-agent/issues/1383
-        metrics.append(('zookeeper.bytes_outstanding', long(value.strip())))
-        metrics.append(('zookeeper.outstanding_requests', long(value.strip())))
+        metrics.append(ZKMetric('zookeeper.bytes_outstanding', long(value.strip())))
+        metrics.append(ZKMetric('zookeeper.outstanding_requests', long(value.strip())))
 
         # Zxid: 0x1034799c7
         _, value = buf.readline().split(':')
@@ -198,8 +210,8 @@ def parse_stat(self, buf):
         # the lower order 4 bytes is the count
         (zxid_count,) = struct.unpack('>i', zxid_bytes[4:8])
 
-        metrics.append(('zookeeper.zxid.epoch', zxid_epoch))
-        metrics.append(('zookeeper.zxid.count', zxid_count))
+        metrics.append(ZKMetric('zookeeper.zxid.epoch', zxid_epoch))
+        metrics.append(ZKMetric('zookeeper.zxid.count', zxid_count))
 
         # Mode: leader
         _, value = buf.readline().split(':')
@@ -208,6 +220,6 @@ def parse_stat(self, buf):
 
         # Node count: 487
         _, value = buf.readline().split(':')
-        metrics.append(('zookeeper.nodes', long(value.strip())))
+        metrics.append(ZKMetric('zookeeper.nodes', long(value.strip())))
 
         return metrics, tags, mode


Test patch:
diff --git a/tests/checks/integration/test_zk.py b/tests/checks/integration/test_zk.py
--- a/tests/checks/integration/test_zk.py
+++ b/tests/checks/integration/test_zk.py
@@ -37,6 +37,8 @@ class ZooKeeperTestCase(AgentCheckTest):
         'zookeeper.latency.max',
         'zookeeper.bytes_received',
         'zookeeper.bytes_sent',
+        'zookeeper.bytes_received_per_second',
+        'zookeeper.bytes_sent_per_second',
         'zookeeper.connections',
         'zookeeper.connections',
         'zookeeper.bytes_outstanding',
@@ -53,7 +55,7 @@ def test_check(self):
         config = {
             'instances': [self.CONFIG]
         }
-        self.run_check(config)
+        self.run_check_twice(config)
 
         # Test metrics
         for mname in self.METRICS:
@@ -67,7 +69,7 @@ def test_check(self):
 
     def test_wrong_expected_mode(self):
         """
-        Raise a 'critical' service check when ZooKeeper is not in the expected mode
+        Raise a 'critical' service check when ZooKeeper is not in the expected mode.
         """
         config = {
             'instances': [self.WRONG_EXPECTED_MODE]
@@ -79,7 +81,7 @@ def test_wrong_expected_mode(self):
 
     def test_error_state(self):
         """
-        Raise a 'critical' service check when ZooKeeper is in an error state
+        Raise a 'critical' service check when ZooKeeper is in an error state.
         """
         config = {
             'instances': [self.CONNECTION_FAILURE_CONFIG]