Repository: DataDog/dd-agent

Issue:
[haproxy][fixme] `collate_status_tags_per_host` option
FIXME: See #2314 



Discussion:

Resolving patch:
diff --git a/checks.d/haproxy.py b/checks.d/haproxy.py
--- a/checks.d/haproxy.py
+++ b/checks.d/haproxy.py
@@ -26,6 +26,17 @@ class Services(object):
     ALL_STATUSES = (
         'up', 'open', 'down', 'maint', 'nolb'
     )
+    AVAILABLE = 'available'
+    UNAVAILABLE = 'unavailable'
+    COLLATED_STATUSES = (AVAILABLE, UNAVAILABLE)
+
+    STATUS_MAP = {
+        'up': AVAILABLE,
+        'open': AVAILABLE,
+        'down': UNAVAILABLE,
+        'maint': UNAVAILABLE,
+        'nolb': UNAVAILABLE
+    }
 
     STATUSES_TO_SERVICE_CHECK = {
         'UP': AgentCheck.OK,
@@ -87,6 +98,10 @@ def check(self, instance):
             instance.get('collect_status_metrics_by_host', False)
         )
 
+        collate_status_tags_per_host = _is_affirmative(
+            instance.get('collate_status_tags_per_host', False)
+        )
+
         count_status_by_service = _is_affirmative(
             instance.get('count_status_by_service', True)
         )
@@ -113,6 +128,7 @@ def check(self, instance):
             tag_service_check_by_host=tag_service_check_by_host,
             services_incl_filter=services_incl_filter,
             services_excl_filter=services_excl_filter,
+            collate_status_tags_per_host=collate_status_tags_per_host,
             count_status_by_service=count_status_by_service,
         )
 
@@ -133,7 +149,8 @@ def _fetch_data(self, url, username, password, verify):
     def _process_data(self, data, collect_aggregates_only, process_events, url=None,
                       collect_status_metrics=False, collect_status_metrics_by_host=False,
                       tag_service_check_by_host=False, services_incl_filter=None,
-                      services_excl_filter=None, count_status_by_service=True):
+                      services_excl_filter=None, collate_status_tags_per_host=False,
+                      count_status_by_service=True):
         ''' Main data-processing loop. For each piece of useful data, we'll
         either save a metric, save an event or both. '''
 
@@ -190,6 +207,7 @@ def _process_data(self, data, collect_aggregates_only, process_events, url=None,
                 self.hosts_statuses, collect_status_metrics_by_host,
                 services_incl_filter=services_incl_filter,
                 services_excl_filter=services_excl_filter,
+                collate_status_tags_per_host=collate_status_tags_per_host,
                 count_status_by_service=count_status_by_service
             )
 
@@ -303,67 +321,55 @@ def _process_backend_hosts_metric(self, hosts_statuses, services_incl_filter=Non
 
     def _process_status_metric(self, hosts_statuses, collect_status_metrics_by_host,
                                services_incl_filter=None, services_excl_filter=None,
-                               count_status_by_service=True):
-        agg_statuses = defaultdict(lambda: {'available': 0, 'unavailable': 0})
+                               collate_status_tags_per_host=False, count_status_by_service=True):
+        agg_statuses_counter = defaultdict(lambda: {status: 0 for status in Services.COLLATED_STATUSES})
 
-        # use a counter unless we have a unique tag set to gauge
-        counter = defaultdict(int)
-        if count_status_by_service and collect_status_metrics_by_host:
-            # `service` and `backend` tags will exist
-            counter = None
+        reported_statuses = Services.ALL_STATUSES
+        if collate_status_tags_per_host:
+            reported_statuses = Services.COLLATED_STATUSES
+        statuses_counter = defaultdict(lambda: {status: 0 for status in reported_statuses})
 
         for host_status, count in hosts_statuses.iteritems():
+            hostname = None
             try:
                 service, hostname, status = host_status
             except Exception:
+                if collect_status_metrics_by_host:
+                    self.warning('`collect_status_metrics_by_host` is enabled but no host info\
+                                 could be extracted from HAProxy stats endpoint for {0}'.format(service))
                 service, status = host_status
             status = status.lower()
 
-            tags = []
-            if count_status_by_service:
-                tags.append('service:%s' % service)
-
             if self._is_service_excl_filtered(service, services_incl_filter, services_excl_filter):
                 continue
 
-            if collect_status_metrics_by_host:
+            tags = []
+            if count_status_by_service:
+                tags.append('service:%s' % service)
+            if hostname:
                 tags.append('backend:%s' % hostname)
 
-            self._gauge_all_statuses(
-                "haproxy.count_per_status",
-                count, status, tags, counter
-            )
-
-            if 'up' in status or 'open' in status:
-                agg_statuses[service]['available'] += count
-            if 'down' in status or 'maint' in status or 'nolb' in status:
-                agg_statuses[service]['unavailable'] += count
+            counter_status = status
+            if collate_status_tags_per_host:
+                counter_status = Services.STATUS_MAP.get(status, status)
+            statuses_counter[tuple(tags)][counter_status] += count
 
-        if counter is not None:
-            # send aggregated counts as gauges
-            for key, count in counter.iteritems():
-                metric_name, tags = key[0], key[1]
-                self.gauge(metric_name, count, tags=tags)
-
-        for service in agg_statuses:
-            for status, count in agg_statuses[service].iteritems():
-                tags = ['status:%s' % status]
+            # Compute aggregates with collated statuses. If collate_status_tags_per_host is enabled we
+            # already send collated statuses with fine-grained tags, so no need to compute/send these aggregates
+            if not collate_status_tags_per_host:
+                agg_tags = []
                 if count_status_by_service:
-                    tags.append('service:%s' % service)
+                    agg_tags.append('service:%s' % service)
+                agg_statuses_counter[tuple(agg_tags)][Services.STATUS_MAP.get(status, status)] += count
 
-                self.gauge("haproxy.count_per_status", count, tags=tags)
-
-    def _gauge_all_statuses(self, metric_name, count, status, tags, counter):
-        if counter is not None:
-            counter_key = tuple([metric_name, tuple(tags + ['status:%s' % status])])
-            counter[counter_key] += count
-        else:
-            # assume we have enough context, just send a gauge
-            self.gauge(metric_name, count, tags + ['status:%s' % status])
+        for tags, count_per_status in statuses_counter.iteritems():
+            for status, count in count_per_status.iteritems():
+                self.gauge('haproxy.count_per_status', count, tags=tags + ('status:%s' % status, ))
 
-        for state in Services.ALL_STATUSES:
-            if state != status:
-                self.gauge(metric_name, 0, tags + ['status:%s' % state.replace(" ", "_")])
+        # Send aggregates
+        for service_tags, service_agg_statuses in agg_statuses_counter.iteritems():
+            for status, count in service_agg_statuses.iteritems():
+                self.gauge("haproxy.count_per_status", count, tags=service_tags + ('status:%s' % status, ))
 
     def _process_metrics(self, data, url, services_incl_filter=None,
                          services_excl_filter=None):


Test patch:
diff --git a/tests/checks/mock/test_haproxy.py b/tests/checks/mock/test_haproxy.py
new file mode 100644
--- /dev/null
+++ b/tests/checks/mock/test_haproxy.py
@@ -0,0 +1,141 @@
+import copy
+
+# 3p
+import mock
+
+# project
+from tests.checks.common import AgentCheckTest
+
+
+MOCK_DATA = """# pxname,svname,qcur,qmax,scur,smax,slim,stot,bin,bout,dreq,dresp,ereq,econ,eresp,wretr,wredis,status,weight,act,bck,chkfail,chkdown,lastchg,downtime,qlimit,pid,iid,sid,throttle,lbtot,tracked,type,rate,rate_lim,rate_max,check_status,check_code,check_duration,hrsp_1xx,hrsp_2xx,hrsp_3xx,hrsp_4xx,hrsp_5xx,hrsp_other,hanafail,req_rate,req_rate_max,req_tot,cli_abrt,srv_abrt,
+a,FRONTEND,,,1,2,12,1,11,11,0,0,0,,,,,OPEN,,,,,,,,,1,1,0,,,,0,1,0,2,,,,0,1,0,0,0,0,,1,1,1,,,
+a,BACKEND,0,0,0,0,12,0,11,11,0,0,,0,0,0,0,UP,0,0,0,,0,1221810,0,,1,1,0,,0,,1,0,,0,,,,0,0,0,0,0,0,,,,,0,0,
+b,FRONTEND,,,1,2,12,11,11,0,0,0,0,,,,,OPEN,,,,,,,,,1,2,0,,,,0,0,0,1,,,,,,,,,,,0,0,0,,,
+b,i-1,0,0,0,1,,1,1,0,,0,,0,0,0,0,UP,1,1,0,0,1,1,30,,1,3,1,,70,,2,0,,1,1,,0,,,,,,,0,,,,0,0,
+b,i-2,0,0,1,1,,1,1,0,,0,,0,0,0,0,UP,1,1,0,0,0,1,0,,1,3,2,,71,,2,0,,1,1,,0,,,,,,,0,,,,0,0,
+b,i-3,0,0,0,1,,1,1,0,,0,,0,0,0,0,UP,1,1,0,0,0,1,0,,1,3,3,,70,,2,0,,1,1,,0,,,,,,,0,,,,0,0,
+b,i-4,0,0,0,1,,1,1,0,,0,,0,0,0,0,DOWN,1,1,0,0,0,1,0,,1,3,3,,70,,2,0,,1,1,,0,,,,,,,0,,,,0,0,
+b,i-5,0,0,0,1,,1,1,0,,0,,0,0,0,0,MAINT,1,1,0,0,0,1,0,,1,3,3,,70,,2,0,,1,1,,0,,,,,,,0,,,,0,0,
+b,BACKEND,0,0,1,2,0,421,1,0,0,0,,0,0,0,0,UP,6,6,0,,0,1,0,,1,3,0,,421,,1,0,,1,,,,,,,,,,,,,,0,0,
+c,i-1,0,0,0,1,,1,1,0,,0,,0,0,0,0,UP,1,1,0,0,1,1,30,,1,3,1,,70,,2,0,,1,1,,0,,,,,,,0,,,,0,0,
+c,i-2,0,0,0,1,,1,1,0,,0,,0,0,0,0,DOWN,1,1,0,0,1,1,30,,1,3,1,,70,,2,0,,1,1,,0,,,,,,,0,,,,0,0,
+c,BACKEND,0,0,1,2,0,421,1,0,0,0,,0,0,0,0,UP,6,6,0,,0,1,0,,1,3,0,,421,,1,0,,1,,,,,,,,,,,,,,0,0,
+"""
+
+AGG_STATUSES_BY_SERVICE = (
+    (['status:available', 'service:a'], 1),
+    (['status:available', 'service:b'], 4),
+    (['status:unavailable', 'service:b'], 2),
+    (['status:available', 'service:c'], 1),
+    (['status:unavailable', 'service:c'], 1)
+)
+
+AGG_STATUSES = (
+    (['status:available'], 6),
+    (['status:unavailable'], 3)
+)
+
+
+class TestCheckHAProxy(AgentCheckTest):
+    CHECK_NAME = 'haproxy'
+
+    BASE_CONFIG = {
+        'init_config': None,
+        'instances': [
+            {
+                'url': 'http://localhost/admin?stats',
+                'collect_status_metrics': True,
+            }
+        ]
+    }
+
+    def _assert_agg_statuses(self, count_status_by_service=True, collate_status_tags_per_host=False):
+        expected_statuses = AGG_STATUSES_BY_SERVICE if count_status_by_service else AGG_STATUSES
+        for tags, value in expected_statuses:
+            if collate_status_tags_per_host:
+                # Assert that no aggregate statuses are sent
+                self.assertMetric('haproxy.count_per_status', tags=tags, count=0)
+            else:
+                self.assertMetric('haproxy.count_per_status', value=value, tags=tags)
+
+    @mock.patch('requests.get', return_value=mock.Mock(content=MOCK_DATA))
+    def test_count_per_status_agg_only(self, mock_requests):
+        config = copy.deepcopy(self.BASE_CONFIG)
+        # with count_status_by_service set to False
+        config['instances'][0]['count_status_by_service'] = False
+        self.run_check(config)
+
+        self.assertMetric('haproxy.count_per_status', value=2, tags=['status:open'])
+        self.assertMetric('haproxy.count_per_status', value=4, tags=['status:up'])
+        self.assertMetric('haproxy.count_per_status', value=2, tags=['status:down'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['status:maint'])
+        self.assertMetric('haproxy.count_per_status', value=0, tags=['status:nolb'])
+
+        self._assert_agg_statuses(count_status_by_service=False)
+
+    @mock.patch('requests.get', return_value=mock.Mock(content=MOCK_DATA))
+    def test_count_per_status_by_service(self, mock_requests):
+        self.run_check(self.BASE_CONFIG)
+
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['status:open', 'service:a'])
+        self.assertMetric('haproxy.count_per_status', value=3, tags=['status:up', 'service:b'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['status:open', 'service:b'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['status:down', 'service:b'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['status:maint', 'service:b'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['status:up', 'service:c'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['status:down', 'service:c'])
+
+        self._assert_agg_statuses()
+
+    @mock.patch('requests.get', return_value=mock.Mock(content=MOCK_DATA))
+    def test_count_per_status_by_service_and_host(self, mock_requests):
+        config = copy.deepcopy(self.BASE_CONFIG)
+        config['instances'][0]['collect_status_metrics_by_host'] = True
+        self.run_check(config)
+
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['backend:FRONTEND', 'status:open', 'service:a'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['backend:FRONTEND', 'status:open', 'service:b'])
+        for backend in ['i-1', 'i-2', 'i-3']:
+            self.assertMetric('haproxy.count_per_status', value=1, tags=['backend:%s' % backend, 'status:up', 'service:b'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['backend:i-4', 'status:down', 'service:b'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['backend:i-5', 'status:maint', 'service:b'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['backend:i-1', 'status:up', 'service:c'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['backend:i-2', 'status:down', 'service:c'])
+
+        self._assert_agg_statuses()
+
+    @mock.patch('requests.get', return_value=mock.Mock(content=MOCK_DATA))
+    def test_count_per_status_by_service_and_collate_per_host(self, mock_requests):
+        config = copy.deepcopy(self.BASE_CONFIG)
+        config['instances'][0]['collect_status_metrics_by_host'] = True
+        config['instances'][0]['collate_status_tags_per_host'] = True
+        self.run_check(config)
+
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['backend:FRONTEND', 'status:available', 'service:a'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['backend:FRONTEND', 'status:available', 'service:b'])
+        for backend in ['i-1', 'i-2', 'i-3']:
+            self.assertMetric('haproxy.count_per_status', value=1, tags=['backend:%s' % backend, 'status:available', 'service:b'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['backend:i-4', 'status:unavailable', 'service:b'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['backend:i-5', 'status:unavailable', 'service:b'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['backend:i-1', 'status:available', 'service:c'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['backend:i-2', 'status:unavailable', 'service:c'])
+
+        self._assert_agg_statuses(collate_status_tags_per_host=True)
+
+    @mock.patch('requests.get', return_value=mock.Mock(content=MOCK_DATA))
+    def test_count_per_status_collate_per_host(self, mock_requests):
+        config = copy.deepcopy(self.BASE_CONFIG)
+        config['instances'][0]['collect_status_metrics_by_host'] = True
+        config['instances'][0]['collate_status_tags_per_host'] = True
+        config['instances'][0]['count_status_by_service'] = False
+        self.run_check(config)
+
+        self.assertMetric('haproxy.count_per_status', value=2, tags=['backend:FRONTEND', 'status:available'])
+        self.assertMetric('haproxy.count_per_status', value=2, tags=['backend:i-1', 'status:available'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['backend:i-2', 'status:available'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['backend:i-2', 'status:unavailable'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['backend:i-3', 'status:available'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['backend:i-4', 'status:unavailable'])
+        self.assertMetric('haproxy.count_per_status', value=1, tags=['backend:i-5', 'status:unavailable'])
+
+        self._assert_agg_statuses(count_status_by_service=False, collate_status_tags_per_host=True)