Repository: DataDog/dd-agent

Issue:
wrong url in elastic.py can.connect monitor used for version > 0.90.10
The new URLS are /_nodes/stats?all=true

But the datadog uses sometimes the old one: /_cluster/nodes/stats?all=true

elastic.py 

_Old_ version: (only interesting parts)
/opt/datadog-agent/agent/checks.d/elastic.py

`````` python
if version >= [0,90,10]:
            if is_external:
                self.STATS_URL = "/_nodes/stats?all=true"
            else:
                self.STATS_URL = "/_nodes/_local/stats?all=true"
else:
            self.STATS_URL = "/_cluster/nodes/stats?all=true"
            self.NODES_URL = "/_cluster/nodes?network=true"```
``````

_fixed version_ should be: (only interesting parts)
/opt/datadog-agent/agent/checks.d/elastic.py

``` python
if version >= [0,90,10]:
            if is_external:
                self.STATS_URL = "/_nodes/stats?all=true"
            else:
                self.STATS_URL = "/_nodes/_local/stats?all=true"
else:
            if is_external:
                self.STATS_URL = "/_nodes/stats?all=true"
            else:
                self.STATS_URL = "/_nodes/_local/stats?all=true"
```



Discussion:
Thanks a lot for the feedback @flutist599 we will investigate.

This bug only occurs if there is a problem while getting the metrics.
Per Default the "version" variable is set to [0,0,0] ... and this is smaller then [0,90,10].
But the script should raise an error if it is not possible to get the metrics ... the current version goes ahead and uses therefore the wrong STATS_URL because the version is still at its default value of [0,0,0]


Resolving patch:
diff --git a/checks.d/elastic.py b/checks.d/elastic.py
--- a/checks.d/elastic.py
+++ b/checks.d/elastic.py
@@ -14,8 +14,8 @@
 from util import headers, Platform
 
 
-class NodeNotFound(Exception): pass
-
+class NodeNotFound(Exception):
+    pass
 
 ESInstanceConfig = namedtuple(
     'ESInstanceConfig', [
@@ -26,7 +26,7 @@ class NodeNotFound(Exception): pass
         'timeout',
         'url',
         'username',
-])
+    ])
 
 
 class ESCheck(AgentCheck):
@@ -35,7 +35,7 @@ class ESCheck(AgentCheck):
 
     DEFAULT_TIMEOUT = 5
 
-    STATS_METRICS = { # Metrics that are common to all Elasticsearch versions
+    STATS_METRICS = {  # Metrics that are common to all Elasticsearch versions
         "elasticsearch.docs.count": ("gauge", "indices.docs.count"),
         "elasticsearch.docs.deleted": ("gauge", "indices.docs.deleted"),
         "elasticsearch.store.size": ("gauge", "indices.store.size_in_bytes"),
@@ -118,6 +118,39 @@ class ESCheck(AgentCheck):
         "jvm.threads.peak_count": ("gauge", "jvm.threads.peak_count"),
     }
 
+    JVM_METRICS_POST_0_90_10 = {
+        "jvm.gc.collectors.young.count": ("gauge", "jvm.gc.collectors.young.collection_count"),
+        "jvm.gc.collectors.young.collection_time": ("gauge", "jvm.gc.collectors.young.collection_time_in_millis", lambda v: float(v)/1000),
+        "jvm.gc.collectors.old.count": ("gauge", "jvm.gc.collectors.old.collection_count"),
+        "jvm.gc.collectors.old.collection_time": ("gauge", "jvm.gc.collectors.old.collection_time_in_millis", lambda v: float(v)/1000)
+    }
+
+    JVM_METRICS_PRE_0_90_10 = {
+        "jvm.gc.concurrent_mark_sweep.count": ("gauge", "jvm.gc.collectors.ConcurrentMarkSweep.collection_count"),
+        "jvm.gc.concurrent_mark_sweep.collection_time": ("gauge", "jvm.gc.collectors.ConcurrentMarkSweep.collection_time_in_millis", lambda v: float(v)/1000),
+        "jvm.gc.par_new.count": ("gauge", "jvm.gc.collectors.ParNew.collection_count"),
+        "jvm.gc.par_new.collection_time": ("gauge", "jvm.gc.collectors.ParNew.collection_time_in_millis", lambda v: float(v)/1000),
+        "jvm.gc.collection_count": ("gauge", "jvm.gc.collection_count"),
+        "jvm.gc.collection_time": ("gauge", "jvm.gc.collection_time_in_millis", lambda v: float(v)/1000),
+    }
+
+    ADDITIONAL_METRICS_POST_0_90_5 = {
+        "elasticsearch.search.fetch.open_contexts": ("gauge", "indices.search.open_contexts"),
+        "elasticsearch.cache.filter.evictions": ("gauge", "indices.filter_cache.evictions"),
+        "elasticsearch.cache.filter.size": ("gauge", "indices.filter_cache.memory_size_in_bytes"),
+        "elasticsearch.id_cache.size": ("gauge", "indices.id_cache.memory_size_in_bytes"),
+        "elasticsearch.fielddata.size": ("gauge", "indices.fielddata.memory_size_in_bytes"),
+        "elasticsearch.fielddata.evictions": ("gauge", "indices.fielddata.evictions"),
+    }
+
+    ADDITIONAL_METRICS_PRE_0_90_5 = {
+        "elasticsearch.cache.field.evictions": ("gauge", "indices.cache.field_evictions"),
+        "elasticsearch.cache.field.size": ("gauge", "indices.cache.field_size_in_bytes"),
+        "elasticsearch.cache.filter.count": ("gauge", "indices.cache.filter_count"),
+        "elasticsearch.cache.filter.evictions": ("gauge", "indices.cache.filter_evictions"),
+        "elasticsearch.cache.filter.size": ("gauge", "indices.cache.filter_size_in_bytes"),
+    }
+
     CLUSTER_HEALTH_METRICS = {
         "elasticsearch.number_of_nodes": ("gauge", "number_of_nodes"),
         "elasticsearch.number_of_data_nodes": ("gauge", "number_of_data_nodes"),
@@ -126,7 +159,7 @@ class ESCheck(AgentCheck):
         "elasticsearch.relocating_shards": ("gauge", "relocating_shards"),
         "elasticsearch.initializing_shards": ("gauge", "initializing_shards"),
         "elasticsearch.unassigned_shards": ("gauge", "unassigned_shards"),
-        "elasticsearch.cluster_status": ("gauge", "status", lambda v: {"red":0, "yellow":1, "green":2}.get(v, -1)),
+        "elasticsearch.cluster_status": ("gauge", "status", lambda v: {"red": 0, "yellow": 1, "green": 2}.get(v, -1)),
     }
 
     SOURCE_TYPE_NAME = 'elasticsearch'
@@ -175,43 +208,45 @@ def get_instance_config(self, instance):
         return config
 
     def check(self, instance):
-        self.curr_config = self.get_instance_config(instance)
+        config = self.get_instance_config(instance)
 
         # Check ES version for this instance and define parameters
         # (URLs and metrics) accordingly
-        version = self._get_es_version()
-        self._define_params(version, self.curr_config.is_external)
+        version = self._get_es_version(config)
+
+        health_url, nodes_url, stats_url, stats_metrics = self._define_params(
+            version, config.is_external)
 
         # Load stats data.
-        stats_url = urlparse.urljoin(self.curr_config.url, self.STATS_URL)
-        stats_data = self._get_data(stats_url)
-        self._process_stats_data(stats_data)
+        stats_url = urlparse.urljoin(config.url, stats_url)
+        stats_data = self._get_data(stats_url, config)
+        self._process_stats_data(stats_data, stats_metrics, config)
 
         # Load the health data.
-        health_url = urlparse.urljoin(self.curr_config.url, self.HEALTH_URL)
-        health_data = self._get_data(health_url)
-        self._process_health_data(health_data)
+        health_url = urlparse.urljoin(config.url, health_url)
+        health_data = self._get_data(health_url, config)
+        self._process_health_data(health_data, config)
 
         # If we're here we did not have any ES conn issues
         self.service_check(
             self.SERVICE_CHECK_CONNECT_NAME,
             AgentCheck.OK,
-            tags=self.curr_config.service_check_tags
+            tags=config.service_check_tags
         )
 
-    def _get_es_version(self):
+    def _get_es_version(self, config):
         """ Get the running version of elasticsearch.
         """
         try:
-            data = self._get_data(self.curr_config.url)
+            data = self._get_data(config.url, config, send_sc=False)
             version = map(int, data['version']['number'].split('.')[0:3])
         except Exception, e:
             self.warning(
                 "Error while trying to get Elasticsearch version "
                 "from %s %s"
-                % (self.curr_config.url, str(e))
+                % (config.url, str(e))
             )
-            version = [0, 0, 0]
+            version = [1, 0, 0]
 
         self.log.debug("Elasticsearch version is %s" % version)
         return version
@@ -220,108 +255,93 @@ def _define_params(self, version, is_external):
         """ Define the set of URLs and METRICS to use depending on the
             running ES version.
         """
-        if version >= [0,90,10]:
+        if version >= [0, 90, 10]:
             # ES versions 0.90.10 and above
-            self.HEALTH_URL = "/_cluster/health?pretty=true"
-            self.NODES_URL = "/_nodes?network=true"
+            health_url = "/_cluster/health?pretty=true"
+            nodes_url = "/_nodes?network=true"
 
             # For "external" clusters, we want to collect from all nodes.
             if is_external:
-                self.STATS_URL = "/_nodes/stats?all=true"
+                stats_url = "/_nodes/stats?all=true"
             else:
-                self.STATS_URL = "/_nodes/_local/stats?all=true"
-
-            additional_metrics = {
-                "jvm.gc.collectors.young.count": ("gauge", "jvm.gc.collectors.young.collection_count"),
-                "jvm.gc.collectors.young.collection_time": ("gauge", "jvm.gc.collectors.young.collection_time_in_millis", lambda v: float(v)/1000),
-                "jvm.gc.collectors.old.count": ("gauge", "jvm.gc.collectors.old.collection_count"),
-                "jvm.gc.collectors.old.collection_time": ("gauge", "jvm.gc.collectors.old.collection_time_in_millis", lambda v: float(v)/1000)
-            }
+                stats_url = "/_nodes/_local/stats?all=true"
+
+            additional_metrics = self.JVM_METRICS_POST_0_90_10
         else:
-            self.HEALTH_URL = "/_cluster/health?pretty=true"
-            self.STATS_URL = "/_cluster/nodes/stats?all=true"
-            self.NODES_URL = "/_cluster/nodes?network=true"
-
-            additional_metrics = {
-                "jvm.gc.concurrent_mark_sweep.count": ("gauge", "jvm.gc.collectors.ConcurrentMarkSweep.collection_count"),
-                "jvm.gc.concurrent_mark_sweep.collection_time": ("gauge", "jvm.gc.collectors.ConcurrentMarkSweep.collection_time_in_millis", lambda v: float(v)/1000),
-                "jvm.gc.par_new.count": ("gauge", "jvm.gc.collectors.ParNew.collection_count"),
-                "jvm.gc.par_new.collection_time": ("gauge", "jvm.gc.collectors.ParNew.collection_time_in_millis", lambda v: float(v)/1000),
-                "jvm.gc.collection_count": ("gauge", "jvm.gc.collection_count"),
-                "jvm.gc.collection_time": ("gauge", "jvm.gc.collection_time_in_millis", lambda v: float(v)/1000),
-            }
-
-        self.STATS_METRICS.update(additional_metrics)
-
-        if version >= [0,90,5]:
+            health_url = "/_cluster/health?pretty=true"
+            nodes_url = "/_cluster/nodes?network=true"
+            if is_external:
+                stats_url = "/_cluster/nodes/stats?all=true"
+            else:
+                stats_url = "/_cluster/nodes/_local/stats?all=true"
+
+            additional_metrics = self.JVM_METRICS_PRE_0_90_10
+
+        stats_metrics = dict(self.STATS_METRICS)
+        stats_metrics.update(additional_metrics)
+
+        if version >= [0, 90, 5]:
             # ES versions 0.90.5 and above
-            additional_metrics = {
-                "elasticsearch.search.fetch.open_contexts": ("gauge", "indices.search.open_contexts"),
-                "elasticsearch.cache.filter.evictions": ("gauge", "indices.filter_cache.evictions"),
-                "elasticsearch.cache.filter.size": ("gauge", "indices.filter_cache.memory_size_in_bytes"),
-                "elasticsearch.id_cache.size": ("gauge","indices.id_cache.memory_size_in_bytes"),
-                "elasticsearch.fielddata.size": ("gauge","indices.fielddata.memory_size_in_bytes"),
-                "elasticsearch.fielddata.evictions": ("gauge","indices.fielddata.evictions"),
-            }
+            additional_metrics = self.ADDITIONAL_METRICS_POST_0_90_5
         else:
             # ES version 0.90.4 and below
-            additional_metrics = {
-                "elasticsearch.cache.field.evictions": ("gauge", "indices.cache.field_evictions"),
-                "elasticsearch.cache.field.size": ("gauge", "indices.cache.field_size_in_bytes"),
-                "elasticsearch.cache.filter.count": ("gauge", "indices.cache.filter_count"),
-                "elasticsearch.cache.filter.evictions": ("gauge", "indices.cache.filter_evictions"),
-                "elasticsearch.cache.filter.size": ("gauge", "indices.cache.filter_size_in_bytes"),
-            }
+            additional_metrics = self.ADDITIONAL_METRICS_PRE_0_90_5
+
+        stats_metrics.update(additional_metrics)
 
-        self.STATS_METRICS.update(additional_metrics)
+        return health_url, nodes_url, stats_url, stats_metrics
 
-    def _get_data(self, url):
+    def _get_data(self, url, config, send_sc=True):
         """ Hit a given URL and return the parsed json
         """
         # Load basic authentication configuration, if available.
-        if self.curr_config.username and self.curr_config.password:
-            auth = (self.curr_config.username, self.curr_config.password)
+        if config.username and config.password:
+            auth = (config.username, config.password)
         else:
             auth = None
 
         try:
             resp = requests.get(
                 url,
-                timeout=self.curr_config.timeout,
+                timeout=config.timeout,
                 headers=headers(self.agentConfig),
                 auth=auth
             )
             resp.raise_for_status()
         except Exception as e:
-            self.service_check(
-                self.SERVICE_CHECK_CONNECT_NAME,
-                AgentCheck.CRITICAL,
-                message="Error {0} when hitting {1}".format(e, url),
-                tags=self.curr_config.service_check_tags
-            )
+            if send_sc:
+                self.service_check(
+                    self.SERVICE_CHECK_CONNECT_NAME,
+                    AgentCheck.CRITICAL,
+                    message="Error {0} when hitting {1}".format(e, url),
+                    tags=config.service_check_tags
+                )
             raise
 
         return resp.json()
 
-    def _process_stats_data(self, data):
-        is_external = self.curr_config.is_external
+    def _process_stats_data(self, data, stats_metrics, config):
+        is_external = config.is_external
         for node_name in data['nodes']:
             node_data = data['nodes'][node_name]
             # On newer version of ES it's "host" not "hostname"
-            node_hostname = node_data.get('hostname', node_data.get('host', None))
-            should_process = is_external \
-                        or self.should_process_node(node_name, node_hostname)
+            node_hostname = node_data.get(
+                'hostname', node_data.get('host', None))
+            should_process = (
+                is_external or self.should_process_node(
+                    node_name, node_hostname, config))
 
-            # Override the metric hostname if we're hitting an external cluster.
+            # Override the metric hostname if we're hitting an external cluster
             metric_hostname = node_hostname if is_external else None
 
             if should_process:
-                for metric in self.STATS_METRICS:
-                    desc = self.STATS_METRICS[metric]
-                    self._process_metric(node_data, metric, *desc,
-                        tags=self.curr_config.tags, hostname=metric_hostname)
+                for metric in stats_metrics:
+                    desc = stats_metrics[metric]
+                    self._process_metric(
+                        node_data, metric, *desc, tags=config.tags,
+                        hostname=metric_hostname)
 
-    def should_process_node(self, node_name, node_hostname):
+    def should_process_node(self, node_name, node_hostname, config):
         """ The node stats API will return stats for every node so we
             want to filter out nodes that we don't care about.
         """
@@ -339,25 +359,26 @@ def should_process_node(self, node_name, node_hostname):
             # Fetch interface address from ifconfig or ip addr and check
             # against the primary IP from ES
             try:
-                nodes_url = urlparse.urljoin(self.curr_config.url, self.NODES_URL)
-                primary_addr = self._get_primary_addr(nodes_url, node_name)
+                nodes_url = urlparse.urljoin(config.url, self.NODES_URL)
+                primary_addr = self._get_primary_addr(
+                    nodes_url, node_name, config)
             except NodeNotFound:
                 # Skip any nodes that aren't found
                 return False
             if self._host_matches_node(primary_addr):
                 return True
 
-    def _get_primary_addr(self, url, node_name):
+    def _get_primary_addr(self, url, node_name, config):
         """ Returns a list of primary interface addresses as seen by ES.
             Used in ES < 0.19
         """
-        data = self._get_data(url)
+        data = self._get_data(url, config)
 
         if node_name in data['nodes']:
             node = data['nodes'][node_name]
-            if 'network' in node\
-            and 'primary_interface' in node['network']\
-            and 'address' in node['network']['primary_interface']:
+            if ('network' in node
+                    and 'primary_interface' in node['network']
+                    and 'address' in node['network']['primary_interface']):
                 return node['network']['primary_interface']['address']
 
         raise NodeNotFound()
@@ -371,8 +392,9 @@ def _host_matches_node(self, primary_addrs):
             ifaces = subprocess.Popen(['ifconfig'], stdout=subprocess.PIPE)
         else:
             ifaces = subprocess.Popen(['ip', 'addr'], stdout=subprocess.PIPE)
-        grepper = subprocess.Popen(['grep', 'inet'], stdin=ifaces.stdout,
-            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        grepper = subprocess.Popen(
+            ['grep', 'inet'], stdin=ifaces.stdout, stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
 
         ifaces.stdout.close()
         out, err = grepper.communicate()
@@ -388,7 +410,7 @@ def _host_matches_node(self, primary_addrs):
         return primary_addrs in ips
 
     def _process_metric(self, data, metric, xtype, path, xform=None,
-        tags=None, hostname=None):
+                        tags=None, hostname=None):
         """data: dictionary containing all the stats
         metric: datadog metric
         path: corresponding path in data, flattened, e.g. thread_pool.bulk.queue
@@ -404,7 +426,8 @@ def _process_metric(self, data, metric, xtype, path, xform=None,
                 break
 
         if value is not None:
-            if xform: value = xform(value)
+            if xform:
+                value = xform(value)
             if xtype == "gauge":
                 self.gauge(metric, value, tags=tags, hostname=hostname)
             else:
@@ -412,22 +435,20 @@ def _process_metric(self, data, metric, xtype, path, xform=None,
         else:
             self._metric_not_found(metric, path)
 
-    def _process_health_data(self, data):
-        if self.cluster_status.get(self.curr_config.url) is None:
-            self.cluster_status[self.curr_config.url] = data['status']
+    def _process_health_data(self, data, config):
+        if self.cluster_status.get(config.url) is None:
+            self.cluster_status[config.url] = data['status']
             if data['status'] in ["yellow", "red"]:
                 event = self._create_event(data['status'])
                 self.event(event)
 
-        if data['status'] != self.cluster_status.get(self.curr_config.url):
-            self.cluster_status[self.curr_config.url] = data['status']
+        if data['status'] != self.cluster_status.get(config.url):
+            self.cluster_status[config.url] = data['status']
             event = self._create_event(data['status'])
             self.event(event)
 
-        for metric in self.CLUSTER_HEALTH_METRICS:
-            # metric description
-            desc = self.CLUSTER_HEALTH_METRICS[metric]
-            self._process_metric(data, metric, *desc, tags=self.curr_config.tags)
+        for metric, desc in self.CLUSTER_HEALTH_METRICS.iteritems():
+            self._process_metric(data, metric, *desc, tags=config.tags)
 
         # Process the service check
         cluster_status = data['status']
@@ -453,7 +474,7 @@ def _process_health_data(self, data):
             self.SERVICE_CHECK_CLUSTER_STATUS,
             status,
             message=msg,
-            tags=self.curr_config.service_check_tags
+            tags=config.service_check_tags
         )
 
     def _metric_not_found(self, metric, path):
@@ -476,14 +497,13 @@ def _create_event(self, status):
 
         msg = "ElasticSearch: %s just reported as %s" % (hostname, status)
 
-        return { 'timestamp': int(time.time()),
-                 'event_type': 'elasticsearch',
-                 'host': hostname,
-                 'msg_text':msg,
-                 'msg_title': msg_title,
-                 'alert_type': alert_type,
-                 'source_type_name': "elasticsearch",
-                 'event_object': hostname
-            }
-
-
+        return {
+            'timestamp': int(time.time()),
+            'event_type': self.SOURCE_TYPE_NAME,
+            'host': hostname,
+            'msg_text': msg,
+            'msg_title': msg_title,
+            'alert_type': alert_type,
+            'source_type_name': self.SOURCE_TYPE_NAME,
+            'event_object': hostname
+        }


Test patch:
diff --git a/tests/common.py b/tests/common.py
--- a/tests/common.py
+++ b/tests/common.py
@@ -7,6 +7,7 @@
 import time
 import traceback
 import unittest
+from pprint import pformat
 
 # project
 from checks import AgentCheck
@@ -172,10 +173,10 @@ def print_current_state(self):
 WARNINGS
     {warnings}
 ++++++++++++++++++++++++++++""".format(
-            metrics=self.metrics,
-            events=self.events,
-            sc=self.service_checks,
-            warnings=self.warnings
+            metrics=pformat(self.metrics),
+            events=pformat(self.events),
+            sc=pformat(self.service_checks),
+            warnings=pformat(self.warnings)
         ))
 
     def coverage_report(self):
@@ -220,11 +221,11 @@ def coverage_report(self):
             tested_metrics=tested_metrics,
             total_metrics=total_metrics,
             coverage_metrics=coverage_metrics,
-            untested_metrics=untested_metrics,
+            untested_metrics=pformat(untested_metrics),
             tested_sc=tested_sc,
             total_sc=total_sc,
             coverage_sc=coverage_sc,
-            untested_sc=untested_sc,
+            untested_sc=pformat(untested_sc),
         ))
 
         if os.getenv('COVERAGE'):
@@ -245,17 +246,8 @@ def _candidates_size_assert(self, candidates, count=None, at_least=1):
             self.print_current_state()
             raise
 
-    def assertMetric(self, metric_name, value=None, tags=None, count=None, at_least=1):
-        log.debug("Looking for metric {0}".format(metric_name))
-        if value is not None:
-            log.debug(" * with value {0}".format(value))
-        if tags is not None:
-            log.debug(" * tagged with {0}".format(tags))
-        if count is not None:
-            log.debug(" * should have exactly {0} data points".format(count))
-        if at_least is not None:
-            log.debug(" * should have at least {0} data points".format(at_least))
-
+    def assertMetric(self, metric_name, value=None, tags=None, count=None,
+        at_least=1, hostname=None):
         candidates = []
         for m_name, ts, val, mdata in self.metrics:
             if m_name == metric_name:
@@ -263,10 +255,19 @@ def assertMetric(self, metric_name, value=None, tags=None, count=None, at_least=
                     continue
                 if tags is not None and sorted(tags) != sorted(mdata.get("tags", [])):
                     continue
+                if hostname is not None and mdata['hostname'] != hostname:
+                    continue
 
                 candidates.append((m_name, ts, val, mdata))
 
-        self._candidates_size_assert(candidates, count=count, at_least=at_least)
+        try:
+            self._candidates_size_assert(candidates, count=count, at_least=at_least)
+        except AssertionError:
+            log.error("Candidates size assertion for {0} (value: {1}, tags: {2}, "
+                "count: {3}, at_least: {4}, hostname: {5}) failed"\
+                .format(metric_name, value, tags, count, at_least, hostname))
+            raise
+
         for mtuple in self.metrics:
             for cmtuple in candidates:
                 if mtuple == cmtuple:
@@ -288,7 +289,14 @@ def assertMetricTagPrefix(self, metric_name, tag_prefix, count=None, at_least=1)
                     continue
                 candidates.append((m_name, ts, val, mdata))
 
-        self._candidates_size_assert(candidates, count=count)
+        try:
+            self._candidates_size_assert(candidates, count=count)
+        except AssertionError:
+            log.error("Candidates size assertion for {0} (tag_prefix: {1}, "
+                "count: {2}, at_least: {3}) failed"\
+                .format(metric_name, tag_prefix, count, at_least))
+            raise
+
         for mtuple in self.metrics:
             for cmtuple in candidates:
                 if mtuple == cmtuple:
@@ -310,14 +318,22 @@ def assertMetricTag(self, metric_name, tag, count=None, at_least=1):
                     continue
                 candidates.append((m_name, ts, val, mdata))
 
-        self._candidates_size_assert(candidates, count=count)
+        try:
+            self._candidates_size_assert(candidates, count=count)
+        except AssertionError:
+            log.error("Candidates size assertion for {0} (tag: {1}, count={2},"
+                " at_least={3}) failed"\
+                .format(metric_name, tag, count, at_least))
+            raise
+
         for mtuple in self.metrics:
             for cmtuple in candidates:
                 if mtuple == cmtuple:
                     mtuple[3]['tested'] = True
         log.debug("FOUND !")
 
-    def assertServiceCheck(self, service_check_name, status=None, tags=None, count=None, at_least=1):
+    def assertServiceCheck(self, service_check_name, status=None, tags=None,
+        count=None, at_least=1):
         log.debug("Looking for service check {0}".format(service_check_name))
         if status is not None:
             log.debug(" * with status {0}".format(status))
@@ -335,7 +351,14 @@ def assertServiceCheck(self, service_check_name, status=None, tags=None, count=N
 
                 candidates.append(sc)
 
-        self._candidates_size_assert(candidates, count=count, at_least=at_least)
+        try:
+            self._candidates_size_assert(candidates, count=count, at_least=at_least)
+        except AssertionError:
+            log.error("Candidates size assertion for {0} (status: {1}, "
+                "tags: {2}, count: {3}, at_least: {4}) failed"\
+                .format(service_check_name, status, tags, count, at_least))
+            raise
+
         for sc in self.service_checks:
             for csc in candidates:
                 if sc == csc:
diff --git a/tests/test_elastic.py b/tests/test_elastic.py
--- a/tests/test_elastic.py
+++ b/tests/test_elastic.py
@@ -1,104 +1,265 @@
 # stdlib
 import socket
-import unittest
+import os
 
 # 3p
 import requests
 from nose.plugins.attrib import attr
 
 # project
-from tests.common import load_check
+from tests.common import AgentCheckTest, load_check
 from checks import AgentCheck
-PORT = 9200
-MAX_WAIT = 150
+from config import get_version
 
+STATS_METRICS = {  # Metrics that are common to all Elasticsearch versions
+    "elasticsearch.docs.count": ("gauge", "indices.docs.count"),
+    "elasticsearch.docs.deleted": ("gauge", "indices.docs.deleted"),
+    "elasticsearch.store.size": ("gauge", "indices.store.size_in_bytes"),
+    "elasticsearch.indexing.index.total": ("gauge", "indices.indexing.index_total"),
+    "elasticsearch.indexing.index.time": ("gauge", "indices.indexing.index_time_in_millis", lambda v: float(v)/1000),
+    "elasticsearch.indexing.index.current": ("gauge", "indices.indexing.index_current"),
+    "elasticsearch.indexing.delete.total": ("gauge", "indices.indexing.delete_total"),
+    "elasticsearch.indexing.delete.time": ("gauge", "indices.indexing.delete_time_in_millis", lambda v: float(v)/1000),
+    "elasticsearch.indexing.delete.current": ("gauge", "indices.indexing.delete_current"),
+    "elasticsearch.get.total": ("gauge", "indices.get.total"),
+    "elasticsearch.get.time": ("gauge", "indices.get.time_in_millis", lambda v: float(v)/1000),
+    "elasticsearch.get.current": ("gauge", "indices.get.current"),
+    "elasticsearch.get.exists.total": ("gauge", "indices.get.exists_total"),
+    "elasticsearch.get.exists.time": ("gauge", "indices.get.exists_time_in_millis", lambda v: float(v)/1000),
+    "elasticsearch.get.missing.total": ("gauge", "indices.get.missing_total"),
+    "elasticsearch.get.missing.time": ("gauge", "indices.get.missing_time_in_millis", lambda v: float(v)/1000),
+    "elasticsearch.search.query.total": ("gauge", "indices.search.query_total"),
+    "elasticsearch.search.query.time": ("gauge", "indices.search.query_time_in_millis", lambda v: float(v)/1000),
+    "elasticsearch.search.query.current": ("gauge", "indices.search.query_current"),
+    "elasticsearch.search.fetch.total": ("gauge", "indices.search.fetch_total"),
+    "elasticsearch.search.fetch.time": ("gauge", "indices.search.fetch_time_in_millis", lambda v: float(v)/1000),
+    "elasticsearch.search.fetch.current": ("gauge", "indices.search.fetch_current"),
+    "elasticsearch.merges.current": ("gauge", "indices.merges.current"),
+    "elasticsearch.merges.current.docs": ("gauge", "indices.merges.current_docs"),
+    "elasticsearch.merges.current.size": ("gauge", "indices.merges.current_size_in_bytes"),
+    "elasticsearch.merges.total": ("gauge", "indices.merges.total"),
+    "elasticsearch.merges.total.time": ("gauge", "indices.merges.total_time_in_millis", lambda v: float(v)/1000),
+    "elasticsearch.merges.total.docs": ("gauge", "indices.merges.total_docs"),
+    "elasticsearch.merges.total.size": ("gauge", "indices.merges.total_size_in_bytes"),
+    "elasticsearch.refresh.total": ("gauge", "indices.refresh.total"),
+    "elasticsearch.refresh.total.time": ("gauge", "indices.refresh.total_time_in_millis", lambda v: float(v)/1000),
+    "elasticsearch.flush.total": ("gauge", "indices.flush.total"),
+    "elasticsearch.flush.total.time": ("gauge", "indices.flush.total_time_in_millis", lambda v: float(v)/1000),
+    "elasticsearch.process.open_fd": ("gauge", "process.open_file_descriptors"),
+    "elasticsearch.transport.rx_count": ("gauge", "transport.rx_count"),
+    "elasticsearch.transport.tx_count": ("gauge", "transport.tx_count"),
+    "elasticsearch.transport.rx_size": ("gauge", "transport.rx_size_in_bytes"),
+    "elasticsearch.transport.tx_size": ("gauge", "transport.tx_size_in_bytes"),
+    "elasticsearch.transport.server_open": ("gauge", "transport.server_open"),
+    "elasticsearch.thread_pool.bulk.active": ("gauge", "thread_pool.bulk.active"),
+    "elasticsearch.thread_pool.bulk.threads": ("gauge", "thread_pool.bulk.threads"),
+    "elasticsearch.thread_pool.bulk.queue": ("gauge", "thread_pool.bulk.queue"),
+    "elasticsearch.thread_pool.flush.active": ("gauge", "thread_pool.flush.active"),
+    "elasticsearch.thread_pool.flush.threads": ("gauge", "thread_pool.flush.threads"),
+    "elasticsearch.thread_pool.flush.queue": ("gauge", "thread_pool.flush.queue"),
+    "elasticsearch.thread_pool.generic.active": ("gauge", "thread_pool.generic.active"),
+    "elasticsearch.thread_pool.generic.threads": ("gauge", "thread_pool.generic.threads"),
+    "elasticsearch.thread_pool.generic.queue": ("gauge", "thread_pool.generic.queue"),
+    "elasticsearch.thread_pool.get.active": ("gauge", "thread_pool.get.active"),
+    "elasticsearch.thread_pool.get.threads": ("gauge", "thread_pool.get.threads"),
+    "elasticsearch.thread_pool.get.queue": ("gauge", "thread_pool.get.queue"),
+    "elasticsearch.thread_pool.index.active": ("gauge", "thread_pool.index.active"),
+    "elasticsearch.thread_pool.index.threads": ("gauge", "thread_pool.index.threads"),
+    "elasticsearch.thread_pool.index.queue": ("gauge", "thread_pool.index.queue"),
+    "elasticsearch.thread_pool.management.active": ("gauge", "thread_pool.management.active"),
+    "elasticsearch.thread_pool.management.threads": ("gauge", "thread_pool.management.threads"),
+    "elasticsearch.thread_pool.management.queue": ("gauge", "thread_pool.management.queue"),
+    "elasticsearch.thread_pool.merge.active": ("gauge", "thread_pool.merge.active"),
+    "elasticsearch.thread_pool.merge.threads": ("gauge", "thread_pool.merge.threads"),
+    "elasticsearch.thread_pool.merge.queue": ("gauge", "thread_pool.merge.queue"),
+    "elasticsearch.thread_pool.percolate.active": ("gauge", "thread_pool.percolate.active"),
+    "elasticsearch.thread_pool.percolate.threads": ("gauge", "thread_pool.percolate.threads"),
+    "elasticsearch.thread_pool.percolate.queue": ("gauge", "thread_pool.percolate.queue"),
+    "elasticsearch.thread_pool.refresh.active": ("gauge", "thread_pool.refresh.active"),
+    "elasticsearch.thread_pool.refresh.threads": ("gauge", "thread_pool.refresh.threads"),
+    "elasticsearch.thread_pool.refresh.queue": ("gauge", "thread_pool.refresh.queue"),
+    "elasticsearch.thread_pool.search.active": ("gauge", "thread_pool.search.active"),
+    "elasticsearch.thread_pool.search.threads": ("gauge", "thread_pool.search.threads"),
+    "elasticsearch.thread_pool.search.queue": ("gauge", "thread_pool.search.queue"),
+    "elasticsearch.thread_pool.snapshot.active": ("gauge", "thread_pool.snapshot.active"),
+    "elasticsearch.thread_pool.snapshot.threads": ("gauge", "thread_pool.snapshot.threads"),
+    "elasticsearch.thread_pool.snapshot.queue": ("gauge", "thread_pool.snapshot.queue"),
+    "elasticsearch.http.current_open": ("gauge", "http.current_open"),
+    "elasticsearch.http.total_opened": ("gauge", "http.total_opened"),
+    "jvm.mem.heap_committed": ("gauge", "jvm.mem.heap_committed_in_bytes"),
+    "jvm.mem.heap_used": ("gauge", "jvm.mem.heap_used_in_bytes"),
+    "jvm.mem.non_heap_committed": ("gauge", "jvm.mem.non_heap_committed_in_bytes"),
+    "jvm.mem.non_heap_used": ("gauge", "jvm.mem.non_heap_used_in_bytes"),
+    "jvm.threads.count": ("gauge", "jvm.threads.count"),
+    "jvm.threads.peak_count": ("gauge", "jvm.threads.peak_count"),
+}
 
-@attr(requires='elasticsearch')
-class TestElastic(unittest.TestCase):
-    def test_bad_config(self):
-        agentConfig = {
-            'version': '0.1',
-            'api_key': 'toto'
-        }
+JVM_METRICS_POST_0_90_10 = {
+    "jvm.gc.collectors.young.count": ("gauge", "jvm.gc.collectors.young.collection_count"),
+    "jvm.gc.collectors.young.collection_time": ("gauge", "jvm.gc.collectors.young.collection_time_in_millis", lambda v: float(v)/1000),
+    "jvm.gc.collectors.old.count": ("gauge", "jvm.gc.collectors.old.collection_count"),
+    "jvm.gc.collectors.old.collection_time": ("gauge", "jvm.gc.collectors.old.collection_time_in_millis", lambda v: float(v)/1000)
+}
 
-        conf = {
-            'instances': [{'url': 'http://losdfsdsdcalhost:%s' % PORT}]
-        }
-        # Initialize the check from checks.d
-        self.check = load_check('elastic', conf, agentConfig)
+JVM_METRICS_PRE_0_90_10 = {
+    "jvm.gc.concurrent_mark_sweep.count": ("gauge", "jvm.gc.collectors.ConcurrentMarkSweep.collection_count"),
+    "jvm.gc.concurrent_mark_sweep.collection_time": ("gauge", "jvm.gc.collectors.ConcurrentMarkSweep.collection_time_in_millis", lambda v: float(v)/1000),
+    "jvm.gc.par_new.count": ("gauge", "jvm.gc.collectors.ParNew.collection_count"),
+    "jvm.gc.par_new.collection_time": ("gauge", "jvm.gc.collectors.ParNew.collection_time_in_millis", lambda v: float(v)/1000),
+    "jvm.gc.collection_count": ("gauge", "jvm.gc.collection_count"),
+    "jvm.gc.collection_time": ("gauge", "jvm.gc.collection_time_in_millis", lambda v: float(v)/1000),
+}
+
+ADDITIONAL_METRICS_POST_0_90_5 = {
+    "elasticsearch.search.fetch.open_contexts": ("gauge", "indices.search.open_contexts"),
+    "elasticsearch.cache.filter.evictions": ("gauge", "indices.filter_cache.evictions"),
+    "elasticsearch.cache.filter.size": ("gauge", "indices.filter_cache.memory_size_in_bytes"),
+    "elasticsearch.id_cache.size": ("gauge", "indices.id_cache.memory_size_in_bytes"),
+    "elasticsearch.fielddata.size": ("gauge", "indices.fielddata.memory_size_in_bytes"),
+    "elasticsearch.fielddata.evictions": ("gauge", "indices.fielddata.evictions"),
+}
+
+ADDITIONAL_METRICS_PRE_0_90_5 = {
+    "elasticsearch.cache.field.evictions": ("gauge", "indices.cache.field_evictions"),
+    "elasticsearch.cache.field.size": ("gauge", "indices.cache.field_size_in_bytes"),
+    "elasticsearch.cache.filter.count": ("gauge", "indices.cache.filter_count"),
+    "elasticsearch.cache.filter.evictions": ("gauge", "indices.cache.filter_evictions"),
+    "elasticsearch.cache.filter.size": ("gauge", "indices.cache.filter_size_in_bytes"),
+}
+
+CLUSTER_HEALTH_METRICS = {
+    "elasticsearch.number_of_nodes": ("gauge", "number_of_nodes"),
+    "elasticsearch.number_of_data_nodes": ("gauge", "number_of_data_nodes"),
+    "elasticsearch.active_primary_shards": ("gauge", "active_primary_shards"),
+    "elasticsearch.active_shards": ("gauge", "active_shards"),
+    "elasticsearch.relocating_shards": ("gauge", "relocating_shards"),
+    "elasticsearch.initializing_shards": ("gauge", "initializing_shards"),
+    "elasticsearch.unassigned_shards": ("gauge", "unassigned_shards"),
+    "elasticsearch.cluster_status": ("gauge", "status", lambda v: {"red": 0, "yellow": 1, "green": 2}.get(v, -1)),
+}
+
+
+def get_es_version():
+    version = os.environ.get("FLAVOR_VERSION")
+    if version is None:
+        return [1, 4, 2]
+    return [int(k) for k in version.split(".")]
 
-        self.assertRaises(requests.exceptions.ConnectionError, self.check.check, conf['instances'][0])
-        service_checks = self.check.get_service_checks()
-        # 2 service checks b/c a connection failed trying to get the version nb first
-        self.assertEquals(len(service_checks), 2, service_checks)
-        self.assertEquals(service_checks[0]['status'], AgentCheck.CRITICAL, service_checks)
-        self.assertEquals(service_checks[1]['status'], AgentCheck.CRITICAL, service_checks)
+
+@attr(requires='elasticsearch')
+class TestElastic(AgentCheckTest):
+    CHECK_NAME = "elastic"
 
     def test_check(self):
-        agentConfig = {
-            'version': '0.1',
-            'api_key': 'toto',
-            'hostname': 'agent-es-test'
+        conf_hostname = "foo"
+        port = 9200
+        bad_port = 9205
+        agent_config = {
+            "hostname": conf_hostname, "version": get_version(),
+            "api_key": "bar"
         }
 
-        conf = {
+        tags = ['foo:bar', 'baz']
+        url = 'http://localhost:{0}'.format(port)
+        bad_url = 'http://localhost:{0}'.format(bad_port)
+
+        config = {
             'instances': [
-                {'url': 'http://localhost:%s' % PORT},
-                {'url': 'http://localhost:%s' % PORT, 'is_external': True}
+                {'url': url, 'tags': tags},  # One with tags not external
+                {'url': url, 'is_external': True},  # One without tags, external
+                {'url': bad_url},  # One bad url
             ]
         }
 
-        # Initialize the check from checks.d
-        self.check = load_check('elastic', conf, agentConfig)
-
-        self.check.check(conf['instances'][0])
-        r = self.check.get_metrics()
-        self.check.get_events()
-
-        self.assertTrue(type(r) == type([]))
-        self.assertTrue(len(r) > 0)
-        self.assertEquals(len([t for t in r if t[0] == "elasticsearch.get.total"]), 1, r)
-        self.assertEquals(len([t for t in r if t[0] == "elasticsearch.search.fetch.total"]), 1, r)
-        self.assertEquals(len([t for t in r if t[0] == "jvm.mem.heap_committed"]), 1, r)
-        self.assertEquals(len([t for t in r if t[0] == "jvm.mem.heap_used"]), 1, r)
-        self.assertEquals(len([t for t in r if t[0] == "jvm.threads.count"]), 1, r)
-        self.assertEquals(len([t for t in r if t[0] == "jvm.threads.peak_count"]), 1, r)
-        self.assertEquals(len([t for t in r if t[0] == "elasticsearch.transport.rx_count"]), 1, r)
-        self.assertEquals(len([t for t in r if t[0] == "elasticsearch.transport.tx_size"]), 1, r)
-        self.assertEquals(len([t for t in r if t[0] == "elasticsearch.transport.server_open"]), 1, r)
-        self.assertEquals(len([t for t in r if t[0] == "elasticsearch.thread_pool.snapshot.queue"]), 1, r)
-        self.assertEquals(len([t for t in r if t[0] == "elasticsearch.active_shards"]), 1, r)
-
-        # Checks enabled for specific ES versions
-        version = self.check._get_es_version()
-        if version >= [0,90,10]:
-            # ES versions 0.90.10 and above
-            pass
+        self.assertRaises(
+            requests.exceptions.ConnectionError,
+            self.run_check, config=config, agent_config=agent_config)
+
+        default_tags = ["url:http://localhost:{0}".format(port)]
+
+        expected_metrics = STATS_METRICS
+        expected_metrics.update(CLUSTER_HEALTH_METRICS)
+
+        instance_config = self.check.get_instance_config(config['instances'][0])
+        es_version = self.check._get_es_version(instance_config)
+
+        self.assertEquals(es_version, get_es_version())
+
+        if es_version >= [0, 90, 5]:
+            expected_metrics.update(ADDITIONAL_METRICS_POST_0_90_5)
+            if es_version >= [0, 90, 10]:
+                expected_metrics.update(JVM_METRICS_POST_0_90_10)
+            else:
+                expected_metrics.update(JVM_METRICS_PRE_0_90_10)
         else:
-            # ES version 0.90.9 and below
-            self.assertEquals(len([t for t in r if t[0] == "jvm.gc.collection_time"]), 1, r)
-
-        # Service checks
-        service_checks = self.check.get_service_checks()
-        service_checks_count = len(service_checks)
-        self.assertTrue(type(service_checks) == type([]))
-        self.assertTrue(service_checks_count > 0)
-        self.assertEquals(len([sc for sc in service_checks if sc['check'] == self.check.SERVICE_CHECK_CLUSTER_STATUS]), 1, service_checks)
-        self.assertEquals(len([sc for sc in service_checks if sc['check'] == self.check.SERVICE_CHECK_CONNECT_NAME]), 1, service_checks)
-        # Assert that all service checks have the proper tags: host and port
-        self.assertEquals(len([sc for sc in service_checks if "host:localhost" in sc['tags']]), service_checks_count, service_checks)
-        self.assertEquals(len([sc for sc in service_checks if "port:%s" % PORT in sc['tags']]), service_checks_count, service_checks)
-
-        self.check.cluster_status[conf['instances'][0].get('url')] = "red"
-        self.check.check(conf['instances'][0])
-        events = self.check.get_events()
-        self.check.get_metrics()
-        self.assertEquals(len(events),1,events)
-
-        # Check an "external" cluster
-        self.check.check(conf['instances'][1])
-        r = self.check.get_metrics()
-        expected_hostname = socket.gethostname()
-        for m in r:
-            if m[0] not in self.check.CLUSTER_HEALTH_METRICS:
-                self.assertEquals(m[3]['hostname'], expected_hostname)
+            expected_metrics.update(ADDITIONAL_METRICS_PRE_0_90_5)
+            expected_metrics.update(JVM_METRICS_PRE_0_90_10)
+
+        contexts = [
+            (conf_hostname, default_tags + tags),
+            (socket.gethostname(), default_tags)
+        ]
+
+        for m_name, desc in expected_metrics.iteritems():
+            for hostname, m_tags in contexts:
+                if (m_name in CLUSTER_HEALTH_METRICS
+                        and hostname == socket.gethostname()):
+                    hostname = conf_hostname
+
+                if desc[0] == "gauge":
+                    self.assertMetric(
+                        m_name, tags=m_tags, count=1, hostname=hostname)
+
+        good_sc_tags = ['host:localhost', 'port:{0}'.format(port)]
+        bad_sc_tags = ['host:localhost', 'port:{0}'.format(bad_port)]
+
+        self.assertServiceCheck('elasticsearch.can_connect',
+                                status=AgentCheck.OK, tags=good_sc_tags,
+                                count=2)
+        self.assertServiceCheck('elasticsearch.can_connect',
+                                status=AgentCheck.CRITICAL, tags=bad_sc_tags,
+                                count=1)
+
+
+        status = AgentCheck.OK if os.environ.get("TRAVIS")\
+            else AgentCheck.CRITICAL
+        # Travis doesn't have any shards in the cluster and consider this as green
+        self.assertServiceCheck('elasticsearch.cluster_health',
+                                status=status, tags=good_sc_tags,
+                                count=2)
+
+        self.coverage_report()
+
+    def test_config_parser(self):
+        check = load_check(self.CHECK_NAME, {}, {})
+        instance = {
+            "username": "user",
+            "password": "pass",
+            "is_external": "yes",
+            "url": "http://foo.bar",
+            "tags": ["a", "b:c"],
+        }
+
+        c = check.get_instance_config(instance)
+        self.assertEquals(c.username, "user")
+        self.assertEquals(c.password, "pass")
+        self.assertEquals(c.is_external, True)
+        self.assertEquals(c.url, "http://foo.bar")
+        self.assertEquals(c.tags, ["url:http://foo.bar", "a", "b:c"])
+        self.assertEquals(c.timeout, check.DEFAULT_TIMEOUT)
+        self.assertEquals(c.service_check_tags, ["host:foo.bar", "port:None"])
+
+        instance = {
+            "url": "http://192.168.42.42:12999",
+            "timeout": 15
+        }
+
+        c = check.get_instance_config(instance)
+        self.assertEquals(c.username, None)
+        self.assertEquals(c.password, None)
+        self.assertEquals(c.is_external, False)
+        self.assertEquals(c.url, "http://192.168.42.42:12999")
+        self.assertEquals(c.tags, ["url:http://192.168.42.42:12999"])
+        self.assertEquals(c.timeout, 15)
+        self.assertEquals(c.service_check_tags,
+                          ["host:192.168.42.42", "port:12999"])