Repository: DataDog/dd-agent

Issue:
[elasticsearch] Add / remove metrics for 2.x
This would involve logic for _not_ adding certain stats if the version of Elasticsearch is >= 2.x.x

A (probably not exhaustive) list of stats changes can be found here: https://www.elastic.co/guide/en/elasticsearch/reference/2.0/breaking_20_stats_info_and_literal_cat_literal_changes.html



Discussion:
Thanks for opening the issue @bdharrington7.

After a quick look it seems that the `fs.*.disk*` are the only affected metrics that are currently fetched by the check, i.e.:

```
elasticsearch.fs.total.disk_reads
elasticsearch.fs.total.disk_writes
elasticsearch.fs.total.disk_io_op
elasticsearch.fs.total.disk_read_size_in_bytes
elasticsearch.fs.total.disk_write_size_in_bytes
elasticsearch.fs.total.disk_io_size_in_bytes
```

Do you have info/experience on metrics that have been added in elasticsearch 2.0+ ?

I haven't played with it, I just opened this issue as more of a bookmark / reminder :)

-Brian 

> On Jan 6, 2016, at 4:29 PM, Olivier Vielpeau notifications@github.com wrote:
> 
> Thanks for opening the issue @bdharrington7.
> 
> After a quick look it seems that the fs._.disk_ are the only affected metrics that are currently fetched by the check, i.e.:
> 
> elasticsearch.fs.total.disk_reads
> elasticsearch.fs.total.disk_writes
> elasticsearch.fs.total.disk_io_op
> elasticsearch.fs.total.disk_read_size_in_bytes
> elasticsearch.fs.total.disk_write_size_in_bytes
> elasticsearch.fs.total.disk_io_size_in_bytes
> Do you have info/experience on metrics that have been added in elasticsearch 2.0+ ?
> 
> —
> Reply to this email directly or view it on GitHub.


Resolving patch:
diff --git a/checks.d/elastic.py b/checks.d/elastic.py
--- a/checks.d/elastic.py
+++ b/checks.d/elastic.py
@@ -135,9 +135,6 @@ class ESCheck(AgentCheck):
         "elasticsearch.thread_pool.management.active": ("gauge", "thread_pool.management.active"),
         "elasticsearch.thread_pool.management.threads": ("gauge", "thread_pool.management.threads"),
         "elasticsearch.thread_pool.management.queue": ("gauge", "thread_pool.management.queue"),
-        "elasticsearch.thread_pool.merge.active": ("gauge", "thread_pool.merge.active"),
-        "elasticsearch.thread_pool.merge.threads": ("gauge", "thread_pool.merge.threads"),
-        "elasticsearch.thread_pool.merge.queue": ("gauge", "thread_pool.merge.queue"),
         "elasticsearch.thread_pool.percolate.active": ("gauge", "thread_pool.percolate.active"),
         "elasticsearch.thread_pool.percolate.threads": ("gauge", "thread_pool.percolate.threads"),
         "elasticsearch.thread_pool.percolate.queue": ("gauge", "thread_pool.percolate.queue"),
@@ -183,11 +180,14 @@ class ESCheck(AgentCheck):
 
     ADDITIONAL_METRICS_POST_0_90_5 = {
         "elasticsearch.search.fetch.open_contexts": ("gauge", "indices.search.open_contexts"),
+        "elasticsearch.fielddata.size": ("gauge", "indices.fielddata.memory_size_in_bytes"),
+        "elasticsearch.fielddata.evictions": ("gauge", "indices.fielddata.evictions"),
+    }
+
+    ADDITIONAL_METRICS_POST_0_90_5_PRE_2_0 = {
         "elasticsearch.cache.filter.evictions": ("gauge", "indices.filter_cache.evictions"),
         "elasticsearch.cache.filter.size": ("gauge", "indices.filter_cache.memory_size_in_bytes"),
         "elasticsearch.id_cache.size": ("gauge", "indices.id_cache.memory_size_in_bytes"),
-        "elasticsearch.fielddata.size": ("gauge", "indices.fielddata.memory_size_in_bytes"),
-        "elasticsearch.fielddata.evictions": ("gauge", "indices.fielddata.evictions"),
     }
 
     ADDITIONAL_METRICS_PRE_0_90_5 = {
@@ -201,6 +201,9 @@ class ESCheck(AgentCheck):
     ADDITIONAL_METRICS_POST_1_0_0 = {
         "elasticsearch.indices.translog.size_in_bytes": ("gauge", "indices.translog.size_in_bytes"),
         "elasticsearch.indices.translog.operations": ("gauge", "indices.translog.operations"),
+    }
+
+    ADDITIONAL_METRICS_1_x = {  # Stats are only valid for v1.x
         "elasticsearch.fs.total.disk_reads": ("rate", "fs.total.disk_reads"),
         "elasticsearch.fs.total.disk_writes": ("rate", "fs.total.disk_writes"),
         "elasticsearch.fs.total.disk_io_op": ("rate", "fs.total.disk_io_op"),
@@ -219,6 +222,12 @@ class ESCheck(AgentCheck):
         "elasticsearch.indices.segments.fixed_bit_set_memory_in_bytes": ("gauge", "indices.segments.fixed_bit_set_memory_in_bytes"),
     }
 
+    ADDITIONAL_METRICS_PRE_2_0 = {
+        "elasticsearch.thread_pool.merge.active": ("gauge", "thread_pool.merge.active"),
+        "elasticsearch.thread_pool.merge.threads": ("gauge", "thread_pool.merge.threads"),
+        "elasticsearch.thread_pool.merge.queue": ("gauge", "thread_pool.merge.queue"),
+    }
+
     CLUSTER_HEALTH_METRICS = {
         "elasticsearch.number_of_nodes": ("gauge", "number_of_nodes"),
         "elasticsearch.number_of_data_nodes": ("gauge", "number_of_data_nodes"),
@@ -393,6 +402,13 @@ def _define_params(self, version, cluster_stats):
         if version >= [1, 0, 0]:
             stats_metrics.update(self.ADDITIONAL_METRICS_POST_1_0_0)
 
+        if version < [2, 0, 0]:
+            stats_metrics.update(self.ADDITIONAL_METRICS_PRE_2_0)
+            if version >= [0, 90, 5]:
+                stats_metrics.update(self.ADDITIONAL_METRICS_POST_0_90_5_PRE_2_0)
+            if version >= [1, 0, 0]:
+                stats_metrics.update(self.ADDITIONAL_METRICS_1_x)
+
         if version >= [1, 3, 0]:
             stats_metrics.update(self.ADDITIONAL_METRICS_POST_1_3_0)
 


Test patch:
diff --git a/tests/checks/integration/test_elastic.py b/tests/checks/integration/test_elastic.py
--- a/tests/checks/integration/test_elastic.py
+++ b/tests/checks/integration/test_elastic.py
@@ -111,9 +111,6 @@
     "elasticsearch.thread_pool.management.active": ("gauge", "thread_pool.management.active"),
     "elasticsearch.thread_pool.management.threads": ("gauge", "thread_pool.management.threads"),
     "elasticsearch.thread_pool.management.queue": ("gauge", "thread_pool.management.queue"),
-    "elasticsearch.thread_pool.merge.active": ("gauge", "thread_pool.merge.active"),
-    "elasticsearch.thread_pool.merge.threads": ("gauge", "thread_pool.merge.threads"),
-    "elasticsearch.thread_pool.merge.queue": ("gauge", "thread_pool.merge.queue"),
     "elasticsearch.thread_pool.percolate.active": ("gauge", "thread_pool.percolate.active"),
     "elasticsearch.thread_pool.percolate.threads": ("gauge", "thread_pool.percolate.threads"),
     "elasticsearch.thread_pool.percolate.queue": ("gauge", "thread_pool.percolate.queue"),
@@ -159,11 +156,14 @@
 
 ADDITIONAL_METRICS_POST_0_90_5 = {
     "elasticsearch.search.fetch.open_contexts": ("gauge", "indices.search.open_contexts"),
+    "elasticsearch.fielddata.size": ("gauge", "indices.fielddata.memory_size_in_bytes"),
+    "elasticsearch.fielddata.evictions": ("gauge", "indices.fielddata.evictions"),
+}
+
+ADDITIONAL_METRICS_POST_0_90_5_PRE_2_0 = {
     "elasticsearch.cache.filter.evictions": ("gauge", "indices.filter_cache.evictions"),
     "elasticsearch.cache.filter.size": ("gauge", "indices.filter_cache.memory_size_in_bytes"),
     "elasticsearch.id_cache.size": ("gauge", "indices.id_cache.memory_size_in_bytes"),
-    "elasticsearch.fielddata.size": ("gauge", "indices.fielddata.memory_size_in_bytes"),
-    "elasticsearch.fielddata.evictions": ("gauge", "indices.fielddata.evictions"),
 }
 
 ADDITIONAL_METRICS_PRE_0_90_5 = {
@@ -177,6 +177,9 @@
 ADDITIONAL_METRICS_POST_1_0_0 = {
     "elasticsearch.indices.translog.size_in_bytes": ("gauge", "indices.translog.size_in_bytes"),
     "elasticsearch.indices.translog.operations": ("gauge", "indices.translog.operations"),
+}
+
+ADDITIONAL_METRICS_1_x = {
     # Currently has issues in test framework:
     # "elasticsearch.fs.total.disk_reads": ("rate", "fs.total.disk_reads"),
     # "elasticsearch.fs.total.disk_writes": ("rate", "fs.total.disk_writes"),
@@ -196,6 +199,12 @@
     "elasticsearch.indices.segments.fixed_bit_set_memory_in_bytes": ("gauge", "indices.segments.fixed_bit_set_memory_in_bytes"),
 }
 
+ADDITIONAL_METRICS_PRE_2_0 = {
+    "elasticsearch.thread_pool.merge.active": ("gauge", "thread_pool.merge.active"),
+    "elasticsearch.thread_pool.merge.threads": ("gauge", "thread_pool.merge.threads"),
+    "elasticsearch.thread_pool.merge.queue": ("gauge", "thread_pool.merge.queue"),
+}
+
 CLUSTER_HEALTH_METRICS = {
     "elasticsearch.number_of_nodes": ("gauge", "number_of_nodes"),
     "elasticsearch.number_of_data_nodes": ("gauge", "number_of_data_nodes"),
@@ -274,21 +283,29 @@ def test_check(self):
         if es_version >= [1, 0, 0]:
             expected_metrics.update(ADDITIONAL_METRICS_POST_1_0_0)
 
+        if es_version < [2, 0, 0]:
+            expected_metrics.update(ADDITIONAL_METRICS_PRE_2_0)
+            if es_version >= [0, 90, 5]:
+                expected_metrics.update(ADDITIONAL_METRICS_POST_0_90_5_PRE_2_0)
+            if es_version >= [1, 0, 0]:
+                expected_metrics.update(ADDITIONAL_METRICS_1_x)
+
         if es_version >= [1, 3, 0]:
             expected_metrics.update(ADDITIONAL_METRICS_POST_1_3_0)
 
         if es_version >= [1, 4, 0]:
             expected_metrics.update(ADDITIONAL_METRICS_POST_1_4_0)
 
+        local_hostname = socket.gethostname() if es_version < [2, 0, 0] else '127.0.0.1'
         contexts = [
             (conf_hostname, default_tags + tags),
-            (socket.gethostname(), default_tags)
+            (local_hostname, default_tags)
         ]
 
         for m_name, desc in expected_metrics.iteritems():
             for hostname, m_tags in contexts:
                 if (m_name in CLUSTER_HEALTH_METRICS
-                        and hostname == socket.gethostname()):
+                        and hostname == local_hostname):
                     hostname = conf_hostname
 
                 if desc[0] == "gauge":