Repository: DataDog/dd-agent

Issue:
Travis Fails Memcache Memory Leak
Travis Build: https://travis-ci.org/DataDog/dd-agent/jobs/6192350
Test results in a gist: https://gist.github.com/pcockwell/5a8a5acc483f8cfe90eb

Investigation: 
- Ran tests on Ubuntu 12.04 VM after reproducing Travis env as closely as possible. 
- Used --pdb-failures to examine garbage collection info. 
- Currently no failures (for memcache tests) on Ubuntu 12.04 VM or on local Mac OSX.

For now, this test will be skipped.



Discussion:

Resolving patch:
diff --git a/checks/db/mcache.py b/checks.d/mcache.py
similarity index 56%
rename from checks/db/mcache.py
rename to checks.d/mcache.py
--- a/checks/db/mcache.py
+++ b/checks.d/mcache.py
@@ -54,78 +54,39 @@
 # http://www.couchbase.org/wiki/display/membase/Membase+Statistics
 # https://github.com/membase/ep-engine/blob/master/docs/stats.org
 
-class Memcache(Check):
+class Memcache(AgentCheck):
     DEFAULT_PORT = 11211
 
-    def __init__(self, logger):
-        Check.__init__(self, logger)
-        self.gauge("memcache.total_items")
-        self.gauge("memcache.curr_items")
-        self.gauge("memcache.limit_maxbytes")
-        self.gauge("memcache.uptime")
-        self.gauge("memcache.bytes")
-        self.gauge("memcache.curr_connections")
-        self.gauge("memcache.connection_structures")
-        self.gauge("memcache.threads")
-        self.gauge("memcache.pointer_size")
-
-        # these two are calculated from other metrics
-        self.gauge("memcache.get_hit_percent")
-        self.gauge("memcache.avg_item_size")
-        self.gauge("memcache.fill_percent")
-
-        self.counter("memcache.rusage_user_rate")
-        self.counter("memcache.rusage_system_rate")
-        self.counter("memcache.cmd_get_rate")
-        self.counter("memcache.cmd_set_rate")
-        self.counter("memcache.cmd_flush_rate")
-        self.counter("memcache.get_hits_rate")
-        self.counter("memcache.get_misses_rate")
-        self.counter("memcache.evictions_rate")
-        self.counter("memcache.bytes_read_rate")
-        self.counter("memcache.bytes_written_rate")
-        self.counter("memcache.total_connections_rate")
-
-    def _load_conf(self, agentConfig):
-
-        # Load the conf according to the old schema
-        memcache_url = agentConfig.get("memcache_server", None)
-        memcache_port = agentConfig.get("memcache_port", None)
-        memcache_urls = []
-        memcache_ports = []
-        tags = []
-        if memcache_url is not None:
-            memcache_urls.append(memcache_url)
-            memcache_ports.append(memcache_port)
-            tags.append(None)
-
-
-        # Load the conf according to the new schema
-        #memcache_instance_1: first_host:first_port:first_tag
-        #memcache_instance_2: second_host:second_port:second_tag
-        #memcache_instance_3: third_host:third_port:third_tag
-        index = 1
-        instance = agentConfig.get("memcache_instance_%s" % index, None)
-        while instance:
-            instance = instance.split(":")
-            memcache_urls.append(instance[0])
-            if len(instance)>1:
-                memcache_ports.append(instance[1])
-            else:
-                memcache_ports.append(self.DEFAULT_PORT)
-            if len(instance)==3:
-                tags.append(instance[2])
-            else:
-                tags.append(None)
-            index = index + 1
-            instance = agentConfig.get("memcache_instance_%s" % index, None)
-
-        return (memcache_urls, memcache_ports, tags)
+    GAUGES = [
+        "total_items",
+        "curr_items",
+        "limit_maxbytes",
+        "uptime",
+        "bytes",
+        "curr_connections",
+        "connection_structures",
+        "threads",
+        "pointer_size"
+    ]
+
+    RATES = [
+        "rusage_user",
+        "rusage_system",
+        "cmd_get",
+        "cmd_set",
+        "cmd_flush",
+        "get_hits",
+        "get_misses",
+        "evictions",
+        "bytes_read",
+        "bytes_written",
+        "total_connections"
+    ]
 
     def _get_metrics(self, server, port, tags, memcache):
-        mc = None # client
+        mc = None  # client
         try:
-            self.logger.debug("Connecting to %s:%s tags:%s", server, port, tags)
+            self.log.debug("Connecting to %s:%s tags:%s", server, port, tags)
             mc = memcache.Client(["%s:%d" % (server, port)])
             raw_stats = mc.get_stats()
 
@@ -133,74 +94,63 @@ def _get_metrics(self, server, port, tags, memcache):
             # Access the dict
             stats = raw_stats[0][1]
             for metric in stats:
-                self.logger.debug("Processing %s: %s", metric, stats[metric])
+                # Check if metric is a gauge or rate
+                if metric in self.GAUGES:
+                    our_metric = self.normalize(metric.lower(), 'memcache')
+                    self.gauge(our_metric, float(stats[metric]), tags=tags)
 
-                our_metric = "memcache." + metric
-                # Tweak the name if it's a counter so that we don't use the exact
+                # Tweak the name if it's a rate so that we don't use the exact
                 # same metric name as the memcache documentation
-                if self.is_counter(our_metric + "_rate"):
-                    our_metric = our_metric + "_rate"
-
-                if self.is_metric(our_metric):
-                    self.save_sample(our_metric, float(stats[metric]), tags=tags)
-                    self.logger.debug("Saved %s: %s", our_metric, stats[metric])
-        except ValueError:
-            self.logger.exception("Cannot convert port value; check your configuration")
-        except CheckException:
-            self.logger.exception("Cannot save sampled data")
-        except Exception:
-            self.logger.exception("Cannot get data from memcache")
+                if metric in self.RATES:
+                    our_metric = self.normalize(metric.lower() + "_rate", 'memcache')
+                    self.rate(our_metric, float(stats[metric]), tags=tags)
 
-        if mc is not None:
             # calculate some metrics based on other metrics.
             # stats should be present, but wrap in try/except
             # and log an exception just in case.
             try:
-                self.save_sample(
+                self.gauge(
                     "memcache.get_hit_percent",
                     100.0 * float(stats["get_hits"]) / float(stats["cmd_get"]),
                     tags=tags,
                 )
             except ZeroDivisionError:
                 pass
-            except Exception: 
-                self.logger.exception("Cannot calculate memcache.get_hit_percent for tags: %s", tags)
 
             try:
-                self.save_sample(
+                self.gauge(
                     "memcache.fill_percent",
                     100.0 * float(stats["bytes"]) / float(stats["limit_maxbytes"]),
                     tags=tags,
                 )
             except ZeroDivisionError:
                 pass
-            except Exception:
-                self.logger.exception("Cannot calculate memcache.fill_percent for tags: %s", tags)
-            
+
             try:
-                self.save_sample(
+                self.gauge(
                     "memcache.avg_item_size",
                     float(stats["bytes"]) / float(stats["curr_items"]),
                     tags=tags,
                 )
             except ZeroDivisionError:
                 pass
-            except Exception:
-                self.logger.exception("Cannot calculate memcache.avg_item_size for tags: %s", tags)
-            
+        except AssertionError:
+            raise Exception("Unable to retrieve stats from memcache instance: " + server + ":" + str(port) + ". Please check your configuration")
+
+        if mc is not None:
             mc.disconnect_all()
-            self.logger.debug("Disconnected from memcached")
+            self.log.debug("Disconnected from memcached")
         del mc
 
-    def check(self, agentConfig):
-        (memcache_urls, memcache_ports, tags) = self._load_conf(agentConfig)
-        if len(memcache_urls) == 0:
-            return False
-        try:        
+    def check(self, instance):
+        server = instance.get('url', None)
+        if not server:
+            raise Exception("Missing or null 'url' in mcache config")
+
+        try:
             import memcache
         except ImportError:
-            self.logger.exception("Cannot import python-based memcache driver")
-            return False
+            raise Exception("Cannot import memcache module. Check the instructions to install this module at https://app.datadoghq.com/account/settings#integrations/mcache")
 
         # Hacky monkeypatch to fix a memory leak in the memcache library.
         # See https://github.com/DataDog/dd-agent/issues/278 for details.
@@ -209,23 +159,59 @@ def check(self, agentConfig):
         except:
             pass
 
-        for i in range(len(memcache_urls)):
-            server = memcache_urls[i]
-            if server is None:
-                continue
-            if memcache_ports[i] is None:
-                memcache_ports[i] = 11211
-            port = int(memcache_ports[i])
-
-            if tags[i] is not None:
-                tag = ["instance:%s" % tags[i]]
-            else:
-                tag=["instance:%s_%s" % (server, port)]
-
-            self._get_metrics(server, port, tag, memcache)
-                
-        metrics = self.get_metrics()
-        self.logger.debug("Memcache samples: %s", metrics)
-        return metrics
-
- 
+        port = int(instance.get('port', self.DEFAULT_PORT))
+        tags = instance.get('tags', None)
+        if not tags:
+            tags = ["instance:%s_%s" % (server, port)]
+
+        self._get_metrics(server, port, tags, memcache)
+
+    @staticmethod
+    def parse_agent_config(agentConfig):
+        all_instances = []
+
+        # Load the conf according to the old schema
+        memcache_url = agentConfig.get("memcache_server", None)
+        memcache_port = agentConfig.get("memcache_port", Memcache.DEFAULT_PORT)
+        if memcache_url is not None:
+            instance = {
+                'url': memcache_url,
+                'port': memcache_port,
+                'tags': None
+            }
+            all_instances.append(instance)
+
+        # Load the conf according to the new schema
+        #memcache_instance_1: first_host:first_port:first_tag
+        #memcache_instance_2: second_host:second_port:second_tag
+        #memcache_instance_3: third_host:third_port:third_tag
+        index = 1
+        instance = agentConfig.get("memcache_instance_%s" % index, None)
+        while instance:
+            instance = instance.split(":")
+
+            url = instance[0]
+            port = Memcache.DEFAULT_PORT
+            tags = None
+
+            if len(instance) > 1:
+                port = instance[1]
+
+            if len(instance) == 3:
+                tags = ["instance:%s" % instance[2]]
+
+            all_instances.append({
+                'url': url,
+                'port': port,
+                'tags': tags
+            })
+
+            index = index + 1
+            instance = agentConfig.get("memcache_instance_%s" % index, None)
+
+        if len(all_instances) == 0:
+            return False
+
+        return {
+            'instances': all_instances
+        }
diff --git a/checks/collector.py b/checks/collector.py
--- a/checks/collector.py
+++ b/checks/collector.py
@@ -17,7 +17,6 @@
 import checks.system.win32 as w32
 from checks.agent_metrics import CollectorMetrics
 from checks.nagios import Nagios
-from checks.db.mcache import Memcache
 from checks.ganglia import Ganglia
 from checks.cassandra import Cassandra
 from checks.datadog import Dogstreams, DdForwarder
@@ -81,10 +80,7 @@ def __init__(self, agentConfig, emitters, systemStats):
         # Agent Metrics
         self._agent_metrics = CollectorMetrics(log)
 
-        # Metric Checks
-        self._metrics_checks = [
-            Memcache(log),
-        ]
+        self._metrics_checks = []
 
         # Custom metric checks
         for module_spec in [s.strip() for s in self.agentConfig.get('custom_checks', '').split(',')]:


Test patch:
diff --git a/tests/test_mcache.py b/tests/test_mcache.py
--- a/tests/test_mcache.py
+++ b/tests/test_mcache.py
@@ -1,19 +1,22 @@
 import unittest
 import logging
 import os
+import time
 from subprocess import Popen, PIPE
-from checks.db.mcache import *
 from nose.plugins.skip import SkipTest
 
+from tests.common import load_check
+
+
 class TestMemCache(unittest.TestCase):
     def setUp(self):
-        self.c = Memcache(logging.getLogger(__file__))
+        self.c = load_check('mcache', {'init_config': {}, 'instances': {}}, None)
         self.agent_config = {
             "memcache_server": "localhost",
             "memcache_instance_1": "localhost:11211:mytag",
-            "memcache_instance_2": "dummy:11211:myothertag",
-            "memcache_instance_3": "localhost:11211:mythirdtag",
+            "memcache_instance_2": "localhost:11211:mythirdtag",
         }
+        self.conf = self.c.parse_agent_config(self.agent_config)
 
     def _countConnections(self, port):
         pid = os.getpid()
@@ -28,27 +31,77 @@ def testConnectionLeaks(self):
         for i in range(3):
             # Count open connections to localhost:11211, should be 0
             self.assertEquals(self._countConnections(11211), 0)
-            r = self.c.check({"memcache_server": "localhost"})
+            new_conf = self.c.parse_agent_config({"memcache_server": "localhost"})
+            self.c.check(new_conf['instances'][0])
             # Verify that the count is still 0
             self.assertEquals(self._countConnections(11211), 0)
 
     def testMetrics(self):
-        raise SkipTest("Test is not working anymore on travis boxes. Needs further investigation")
-        self.c.check(self.agent_config)
-        r = self.c.check(self.agent_config)
+        for instance in self.conf['instances']:
+            self.c.check(instance)
+            # Sleep for 1 second so the rate interval >=1
+            time.sleep(1)
+            self.c.check(instance)
+
+        r = self.c.get_metrics()
 
+        # Check that we got metrics from 3 hosts (aka all but the dummy host)
         self.assertEquals(len([t for t in r if t[0] == "memcache.total_items"]), 3, r)
-        self.assertEquals(len([t for t in r if t[3].get('tags') == ["instance:mythirdtag"]]), 20, r)
+
+        # Check that we got 21 metrics for a specific host
+        self.assertEquals(len([t for t in r if t[3].get('tags') == ["instance:mythirdtag"]]), 21, r)
+
+    def testTagging(self):
+        instance = {
+            'url': 'localhost',
+            'port': 11211,
+            'tags': ['instance:mytag', 'regular_old_tag']
+        }
+
+        self.c.check(instance)
+        # Sleep for 1 second so the rate interval >=1
+        time.sleep(1)
+        self.c.check(instance)
+
+        r = self.c.get_metrics()
+
+        # Check the tags
+        self.assertEquals(len([t for t in r if t[3].get('tags') == ["instance:mytag", "regular_old_tag"]]), 21, r)
+
+        instance = {
+            'url': 'localhost',
+            'port': 11211
+        }
+
+        self.c.check(instance)
+        # Sleep for 1 second so the rate interval >=1
+        time.sleep(1)
+        self.c.check(instance)
+
+        r = self.c.get_metrics()
+
+        # Check the tags
+        self.assertEquals(len([t for t in r if t[3].get('tags') == ["instance:localhost_11211"]]), 21, r)
+
+    def testDummyHost(self):
+        new_conf = self.c.parse_agent_config({"memcache_instance_1": "dummy:11211:myothertag"})
+        self.assertRaises(Exception, self.c.check, new_conf['instances'][0])
 
     def testMemoryLeak(self):
-        raise SkipTest("Test is not working anymore on travis boxes. Needs further investigation")
-        self.c.check(self.agent_config)
+        for instance in self.conf['instances']:
+            self.c.check(instance)
+        self.c.get_metrics()
+
         import gc
         gc.set_debug(gc.DEBUG_LEAK)
         try:
             start = len(gc.garbage)
             for i in range(10):
-                self.c.check(self.agent_config)
+                for instance in self.conf['instances']:
+                    self.c.check(instance)
+                time.sleep(1)
+                self.c.get_metrics()
+
             end = len(gc.garbage)
             self.assertEquals(end - start, 0, gc.garbage)
         finally: