Repository: DataDog/dd-agent

Issue:
[core] Memory/handle leak in Windows agent
It seems there's a memory and handle leak in v5.7.1 of the Windows agent. Please see the attached Process Explorer screenshot. Note that the amount of memory and handles used were built up in less than 24 hours. The exact same thing happened in three different servers.
The vast majority of the handles are file handles for `c:\windows\system32\wbem\wbemdisp.tlb` which is used in WMI yet I am not using `wmi_check`.

I'm running v5.6.1 in two other servers and it's not displaying this behaviour. Memory usage is stable around 20MB with only a couple hunded handles.

I made a full dump of the ddagent.exe process, let me know if that's useful, it's 3.5GB.

![datadog-leak](https://cloud.githubusercontent.com/assets/95194/13849585/b3399c22-ec4e-11e5-95af-89df8f29d056.png)



Discussion:
Thanks a lot for this report @mausch

We use WMI to capture system metrics (cpu, memory, etc), so even if the WMI check is not enabled, WMI is being used.

Is there a convenient way for you to share your memory dump ? Could you reach out to support@datadoghq.com please ?

Could you attach a flare (https://help.datadoghq.com/hc/en-us/articles/204991415-Send-logs-and-configs-to-Datadog-via-flare-command) as well ?

Thanks a lot for this feedback. We'll investigate that shortly.


Resolving patch:
diff --git a/checks/libs/wmi/sampler.py b/checks/libs/wmi/sampler.py
--- a/checks/libs/wmi/sampler.py
+++ b/checks/libs/wmi/sampler.py
@@ -19,18 +19,18 @@
 """
 
 # stdlib
+from collections import defaultdict
 from contextlib import contextmanager
 from copy import deepcopy
 from itertools import izip
+from utils.timeout import TimeoutException
 import pywintypes
 
 # 3p
 from win32com.client import Dispatch
-import pythoncom
 
 # project
 from checks.libs.wmi.counter_type import get_calculator, get_raw, UndefinedCalculator
-from utils.timeout import timeout, TimeoutException
 
 
 class CaseInsensitiveDict(dict):
@@ -52,8 +52,7 @@ class WMISampler(object):
     WMI Sampler.
     """
     # Shared resources
-    _wmi_locators = {}
-    _wmi_last_connections = {}
+    _wmi_connections = defaultdict(list)
 
     def __init__(self, logger, class_name, property_names, filters="", host="localhost",
                  namespace="root\\cimv2", username="", password="", and_props=[], timeout_duration=10):
@@ -91,7 +90,9 @@ def __init__(self, logger, class_name, property_names, filters="", host="localho
         self._formatted_filters = None
         self.property_counter_types = None
         self._timeout_duration = timeout_duration
-        self._query = timeout(timeout_duration)(self._query)
+
+        # FIXME
+        # self._query = timeout(timeout_duration)(self._query)
 
         # Samples
         self.current_sample = None
@@ -262,31 +263,43 @@ def _format_property_values(self, previous, current):
     def get_connection(self):
         """
         Return an existing, available WMI connection or create a new one.
-
         Release, i.e. mark as available at exit.
         """
-        self.logger.debug(
-            u"Connecting to WMI server "
-            u"(host={host}, namespace={namespace}, username={username}).".format(
-                host=self.host,
-                namespace=self.namespace,
-                username=self.username
-            )
-        )
+        connection = None
 
-        pythoncom.CoInitialize()
-        locator = Dispatch("WbemScripting.SWbemLocator")
-        connection = locator.ConnectServer(
-            self.host, self.namespace,
-            self.username, self.password
-        )
+        # Fetch an existing connection or create a new one
+        available_connections = self._wmi_connections[self.connection_key]
+
+        if available_connections:
+            self.logger.debug(
+                u"Using cached connection "
+                u"(host={host}, namespace={namespace}, username={username}).".format(
+                    host=self.host,
+                    namespace=self.namespace,
+                    username=self.username
+                )
+            )
+            connection = available_connections.pop()
+        else:
+            self.logger.debug(
+                u"Connecting to WMI server "
+                u"(host={host}, namespace={namespace}, username={username}).".format(
+                    host=self.host,
+                    namespace=self.namespace,
+                    username=self.username
+                )
+            )
+            locator = Dispatch("WbemScripting.SWbemLocator")
+            connection = locator.ConnectServer(
+                self.host, self.namespace,
+                self.username, self.password
+            )
 
         # Yield the connection
         yield connection
-        self._wmi_last_connections[self.connection_key] = connection
 
-    def get_last_connection(self):
-        return self._wmi_last_connections.get(self.connection_key, None)
+        # Release it
+        self._wmi_connections[self.connection_key].append(connection)
 
     @staticmethod
     def _format_filter(filters, and_props=[]):


Test patch:
diff --git a/tests/checks/mock/test_wmi_check.py b/tests/checks/mock/test_wmi_check.py
--- a/tests/checks/mock/test_wmi_check.py
+++ b/tests/checks/mock/test_wmi_check.py
@@ -3,7 +3,7 @@
 
 # project
 from tests.checks.common import AgentCheckTest
-from tests.core.test_wmi import SWbemServices, TestCommonWMI
+from tests.core.test_wmi import TestCommonWMI
 
 
 class WMITestCase(AgentCheckTest, TestCommonWMI):
@@ -217,35 +217,35 @@ def test_warnings_on_non_digit(self):
         self.run_check(config, mocks={'log': logger})
         self.assertEquals(logger.warning.call_count, 1)
 
-    def test_query_timeouts(self):
-        """
-        Gracefully handle WMI query timeouts.
-        """
-        def __patched_init__(*args, **kwargs):
-            """
-            Force `timeout_duration` value.
-            """
-            kwargs['timeout_duration'] = 0.1
-            return wmi_constructor(*args, **kwargs)
-
-        # Increase WMI queries' runtime
-        SWbemServices._exec_query_run_time = 0.2
-
-        # Patch WMISampler to decrease timeout tolerance
-        from checks.libs.wmi.sampler import WMISampler
-
-        wmi_constructor = WMISampler.__init__
-        WMISampler.__init__ = __patched_init__
-
-        # Set up the check
-        config = {
-            'instances': [self.WMI_CONFIG]
-        }
-        logger = Mock()
-
-        # No exception is raised but a WARNING is logged
-        self.run_check(config, mocks={'log': logger})
-        self.assertTrue(logger.warning.called)
+    # def test_query_timeouts(self):
+    #     """
+    #     Gracefully handle WMI query timeouts.
+    #     """
+    #     def __patched_init__(*args, **kwargs):
+    #         """
+    #         Force `timeout_duration` value.
+    #         """
+    #         kwargs['timeout_duration'] = 0.1
+    #         return wmi_constructor(*args, **kwargs)
+
+    #     # Increase WMI queries' runtime
+    #     SWbemServices._exec_query_run_time = 0.2
+
+    #     # Patch WMISampler to decrease timeout tolerance
+    #     from checks.libs.wmi.sampler import WMISampler
+
+    #     wmi_constructor = WMISampler.__init__
+    #     WMISampler.__init__ = __patched_init__
+
+    #     # Set up the check
+    #     config = {
+    #         'instances': [self.WMI_CONFIG]
+    #     }
+    #     logger = Mock()
+
+    #     # No exception is raised but a WARNING is logged
+    #     self.run_check(config, mocks={'log': logger})
+    #     self.assertTrue(logger.warning.called)
 
     def test_mandatory_tag_by(self):
         """
diff --git a/tests/core/test_wmi.py b/tests/core/test_wmi.py
--- a/tests/core/test_wmi.py
+++ b/tests/core/test_wmi.py
@@ -10,7 +10,6 @@
 
 # project
 from tests.checks.common import Fixtures
-from utils.timeout import TimeoutException
 
 
 log = logging.getLogger(__name__)
@@ -265,27 +264,27 @@ def assertWMIConn(self, wmi_sampler, param=None):
         """
 
         if param:
-            connection = wmi_sampler.get_last_connection()
-            wmi_conn_args, wmi_conn_kwargs = connection.get_conn_args()
-            if isinstance(param, tuple):
-                key, value = param
-                self.assertIn(key, wmi_conn_kwargs)
-                self.assertEquals(wmi_conn_kwargs[key], value)
-            else:
-                self.assertIn(param, wmi_conn_args)
+            with wmi_sampler.get_connection() as connection:
+                wmi_conn_args, wmi_conn_kwargs = connection.get_conn_args()
+                if isinstance(param, tuple):
+                    key, value = param
+                    self.assertIn(key, wmi_conn_kwargs)
+                    self.assertEquals(wmi_conn_kwargs[key], value)
+                else:
+                    self.assertIn(param, wmi_conn_args)
 
     def assertWMIQuery(self, wmi_sampler, query=None, flags=None):
         """
         Helper, assert that the given WMI query and flags were submitted.
         """
-        connection = wmi_sampler.get_last_connection()
-        if query:
-            last_wmi_query = connection.get_last_wmi_query()
-            self.assertEquals(last_wmi_query, query)
+        with wmi_sampler.get_connection() as connection:
+            if query:
+                last_wmi_query = connection.get_last_wmi_query()
+                self.assertEquals(last_wmi_query, query)
 
-        if flags:
-            last_wmi_flags = connection.get_last_wmi_flags()
-            self.assertEquals(last_wmi_flags, flags)
+            if flags:
+                last_wmi_flags = connection.get_last_wmi_flags()
+                self.assertEquals(last_wmi_flags, flags)
 
     def assertWMIObject(self, wmi_obj, properties):
         """
@@ -386,13 +385,11 @@ def test_wmi_connection_pooling(self):
         wmi_sampler_1.sample()
         wmi_sampler_2.sample()
 
-        # one connection, two samples
-        self.assertEquals(Dispatch.ConnectServer.call_count, 3, Dispatch.ConnectServer.call_count)
+        self.assertEquals(Dispatch.ConnectServer.call_count, 1, Dispatch.ConnectServer.call_count)
 
         wmi_sampler_3.sample()
 
-        # two connection, three samples
-        self.assertEquals(Dispatch.ConnectServer.call_count, 5, Dispatch.ConnectServer.call_count)
+        self.assertEquals(Dispatch.ConnectServer.call_count, 2, Dispatch.ConnectServer.call_count)
 
     def test_wql_filtering(self):
         """
@@ -610,38 +607,38 @@ def test_wmi_sampler_iterator_getter(self):
         for index in xrange(0, 2):
             self.assertWMIObject(wmi_sampler[index], ["AvgDiskBytesPerWrite", "FreeMegabytes", "name"])
 
-    def test_wmi_sampler_timeout(self):
-        """
-        Gracefully handle WMI query timeouts.
-        """
-        from checks.libs.wmi.sampler import WMISampler
-        logger = Mock()
-
-        # Create a sampler that timeouts
-        wmi_sampler = WMISampler(logger, "Win32_PerfFormattedData_PerfDisk_LogicalDisk",
-                                 ["AvgDiskBytesPerWrite", "FreeMegabytes"],
-                                 timeout_duration=0.1)
-        SWbemServices._exec_query_run_time = 0.11
-
-        # `TimeoutException` exception is raised, DEBUG message logged
-        self.assertRaises(TimeoutException, wmi_sampler.sample)
-        self.assertTrue(wmi_sampler._sampling)
-        self.assertTrue(logger.debug.called)
-
-        # Cannot iterate on data
-        self.assertRaises(TypeError, lambda: len(wmi_sampler))
-        self.assertRaises(TypeError, lambda: sum(1 for _ in wmi_sampler))
-
-        # Recover from timeout at next iteration
-        wmi_sampler.sample()
-        self.assertFalse(wmi_sampler._sampling)
-
-        # The existing query was retrieved
-        self.assertEquals(SWbemServices.ExecQuery.call_count, 1, SWbemServices.ExecQuery.call_count)
-
-        # Data is populated
-        self.assertEquals(len(wmi_sampler), 2)
-        self.assertEquals(sum(1 for _ in wmi_sampler), 2)
+    # def test_wmi_sampler_timeout(self):
+    #     """
+    #     Gracefully handle WMI query timeouts.
+    #     """
+    #     from checks.libs.wmi.sampler import WMISampler
+    #     logger = Mock()
+
+    #     # Create a sampler that timeouts
+    #     wmi_sampler = WMISampler(logger, "Win32_PerfFormattedData_PerfDisk_LogicalDisk",
+    #                              ["AvgDiskBytesPerWrite", "FreeMegabytes"],
+    #                              timeout_duration=0.1)
+    #     SWbemServices._exec_query_run_time = 0.11
+
+    #     # `TimeoutException` exception is raised, DEBUG message logged
+    #     self.assertRaises(TimeoutException, wmi_sampler.sample)
+    #     self.assertTrue(wmi_sampler._sampling)
+    #     self.assertTrue(logger.debug.called)
+
+    #     # Cannot iterate on data
+    #     self.assertRaises(TypeError, lambda: len(wmi_sampler))
+    #     self.assertRaises(TypeError, lambda: sum(1 for _ in wmi_sampler))
+
+    #     # Recover from timeout at next iteration
+    #     wmi_sampler.sample()
+    #     self.assertFalse(wmi_sampler._sampling)
+
+    #     # The existing query was retrieved
+    #     self.assertEquals(SWbemServices.ExecQuery.call_count, 1, SWbemServices.ExecQuery.call_count)
+
+    #     # Data is populated
+    #     self.assertEquals(len(wmi_sampler), 2)
+    #     self.assertEquals(sum(1 for _ in wmi_sampler), 2)
 
     def test_raw_perf_properties(self):
         """