Repository: DataDog/dd-agent

Issue:
High CPU Load  and intermediate image count failure
When there are lot of intermediate docker images, the following line leads to high cpu utilization by docker daemon:

https://github.com/DataDog/dd-agent/blob/5.1.x/checks.d/docker.py#L150

Also the call fails due to timeout .

Can we make intermediate image counter configurable (and turn it off by default ?).  

```
Feb 03 06:32:23 ip-10-102-169-13.ec2.internal dockerd[21250]: time="2015-02-03T06:32:23Z" level="info" msg="GET /containers/json?all=False&size=False"
Feb 03 06:32:23 ip-10-102-169-13.ec2.internal dockerd[21250]: time="2015-02-03T06:32:23Z" level="info" msg="+job containers()"
Feb 03 06:32:23 ip-10-102-169-13.ec2.internal dockerd[21250]: time="2015-02-03T06:32:23Z" level="info" msg="-job containers() = OK (0)"
Feb 03 06:32:23 ip-10-102-169-13.ec2.internal dockerd[21250]: time="2015-02-03T06:32:23Z" level="info" msg="GET /containers/json?all=True&size=False"
Feb 03 06:32:23 ip-10-102-169-13.ec2.internal dockerd[21250]: time="2015-02-03T06:32:23Z" level="info" msg="+job containers()"
Feb 03 06:32:23 ip-10-102-169-13.ec2.internal dockerd[21250]: time="2015-02-03T06:32:23Z" level="info" msg="-job containers() = OK (0)"
Feb 03 06:32:23 ip-10-102-169-13.ec2.internal dockerd[21250]: time="2015-02-03T06:32:23Z" level="info" msg="GET /events?since=1422945112&until=1422945143"
Feb 03 06:32:23 ip-10-102-169-13.ec2.internal dockerd[21250]: time="2015-02-03T06:32:23Z" level="info" msg="+job events()"
Feb 03 06:32:23 ip-10-102-169-13.ec2.internal dockerd[21250]: time="2015-02-03T06:32:23Z" level="info" msg="-job events() = OK (0)"
Feb 03 06:32:26 ip-10-102-169-13.ec2.internal dockerd[21250]: write unix @: broken pipe
Feb 03 06:32:26 ip-10-102-169-13.ec2.internal dockerd[21250]: time="2015-02-03T06:32:26Z" level="info" msg="-job images() = ERR (1)"
Feb 03 06:32:26 ip-10-102-169-13.ec2.internal dockerd[21250]: time="2015-02-03T06:32:26Z" level="error" msg="Handler for GET /images/json returned error: write unix @: broken pipe"
Feb 03 06:32:26 ip-10-102-169-13.ec2.internal dockerd[21250]: time="2015-02-03T06:32:26Z" level="error" msg="HTTP Error: statusCode=500 write unix @: broken pipe"
```

**Env Info**:
Docker version: `Docker version 1.4.1, build 5bc2ff8-dirty`
Agent Image: `datadog/docker-dd-agent:latest  image-id: 8f65aeca1481`



Discussion:
Hi @sukrit007 ,

Can you try to use the `socket_timeout` option and increase it?
https://github.com/DataDog/dd-agent/blob/5.1.x/conf.d/docker.yaml.example#L14

A big list of images may take longer than the default timeout (5 seconds).
If it doesn't help, please share your collector logs with the `log_level: DEBUG`.

In theory I think that should get rid of timeouts. However,  this call seems a bit expensive  when used with all=true parameter. (results in CPU spikes).  Wondering if this can be made configurable. 

 Ideally I would have hoped docker to provide api to publish the counts. But intermediate image layers might not have much relevance. So would prefer to turn it off.


Resolving patch:
diff --git a/checks.d/docker.py b/checks.d/docker.py
--- a/checks.d/docker.py
+++ b/checks.d/docker.py
@@ -12,6 +12,7 @@
 
 # project
 from checks import AgentCheck
+from config import _is_affirmative
 
 EVENT_TYPE = SOURCE_TYPE_NAME = 'docker'
 
@@ -108,8 +109,8 @@ def unix_open(self, req):
 class Docker(AgentCheck):
     """Collect metrics and events from Docker API and cgroups"""
 
-    def __init__(self, name, init_config, agentConfig):
-        AgentCheck.__init__(self, name, init_config, agentConfig)
+    def __init__(self, name, init_config, agentConfig, instances=None):
+        AgentCheck.__init__(self, name, init_config, agentConfig, instances)
 
         # Initialize a HTTP opener with Unix socket support
         socket_timeout = int(init_config.get('socket_timeout', 0)) or DEFAULT_SOCKET_TIMEOUT
@@ -127,7 +128,8 @@ def __init__(self, name, init_config, agentConfig):
 
     def check(self, instance):
         # Report image metrics
-        self._count_images(instance)
+        if _is_affirmative(instance.get('collect_images_stats', True)):
+            self._count_images(instance)
 
         # Get the list of containers and the index of their names
         containers, ids_to_names = self._get_and_count_containers(instance)
@@ -136,7 +138,7 @@ def check(self, instance):
         skipped_container_ids = self._report_containers_metrics(containers, instance)
 
         # Send events from Docker API
-        if instance.get('collect_events', True):
+        if _is_affirmative(instance.get('collect_events', True)):
             self._process_events(instance, ids_to_names, skipped_container_ids)
 
 
@@ -156,7 +158,7 @@ def _count_images(self, instance):
 
     def _get_and_count_containers(self, instance):
         tags = instance.get("tags", [])
-        with_size = instance.get('collect_container_size', False)
+        with_size = _is_affirmative(instance.get('collect_container_size', False))
 
         service_check_name = 'docker.service_up'
         try:
@@ -215,7 +217,7 @@ def _tags_match_patterns(self, tags, filters):
 
     def _report_containers_metrics(self, containers, instance):
         skipped_container_ids = []
-        collect_uncommon_metrics = instance.get("collect_all_metrics", False)
+        collect_uncommon_metrics = _is_affirmative(instance.get("collect_all_metrics", False))
         tags = instance.get("tags", [])
 
         # Pre-compile regex to include/exclude containers
@@ -413,11 +415,13 @@ def _get_cgroup_file(self, cgroup, container_id, filename):
     def _find_cgroup(self, hierarchy, docker_root):
         """Finds the mount point for a specified cgroup hierarchy. Works with
         old style and new style mounts."""
+        fp = None
         try:
             fp = open(os.path.join(docker_root, "/proc/mounts"))
             mounts = map(lambda x: x.split(), fp.read().splitlines())
         finally:
-            fp.close()
+            if fp is not None:
+                fp.close()
         cgroup_mounts = filter(lambda x: x[2] == "cgroup", mounts)
         if len(cgroup_mounts) == 0:
             raise Exception("Can't find mounted cgroups. If you run the Agent inside a container,"


Test patch:
diff --git a/tests/common.py b/tests/common.py
--- a/tests/common.py
+++ b/tests/common.py
@@ -11,6 +11,26 @@
 from config import get_checksd_path
 from util import get_os, get_hostname
 
+def get_check_class(name):
+    checksd_path = get_checksd_path(get_os())
+    if checksd_path not in sys.path:
+        sys.path.append(checksd_path)
+
+    check_module = __import__(name)
+    check_class = None
+    classes = inspect.getmembers(check_module, inspect.isclass)
+    for _, clsmember in classes:
+        if clsmember == AgentCheck:
+            continue
+        if issubclass(clsmember, AgentCheck):
+            check_class = clsmember
+            if AgentCheck in clsmember.__bases__:
+                continue
+            else:
+                break
+
+    return check_class
+
 def load_check(name, config, agentConfig):
     checksd_path = get_checksd_path(get_os())
     if checksd_path not in sys.path:
diff --git a/tests/test_docker.py b/tests/test_docker.py
new file mode 100644
--- /dev/null
+++ b/tests/test_docker.py
@@ -0,0 +1,77 @@
+import unittest
+
+from mock import patch
+
+from tests.common import get_check_class
+
+def _mocked_find_cgroup(*args, **kwargs):
+    return
+
+class DockerCheckTest(unittest.TestCase):
+    def test_tag_exclude_all(self):
+        """ exclude all, except ubuntu and debian. """
+        instance = {
+            'include': [
+                'docker_image:ubuntu',
+                'docker_image:debian',
+            ],
+            'exclude': ['.*'],
+        }
+
+        klass = get_check_class('docker')
+        # NO-OP but loads the check
+        with patch.object(klass, '_find_cgroup', _mocked_find_cgroup):
+            check = klass('docker', {}, {})
+
+        check._prepare_filters(instance)
+        self.assertEquals(len(instance['exclude_patterns']), 1)
+        self.assertEquals(len(instance['include_patterns']), 2)
+
+        truth_table_exclusion = {
+            'some_tag': True,
+            'debian:ubuntu': True,
+            'docker_image:centos': True,
+            'docker_image:ubuntu': False,
+            'docker_image:debian': False,
+        }
+
+        for tag, val in truth_table_exclusion.iteritems():
+            self.assertEquals(
+                check._is_container_excluded(instance, [tag]),
+                val,
+                "{0} expected {1} but is not".format(tag, val)
+            )
+
+    def test_tag_include_all(self):
+        """ exclude all, except ubuntu and debian. """
+        instance = {
+            'include': [],
+            'exclude': [
+                'docker_image:ubuntu',
+                'docker_image:debian',
+            ],
+        }
+
+        klass = get_check_class('docker')
+        # NO-OP but loads the check
+        with patch.object(klass, '_find_cgroup', _mocked_find_cgroup):
+            check = klass('docker', {}, {})
+
+        check._prepare_filters(instance)
+        self.assertEquals(len(instance['exclude_patterns']), 2)
+        self.assertEquals(len(instance['include_patterns']), 0)
+
+        truth_table_exclusion = {
+            'some_tag': False,
+            'debian:ubuntu': False,
+            'docker_image:centos': False,
+            'docker_image:ubuntu': True,
+            'docker_image:debian': True,
+        }
+
+        for tag, val in truth_table_exclusion.iteritems():
+            self.assertEquals(
+                check._is_container_excluded(instance, [tag]),
+                val,
+                "{0} expected {1} but is not".format(tag, val)
+            )