Repository: DataDog/dd-agent

Issue:
Docker integration timeout
Hi,

I got some timeouts using the docker check, resulting in an error of the check.

It happened:
- With a default setup of a Ubuntu 14.04 inside a Vagrant, using package version of Docker with 2 running containers. It was timing out on the event endpoint.
- I was able to make it work by upgrading to Docker 0.11, but the bug appeared again when I was running 8+ containers. It was timing out on both event and cgroup metrics.

```
 Traceback (most recent call last):
  File "docker.py", line 292, in <module>
    check.check(instance)
  File "docker.py", line 142, in check
    containers = self._get_containers(instance)
  File "docker.py", line 216, in _get_containers
    return self._get_json("%(url)s/containers/json" % instance, params={"size": 1})
  File "docker.py", line 241, in _get_json
    request = urllib2.urlopen(req)
  File "/usr/lib/python2.7/urllib2.py", line 127, in urlopen
    return _opener.open(url, data, timeout)
  File "/usr/lib/python2.7/urllib2.py", line 404, in open
    response = self._open(req, data)
  File "/usr/lib/python2.7/urllib2.py", line 422, in _open
    '_open', req)
  File "/usr/lib/python2.7/urllib2.py", line 382, in _call_chain
    result = func(*args)
  File "docker.py", line 105, in unix_open
    return self.do_open(UnixHTTPConnection(unix_socket), new_req)
  File "/usr/lib/python2.7/urllib2.py", line 1187, in do_open
    r = h.getresponse(buffering=True)
  File "/usr/lib/python2.7/httplib.py", line 1045, in getresponse
    response.begin()
  File "/usr/lib/python2.7/httplib.py", line 409, in begin
    version, status, reason = self._read_status()
  File "/usr/lib/python2.7/httplib.py", line 365, in _read_status
    line = self.fp.readline(_MAXLINE + 1)
  File "/usr/lib/python2.7/socket.py", line 476, in readline
    data = self._sock.recv(self._rbufsize)
socket.timeout: timed out
```



Discussion:
@LotharSee It was fixed with 4.3.1 right ?

Partially.
The timeout can occur while getting events and while getting the container list.

I caught the one with the events (to allow trying to get containers and metrics after), but it is raised with the list of containers (if we don't have it, there is nothing else to do).

So it does not solve the real issue: the timeout itself.

As discussed, 2 things to check with a lot of containers
1. run the collector in the foreground with a patch of socket.py (drop in the debugger when you get a timeout)
2. see why the opener has to be reinstalled each time it runs.


Resolving patch:
diff --git a/checks.d/docker.py b/checks.d/docker.py
--- a/checks.d/docker.py
+++ b/checks.d/docker.py
@@ -70,6 +70,8 @@
     "Image",
 ]
 
+SOCKET_TIMEOUT = 5
+
 class UnixHTTPConnection(httplib.HTTPConnection, object):
     """Class used in conjuction with UnixSocketHandler to make urllib2
     compatible with Unix sockets."""
@@ -79,6 +81,7 @@ def __init__(self, unix_socket):
     def connect(self):
         sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
         sock.connect(self._unix_socket)
+        sock.settimeout(SOCKET_TIMEOUT)
         self.sock = sock
 
     def __call__(self, *args, **kwargs):
@@ -115,6 +118,8 @@ def __init__(self, *args, **kwargs):
             self._mountpoints[metric["cgroup"]] = self._find_cgroup(metric["cgroup"])
         self._path_prefix = None
         self._last_event_collection_ts = defaultdict(lambda: None)
+        self.url_opener = urllib2.build_opener(UnixSocketHandler())
+        self.should_get_size = True
 
     @property
     def path_prefix(self):
@@ -133,7 +138,6 @@ def path_prefix(self):
         return self._path_prefix
 
     def check(self, instance):
-        urllib2.install_opener(urllib2.build_opener(UnixSocketHandler())) # We need to reinstall the opener every time as it gets uninstalled
         tags = instance.get("tags") or []
 
         try:
@@ -141,10 +145,17 @@ def check(self, instance):
         except socket.timeout:
             self.warning('Timeout during socket connection. Events will be missing.')
 
-        try:
-            containers = self._get_containers(instance)
-        except socket.timeout:
-            raise Exception('Cannot get containers list: timeout during socket connection. Try to refine the containers to collect by editing the configuration file.')
+        if self.should_get_size:
+            try:
+                containers = self._get_containers(instance, with_size=True)
+            except socket.timeout:
+                # Probably because of: https://github.com/DataDog/dd-agent/issues/963
+                # Then we should stop trying to get size info
+                self.log.info('Cannot get container size because of API timeout. Turn size flag off.')
+                self.should_get_size = False
+
+        if not self.should_get_size:
+            containers = self._get_containers(instance, with_size=False)
 
         if not containers:
             self.gauge("docker.containers.running", 0)
@@ -219,13 +230,9 @@ def _is_tag_included(tag):
                 return True
         return False
 
-    def _get_containers(self, instance):
+    def _get_containers(self, instance, with_size=True):
         """Gets the list of running containers in Docker."""
-        return self._get_json("%(url)s/containers/json" % instance, params={"size": 1})
-
-    def _get_container(self, instance, cid):
-        """Get container information from Docker, gived a container Id."""
-        return self._get_json("%s/containers/%s/json" % (instance["url"], cid))
+        return self._get_json("%(url)s/containers/json" % instance, params={'size': with_size})
 
     def _get_events(self, instance):
         """Get the list of events """
@@ -246,7 +253,7 @@ def _get_json(self, uri, params=None, multi=False):
         self.log.debug("Connecting to: %s" % uri)
         req = urllib2.Request(uri, None)
         try:
-            request = urllib2.urlopen(req)
+            request = self.url_opener.open(req)
         except urllib2.URLError, e:
             if "Errno 13" in str(e):
                 raise Exception("Unable to connect to socket. dd-agent user must be part of the 'docker' group")
diff --git a/test-docker.py b/test-docker.py
new file mode 100644
--- /dev/null
+++ b/test-docker.py
@@ -0,0 +1,53 @@
+import os
+import httplib
+import urllib2
+import socket
+from urlparse import urlsplit
+
+class UnixHTTPConnection(httplib.HTTPConnection, object):
+    """Class used in conjuction with UnixSocketHandler to make urllib2
+    compatible with Unix sockets."""
+    def __init__(self, unix_socket):
+        self._unix_socket = unix_socket
+    def connect(self):
+        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        sock.connect(self._unix_socket)
+        self.sock = sock
+    def __call__(self, *args, **kwargs):
+        httplib.HTTPConnection.__init__(self, *args, **kwargs)
+        return self
+
+
+class UnixSocketHandler(urllib2.AbstractHTTPHandler):
+    """Class that makes Unix sockets work with urllib2 without any additional
+    dependencies."""
+    def unix_open(self, req):
+        full_path = "%s%s" % urlsplit(req.get_full_url())[1:3]
+        path = os.path.sep
+        for part in full_path.split("/"):
+            path = os.path.join(path, part)
+            if not os.path.exists(path):
+                break
+            unix_socket = path
+        # add a host or else urllib2 complains
+        url = req.get_full_url().replace(unix_socket, "/localhost")
+        new_req = urllib2.Request(url, req.get_data(), dict(req.header_items()))
+        new_req.timeout = req.timeout
+        return self.do_open(UnixHTTPConnection(unix_socket), new_req)
+    unix_request = urllib2.AbstractHTTPHandler.do_request_
+
+urllib2.install_opener(urllib2.build_opener(UnixSocketHandler()))
+
+uri = 'unix://var/run/docker.sock/containers/json'
+
+
+
+if True:
+    req = urllib2.Request(uri, None)
+    try:
+        request = urllib2.urlopen(req)
+    except urllib2.URLError, e:
+        if "Errno 13" in str(e):
+            raise Exception("Unable to connect to socket. dd-agent user must be part of the 'docker' group")
+        raise
+    response = request.read()
\ No newline at end of file


Test patch: