Repository: DataDog/dd-agent

Issue:
Exception in docker.py when containers are exiting
I think there's a race in collecting cgroup stats when a container exits, at which point the memory.stat files will disappear (afiak).  Currently the logs identify this a Docker 0.9 / 0.10 issue which I don't think is the case any longer as we're on the Docker 1.0 release:

```
Client version: 1.0.0
Client API version: 1.12
Go version (client): go1.2.1
Git commit (client): 63fe64c
Server version: 1.0.0
Server API version: 1.12
Go version (server): go1.2.1
Git commit (server): 63fe64c
```

The exception winds up looking like this.  Could probably do a simple check on the container to see if it is still running before throwing.  This will interrupt the current check cycle, maybe catching and going on with the next container would make sense.

```
IOError: Can't open /sys/fs/cgroup/memory/docker/02b07c9acbd8d936fefe7fc54d1f92982588eccdfed3acab56baaa49de1cf7d5/memory.stat. If you are using Docker 0.9 or 0.10, it is a known bug in Do
cker fixed in Docker 0.11
2014-06-17 14:40:43 UTC | INFO | dd.collector | checks.collector(collector.py:347) | Finished run #60. Collection time: 4.13s. Emit time: 0.03s
2014-06-17 14:42:20 UTC | ERROR | dd.collector | checks.docker(__init__.py:515) | Check 'docker' instance #0 failed
Traceback (most recent call last):
  File "/usr/share/datadog/agent/checks/__init__.py", line 506, in run
    self.check(copy.deepcopy(instance))
  File "/usr/share/datadog/agent/checks.d/docker.py", line 185, in check
    stats = self._parse_cgroup_file(stat_file)
  File "/usr/share/datadog/agent/checks.d/docker.py", line 287, in _parse_cgroup_file
    raise IOError("Can't open %s. If you are using Docker 0.9 or 0.10, it is a known bug in Docker fixed in Docker 0.11" % file_)
IOError: Can't open /sys/fs/cgroup/memory/docker/969fc2e6ced1697aa2cd139be09ecc6c359d76edc20f8010b56f546de03c48ce/memory.stat. If you are using Docker 0.9 or 0.10, it is a known bug in Do
cker fixed in Docker 0.11
```



Discussion:

Resolving patch:
diff --git a/checks.d/docker.py b/checks.d/docker.py
--- a/checks.d/docker.py
+++ b/checks.d/docker.py
@@ -135,6 +135,7 @@ def path_prefix(self):
     def check(self, instance):
         urllib2.install_opener(urllib2.build_opener(UnixSocketHandler())) # We need to reinstall the opener every time as it gets uninstalled
         tags = instance.get("tags") or []
+        skipped_cgroup = 0
 
         try:
             self._process_events(self._get_events(instance))
@@ -147,8 +148,9 @@ def check(self, instance):
             raise Exception('Cannot get containers list: timeout during socket connection. Try to refine the containers to collect by editing the configuration file.')
 
         if not containers:
-            self.gauge("docker.containers.running", 0)
-            raise Exception("No containers are running.")
+            containers = []
+            self.warning("No containers are running.")
+            return
 
         self.gauge("docker.containers.running", len(containers))
 
@@ -171,11 +173,6 @@ def check(self, instance):
             if not self._is_container_included(instance, container_tags):
                 continue
 
-            collected_containers += 1
-            if collected_containers > max_containers:
-                self.warning("Too many containers are matching the current configuration. Some containers will not be collected. Please refine your configuration")
-                break
-
             for key, (dd_key, metric_type) in DOCKER_METRICS.items():
                 if key in container:
                     getattr(self, metric_type)(dd_key, int(container[key]), tags=container_tags)
@@ -183,11 +180,22 @@ def check(self, instance):
                 mountpoint = self._mountpoints[metric["cgroup"]]
                 stat_file = os.path.join(mountpoint, metric["file"] % (self.path_prefix, container["Id"]))
                 stats = self._parse_cgroup_file(stat_file)
-                for key, (dd_key, metric_type) in metric["metrics"].items():
-                    if key.startswith("total_") and not instance.get("collect_total"):
-                        continue
-                    if key in stats:
-                        getattr(self, metric_type)(dd_key, int(stats[key]), tags=container_tags)
+                if stats:
+                    for key, (dd_key, metric_type) in metric["metrics"].items():
+                        if key.startswith("total_") and not instance.get("collect_total"):
+                            continue
+                        if key in stats:
+                            getattr(self, metric_type)(dd_key, int(stats[key]), tags=container_tags)
+                else:
+                    skipped_cgroup += 1
+
+            collected_containers += 1
+            if collected_containers >= max_containers:
+                self.warning("Too many containers are matching the current configuration. Some containers will not be collected. Please refine your configuration")
+                break
+
+        if skipped_cgroup and skipped_cgroup == collected_containers * len(LXC_METRICS):
+            raise IOError("We were unable to open cgroup files. If you are using Docker 0.9 or 0.10, it is a known bug in Docker fixed in Docker 0.11")
 
     def _process_events(self, events):
         for ev in events:
@@ -279,14 +287,16 @@ def _find_cgroup(self, hierarchy):
     def _parse_cgroup_file(self, file_):
         """Parses a cgroup pseudo file for key/values."""
         fp = None
+        self.log.debug("Opening file: %s" % file_)
         try:
-            self.log.debug("Opening file: %s" % file_)
             try:
                 fp = open(file_)
+                return dict(map(lambda x: x.split(), fp.read().splitlines()))
             except IOError:
-                raise IOError("Can't open %s. If you are using Docker 0.9 or 0.10, it is a known bug in Docker fixed in Docker 0.11" % file_)
-            return dict(map(lambda x: x.split(), fp.read().splitlines()))
-
+                # Can be because of Docker 0.9/0.10 bug or because the container got stopped
+                # Count this kind of exception, if it happens to often it is because of the bug
+                self.log.info("Can't open %s. Metrics for this container are skipped." % file_)
+                return None
         finally:
             if fp is not None:
                 fp.close()


Test patch: