Repository: DataDog/dd-agent

Issue:
[core] Service discovery strips important information from image name
According to the service discovery guide, it works by matching the name of the image to a folder in etcd. However, when looking at the code (https://github.com/DataDog/dd-agent/blob/fd22e3c57d4d690e8bda902ed4dccdfb926fa71b/utils/service_discovery/sd_docker_backend.py#L138), this is not the case. It is only taking a small portion of the image name. This is causing conflicts for me that block me from using the feature. As an example, let's assume I have images `myregistry.com/foo/web` and `myregistry.com/bar/web`. These two projects have very different checks. As another example, assume I have images `myregistry.com/foo:web` and `myregistry.com/foo:worker` which share all layers except the command used to execute them. Obviously, the worker check will be much different from the web check.

In both cases, the two images look identical to the linked line of code. To fix this, I propose using labels (https://docs.docker.com/engine/userguide/labels-custom-metadata/) or environment variables. Labels were added to attach extra metadata, which is perfect for this use case. I also propose environment variables since that's what a lot of other tools still use as an artifact of history. I would be willing to write the code necessary to do this, but I wanted to bring this up for discussion beforehand. Let me know your thoughts.

Thanks,
George



Discussion:
I realized I wasn't clear how the labels and environment variables would work. I'm not suggesting storing the check config data in labels. I'd like to use labels and environment variables to store the key the agent uses to march in etcd. For instance, `docker run -e DATADOG_ID=foo-web myregistry.com/foo/web`. This way, instead of deducing "web" as the key to check in etcd, the agent would pull the environment variable DATADOG_ID and check etcd for `/datadog/check_configs/foo-web`, or more generically, `/datadog/check_configs/$DATADOG_ID`. This strategy would allow the agent to fall back on the current scheme for those that changed their image naming scheme to match what the agent is looking for.

Running the REPL, I've found that the environment variables are not getting pulled into the container dictionary. Labels are. 

```
>>> docker_client.containers()
[{u'Status': u'Up 2 minutes', u'Created': 1465399125, u'Image': u'nginx', u'Labels': {u'datadog_id': u'custom-nginx'}, u'NetworkSettings': {u'Networks': {u'bridge': {u'NetworkID': u'', u'MacAddress': u'', u'GlobalIPv6PrefixLen': 0, u'Links': None, u'GlobalIPv6Address': u'', u'IPv6Gateway': u'', u'IPAMConfig': None, u'EndpointID':', u'IPPrefixLen': 16, u'IPAddress': u'172.17.0.2', u'Gateway': u'172.17.0.1', u'Aliases': None}}}, u'HostConfig': {u'NetworkMode': u'default'}, u'ImageID': u'sha256:6f8d099c3adc0f9c14bdc11d6c698a647c2f0457aa1ae76a592d6f7dc9bdb5b8', u'State': u'running', u'Command': u"nginx -g 'daemon off;'", u'Names': [u'/stoic_allen'], u'Mounts': [], u'Id': u'445272bc04e4ad5f0d0a480e7a56af36aeef57053601835ef3579d98cdf3e567', u'Ports': [{u'IP': u'0.0.0.0', u'Type': u'tcp', u'PublicPort': 32771, u'PrivatePort': 80}, {u'IP': u'0.0.0.0', u'Type': u'tcp', u'PublicPort': 32770, u'PrivatePort': 443}]}]
```

I'm closing this in favor of my PR, #2583, implementing what I've mentioned.


Resolving patch:
diff --git a/utils/service_discovery/abstract_config_store.py b/utils/service_discovery/abstract_config_store.py
--- a/utils/service_discovery/abstract_config_store.py
+++ b/utils/service_discovery/abstract_config_store.py
@@ -20,6 +20,9 @@
 CONFIG_FROM_FILE = 'YAML file'
 CONFIG_FROM_TEMPLATE = 'template'
 TRACE_CONFIG = 'trace_config'  # used for tracing config load by service discovery
+CHECK_NAMES = 'check_names'
+INIT_CONFIGS = 'init_configs'
+INSTANCES = 'instances'
 
 
 class KeyNotFound(Exception):
@@ -59,7 +62,8 @@ def dump_directory(self, path, **kwargs):
         raise NotImplementedError()
 
     def _get_auto_config(self, image_name):
-        if image_name in self.auto_conf_images:
+        # use the image name, ignore the tag
+        if image_name.split(':')[0] in self.auto_conf_images:
             check_name = self.auto_conf_images[image_name]
 
             # get the check class to verify it matches
@@ -75,26 +79,26 @@ def _get_auto_config(self, image_name):
 
         return None
 
-    def get_check_tpls(self, image, **kwargs):
-        """Retrieve template configs for an image from the config_store or auto configuration."""
-        # TODO: make mixing both sources possible
+    def get_check_tpls(self, identifier, **kwargs):
+        """Retrieve template configs for an identifier from the config_store or auto configuration."""
         templates = []
         trace_config = kwargs.get(TRACE_CONFIG, False)
 
         # this flag is used when no valid configuration store was provided
         # it makes the method skip directly to the auto_conf
         if kwargs.get('auto_conf') is True:
-            auto_config = self._get_auto_config(image)
+            # in auto config mode, identifier is the image name
+            auto_config = self._get_auto_config(identifier)
             if auto_config is not None:
                 source = CONFIG_FROM_AUTOCONF
                 if trace_config:
                     return [(source, auto_config)]
                 return [auto_config]
             else:
-                log.debug('No auto config was found for image %s, leaving it alone.' % image)
+                log.debug('No auto config was found for image %s, leaving it alone.' % identifier)
                 return []
         else:
-            config = self.read_config_from_store(image)
+            config = self.read_config_from_store(identifier)
             if config:
                 source, check_names, init_config_tpls, instance_tpls = config
             else:
@@ -102,8 +106,8 @@ def get_check_tpls(self, image, **kwargs):
 
         if len(check_names) != len(init_config_tpls) or len(check_names) != len(instance_tpls):
             log.error('Malformed configuration template: check_names, init_configs '
-                      'and instances are not all the same length. Image {0} '
-                      'will not be configured by the service discovery'.format(image))
+                      'and instances are not all the same length. Container with identifier {} '
+                      'will not be configured by the service discovery'.format(identifier))
             return []
 
         for idx, c_name in enumerate(check_names):
@@ -113,45 +117,72 @@ def get_check_tpls(self, image, **kwargs):
                 templates.append((c_name, init_config_tpls[idx], instance_tpls[idx]))
         return templates
 
-    def read_config_from_store(self, image):
+    def read_config_from_store(self, identifier):
         """Try to read from the config store, falls back to auto-config in case of failure."""
         try:
-            check_names = json.loads(
-                self.client_read(path.join(self.sd_template_dir, image, 'check_names').lstrip('/')))
-            init_config_tpls = json.loads(
-                self.client_read(path.join(self.sd_template_dir, image, 'init_configs').lstrip('/')))
-            instance_tpls = json.loads(
-                self.client_read(path.join(self.sd_template_dir, image, 'instances').lstrip('/')))
-            source = CONFIG_FROM_TEMPLATE
-        except (KeyNotFound, TimeoutError, json.JSONDecodeError) as ex:
-            # first case is kind of expected, it means that no template was provided for this container
-            if isinstance(ex, KeyNotFound):
-                log.debug("Could not find directory {0} in the config store, "
-                          "trying to auto-configure a check...".format(image))
+            try:
+                res = self._issue_read(identifier)
+            except KeyNotFound:
+                log.debug("Could not find directory {} in the config store, "
+                          "trying to convert to the old format.".format(identifier))
+                legacy_ident = identifier.split(':')[0].split('/')[-1]
+                res = self._issue_read(legacy_ident)
+
+            if res and len(res) == 3:
+                source = CONFIG_FROM_TEMPLATE
+                check_names, init_config_tpls, instance_tpls = res
+            else:
+                log.debug("Could not find directory {} in the config store, "
+                          "trying to convert to the old format...".format(identifier))
+                legacy_ident = identifier.split(':')[0].split('/')[-1]
+                res = self._issue_read(legacy_ident)
+                if res and len(res) == 3:
+                    source = CONFIG_FROM_TEMPLATE
+                    check_names, init_config_tpls, instance_tpls = res
+                else:
+                    raise KeyError
+        except (KeyError, KeyNotFound, TimeoutError, json.JSONDecodeError) as ex:
+            # this is kind of expected, it means that no template was provided for this container
+            if isinstance(ex, KeyError) or isinstance(ex, KeyNotFound):
+                log.debug("Could not find directory {} in the config store, "
+                          "trying to auto-configure a check...".format(identifier))
             # this case is not expected, the agent can't reach the config store
-            elif isinstance(ex, TimeoutError):
+            if isinstance(ex, TimeoutError):
                 log.warning("Connection to the config backend timed out. Is it reachable?\n"
-                            "Trying to auto-configure a check for the image %s..." % image)
+                            "Trying to auto-configure a check for the container with ident %s." % identifier)
             # the template is reachable but invalid
             elif isinstance(ex, json.JSONDecodeError):
-                log.error('Could not decode the JSON configuration template. '
-                          'Trying to auto-configure a check for the image %s...' % image)
-            # In any case cases we try to read from auto-config templates
-            auto_config = self._get_auto_config(image)
+                log.error('Could not decode the JSON configuration template '
+                          'for the container with ident %s...' % identifier)
+                return []
+            # try to read from auto-config templates
+            auto_config = self._get_auto_config(identifier)
             if auto_config is not None:
                 # create list-format config based on an autoconf template
                 check_names, init_config_tpls, instance_tpls = map(lambda x: [x], auto_config)
                 source = CONFIG_FROM_AUTOCONF
             else:
-                log.debug('No auto config was found for image %s, leaving it alone.' % image)
+                log.debug('No config was found for container with ident %s, leaving it alone.' % identifier)
                 return []
         except Exception as ex:
             log.warning(
                 'Fetching the value for {0} in the config store failed, this check '
-                'will not be configured by the service discovery. Error: {1}'.format(image, str(ex)))
+                'will not be configured by the service discovery. Error: {1}'.format(identifier, str(ex)))
             return []
         return source, check_names, init_config_tpls, instance_tpls
 
+    def _issue_read(self, identifier):
+        try:
+            check_names = json.loads(
+                self.client_read(path.join(self.sd_template_dir, identifier, CHECK_NAMES).lstrip('/')))
+            init_config_tpls = json.loads(
+                self.client_read(path.join(self.sd_template_dir, identifier, INIT_CONFIGS).lstrip('/')))
+            instance_tpls = json.loads(
+                self.client_read(path.join(self.sd_template_dir, identifier, INSTANCES).lstrip('/')))
+            return [check_names, init_config_tpls, instance_tpls]
+        except KeyError:
+            return None
+
     def crawl_config_template(self):
         """Return whether or not configuration templates have changed since the previous crawl"""
         try:
diff --git a/utils/service_discovery/sd_docker_backend.py b/utils/service_discovery/sd_docker_backend.py
--- a/utils/service_discovery/sd_docker_backend.py
+++ b/utils/service_discovery/sd_docker_backend.py
@@ -12,6 +12,7 @@
 from utils.service_discovery.abstract_sd_backend import AbstractSDBackend
 from utils.service_discovery.config_stores import get_config_store, TRACE_CONFIG
 
+DATADOG_ID = 'com.datadoghq.sd.check.id'
 log = logging.getLogger(__name__)
 
 
@@ -135,7 +136,7 @@ def get_configs(self):
         """Get the config for all docker containers running on the host."""
         configs = {}
         containers = [(
-            container.get('Image').split(':')[0].split('/')[-1],
+            container.get('Image'),
             container.get('Id'), container.get('Labels')
         ) for container in self.docker_client.containers()]
 
@@ -144,7 +145,9 @@ def get_configs(self):
 
         for image, cid, labels in containers:
             try:
-                check_configs = self._get_check_configs(cid, image, trace_config=trace_config) or []
+                # value of the DATADOG_ID tag or the image name if the label is missing
+                identifier = self.get_config_id(image, labels)
+                check_configs = self._get_check_configs(cid, identifier, trace_config=trace_config) or []
                 for conf in check_configs:
                     if trace_config and conf is not None:
                         source, conf = conf
@@ -157,7 +160,7 @@ def get_configs(self):
                         else:
                             configs[check_name] = (init_config, [instance])
                     else:
-                        conflict_init_msg = 'Different versions of `init_config` found for check {0}. ' \
+                        conflict_init_msg = 'Different versions of `init_config` found for check {}. ' \
                             'Keeping the first one found.'
                         if trace_config:
                             if configs[check_name][1][0] != init_config:
@@ -172,13 +175,17 @@ def get_configs(self):
                               ' discovery failed, leaving it alone.' % (cid[:12], image))
         return configs
 
-    def _get_check_configs(self, c_id, image, trace_config=False):
+    def get_config_id(self, image, labels):
+        """Look for a DATADOG_ID label, return its value or the image name if missing"""
+        return labels.get(DATADOG_ID) or image
+
+    def _get_check_configs(self, c_id, identifier, trace_config=False):
         """Retrieve configuration templates and fill them with data pulled from docker and tags."""
         inspect = self.docker_client.inspect_container(c_id)
-        config_templates = self._get_config_templates(image, trace_config=trace_config)
+        config_templates = self._get_config_templates(identifier, trace_config=trace_config)
         if not config_templates:
-            log.debug('No config template for container %s with image %s. '
-                      'It will be left unconfigured.' % (c_id[:12], image))
+            log.debug('No config template for container %s with identifier %s. '
+                      'It will be left unconfigured.' % (c_id[:12], identifier))
             return None
 
         check_configs = []
@@ -201,8 +208,8 @@ def _get_check_configs(self, c_id, image, trace_config=False):
 
         return check_configs
 
-    def _get_config_templates(self, image_name, trace_config=False):
-        """Extract config templates for an image from a K/V store and returns it as a dict object."""
+    def _get_config_templates(self, identifier, trace_config=False):
+        """Extract config templates for an identifier from a K/V store and returns it as a dict object."""
         config_backend = self.agentConfig.get('sd_config_backend')
         templates = []
         if config_backend is None:
@@ -211,9 +218,10 @@ def _get_config_templates(self, image_name, trace_config=False):
         else:
             auto_conf = False
 
-        # format: [('image', {init_tpl}, {instance_tpl})] without trace_config
-        # or      [(source, ('image', {init_tpl}, {instance_tpl}))] with trace_config
-        raw_tpls = self.config_store.get_check_tpls(image_name, auto_conf=auto_conf, trace_config=trace_config)
+        # format: [('ident', {init_tpl}, {instance_tpl})] without trace_config
+        # or      [(source, ('ident', {init_tpl}, {instance_tpl}))] with trace_config
+        raw_tpls = self.config_store.get_check_tpls(
+            identifier, auto_conf=auto_conf, trace_config=trace_config)
         for tpl in raw_tpls:
             if trace_config and tpl is not None:
                 # each template can come from either auto configuration or user-supplied templates
@@ -221,7 +229,7 @@ def _get_config_templates(self, image_name, trace_config=False):
             if tpl is not None and len(tpl) == 3:
                 check_name, init_config_tpl, instance_tpl = tpl
             else:
-                log.debug('No template was found for image %s, leaving it alone.' % image_name)
+                log.debug('No template was found for identifier %s, leaving it alone.' % identifier)
                 return None
             try:
                 # build a list of all variables to replace in the template
@@ -234,7 +242,7 @@ def _get_config_templates(self, image_name, trace_config=False):
                     instance_tpl = json.loads(instance_tpl or '{}')
             except json.JSONDecodeError:
                 log.exception('Failed to decode the JSON template fetched for check {0}. Its configuration'
-                              ' by service discovery failed for {1}.'.format(check_name, image_name))
+                              ' by service discovery failed for ident  {1}.'.format(check_name, identifier))
                 return None
 
             if trace_config:


Test patch:
diff --git a/tests/core/test_service_discovery.py b/tests/core/test_service_discovery.py
--- a/tests/core/test_service_discovery.py
+++ b/tests/core/test_service_discovery.py
@@ -60,30 +60,42 @@ def client_read(path):
     return TestServiceDiscovery.mock_tpls.get(image)[0][config_parts.index(config_part)]
 
 
+def issue_read(identifier):
+    return TestServiceDiscovery.mock_tpls.get(identifier)
+
+
 class TestServiceDiscovery(unittest.TestCase):
     docker_container_inspect = {
         u'Id': u'69ff25598b2314d1cdb7752cc3a659fb1c1352b32546af4f1454321550e842c0',
-        u'Image': u'6ffc02088cb870652eca9ccd4c4fb582f75b29af2879792ed09bb46fd1c898ef',
+        u'Image': u'nginx',
         u'Name': u'/nginx',
         u'NetworkSettings': {u'IPAddress': u'172.17.0.21', u'Ports': {u'443/tcp': None, u'80/tcp': None}}
     }
+    docker_container_inspect_with_label = {
+        u'Id': u'69ff25598b2314d1cdb7752cc3a659fb1c1352b32546af4f1454321550e842c0',
+        u'Image': u'nginx',
+        u'Name': u'/nginx',
+        u'NetworkSettings': {u'IPAddress': u'172.17.0.21', u'Ports': {u'443/tcp': None, u'80/tcp': None}},
+        u'Labels': {'com.datadoghq.sd.check.id': 'custom-nginx'}
+    }
     kubernetes_container_inspect = {
         u'Id': u'389dc8a4361f3d6c866e9e9a7b6972b26a31c589c4e2f097375d55656a070bc9',
-        u'Image': u'de309495e6c7b2071bc60c0b7e4405b0d65e33e3a4b732ad77615d90452dd827',
+        u'Image': u'foo',
         u'Name': u'/k8s_sentinel.38057ab9_redis-master_default_27b84e1e-a81c-11e5-8347-42010af00002_f70875a1',
         u'Config': {u'ExposedPorts': {u'6379/tcp': {}}},
         u'NetworkSettings': {u'IPAddress': u'', u'Ports': None}
     }
     malformed_container_inspect = {
         u'Id': u'69ff25598b2314d1cdb7752cc3a659fb1c1352b32546af4f1454321550e842c0',
-        u'Image': u'6ffc02088cb870652eca9ccd4c4fb582f75b29af2879792ed09bb46fd1c898ef',
+        u'Image': u'foo',
         u'Name': u'/nginx'
     }
     container_inspects = [
-        # (inspect_dict, expected_ip, expected_port)
-        (docker_container_inspect, '172.17.0.21', ['80', '443']),
-        (kubernetes_container_inspect, None, ['6379']),  # arbitrarily defined in the mocked pod_list
-        (malformed_container_inspect, None, KeyError)
+        # (inspect_dict, expected_ip, expected_port, expected_ident)
+        (docker_container_inspect, '172.17.0.21', ['80', '443'], 'nginx'),
+        (docker_container_inspect_with_label, '172.17.0.21', ['80', '443'], 'custom-nginx'),
+        (kubernetes_container_inspect, None, ['6379'], 'foo'),  # arbitrarily defined in the mocked pod_list
+        (malformed_container_inspect, None, KeyError, 'foo')
     ]
 
     # templates with variables already extracted
@@ -114,7 +126,12 @@ class TestServiceDiscovery(unittest.TestCase):
             [('check_2', {}, {"host": "%%host%%", "port": "%%port%%"})]),
         'bad_image_0': ((['invalid template']), []),
         'bad_image_1': (('invalid template'), []),
-        'bad_image_2': (None, [])
+        'bad_image_2': (None, []),
+        'nginx': ('["nginx"]', '[{}]', '[{"host": "localhost"}]'),
+        'nginx:latest': ('["nginx"]', '[{}]', '[{"host": "localhost", "tags": ["foo"]}]'),
+        'custom-nginx': ('["nginx"]', '[{}]', '[{"host": "localhost"}]'),
+        'repo/custom-nginx': ('["nginx"]', '[{}]', '[{"host": "localhost", "tags": ["bar"]}]'),
+        'repo/dir:5000/custom-nginx:latest': ('["nginx"]', '[{}]', '[{"host": "local", "tags": ["foobar"]}]')
     }
 
     bad_mock_templates = {
@@ -168,7 +185,7 @@ def test_get_host(self, mock_check_yaml, mock_get):
         mock_check_yaml.return_value = kubernetes_config
         mock_get.return_value = Response(pod_list)
 
-        for c_ins, expected_ip, _ in self.container_inspects:
+        for c_ins, expected_ip, _, _ in self.container_inspects:
             with mock.patch.object(AbstractConfigStore, '__init__', return_value=None):
                 with mock.patch('utils.dockerutil.DockerUtil.client', return_value=None):
                     with mock.patch('utils.kubeutil.get_conf_path', return_value=None):
@@ -178,7 +195,7 @@ def test_get_host(self, mock_check_yaml, mock_get):
 
     def test_get_ports(self):
         with mock.patch('utils.dockerutil.DockerUtil.client', return_value=None):
-            for c_ins, _, expected_ports in self.container_inspects:
+            for c_ins, _, expected_ports, _ in self.container_inspects:
                 sd_backend = get_sd_backend(agentConfig=self.auto_conf_agentConfig)
                 if isinstance(expected_ports, list):
                     self.assertEqual(sd_backend._get_ports(c_ins), expected_ports)
@@ -318,3 +335,29 @@ def test_get_check_tpls(self, mock_client_read):
         for image in invalid_config:
             tpl = self.mock_tpls.get(image)[1]
             self.assertEquals(tpl, config_store.get_check_tpls(image))
+
+    def test_get_config_id(self):
+        """Test get_config_id"""
+        with mock.patch('utils.dockerutil.DockerUtil.client', return_value=None):
+            for c_ins, _, _, expected_ident in self.container_inspects:
+                sd_backend = get_sd_backend(agentConfig=self.auto_conf_agentConfig)
+                self.assertEqual(
+                    sd_backend.get_config_id(c_ins.get('Image'), c_ins.get('Labels', {})),
+                    expected_ident)
+                clear_singletons(self.auto_conf_agentConfig)
+
+    @mock.patch.object(AbstractConfigStore, '_issue_read', side_effect=issue_read)
+    def test_read_config_from_store(self, issue_read):
+        """Test read_config_from_store"""
+        valid_idents = [('nginx', 'nginx'), ('nginx:latest', 'nginx:latest'),
+                        ('custom-nginx', 'custom-nginx'), ('custom-nginx:latest', 'custom-nginx'),
+                        ('repo/custom-nginx:latest', 'custom-nginx'),
+                        ('repo/dir:5000/custom-nginx:latest', 'repo/dir:5000/custom-nginx:latest')]
+        invalid_idents = ['foo']
+        config_store = get_config_store(self.auto_conf_agentConfig)
+        for ident, expected_key in valid_idents:
+            tpl = config_store.read_config_from_store(ident)
+            # source is added after reading from the store
+            self.assertEquals(tpl, ('template',) + self.mock_tpls.get(expected_key))
+        for ident in invalid_idents:
+            self.assertEquals(config_store.read_config_from_store(ident), [])