Repository: DataDog/dd-agent

Issue:
Network check instance tag mismatch in 5.4
Instance tags used to be a part of the check name and hence normalized on the agent before sending to DD. However, since the new change in 5.4 where instance is separated as tag it doesn't get normalized on the agent. The problem is that there is a difference between the agent's normalize and sobotka's tag normalize and hence there are cases when instance tag names don't match up. 

eg: `instance:tag-tag`  in 5.3 would get normalized to `instance:tag_tag` in 5.3 but remains `instance:tag-tag` in 5.4 

I've created a fix (https://github.com/DataDog/dogweb/pull/9031) in sobotka for now but this might be better fixed on the agent itself for tcp and http checks. 



Discussion:
Thanks for reporting @amankapur. Let's plan a fix for the 5.5.0 agent release.


Resolving patch:
diff --git a/checks.d/http_check.py b/checks.d/http_check.py
--- a/checks.d/http_check.py
+++ b/checks.d/http_check.py
@@ -72,7 +72,6 @@ def _check(self, instance):
         start = time.time()
 
         service_checks = []
-
         try:
             self.log.debug("Connecting to %s" % addr)
             if disable_ssl_validation and urlparse(addr)[0] == "https":
@@ -240,7 +239,7 @@ def _create_status_event(self, sc_name, status, msg, instance):
         }
 
     def report_as_service_check(self, sc_name, status, instance, msg=None):
-        instance_name = instance['name']
+        instance_name = self.normalize(instance['name'])
         url = instance.get('url', None)
         sc_tags = ['url:{0}'.format(url), "instance:{0}".format(instance_name)]
         custom_tags = instance.get('tags', [])
diff --git a/checks.d/tcp_check.py b/checks.d/tcp_check.py
--- a/checks.d/tcp_check.py
+++ b/checks.d/tcp_check.py
@@ -152,7 +152,7 @@ def _create_status_event(self, sc_name, status, msg, instance):
         }
 
     def report_as_service_check(self, sc_name, status, instance, msg=None):
-        instance_name = instance['name']
+        instance_name = self.normalize(instance['name'])
         host = instance.get('host', None)
         port = instance.get('port', None)
         custom_tags = instance.get('tags', [])
diff --git a/checks/__init__.py b/checks/__init__.py
--- a/checks/__init__.py
+++ b/checks/__init__.py
@@ -744,7 +744,7 @@ def from_yaml(cls, path_to_yaml=None, agentConfig=None, yaml_text=None, check_na
             check = cls(check_name, config.get('init_config') or {}, agentConfig or {})
         return check, config.get('instances', [])
 
-    def normalize(self, metric, prefix=None, fix_case = False):
+    def normalize(self, metric, prefix=None, fix_case=False):
         """
         Turn a metric into a well-formed metric name
         prefix.b.c


Test patch:
diff --git a/tests/checks/integration/test_http_check.py b/tests/checks/integration/test_http_check.py
--- a/tests/checks/integration/test_http_check.py
+++ b/tests/checks/integration/test_http_check.py
@@ -3,11 +3,19 @@
 
 # 3p
 import mock
+from nose.plugins.attrib import attr
 
 # project
+from config import AGENT_VERSION
 from tests.checks.common import AgentCheckTest
+from util import headers as agent_headers
 
-RESULTS_TIMEOUT = 5
+RESULTS_TIMEOUT = 10
+
+AGENT_CONFIG = {
+    'version': AGENT_VERSION,
+    'api_key': 'toto'
+}
 
 CONFIG = {
     'instances': [{
@@ -77,31 +85,77 @@
     ]
 }
 
+CONFIG_UNORMALIZED_INSTANCE_NAME = {
+    'instances': [{
+        'name': '_need-to__be_normalized-',
+        'url': 'https://github.com',
+        'timeout': 1,
+        'check_certificate_expiration': True,
+        'days_warning': 14
+    },
+    ]
+}
+
+CONFIG_HTTP_HEADERS = {
+    'instances': [{
+        'url': 'https://google.com',
+        'name': 'UpService',
+        'timeout': 1,
+        'headers': {"X-Auth-Token": "SOME-AUTH-TOKEN"}
+    }]
+}
+
 
 FAKE_CERT = {'notAfter': 'Apr 12 12:00:00 2006 GMT'}
 
 
+@attr(requires='network')
 class HTTPCheckTest(AgentCheckTest):
     CHECK_NAME = 'http_check'
 
     def tearDown(self):
-        self.check.stop()
+        if self.check:
+            self.check.stop()
 
-    def wait_for_async_service_checks(self, count):
+    def wait_for_async(self, method, attribute, count):
+        """
+        Loop on `self.check.method` until `self.check.attribute >= count`.
+
+        Raise after
+        """
         i = 0
         while i < RESULTS_TIMEOUT:
             self.check._process_results()
-            if len(self.check.service_checks) >= count:
-                return self.check.get_service_checks()
+            if len(getattr(self.check, attribute)) >= count:
+                return getattr(self.check, method)()
             time.sleep(1)
             i += 1
         raise Exception("Didn't get the right count of service checks in time {0}"
-                        .format(self.check.service_checks))
+                        .format(getattr(self.check, attribute)))
+
+    def test_http_headers(self):
+        """
+        Headers format.
+        """
+        # Run the check
+        self.load_check(CONFIG_HTTP_HEADERS, AGENT_CONFIG)
+
+        url, username, password, http_response_status_code, timeout,\
+            include_content, headers, response_time, content_match,\
+            tags, ssl, ssl_expiration,\
+            instance_ca_certs = self.check._load_conf(CONFIG_HTTP_HEADERS['instances'][0])
+
+        self.assertEqual(headers["X-Auth-Token"], "SOME-AUTH-TOKEN", headers)
+        expected_headers = agent_headers(AGENT_CONFIG).get('User-Agent')
+        self.assertEqual(expected_headers, headers.get('User-Agent'), headers)
 
     def test_check(self):
+        """
+        Check coverage.
+        """
         self.run_check(CONFIG)
         # Overrides self.service_checks attribute when values are available\
-        self.service_checks = self.wait_for_async_service_checks(5)
+        self.service_checks = self.wait_for_async('get_service_checks', 'service_checks', 5)
 
         # HTTP connection error
         tags = ['url:https://thereisnosuchlink.com', 'instance:conn_error']
@@ -130,7 +184,7 @@ def test_check(self):
     def test_check_ssl(self):
         self.run_check(CONFIG_SSL_ONLY)
         # Overrides self.service_checks attribute when values are available
-        self.service_checks = self.wait_for_async_service_checks(6)
+        self.service_checks = self.wait_for_async('get_service_checks', 'service_checks', 6)
         tags = ['url:https://github.com', 'instance:good_cert']
         self.assertServiceCheckOK("http.can_connect", tags=tags)
         self.assertServiceCheckOK("http.ssl_cert", tags=tags)
@@ -150,8 +204,52 @@ def test_mock_case(self, getpeercert_func):
         self.run_check(CONFIG_EXPIRED_SSL)
         # Overrides self.service_checks attribute when values are av
         # Needed for the HTTP headers
-        self.service_checks = self.wait_for_async_service_checks(2)
+        self.service_checks = self.wait_for_async('get_service_checks', 'service_checks', 2)
         tags = ['url:https://github.com', 'instance:expired_cert']
         self.assertServiceCheckOK("http.can_connect", tags=tags)
         self.assertServiceCheckCritical("http.ssl_cert", tags=tags)
         self.coverage_report()
+
+    def test_service_check_instance_name_normalization(self):
+        """
+        Service check `instance` tag value is normalized.
+
+        Note: necessary to avoid mismatch and backward incompatiblity.
+        """
+        # Run the check
+        self.run_check(CONFIG_UNORMALIZED_INSTANCE_NAME)
+
+        # Overrides self.service_checks attribute when values are available
+        self.service_checks = self.wait_for_async('get_service_checks', 'service_checks', 2)
+
+        # Assess instance name normalization
+        tags = ['url:https://github.com', 'instance:need_to_be_normalized']
+        self.assertServiceCheckOK("http.can_connect", tags=tags)
+        self.assertServiceCheckOK("http.ssl_cert", tags=tags)
+
+    def test_warnings(self):
+        """
+        Deprecate events usage for service checks.
+        """
+        self.run_check(CONFIG)
+
+        # Overrides self.service_checks attribute when values are available\
+        self.warnings = self.wait_for_async('get_warnings', 'warnings', 8)
+
+        # Assess warnings
+        self.assertWarning(
+            "Skipping SSL certificate validation for "
+            "https://github.com based on configuration",
+            count=2
+        )
+        self.assertWarning(
+            "Skipping SSL certificate validation for "
+            "https://thereisnosuchlink.com based on configuration",
+            count=1
+        )
+        self.assertWarning(
+            "Using events for service checks is deprecated in "
+            "favor of monitors and will be removed in future versions of the "
+            "Datadog Agent.",
+            count=5
+        )
diff --git a/tests/checks/integration/test_tcp_check.py b/tests/checks/integration/test_tcp_check.py
new file mode 100644
--- /dev/null
+++ b/tests/checks/integration/test_tcp_check.py
@@ -0,0 +1,93 @@
+# stdlibb
+import time
+
+# 3p
+from nose.plugins.attrib import attr
+
+# project
+from tests.checks.common import AgentCheckTest
+
+RESULTS_TIMEOUT = 5
+
+CONFIG = {
+    'init_config': {},
+    'instances': [{
+        'host': '127.0.0.1',
+        'port': 65530,
+        'timeout': 1,
+        'name': 'DownService'
+    }, {
+        'host': '126.0.0.1',
+        'port': 65530,
+        'timeout': 1,
+        'name': 'DownService2'
+    }, {
+        'host': 'datadoghq.com',
+        'port': 80,
+        'timeout': 1,
+        'name': 'UpService'
+    }]
+}
+
+
+@attr(requires='network')
+class TCPCheckTest(AgentCheckTest):
+    CHECK_NAME = 'tcp_check'
+
+    def tearDown(self):
+        self.check.stop()
+
+    def wait_for_async(self, method, attribute, count):
+        """
+        Loop on `self.check.method` until `self.check.attribute >= count`.
+
+        Raise after
+        """
+        i = 0
+        while i < RESULTS_TIMEOUT:
+            self.check._process_results()
+            if len(getattr(self.check, attribute)) >= count:
+                return getattr(self.check, method)()
+            time.sleep(1)
+            i += 1
+        raise Exception("Didn't get the right count of service checks in time {0}"
+                        .format(getattr(self.check, attribute)))
+
+    def test_event_deprecation(self):
+        """
+        Deprecate events usage for service checks.
+        """
+        # Run the check
+        self.run_check(CONFIG)
+
+        # Overrides self.service_checks attribute when values are available
+        self.warnings = self.wait_for_async('get_warnings', 'warnings', 3)
+
+        # Assess warnings
+        self.assertWarning(
+            "Using events for service checks is deprecated in "
+            "favor of monitors and will be removed in future versions of the "
+            "Datadog Agent.",
+            count=3
+        )
+
+    def test_check(self):
+        """
+        Check coverage.
+        """
+        # Run the check
+        self.run_check(CONFIG)
+
+        # Overrides self.service_checks attribute when values are available
+        self.service_checks = self.wait_for_async('get_service_checks', 'service_checks', 3)
+
+        expected_tags = ["instance:DownService", "target_host:127.0.0.1", "port:65530"]
+        self.assertServiceCheckCritical("tcp.can_connect", tags=expected_tags)
+
+        expected_tags = ["instance:DownService2", "target_host:126.0.0.1", "port:65530"]
+        self.assertServiceCheckCritical("tcp.can_connect", tags=expected_tags)
+
+        expected_tags = ["instance:UpService", "target_host:datadoghq.com", "port:80"]
+        self.assertServiceCheckOK("tcp.can_connect", tags=expected_tags)
+
+        self.coverage_report()
diff --git a/tests/core/test_service_checks.py b/tests/core/test_service_checks.py
deleted file mode 100644
--- a/tests/core/test_service_checks.py
+++ /dev/null
@@ -1,185 +0,0 @@
-# stdlib
-from Queue import Empty
-import time
-
-# 3p
-from nose.plugins.attrib import attr
-
-# project
-from config import AGENT_VERSION
-from tests.checks.common import AgentCheckTest
-from util import headers as agent_headers
-
-
-@attr(requires='core_integration')
-class ServiceCheckTestCase(AgentCheckTest):
-
-    CHECK_NAME = "http_check"
-
-    def testHTTPHeaders(self):
-        agentConfig = {
-            'version': AGENT_VERSION,
-            'api_key': 'toto'
-        }
-
-        config = {
-            'init_config': {},
-            'instances': [{
-                'url': 'https://google.com',
-                'name': 'UpService',
-                'timeout': 1,
-                'headers': {"X-Auth-Token": "SOME-AUTH-TOKEN"}
-            }]
-        }
-
-        self.load_check(config, agentConfig)
-        url, username, password, http_response_status_code, timeout,\
-            include_content, headers, response_time, content_match,\
-            tags, ssl, ssl_expiration,\
-            instance_ca_certs = self.check._load_conf(config['instances'][0])
-
-        self.assertEqual(headers["X-Auth-Token"], "SOME-AUTH-TOKEN", headers)
-        expected_headers = agent_headers(agentConfig).get('User-Agent')
-        self.assertEqual(expected_headers, headers.get('User-Agent'), headers)
-
-
-    def testHTTPWarning(self):
-        self.CHECK_NAME = "http_check"
-
-        config = {
-            'init_config': {},
-            'instances': [{
-                'url': 'http://127.0.0.1:55555',
-                'name': 'DownService',
-                'timeout': 1
-            },{
-                'url': 'https://google.com',
-                'name': 'UpService',
-                'timeout': 1
-            }]
-        }
-
-
-        self.run_check(config, force_reload=True)
-        time.sleep(2)
-        # This would normally be called during the next run(), it is what
-        # flushes the results of the check
-
-        self.check._process_results()
-        self.warnings = self.check.get_warnings()
-
-        self.assertEqual(len(self.warnings), 4, self.warnings)
-        self.assertWarning("Skipping SSL certificate validation for "
-            "https://google.com based on configuration", count=1)
-        self.assertWarning("Using events for service checks is deprecated in "
-            "favor of monitors and will be removed in future versions of the "
-            "Datadog Agent.", count=3)
-
-        self.check.stop()
-
-    def testHTTP(self):
-        self.CHECK_NAME = "http_check"
-
-        # No passwords this time
-        config = {
-            'init_config': {},
-            'instances': [{
-                'url': 'http://127.0.0.1:55555',
-                'name': 'DownService',
-                'timeout': 1
-            },{
-                'url': 'http://google.com',
-                'name': 'UpService',
-                'timeout': 1
-            }]
-        }
-
-        self.run_check(config, force_reload=True)
-        time.sleep(2)
-        self.assertEqual(self.check.pool.get_nworkers(), 2)
-
-        self.check._process_results()
-
-        self.events = self.check.get_events()
-        self.service_checks = self.check.get_service_checks()
-
-        self.assertEqual(type(self.events), type([]))
-        self.assertEqual(len(self.events), 1, self.events)
-        self.assertEqual(self.events[0]['event_object'], 'DownService')
-
-        self.assertEqual(type(self.service_checks), type([]))
-        self.assertEqual(len(self.service_checks), 2, self.service_checks) # 1 per instance
-
-        expected_tags = ["instance:DownService", "url:http://127.0.0.1:55555"]
-        self.assertServiceCheck("http.can_connect", status=2, tags=expected_tags)
-
-        expected_tags = ["instance:UpService", "url:http://google.com"]
-        self.assertServiceCheck("http.can_connect", status=0, tags=expected_tags)
-
-        events = self.check.get_events()
-        service_checks = self.check.get_service_checks()
-        self.assertEqual(type(events), type([]))
-        self.assertEqual(len(events), 0)
-        self.assertEqual(type(service_checks), type([]))
-        self.assertEqual(len(service_checks), 0)
-        # result Q should be empty here
-        self.assertRaises(Empty, self.check.resultsq.get_nowait)
-
-        # Cleanup the threads
-        self.check.stop()
-
-    def testTCP(self):
-        # No passwords this time
-        config = {
-            'init_config': {
-            },
-            'instances': [{
-                'host': '127.0.0.1',
-                'port': 65530,
-                'timeout': 1,
-                'name': 'DownService'
-            },{
-                'host': '126.0.0.1',
-                'port': 65530,
-                'timeout': 1,
-                'name': 'DownService2'
-            },{
-                'host': 'datadoghq.com',
-                'port': 80,
-                'timeout': 1,
-                'name': 'UpService'
-
-            }]
-        }
-
-
-        self.CHECK_NAME = "tcp_check"
-        self.run_check(config, force_reload=True)
-        time.sleep(2)
-        self.assertEqual(self.check.pool.get_nworkers(), 3)
-        # This would normally be called during the next run(), it is what
-        # flushes the results of the check
-        self.check._process_results()
-
-        self.events = self.check.get_events()
-        self.service_checks = self.check.get_service_checks()
-
-
-        self.assertEqual(type(self.events), type([]))
-        self.assertEqual(len(self.events), 2, self.events)
-        for event in self.events:
-            self.assertTrue(event['event_object'][:11] == 'DownService')
-
-        self.assertEqual(type(self.service_checks), type([]))
-        self.assertEqual(len(self.service_checks), 3, self.service_checks) # 1 per instance
-
-        expected_tags = ["instance:DownService", "target_host:127.0.0.1", "port:65530"]
-        self.assertServiceCheck("tcp.can_connect", status=2, tags=expected_tags)
-
-        expected_tags = ["instance:DownService2", "target_host:126.0.0.1", "port:65530"]
-        self.assertServiceCheck("tcp.can_connect", status=2, tags=expected_tags)
-
-        expected_tags = ["instance:UpService", "target_host:datadoghq.com", "port:80"]
-        self.assertServiceCheck("tcp.can_connect", status=0, tags=expected_tags)
-
-        self.check.stop()