Repository: DataDog/dd-agent

Issue:
http_check - define success
Sometimes you need to check the negativity of a request, e.g. to see if a login form appears using HTTP 403 - this currently fails because the check uses `requests.exceptions.HTTPError` and there is no configuration option to customize it.

I wish there was an option to specify the explicit HTTP return code per http_check.



Discussion:

Resolving patch:
diff --git a/checks.d/http_check.py b/checks.d/http_check.py
--- a/checks.d/http_check.py
+++ b/checks.d/http_check.py
@@ -4,6 +4,7 @@
 import socket
 import ssl
 import time
+import re
 from urlparse import urlparse
 
 # 3rd party
@@ -44,66 +45,60 @@ def __init__(self, name, init_config, agentConfig, instances):
     def _load_conf(self, instance):
         # Fetches the conf
         tags = instance.get('tags', [])
-        username = instance.get('username', None)
-        password = instance.get('password', None)
+        username = instance.get('username')
+        password = instance.get('password')
+        http_response_status_code = instance.get('http_response_status_code', "(1|2|3)\d\d")
         timeout = int(instance.get('timeout', 10))
         config_headers = instance.get('headers', {})
         headers = agent_headers(self.agentConfig)
         headers.update(config_headers)
-        url = instance.get('url', None)
+        url = instance.get('url')
+        content_match = instance.get('content_match')
         response_time = _is_affirmative(instance.get('collect_response_time', True))
-        if url is None:
+        if not url:
             raise Exception("Bad configuration. You must specify a url")
         include_content = _is_affirmative(instance.get('include_content', False))
         ssl = _is_affirmative(instance.get('disable_ssl_validation', True))
         ssl_expire = _is_affirmative(instance.get('check_certificate_expiration', True))
 
-        return url, username, password, timeout, include_content, headers, response_time, tags, ssl, ssl_expire
+        return url, username, password, http_response_status_code, timeout, include_content,\
+            headers, response_time, content_match, tags, ssl, ssl_expire
 
     def _check(self, instance):
-        addr, username, password, timeout, include_content, headers, response_time, tags, disable_ssl_validation, ssl_expire = self._load_conf(instance)
-        content = ''
+        addr, username, password, http_response_status_code, timeout, include_content, headers,\
+            response_time, content_match, tags, disable_ssl_validation,\
+            ssl_expire = self._load_conf(instance)
         start = time.time()
 
         service_checks = []
-        resp = None
 
         try:
             self.log.debug("Connecting to %s" % addr)
             if disable_ssl_validation and urlparse(addr)[0] == "https":
-                self.warning("Skipping SSL certificate validation for %s based on configuration" % addr)
-            
+                self.warning("Skipping SSL certificate validation for %s based on configuration"
+                             % addr)
+
             auth = None
             if username is not None and password is not None:
                 auth = (username, password)
 
-            r = requests.get(addr, auth=auth,timeout=timeout, headers=headers,
-                verify=not disable_ssl_validation)
-            r.raise_for_status()
-            
+            r = requests.get(addr, auth=auth, timeout=timeout, headers=headers,
+                             verify=not disable_ssl_validation)
 
         except socket.timeout, e:
             length = int((time.time() - start) * 1000)
-            self.log.info("%s is DOWN, error: %s. Connection failed after %s ms" % (addr, str(e), length))
+            self.log.info("%s is DOWN, error: %s. Connection failed after %s ms"
+                          % (addr, str(e), length))
             service_checks.append((
                 self.SC_STATUS,
                 Status.DOWN,
                 "%s. Connection failed after %s ms" % (str(e), length)
             ))
 
-        except requests.exceptions.HTTPError, r:
-            length = int((time.time() - start) * 1000)
-            self.log.info("%s is DOWN, error code: %s" % (addr, str(r.status_code)))
-
-            content = r.content if include_content else ''
-
-            service_checks.append((
-                self.SC_STATUS, Status.DOWN, (r.status_code, r.reason, content or '')
-            ))
-
         except requests.exceptions.ConnectionError, e:
             length = int((time.time() - start) * 1000)
-            self.log.info("%s is DOWN, error: %s. Connection failed after %s ms" % (addr, str(e), length))
+            self.log.info("%s is DOWN, error: %s. Connection failed after %s ms"
+                          % (addr, str(e), length))
             service_checks.append((
                 self.SC_STATUS,
                 Status.DOWN,
@@ -112,7 +107,8 @@ def _check(self, instance):
 
         except socket.error, e:
             length = int((time.time() - start) * 1000)
-            self.log.info("%s is DOWN, error: %s. Connection failed after %s ms" % (addr, repr(e), length))
+            self.log.info("%s is DOWN, error: %s. Connection failed after %s ms"
+                          % (addr, repr(e), length))
             service_checks.append((
                 self.SC_STATUS,
                 Status.DOWN,
@@ -121,7 +117,8 @@ def _check(self, instance):
 
         except Exception, e:
             length = int((time.time() - start) * 1000)
-            self.log.error("Unhandled exception %s. Connection failed after %s ms" % (str(e), length))
+            self.log.error("Unhandled exception %s. Connection failed after %s ms"
+                           % (str(e), length))
             raise
 
         # Only report this metric if the site is not down
@@ -133,12 +130,42 @@ def _check(self, instance):
             tags_list.append('url:%s' % addr)
             self.gauge('network.http.response_time', running_time, tags=tags_list)
 
-        if not service_checks:
-            self.log.debug("%s is UP" % addr)
+        # Check HTTP response status code
+        if not (service_checks or re.match(http_response_status_code, str(r.status_code))):
+            self.log.info("Incorrect HTTP return code. Expected %s, got %s"
+                          % (http_response_status_code, str(r.status_code)))
+
             service_checks.append((
-                self.SC_STATUS, Status.UP, "UP"
+                self.SC_STATUS,
+                Status.DOWN,
+                "Incorrect HTTP return code. Expected %s, got %s"
+                % (http_response_status_code, str(r.status_code))
             ))
 
+        if not service_checks:
+            # Host is UP
+            # Check content matching is set
+            if content_match:
+                content = r.content
+                if re.search(content_match, content):
+                    self.log.debug("%s is found in return content" % content_match)
+                    service_checks.append((
+                        self.SC_STATUS, Status.UP, "UP"
+                    ))
+                else:
+                    self.log.info("%s not found in content" % content_match)
+                    self.log.debug("Content returned:\n%s" % content)
+                    service_checks.append((
+                        self.SC_STATUS,
+                        Status.DOWN,
+                        "Content %s not found in response" % content_match
+                    ))
+            else:
+                self.log.debug("%s is UP" % addr)
+                service_checks.append((
+                    self.SC_STATUS, Status.UP, "UP"
+                ))
+
         if ssl_expire and urlparse(addr)[0] == "https":
             status, msg = self.check_cert_expiration(instance)
             service_checks.append((
@@ -174,7 +201,6 @@ def _create_status_event(self, sc_name, status, msg, instance):
         else:
             source_type = "%s.%s" % (NetworkCheck.SOURCE_TYPE_NAME, instance_source_type_name)
 
-
         # Get the handles you want to notify
         notify = instance.get('notify', self.init_config.get('notify', []))
         notify_message = ""
@@ -198,27 +224,28 @@ def _create_status_event(self, sc_name, status, msg, instance):
 
             title = "[Alert] %s reported that %s is down" % (self.hostname, name)
             alert_type = "error"
-            msg = "%s %s %s reported that %s (%s) failed %s time(s) within %s last attempt(s). Last error: %s" % (notify_message,
-                custom_message, self.hostname, name, url, nb_failures, nb_tries, msg)
+            msg = "%s %s %s reported that %s (%s) failed %s time(s) within %s last attempt(s)."\
+                " Last error: %s" % (notify_message, custom_message, self.hostname,
+                                     name, url, nb_failures, nb_tries, msg)
             event_type = EventType.DOWN
 
-        else: # Status is UP
+        else:  # Status is UP
             title = "[Recovered] %s reported that %s is up" % (self.hostname, name)
             alert_type = "success"
-            msg = "%s %s %s reported that %s (%s) recovered" % (notify_message,
-                custom_message, self.hostname, name,url)
+            msg = "%s %s %s reported that %s (%s) recovered" \
+                % (notify_message, custom_message, self.hostname, name, url)
             event_type = EventType.UP
 
         return {
-             'timestamp': int(time.time()),
-             'event_type': event_type,
-             'host': self.hostname,
-             'msg_text': msg,
-             'msg_title': title,
-             'alert_type': alert_type,
-             "source_type_name": source_type,
-             "event_object": name,
-             "tags": tags_list
+            'timestamp': int(time.time()),
+            'event_type': event_type,
+            'host': self.hostname,
+            'msg_text': msg,
+            'msg_title': title,
+            'alert_type': alert_type,
+            "source_type_name": source_type,
+            "event_object": name,
+            "tags": tags_list
         }
 
     def report_as_service_check(self, sc_name, status, instance, msg=None):
@@ -258,7 +285,7 @@ def check_cert_expiration(self, instance):
             sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             sock.connect((host, port))
             ssl_sock = ssl.wrap_socket(sock, cert_reqs=ssl.CERT_REQUIRED,
-                                           ca_certs=self.ca_certs)
+                                       ca_certs=self.ca_certs)
             cert = ssl_sock.getpeercert()
 
         except Exception as e:
@@ -271,7 +298,8 @@ def check_cert_expiration(self, instance):
             return Status.DOWN, "Expired by {0} days".format(days_left.days)
 
         elif days_left.days < warning_days:
-            return Status.WARNING, "This cert is almost expired, only {0} days left".format(days_left.days)
+            return Status.WARNING, "This cert is almost expired, only {0} days left"\
+                .format(days_left.days)
 
         else:
             return Status.UP, "Days left: {0}".format(days_left.days)


Test patch:
diff --git a/tests/test_http_check.py b/tests/test_http_check.py
new file mode 100644
--- /dev/null
+++ b/tests/test_http_check.py
@@ -0,0 +1,142 @@
+import time
+import mock
+
+from checks import AgentCheck
+from tests.common import AgentCheckTest
+
+RESULTS_TIMEOUT = 5
+
+CONFIG = {
+    'instances': [{
+        'name': 'conn_error',
+        'url': 'https://thereisnosuchlink.com',
+        'check_certificate_expiration': False,
+        'timeout': 1,
+    }, {
+        'name': 'http_error_status_code',
+        'url': 'http://httpbin.org/404',
+        'check_certificate_expiration': False,
+        'timeout': 1,
+    }, {
+        'name': 'status_code_match',
+        'url': 'http://httpbin.org/404',
+        'http_response_status_code': '4..',
+        'check_certificate_expiration': False,
+        'timeout': 1,
+    }, {
+        'name': 'cnt_mismatch',
+        'url': 'https://github.com',
+        'timeout': 1,
+        'check_certificate_expiration': False,
+        'content_match': 'thereisnosuchword'
+    }, {
+        'name': 'cnt_match',
+        'url': 'https://github.com',
+        'timeout': 1,
+        'check_certificate_expiration': False,
+        'content_match': '(thereisnosuchword|github)'
+    }
+    ]
+}
+
+CONFIG_SSL_ONLY = {
+    'instances': [{
+        'name': 'good_cert',
+        'url': 'https://github.com',
+        'timeout': 1,
+        'check_certificate_expiration': True,
+        'days_warning': 14
+    }, {
+        'name': 'cert_exp_soon',
+        'url': 'https://github.com',
+        'timeout': 1,
+        'check_certificate_expiration': True,
+        'days_warning': 9999
+    }, {
+        'name': 'conn_error',
+        'url': 'https://thereisnosuchlink.com',
+        'timeout': 1,
+        'check_certificate_expiration': True,
+        'days_warning': 14
+    }
+    ]
+}
+
+CONFIG_EXPIRED_SSL = {
+    'instances': [{
+        'name': 'expired_cert',
+        'url': 'https://github.com',
+        'timeout': 1,
+        'check_certificate_expiration': True,
+        'days_warning': 14
+    },
+    ]
+}
+
+
+FAKE_CERT = {'notAfter': 'Apr 12 12:00:00 2006 GMT'}
+
+
+class HTTPCheckTest(AgentCheckTest):
+    CHECK_NAME = 'http_check'
+
+    def tearDown(self):
+        self.check.stop()
+
+    def wait_for_async_service_checks(self, count):
+        i = 0
+        while i < RESULTS_TIMEOUT:
+            self.check._process_results()
+            if len(self.check.service_checks) >= count:
+                return self.check.get_service_checks()
+            time.sleep(1)
+            i += 1
+        raise Exception("Didn't get the right count of service checks in time {0}"
+                        .format(self.check.service_checks))
+
+    def test_check(self):
+        self.run_check(CONFIG)
+        # Overrides self.service_checks attribute when values are available\
+        self.service_checks = self.wait_for_async_service_checks(5)
+
+        # HTTP connection error
+        self.assertServiceCheck("http_check.conn_error", status=AgentCheck.CRITICAL,
+                                tags=['url:https://thereisnosuchlink.com'])
+
+        # Wrong HTTP response status code
+        self.assertServiceCheck("http_check.http_error_status_code", status=AgentCheck.CRITICAL,
+                                tags=['url:http://httpbin.org/404'])
+        self.assertServiceCheck("http_check.http_error_status_code", status=AgentCheck.OK,
+                                tags=['url:http://httpbin.org/404'], count=0)
+
+        # HTTP response status code match
+        self.assertServiceCheck("http_check.status_code_match", status=AgentCheck.OK,
+                                tags=['url:http://httpbin.org/404'])
+
+        # Content match & mismatching
+        self.assertServiceCheck("http_check.cnt_mismatch", status=AgentCheck.CRITICAL,
+                                tags=['url:https://github.com'])
+        self.assertServiceCheck("http_check.cnt_mismatch", status=AgentCheck.OK,
+                                tags=['url:https://github.com'], count=0)
+        self.assertServiceCheck("http_check.cnt_match", status=AgentCheck.OK,
+                                tags=['url:https://github.com'])
+
+    def test_check_ssl(self):
+        self.run_check(CONFIG_SSL_ONLY)
+        # Overrides self.service_checks attribute when values are available
+        self.service_checks = self.wait_for_async_service_checks(6)
+        self.assertServiceCheck("http_check.ssl_cert.good_cert", status=AgentCheck.OK,
+                                tags=['url:https://github.com'])
+        self.assertServiceCheck("http_check.ssl_cert.cert_exp_soon", status=AgentCheck.WARNING,
+                                tags=['url:https://github.com'])
+        self.assertServiceCheck("http_check.ssl_cert.conn_error", status=AgentCheck.CRITICAL,
+                                tags=['url:https://thereisnosuchlink.com'])
+
+    @mock.patch('ssl.SSLSocket.getpeercert', return_value=FAKE_CERT)
+    def test_mock_case(self, getpeercert_func):
+        self.run_check(CONFIG_EXPIRED_SSL)
+        # Overrides self.service_checks attribute when values are av
+        # Needed for the HTTP headers
+        self.service_checks = self.wait_for_async_service_checks(2)
+        self.assertServiceCheck("http_check.ssl_cert.expired_cert", status=AgentCheck.CRITICAL,
+                                tags=['url:https://github.com'])
diff --git a/tests/test_http_check_ssl.py b/tests/test_http_check_ssl.py
deleted file mode 100644
--- a/tests/test_http_check_ssl.py
+++ /dev/null
@@ -1,97 +0,0 @@
-import unittest
-from tests.common import load_check
-from checks import AgentCheck
-import time
-import mock
-
-RESULTS_TIMEOUT = 5
-
-class HttpSslTestCase(unittest.TestCase):
-
-    def tearDown(self):
-        self.check.stop()
-
-    def wait_for_async_service_checks(self, count):
-        i = 0
-        while i < RESULTS_TIMEOUT:
-            self.check._process_results()
-            if len(self.check.service_checks) >= count:
-                return self.check.get_service_checks()
-            time.sleep(1)
-            i += 1
-        raise Exception("Didn't get the right count of service checks in time {}".format(self.check.service_checks))
-
-    def test_http_ssl(self):
-
-        config = {
-            'instances': [{
-            'name' : 'Good Cert',
-            'url' : 'https://github.com',
-            'timeout' : 1,
-            'check_certificate_expiration': True,
-            'days_warning': 14
-            },
-            {
-            'name' : 'Warning about Cert expiring',
-            'url' : 'https://github.com',
-            'timeout' : 1,
-            'check_certificate_expiration': True,
-            'days_warning': 9999
-            },
-            {
-            'name' : 'Bad Url',
-            'url' : 'https://thereisnosuchlink.com',
-            'timeout' : 1,
-            'check_certificate_expiration': True,
-            'days_warning': 14
-            }
-            ]
-        }
-
-        # Needed for the HTTP headers
-        agentConfig = {'version': '5.1'}
-        self.check = load_check('http_check', config, agentConfig)
-
-        # OK Status
-        self.check.check(config['instances'][0])
-        # status and ssl_cert
-        sc = self.wait_for_async_service_checks(2)
-        self.assertEqual(sc[1].get('status'), AgentCheck.OK)
-
-        # Warning Status due to close to expiration date
-        self.check.check(config['instances'][1])
-        # status and ssl_cert
-        sc = self.wait_for_async_service_checks(2)
-        self.assertEqual(sc[1].get('status'), AgentCheck.WARNING)
-
-        # Warning Status due to bad link
-        self.check.check(config['instances'][2])
-        # status and ssl_cert
-        sc = self.wait_for_async_service_checks(2)
-        self.assertEqual(sc[1].get('status'), AgentCheck.CRITICAL)
-
-    fake_cert = {'notAfter': 'Apr 12 12:00:00 2006 GMT'}
-    @mock.patch('ssl.SSLSocket.getpeercert', return_value=fake_cert)
-    def test_mock_case(self, getpeercert_func):
-
-        # assert getpeercert_func.called
-        config = {
-            'instances': [{
-            'name' : 'Expired Cert',
-            'url' : 'https://github.com',
-            'timeout' : 1,
-            'check_certificate_expiration': True,
-            'days_warning': 14
-            },
-            ]
-        }
-        # Needed for the HTTP headers
-        agentConfig = {'version' : '5.1'}
-        self.check = load_check('http_check', config, agentConfig)
-
-        #Failed Status
-        self.check.check(config['instances'][0])
-        time.sleep(2)
-        self.check._process_results()
-        service = self.check.get_service_checks()
-        self.assertEqual(service[1].get('status'), AgentCheck.CRITICAL)
diff --git a/tests/test_service_checks.py b/tests/test_service_checks.py
--- a/tests/test_service_checks.py
+++ b/tests/test_service_checks.py
@@ -2,11 +2,11 @@
 import unittest
 import time
 from tests.common import load_check
-import logging
 import nose.tools as nt
 from config import AGENT_VERSION
 from util import headers as agent_headers
 
+
 class ServiceCheckTestCase(unittest.TestCase):
 
     def setUp(self):
@@ -28,18 +28,18 @@ def testHTTPHeaders(self):
                 'url': 'https://google.com',
                 'name': 'UpService',
                 'timeout': 1,
-                'headers': { "X-Auth-Token": "SOME-AUTH-TOKEN"}
+                'headers': {"X-Auth-Token": "SOME-AUTH-TOKEN"}
             }]
         }
 
         self.init_check(config, 'http_check')
-        url, username, password, timeout, include_content, headers, response_time, tags, ssl, ssl_expiration = self.check._load_conf(config['instances'][0])
+        url, username, password, timeout, http_response_status_code, include_content, headers, \
+            response_time, content_match, tags, ssl, \
+            ssl_expiration = self.check._load_conf(config['instances'][0])
 
         self.assertTrue(headers["X-Auth-Token"] == "SOME-AUTH-TOKEN", headers)
         self.assertTrue(headers.get('User-Agent') == agent_headers(self.agentConfig).get('User-Agent'), headers)
 
-
-
     def testHTTPWarning(self):
         config = {
             'init_config': {},
@@ -47,7 +47,7 @@ def testHTTPWarning(self):
                 'url': 'http://127.0.0.1:55555',
                 'name': 'DownService',
                 'timeout': 1
-            },{
+            }, {
                 'url': 'https://google.com',
                 'name': 'UpService',
                 'timeout': 1
@@ -63,7 +63,7 @@ def testHTTPWarning(self):
         warnings = self.check.get_warnings()
 
         self.assertTrue(len(warnings) == 4, warnings)
-        self.assertTrue(len([k for k in warnings if "Skipping SSL certificate validation" in k])==1, warnings)
+        self.assertTrue(len([k for k in warnings if "Skipping SSL certificate validation" in k]) == 1, warnings)
 
     def testHTTP(self):
         # No passwords this time
@@ -108,7 +108,7 @@ def verify_service_checks(service_checks):
         self.assertEqual(events[0]['event_object'], 'DownService')
         assert service_checks
         self.assertEqual(type(service_checks), type([]))
-        self.assertEqual(len(service_checks), 2, service_checks) # 1 per instance
+        self.assertEqual(len(service_checks), 2, service_checks)  # 1 per instance
         verify_service_checks(service_checks)
 
         events = self.check.get_events()
@@ -124,7 +124,6 @@ def verify_service_checks(service_checks):
         self.check.log.warning(self.check.notified)
         self.check.notified[('UpService', 'http_check')] = "DOWN"
 
-
         time.sleep(1)
         self.check.run()
         time.sleep(1)
@@ -139,7 +138,7 @@ def verify_service_checks(service_checks):
         self.assertEqual(events[0]['event_object'], 'UpService', events)
         self.assertEqual(type(service_checks), type([]))
         # FIXME: sometimes it's 3 instead of 2
-        self.assertTrue(len(service_checks) >= 2, service_checks) # Only 2 because the second run wasn't flushed
+        self.assertTrue(len(service_checks) >= 2, service_checks)  # Only 2 because the second run wasn't flushed
         verify_service_checks(service_checks)
 
         # Cleanup the threads
@@ -155,12 +154,12 @@ def testTCP(self):
                 'port': 65530,
                 'timeout': 1,
                 'name': 'DownService'
-            },{
+            }, {
                 'host': '126.0.0.1',
                 'port': 65530,
                 'timeout': 1,
                 'name': 'DownService2'
-            },{
+            }, {
                 'host': 'datadoghq.com',
                 'port': 80,
                 'timeout': 1,
@@ -180,7 +179,6 @@ def verify_service_checks(service_checks):
                 else:
                     raise Exception('Bad check name %s' % service_check['check'])
 
-
         self.check.run()
         time.sleep(2)
         nt.assert_equals(self.check.pool.get_nworkers(), 3)
@@ -198,7 +196,7 @@ def verify_service_checks(service_checks):
             self.assertTrue(event['event_object'][:11] == 'DownService')
         assert service_checks
         self.assertEqual(type(service_checks), type([]))
-        self.assertEqual(len(service_checks), 3, service_checks) # 1 per instance
+        self.assertEqual(len(service_checks), 3, service_checks)  # 1 per instance
         verify_service_checks(service_checks)
 
         events = self.check.get_events()
@@ -228,7 +226,7 @@ def verify_service_checks(service_checks):
         assert service_checks
         self.assertEqual(type(service_checks), type([]))
         # FIXME: sometimes it's 4 instead of 3
-        self.assertTrue(len(service_checks) >= 3, service_checks) # Only 3 because the second run wasn't flushed
+        self.assertTrue(len(service_checks) >= 3, service_checks)  # Only 3 because the second run wasn't flushed
         verify_service_checks(service_checks)
 
     def tearDown(self):