Repository: DataDog/dd-agent

Issue:
DNS query response monitoring
It would be great to be able to monitor DNS responses. We had issues where we randomly got SERVFAIL status responses from our DNS and as a result this host could not be resolved, and it caused some trouble. 

Would be great to have DNS monitoring bundled to datadog that would do following:
- You can define which host to query for
- You can define which DNS server to use
- Define timeout of the DNS query
- Store response time in milliseconds as gauge metric

This script would create Datadog event if:
- We get timeout
- Hostname is not found
- We get any other status than "NOERROR"

There is DataDog agent check script that someone has created that does almost everything we need, but some features are missing:
- You cannot define DNS server you use
- Response time should be stored in milliseconds (using seconds is not practical because most DNS responses on LAN are on range of 1-30 milliseconds)
- Doesn't validate the status of the response (SERVFAIL or NOERROR etc.) so our errors went unnoticed.

Here is the script for reference:
- dns.py: https://gist.github.com/srkiNZ84/ef5a099bc4787b698fd7
- dns.yaml: https://gist.github.com/srkiNZ84/7bed4a5bd41ac602ad72



Discussion:
We should turn this into a service check. This will let us build monitors on top of this check.

Thanks a lot @eeroniemi . We are scheduling this for our 5.8.0 release


Resolving patch:
diff --git a/checks.d/dns_check.py b/checks.d/dns_check.py
new file mode 100644
--- /dev/null
+++ b/checks.d/dns_check.py
@@ -0,0 +1,70 @@
+# stdlib
+import time
+
+# 3p
+import dns.resolver
+
+# project
+from checks import AgentCheck
+
+
+class DNSCheck(AgentCheck):
+    SERVICE_CHECK_NAME = 'dns.can_resolve'
+    DEFAULT_TIMEOUT = 5
+
+    def __init__(self, name, init_config, agentConfig, instances=None):
+        AgentCheck.__init__(self, name, init_config, agentConfig, instances)
+        self.default_timeout = init_config.get('default_timeout', self.DEFAULT_TIMEOUT)
+
+    def check(self, instance):
+        if 'hostname' not in instance:
+            self.log.info("Skipping instance, no hostname found.")
+            return
+
+        timeout = float(instance.get('timeout', self.default_timeout))
+        self.query_dns(instance, timeout)
+
+    def query_dns(self, instance, timeout):
+        """Perform the DNS query, and report its duration as a gauge"""
+        start_time = end_time = 0.0
+        resolver = dns.resolver.Resolver()
+        resolver.lifetime = timeout
+
+        hostname = instance.get('hostname')
+        tags = self._get_tags(instance)
+
+        # if a specific DNS server was defined use it, else use the system default
+        nameserver = instance.get('nameserver')
+        if nameserver is not None:
+            resolver.nameservers = [nameserver]
+
+        status = AgentCheck.CRITICAL
+        start_time = time.time()
+        try:
+            self.log.debug('Resolving hostname %s...' % hostname)
+            answer = resolver.query(hostname)
+            assert(answer.rrset.items[0].address)
+            end_time = time.time()
+        except dns.exception.Timeout:
+            self.log.error('DNS resolution of %s timed out' % hostname)
+            self.service_check(self.SERVICE_CHECK_NAME, status, tags=self._get_tags(instance))
+            raise
+        except Exception:
+            self.log.exception('DNS resolution of %s has failed.' % hostname)
+            self.service_check(self.SERVICE_CHECK_NAME, status, tags=self._get_tags(instance))
+            raise
+        else:
+            if end_time - start_time > 0:
+                self.gauge('dns.response_time', end_time - start_time, tags=tags)
+                self.service_check(self.SERVICE_CHECK_NAME, AgentCheck.OK, tags=self._get_tags(instance))
+
+    def _get_tags(self, instance):
+        tags = []
+        try:
+            nameserver = instance.get('nameserver') or dns.resolver.Resolver().nameservers[0]
+            tags.append('nameserver:%s' % nameserver)
+        except IndexError:
+            self.log.error('No DNS server was found on this host.')
+
+        tags.append('resolved_hostname:%s' % instance.get('hostname'))
+        return tags


Test patch:
diff --git a/tests/checks/mock/test_dns_check.py b/tests/checks/mock/test_dns_check.py
new file mode 100644
--- /dev/null
+++ b/tests/checks/mock/test_dns_check.py
@@ -0,0 +1,85 @@
+# stdlib
+import mock
+
+# 3p
+from dns.resolver import Resolver, Timeout, NXDOMAIN
+
+# project
+from tests.checks.common import AgentCheckTest
+from checks import AgentCheck
+
+
+METRICS = [
+    'dns.response_time'
+]
+
+SERVICE_CHECK_NAME = 'dns.can_resolve'
+
+
+class MockDNSAnswer:
+    def __init__(self, address):
+        self.rrset = MockDNSAnswer.MockRrset(address)
+
+    class MockRrset:
+        def __init__(self, address):
+            self.items = [MockDNSAnswer.MockItem()]
+
+    class MockItem:
+        def __getattr__(self, name):
+            return '127.0.0.1'
+
+
+def success_query_mock(d_name):
+    return MockDNSAnswer('127.0.0.1')
+
+
+def timeout_query_mock(d_name):
+    raise Timeout()
+
+
+class TestDns(AgentCheckTest):
+    CHECK_NAME = 'dns_check'
+
+    @mock.patch.object(Resolver, 'query', side_effect=success_query_mock)
+    def test_success(self, mocked_query):
+        config = {
+            'instances': [{'hostname': 'www.example.org', 'nameserver': '127.0.0.1'}]
+        }
+        self.run_check(config)
+        self.assertMetric('dns.response_time', count=1,
+                          tags=['nameserver:127.0.0.1', 'resolved_hostname:www.example.org'])
+        self.assertServiceCheck(SERVICE_CHECK_NAME, status=AgentCheck.OK,
+                                tags=['resolved_hostname:www.example.org', 'nameserver:127.0.0.1'])
+        self.coverage_report()
+
+    @mock.patch.object(Resolver, 'query', side_effect=Timeout())
+    def test_timeout(self, mocked_query):
+        configs = [
+            # short default timeout
+            {'init_config': {'default_timeout': 0.1},
+             'instances': [{'hostname': 'www.example.org', 'nameserver': '127.0.0.1'}]},
+            # short timeout
+            {'instances': [{'hostname': 'www.example.org', 'timeout': 0.1, 'nameserver': '127.0.0.1'}]},
+        ]
+        for config in configs:
+            self.assertRaises(
+                Timeout,
+                lambda: self.run_check(config)
+            )
+            self.assertEquals(len(self.metrics), 0)
+            self.assertServiceCheck(SERVICE_CHECK_NAME, status=AgentCheck.CRITICAL,
+                                    tags=['resolved_hostname:www.example.org', 'nameserver:127.0.0.1'])
+            self.coverage_report()
+
+    def test_invalid_config(self):
+        configs = [
+            # invalid hostname
+            {'instances': [{'hostname': 'example'}]},
+            # invalid nameserver
+            {'instances': [{'hostname': 'www.example.org', 'nameserver': '0.0.0.0'}]}
+        ]
+        for config in configs:
+            self.assertRaises(NXDOMAIN, lambda: self.run_check(config))
+            self.assertEquals(len(self.metrics), 0)
+            self.assertServiceCheck(SERVICE_CHECK_NAME, status=AgentCheck.CRITICAL)
+            self.coverage_report()