Repository: DataDog/dd-agent

Issue:
Failures with the new NTP checking
The NTP checking introduced 5.3.0 itself works fairly well, but [`get_ntp_info`](https://github.com/DataDog/dd-agent/blob/master/checks/check_status.py#L98-L104) in `checks/check_status.py` is broken in some configurations.

The NTP checking pulls its configuration, including peers/servers to check against, from a configuration file.  But `get_ntp_info` checks NTP offset against a hard-coded `pool.ntp.org`.  It's not uncommon for datacenters to only allow a small number of servers to poll Internet-based NTP sources, and then to use those servers to re-distribute time internal to the local networks.  In such a configuration, the `ntplib.NTPClient().request` call will always fail.



Discussion:
Thanks @mutemule for reporting this, we probably need to have better error handling and:
- to allow overriding this default `pool.ntp.org` (which will be changed soon by a `datadog` subdomain of the pool cc @remh)
- look at the NTP check settings if there are some?

Thanks, @LeoCavaille!

I took a quick look at fixing this before I submitted the issue, but I'm not familiar enough with the DD internals.  It seemed reasonable to me to use the check settings if they exist, so there's consistency in the upstream queries, and just set defaults in there.


Resolving patch:
diff --git a/checks.d/ntp.py b/checks.d/ntp.py
--- a/checks.d/ntp.py
+++ b/checks.d/ntp.py
@@ -3,12 +3,10 @@
 
 # project
 from checks import AgentCheck
-from utils.ntp import get_ntp_datadog_host
+from utils.ntp import get_ntp_args, set_user_ntp_settings
 
 DEFAULT_OFFSET_THRESHOLD = 60 # in seconds
-DEFAULT_NTP_VERSION = 3
-DEFAULT_TIMEOUT = 1 # in seconds
-DEFAULT_PORT = "ntp"
+
 
 
 class NtpCheck(AgentCheck):
@@ -23,18 +21,18 @@ def check(self, instance):
         except (TypeError, ValueError):
             raise Exception('Must specify an integer value for offset_threshold. Configured value is %s' % repr(offset_threshold))
 
-        host = instance.get('host') or get_ntp_datadog_host()
-        self.log.debug("Using ntp host: {0}".format(host))
-        req_args = {
-            'host':    host,
-            'port':    instance.get('port', DEFAULT_PORT),
-            'version': int(instance.get('version', DEFAULT_NTP_VERSION)),
-            'timeout': float(instance.get('timeout', DEFAULT_TIMEOUT)),
-        }
+
+        set_user_ntp_settings(dict(instance))
+
+        req_args = get_ntp_args()
+
+        self.log.debug("Using ntp host: {0}".format(req_args['host']))
+
         try:
             ntp_stats = ntplib.NTPClient().request(**req_args)
         except ntplib.NTPException:
-            self.log.debug("Could not connect to NTP Server {0}".format(req_args['host']))
+            self.log.debug("Could not connect to NTP Server {0}".format(
+                req_args['host']))
             status  = AgentCheck.UNKNOWN
             ntp_ts = None
         else:
diff --git a/checks/check_status.py b/checks/check_status.py
--- a/checks/check_status.py
+++ b/checks/check_status.py
@@ -21,7 +21,7 @@
 import config
 from config import get_config, get_jmx_status_path, _windows_commondata_path
 from util import plural
-from utils.ntp import get_ntp_datadog_host
+from utils.ntp import get_ntp_args
 from utils.pidfile import PidFile
 from utils.platform import Platform
 from utils.profile import pretty_statistics
@@ -31,7 +31,7 @@
 STATUS_ERROR = 'ERROR'
 STATUS_WARNING = 'WARNING'
 
-NTP_OFFSET_THRESHOLD = 600
+NTP_OFFSET_THRESHOLD = 60
 
 
 log = logging.getLogger(__name__)
@@ -98,8 +98,8 @@ def logger_info():
 
 
 def get_ntp_info():
-    ntp_host = get_ntp_datadog_host()
-    ntp_offset = ntplib.NTPClient().request(ntp_host, version=3).offset
+    req_args = get_ntp_args()
+    ntp_offset = ntplib.NTPClient().request(**req_args).offset
     if abs(ntp_offset) > NTP_OFFSET_THRESHOLD:
         ntp_styles = ['red', 'bold']
     else:
diff --git a/utils/ntp.py b/utils/ntp.py
--- a/utils/ntp.py
+++ b/utils/ntp.py
@@ -1,11 +1,42 @@
 # stdlib
 import random
 
+user_ntp_settings = {}
 
-def get_ntp_datadog_host(subpool=None):
+DEFAULT_VERSION = 3
+DEFAULT_TIMEOUT = 1 # in seconds
+DEFAULT_PORT = "ntp"
+
+
+def set_user_ntp_settings(instance):
+    global user_ntp_settings
+    user_ntp_settings = instance
+
+def get_ntp_host(subpool=None):
     """
     Returns randomly a NTP hostname of our vendor pool. Or
     a given subpool if given in input.
     """
+
+    if user_ntp_settings.get('host') is not None:
+        return user_ntp_settings['host']
+
     subpool = subpool or random.randint(0, 3)
     return "{0}.datadog.pool.ntp.org".format(subpool)
+
+def get_ntp_port():
+    return user_ntp_settings.get('port') or DEFAULT_PORT
+
+def get_ntp_version():
+    return int(user_ntp_settings.get("version") or DEFAULT_VERSION)
+
+def get_ntp_timeout():
+    return float(user_ntp_settings.get('timeout') or DEFAULT_TIMEOUT)
+
+def get_ntp_args():
+    return {
+        'host':    get_ntp_host(),
+        'port':    get_ntp_port(),
+        'version': get_ntp_version(),
+        'timeout': get_ntp_timeout(),
+    }


Test patch:
diff --git a/tests/core/test_common.py b/tests/core/test_common.py
--- a/tests/core/test_common.py
+++ b/tests/core/test_common.py
@@ -4,9 +4,6 @@
 import time
 import unittest
 
-# 3p
-from nose.plugins.skip import SkipTest
-
 # project
 from aggregator import MetricsAggregator
 from checks import (
@@ -19,7 +16,7 @@
 from checks.collector import Collector
 from tests.checks.common import load_check
 from util import get_hostname
-from utils.ntp import get_ntp_datadog_host
+from utils.ntp import get_ntp_args
 from utils.proxy import get_proxy
 
 logger = logging.getLogger()
@@ -284,9 +281,7 @@ def test_get_proxy(self):
 
 
     def test_min_collection_interval(self):
-        if os.environ.get('TRAVIS', False):
-            raise SkipTest('ntp server times out too often on Travis')
-        config = {'instances': [{'host': get_ntp_datadog_host(), 'timeout': 1}], 'init_config': {}}
+        config = {'instances': [{}], 'init_config': {}}
 
         agentConfig = {
             'version': '0.1',
@@ -320,7 +315,7 @@ def test_min_collection_interval(self):
         metrics = check.get_metrics()
         self.assertTrue(len(metrics) > 0, metrics)
 
-        config = {'instances': [{'host': get_ntp_datadog_host(), 'timeout': 1, 'min_collection_interval':3}], 'init_config': {}}
+        config = {'instances': [{'min_collection_interval':3}], 'init_config': {}}
         check = load_check('ntp', config, agentConfig)
         check.run()
         metrics = check.get_metrics()
@@ -333,7 +328,7 @@ def test_min_collection_interval(self):
         metrics = check.get_metrics()
         self.assertTrue(len(metrics) > 0, metrics)
 
-        config = {'instances': [{'host': get_ntp_datadog_host(), 'timeout': 1, 'min_collection_interval': 12}], 'init_config': {'min_collection_interval':3}}
+        config = {'instances': [{'min_collection_interval': 12}], 'init_config': {'min_collection_interval':3}}
         check = load_check('ntp', config, agentConfig)
         check.run()
         metrics = check.get_metrics()
@@ -350,6 +345,52 @@ def test_min_collection_interval(self):
         metrics = check.get_metrics()
         self.assertTrue(len(metrics) > 0, metrics)
 
+    def test_ntp_global_settings(self):
+        config = {'instances': [{
+            "host": "foo.com",
+            "port": "bar",
+            "version": 42,
+            "timeout": 13.37}],
+            'init_config': {}}
+
+        agentConfig = {
+            'version': '0.1',
+            'api_key': 'toto'
+        }
+
+        # default min collection interval for that check was 20sec
+        check = load_check('ntp', config, agentConfig)
+        check.run()
+
+        ntp_args = get_ntp_args()
+
+        self.assertEqual(ntp_args["host"], "foo.com")
+        self.assertEqual(ntp_args["port"], "bar")
+        self.assertEqual(ntp_args["version"], 42)
+        self.assertEqual(ntp_args["timeout"], 13.37)
+
+        config = {'instances': [{}], 'init_config': {}}
+
+        agentConfig = {
+            'version': '0.1',
+            'api_key': 'toto'
+        }
+
+        # default min collection interval for that check was 20sec
+        check = load_check('ntp', config, agentConfig)
+        try:
+            check.run()
+        except Exception:
+            pass
+
+        ntp_args = get_ntp_args()
+
+        self.assertTrue(ntp_args["host"].endswith("datadog.pool.ntp.org"))
+        self.assertEqual(ntp_args["port"], "ntp")
+        self.assertEqual(ntp_args["version"], 3)
+        self.assertEqual(ntp_args["timeout"], 1.0)
+
+
 
 class TestAggregator(unittest.TestCase):
     def setUp(self):