Repository: DataDog/dd-agent

Issue:
Submitting a gauge with a timestamp disregards timestamp in AgentChecks
https://github.com/DataDog/dd-agent/blob/master/aggregator.py#L310-L324

Timestamp is ignored in submit_metric()



Discussion:
FYI it's not only gauges but all types of metrics
Good catch @pcockwell 


Resolving patch:
diff --git a/aggregator.py b/aggregator.py
--- a/aggregator.py
+++ b/aggregator.py
@@ -3,9 +3,18 @@
 
 log = logging.getLogger(__name__)
 
+# This is used to ensure that metrics with a timestamp older than
+# RECENT_POINT_THRESHOLD_DEFAULT seconds (or the value passed in to
+# the MetricsAggregator constructor) get discarded rather than being
+# input into the incorrect bucket. Currently, the MetricsAggregator
+# does not support submitting values for the past, and all values get
+# submitted for the timestamp passed into the flush() function.
+RECENT_POINT_THRESHOLD_DEFAULT = 30
+
 class Infinity(Exception): pass
 class UnknownValue(Exception): pass
 
+
 class Metric(object):
     """
     A base metric class that accepts points, slices them into time intervals
@@ -228,7 +237,6 @@ def flush(self, timestamp, interval):
             self.samples = self.samples[-1:]
 
 
-
 class MetricsAggregator(object):
     """
     A metric aggregator class.
@@ -237,7 +245,7 @@ class MetricsAggregator(object):
     # Types of metrics that allow strings
     ALLOW_STRINGS = ['s', ]
 
-    def __init__(self, hostname, interval=1.0, expiry_seconds=300, formatter=None):
+    def __init__(self, hostname, interval=1.0, expiry_seconds=300, formatter=None, recent_point_threshold=None):
         self.metrics = {}
         self.total_count = 0
         self.count = 0
@@ -245,8 +253,8 @@ def __init__(self, hostname, interval=1.0, expiry_seconds=300, formatter=None):
             'g': Gauge,
             'c': Counter,
             'h': Histogram,
-            'ms' : Histogram,
-            's'  : Set,
+            'ms': Histogram,
+            's': Set,
             '_dd-r': Rate,
         }
         self.hostname = hostname
@@ -254,6 +262,10 @@ def __init__(self, hostname, interval=1.0, expiry_seconds=300, formatter=None):
         self.formatter = formatter or api_formatter
         self.interval = float(interval)
 
+        recent_point_threshold = recent_point_threshold or RECENT_POINT_THRESHOLD_DEFAULT
+        self.recent_point_threshold = int(recent_point_threshold)
+        self.num_discarded_old_points = 0
+
     def packets_per_second(self, interval):
         return round(float(self.count)/interval, 2)
 
@@ -318,7 +330,11 @@ def submit_metric(self, name, value, mtype, tags=None, hostname=None,
             metric_class = self.metric_type_to_class[mtype]
             self.metrics[context] = metric_class(self.formatter, name, tags,
                 hostname or self.hostname, device_name)
-        self.metrics[context].sample(value, sample_rate)
+        cur_time = time()
+        if timestamp is not None and cur_time - int(timestamp) > self.recent_point_threshold:
+            self.num_discarded_old_points += 1
+        else:
+            self.metrics[context].sample(value, sample_rate)
 
     def gauge(self, name, value, tags=None, hostname=None, device_name=None, timestamp=None):
         self.submit_metric(name, value, 'g', tags, hostname, device_name, timestamp)
@@ -352,6 +368,11 @@ def flush(self):
             else:
                 metrics += metric.flush(timestamp, self.interval)
 
+        # Log a warning regarding metrics with old timestamps being submitted
+        if self.num_discarded_old_points > 0:
+            log.warn('%s points were discarded as a result of having an old timestamp' % self.num_discarded_old_points)
+            self.num_discarded_old_points = 0
+
         # Save some stats.
         log.debug("received %s payloads since last flush" % self.count)
         self.total_count += self.count
@@ -361,6 +382,7 @@ def flush(self):
     def send_packet_count(self, metric_name):
         self.submit_metric(metric_name, self.count, 'g')
 
+
 def api_formatter(metric, value, timestamp, tags, hostname, device_name=None):
 
     # Workaround for a bug in minjson serialization
@@ -368,9 +390,9 @@ def api_formatter(metric, value, timestamp, tags, hostname, device_name=None):
     if tags is not None and isinstance(tags, tuple) and len(tags) == 1:
         tags = list(tags)
     return {
-        'metric' : metric,
-        'points' : [(timestamp, value)],
-        'tags' : tags,
-        'host' : hostname,
+        'metric': metric,
+        'points': [(timestamp, value)],
+        'tags': tags,
+        'host': hostname,
         'device_name': device_name
     }
diff --git a/checks/__init__.py b/checks/__init__.py
--- a/checks/__init__.py
+++ b/checks/__init__.py
@@ -278,7 +278,9 @@ def __init__(self, name, init_config, agentConfig, instances=None):
         self.agentConfig = agentConfig
         self.hostname = get_hostname(agentConfig)
         self.log = logging.getLogger('%s.%s' % (__name__, name))
-        self.aggregator = MetricsAggregator(self.hostname, formatter=agent_formatter)
+
+        self.aggregator = MetricsAggregator(self.hostname, formatter=agent_formatter, recent_point_threshold=agentConfig.get('recent_point_threshold', None))
+
         self.events = []
         self.instances = instances or []
 
diff --git a/dogstatsd.py b/dogstatsd.py
--- a/dogstatsd.py
+++ b/dogstatsd.py
@@ -253,7 +253,8 @@ def init(config_path=None, use_watchdog=False, use_forwarder=False):
     # Create the aggregator (which is the point of communication between the
     # server and reporting threads.
     assert 0 < interval
-    aggregator = MetricsAggregator(hostname, interval)
+
+    aggregator = MetricsAggregator(hostname, interval, recent_point_threshold=c.get('recent_point_threshold', None))
 
     # Start the reporting thread.
     reporter = Reporter(interval, aggregator, target, api_key, use_watchdog)


Test patch:
diff --git a/tests/test_dogstatsd.py b/tests/test_dogstatsd.py
--- a/tests/test_dogstatsd.py
+++ b/tests/test_dogstatsd.py
@@ -160,6 +160,19 @@ def test_gauge(self):
         nt.assert_equals(second['metric'], 'my.second.gauge')
         nt.assert_equals(second['points'][0][1], 1.5)
 
+        # Ensure that old gauges get dropped due to old timestamps
+        stats.gauge('my.first.gauge', 5)
+        stats.gauge('my.first.gauge', 1, timestamp=1000000000)
+        stats.gauge('my.second.gauge', 20, timestamp=1000000000)
+
+        metrics = self.sort_metrics(stats.flush())
+        assert len(metrics) == 1
+
+        first = metrics[0]
+
+        nt.assert_equals(first['metric'], 'my.first.gauge')
+        nt.assert_equals(first['points'][0][1], 5)
+        nt.assert_equals(first['host'], 'myhost')
 
     def test_sets(self):
         stats = MetricsAggregator('myhost')