Repository: DataDog/dd-agent

Issue:
WARNING | dd.collector | aggregator(aggregator.py:336) | Metric postgresql.commits has an interval of 0. Not flushing.
For some unclear reason datadog-agent is filling my logs with WARNING messages like these:

```
2014-11-20 13:38:19 GMT | WARNING | dd.collector | aggregator(aggregator.py:336) | Metric postgresql.commits has an interval of 0. Not flushing.
2014-11-20 13:38:19 GMT | WARNING | dd.collector | aggregator(aggregator.py:336) | Metric postgresql.rows_inserted has an interval of 0. Not flushing.
2014-11-20 13:38:19 GMT | WARNING | dd.collector | aggregator(aggregator.py:336) | Metric postgresql.rows_fetched has an interval of 0. Not flushing.
2014-11-20 13:38:19 GMT | WARNING | dd.collector | aggregator(aggregator.py:336) | Metric postgresql.rollbacks has an interval of 0. Not flushing.
2014-11-20 13:38:19 GMT | WARNING | dd.collector | aggregator(aggregator.py:336) | Metric postgresql.rows_updated has an interval of 0. Not flushing.
2014-11-20 13:38:19 GMT | WARNING | dd.collector | aggregator(aggregator.py:336) | Metric postgresql.rows_returned has an interval of 0. Not flushing.
2014-11-20 13:38:19 GMT | WARNING | dd.collector | aggregator(aggregator.py:336) | Metric postgresql.rows_fetched has an interval of 0. Not flushing.
2014-11-20 13:38:19 GMT | WARNING | dd.collector | aggregator(aggregator.py:336) | Metric postgresql.rows_deleted has an interval of 0. Not flushing.
2014-11-20 13:38:19 GMT | WARNING | dd.collector | aggregator(aggregator.py:336) | Metric postgresql.rollbacks has an interval of 0. Not flushing.
2014-11-20 13:38:19 GMT | WARNING | dd.collector | aggregator(aggregator.py:336) | Metric postgresql.rows_inserted has an interval of 0. Not flushing.
2014-11-20 13:38:19 GMT | WARNING | dd.collector | aggregator(aggregator.py:336) | Metric postgresql.commits has an interval of 0. Not flushing.
2014-11-20 13:38:19 GMT | WARNING | dd.collector | aggregator(aggregator.py:336) | Metric postgresql.rollbacks has an interval of 0. Not flushing.
2014-11-20 13:38:19 GMT | WARNING | dd.collector | aggregator(aggregator.py:336) | Metric postgresql.disk_read has an interval of 0. Not flushing.
```

The postgresql configuration is as simple as it can be:

```
init_config:

instances:
   -   host: localhost
       port: 5432
       username: confluence
       dbname: confluence
       password: xxx
   -   host: localhost
       port: 5432
       username: jira
       dbname: jira
       password: xxx
   -   host: localhost
       port: 5432
       username: crowd
       dbname: crowd
       password: xxx
```

Agent is version 5.0.5



Discussion:
Thanks for the feedback @ssbarnea .
This is indeed a bug when you are setting up multiple instances to the same server but to different databases.

However, if you don't need relation specific metrics (e.g. for a particular table) you shouldn't have to specify different instances to different tables. 

By setting up just one instance you'll get the same exact metrics as if you were setting up an instance for each one of your database as we'll be querying "pg_stat_database" that will return metrics for all your database. (See: https://github.com/DataDog/dd-agent/blob/5.0.5/checks.d/postgres.py#L24-L28 )

So you should get rid of your last 2 instances in your postgres.yaml file and these warnings should go away and your metrics should get the proper values.

I agree that this behavior is not really clear and should be improved.
Thanks for pointing it to us!
Remi

Adding this to triage for 5.3.0 release.


Resolving patch:
diff --git a/checks.d/postgres.py b/checks.d/postgres.py
--- a/checks.d/postgres.py
+++ b/checks.d/postgres.py
@@ -203,6 +203,8 @@ def __init__(self, name, init_config, agentConfig, instances=None):
         self.versions = {}
         self.instance_metrics = {}
         self.bgw_metrics = {}
+        self.db_instance_metrics = []
+        self.db_bgw_metrics = []
 
     def _get_version(self, key, db):
         if key not in self.versions:
@@ -237,7 +239,22 @@ def _get_instance_metrics(self, key, db):
         """
         # Extended 9.2+ metrics if needed
         metrics = self.instance_metrics.get(key)
+        
         if metrics is None:
+
+            # Hack to make sure that if we have multiple instances that connect to
+            # the same host, port, we don't collect metrics twice
+            # as it will result in https://github.com/DataDog/dd-agent/issues/1211
+            sub_key = key[:2]
+            if sub_key in self.db_instance_metrics:
+                self.instance_metrics[key] = {}
+                self.log.debug("Not collecting instance metrics for key: {0} as"\
+                    " they are already collected by another instance".format(key))
+                return {}
+
+            self.db_instance_metrics.append(sub_key)
+
+            
             if self._is_9_2_or_above(key, db):
                 self.instance_metrics[key] = dict(self.COMMON_METRICS, **self.NEWER_92_METRICS)
             else:
@@ -252,7 +269,21 @@ def _get_bgw_metrics(self, key, db):
         """
         # Extended 9.2+ metrics if needed
         metrics = self.bgw_metrics.get(key)
+
         if metrics is None:
+
+            # Hack to make sure that if we have multiple instances that connect to
+            # the same host, port, we don't collect metrics twice
+            # as it will result in https://github.com/DataDog/dd-agent/issues/1211
+            sub_key = key[:2]
+            if sub_key in self.db_bgw_metrics:
+                self.bgw_metrics[key] = {}
+                self.log.debug("Not collecting bgw metrics for key: {0} as"\
+                    " they are already collected by another instance".format(key))
+                return {}
+
+            self.db_bgw_metrics.append(sub_key)
+
             self.bgw_metrics[key] = dict(self.COMMON_BGW_METRICS)
             if self._is_9_1_or_above(key, db):
                 self.bgw_metrics[key].update(self.NEWER_91_BGW_METRICS)
@@ -452,7 +483,7 @@ def check(self, instance):
         if dbname is None:
             dbname = 'postgres'
 
-        key = '%s:%s:%s' % (host, port, dbname)
+        key = (host, port, dbname)
 
         # Clean up tags in case there was a None entry in the instance
         # e.g. if the yaml contains tags: but no actual tags


Test patch:
diff --git a/tests/test_postgresql.py b/tests/test_postgresql.py
--- a/tests/test_postgresql.py
+++ b/tests/test_postgresql.py
@@ -35,10 +35,16 @@ def test_checks(self):
                         },
                         "query": "SELECT datname, %s FROM pg_stat_database WHERE datname = 'datadog_test' LIMIT(1)", 
                         "relation": False,
-                    }
-                ]
+                    }]
+                },
+                {
+                'host': host,
+                'port': port,
+                'username': 'datadog',
+                'password': 'datadog',
+                'dbname': 'postgres'
                 }
-            ]
+                ]
         }
         agentConfig = {
             'version': '0.1',
@@ -51,7 +57,7 @@ def test_checks(self):
         self.check.run()
 
         # FIXME: Not great, should have a function like that available
-        key = '%s:%s:%s' % (host, port, dbname)
+        key = (host, port, dbname)
         db = self.check.dbs[key]
 
         metrics = self.check.get_metrics()
@@ -97,11 +103,11 @@ def test_checks(self):
         service_checks_count = len(service_checks)
         self.assertTrue(type(service_checks) == type([]))
         self.assertTrue(service_checks_count > 0)
-        self.assertEquals(len([sc for sc in service_checks if sc['check'] == "postgres.can_connect"]), 2, service_checks)
+        self.assertEquals(len([sc for sc in service_checks if sc['check'] == "postgres.can_connect"]), 4, service_checks)
         # Assert that all service checks have the proper tags: host, port and db
         self.assertEquals(len([sc for sc in service_checks if "host:localhost" in sc['tags']]), service_checks_count, service_checks)
         self.assertEquals(len([sc for sc in service_checks if "port:%s" % config['instances'][0]['port'] in sc['tags']]), service_checks_count, service_checks)
-        self.assertEquals(len([sc for sc in service_checks if "db:%s" % config['instances'][0]['dbname'] in sc['tags']]), service_checks_count, service_checks)
+        self.assertEquals(len([sc for sc in service_checks if "db:%s" % config['instances'][0]['dbname'] in sc['tags']]), service_checks_count/2, service_checks)
 
         time.sleep(1)
         self.check.run()
@@ -112,7 +118,7 @@ def test_checks(self):
         self.assertEquals(len([m for m in metrics if 'table:persons' in str(m[3].get('tags', [])) ]), 11, metrics)
 
         self.metrics = metrics
-        self.assertMetric("custom.numbackendss")
+        self.assertMetric("custom.numbackends")
 
 if __name__ == '__main__':
     unittest.main()