Repository: DataDog/integrations-core

Issue:
MySQL Replication Lag doesn't tag by channel name
MySQL 5.7 introduces "replication channels", which means a slave can replicate from multiple masters ("sources") at any given time.

To measure replication lag across `n` channels, it'd be great if DataDog could tag the `mysql.replication.seconds_behind_master` metric with `channel:<CHANNEL_NAME>`. That way users could get an average across the channels, or have monitors alert when one of the channels falls behind.

Currently, the check fetches only the _first_ replication channel (https://github.com/DataDog/integrations-core/blob/master/mysql/check.py#L855)


Discussion:

Resolving patch:
diff --git a/mysql/check.py b/mysql/check.py
--- a/mysql/check.py
+++ b/mysql/check.py
@@ -543,12 +543,12 @@ def _collect_metrics(self, host, db, tags, options, queries):
             # MySQL 5.7.x might not have 'Slave_running'. See: https://bugs.mysql.com/bug.php?id=78544
             # look at replica vars collected at the top of if-block
             if self._version_compatible(db, host, (5, 7, 0)):
-                slave_io_running = self._collect_string('Slave_IO_Running', results)
-                slave_sql_running = self._collect_string('Slave_SQL_Running', results)
+                slave_io_running = self._collect_type('Slave_IO_Running', results, dict)
+                slave_sql_running = self._collect_type('Slave_SQL_Running', results, dict)
                 if slave_io_running:
-                    slave_io_running = (slave_io_running.lower().strip() == "yes")
+                    slave_io_running = any(v.lower().strip() == 'yes' for v in slave_io_running.itervalues())
                 if slave_sql_running:
-                    slave_sql_running = (slave_sql_running.lower().strip() == "yes")
+                    slave_sql_running = any(v.lower().strip() == 'yes' for v in slave_sql_running.itervalues())
 
                 if not (slave_io_running is None and slave_sql_running is None):
                     if slave_io_running and slave_sql_running:
@@ -851,10 +851,20 @@ def _get_replica_stats(self, db):
         try:
             with closing(db.cursor(pymysql.cursors.DictCursor)) as cursor:
                 replica_results = {}
+
                 cursor.execute("SHOW SLAVE STATUS;")
-                slave_results = cursor.fetchone()
-                if slave_results:
-                    replica_results.update(slave_results)
+                slave_results = cursor.fetchall()
+                if len(slave_results) > 0:
+                    for slave_result in slave_results:
+                        # MySQL <5.7 does not have Channel_Name.
+                        # For MySQL >=5.7 'Channel_Name' is set to an empty string by default
+                        channel = slave_result.get('Channel_Name') or 'default'
+                        for key in slave_result:
+                            if slave_result[key] is not None:
+                                if key not in replica_results:
+                                    replica_results[key] = {}
+                                replica_results[key]["channel:{0}".format(channel)] = slave_result[key]
+
                 cursor.execute("SHOW MASTER STATUS;")
                 binlog_results = cursor.fetchone()
                 if binlog_results:


Test patch:
diff --git a/mysql/test_mysql.py b/mysql/test_mysql.py
--- a/mysql/test_mysql.py
+++ b/mysql/test_mysql.py
@@ -400,6 +400,8 @@ def test_complex_config_replica(self):
         config['instances'][0]['port'] = 13307
         self.run_check_twice(config)
 
+        self.assertMetricTag('mysql.replication.seconds_behind_master', 'channel:default')
+
         # Test service check
         self.assertServiceCheck('mysql.can_connect', status=AgentCheck.OK,
                                 tags=self.SC_TAGS_REPLICA, count=1)