Repository: cloud-custodian/cloud-custodian

Issue:
need tag augment added to RDS aurora



Discussion:
Tags are not present for rdscluster for aurora
@kapilt or @ocampocj - should the rds-cluster augment with tags from the RDS cluster or should it be looking at the tags of its `DBClusterMembers` DB instances?  

It seems like it should be the cluster, but I am asking because I can't figure out how to set tags on an RDS cluster through the web interface.  It is possible to set tags through the command line, as you can see here:

```
(Pdb) client.add_tags_to_resource(ResourceName='arn:aws:rds:us-east-1:xxx:cluster:scot-test-cluster', Tags=[{'Key': 'cfoo', 'Value': 'cbar'}])
{...}

(Pdb) client.list_tags_for_resource(ResourceName='arn:aws:rds:us-east-1:xxx:cluster:scot-test-cluster')
{..., u'TagList': [{u'Value': 'cbar', u'Key': 'cfoo'}]}
```

Just wanted to be sure I was augmenting with the correct tags.
@scotwk - should be the cluster, and though I don't see them in the UI right now either, they appear to be available:  

http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-rds-dbcluster.html#cfn-rds-dbcluster-tags


Resolving patch:
diff --git a/c7n/resources/rdscluster.py b/c7n/resources/rdscluster.py
--- a/c7n/resources/rdscluster.py
+++ b/c7n/resources/rdscluster.py
@@ -24,7 +24,7 @@
 from c7n.manager import resources
 from c7n.query import QueryResourceManager
 from c7n.utils import (
-    type_schema, local_session, snapshot_identifier, chunks)
+    type_schema, local_session, snapshot_identifier, chunks, get_retry)
 
 log = logging.getLogger('custodian.rds-cluster')
 
@@ -40,7 +40,7 @@ class RDSCluster(QueryResourceManager):
     class resource_type(object):
 
         service = 'rds'
-        type = 'rds-cluster'
+        type = 'cluster'
         enum_spec = ('describe_db_clusters', 'DBClusters', None)
         name = id = 'DBClusterIdentifier'
         filter_name = None
@@ -50,6 +50,35 @@ class resource_type(object):
 
     filter_registry = filters
     action_registry = actions
+    retry = staticmethod(get_retry(('Throttled',)))
+
+    def augment(self, dbs):
+        filter(None, _rds_cluster_tags(
+            self.get_model(),
+            dbs, self.session_factory, self.executor_factory,
+            self.generate_arn, self.retry))
+        return dbs
+
+
+def _rds_cluster_tags(model, dbs, session_factory, executor_factory, generator, retry):
+    """Augment rds clusters with their respective tags."""
+
+    def process_tags(db):
+        client = local_session(session_factory).client('rds')
+        arn = generator(db[model.id])
+        tag_list = None
+        try:
+            tag_list = retry(client.list_tags_for_resource, ResourceName=arn)['TagList']
+        except ClientError as e:
+            if e.response['Error']['Code'] != 'DBClusterNotFoundFault':
+                log.warning("Exception getting rdscluster tags\n %s", e)
+            return None
+        db['Tags'] = tag_list or []
+        return db
+
+    # Rds maintains a low api call limit, so this can take some time :-(
+    with executor_factory(max_workers=1) as w:
+        return list(w.map(process_tags, dbs))
 
 
 @filters.register('security-group')


Test patch:
diff --git a/tests/data/placebo/test_rdscluster_tag_augment/rds.DescribeDBClusters_1.json b/tests/data/placebo/test_rdscluster_tag_augment/rds.DescribeDBClusters_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_rdscluster_tag_augment/rds.DescribeDBClusters_1.json
@@ -0,0 +1,92 @@
+{
+    "status_code": 200, 
+    "data": {
+        "DBClusters": [
+            {
+                "MasterUsername": "root", 
+                "ReaderEndpoint": "scot-test-cluster.cluster-ro-cz6wy8cci3uf.us-east-1.rds.amazonaws.com", 
+                "ReadReplicaIdentifiers": [], 
+                "VpcSecurityGroups": [
+                    {
+                        "Status": "active", 
+                        "VpcSecurityGroupId": "sg-0b3d3377"
+                    }
+                ], 
+                "HostedZoneId": "Z2R2ITUGPM61AM", 
+                "Status": "available", 
+                "MultiAZ": false, 
+                "LatestRestorableTime": {
+                    "hour": 18, 
+                    "__class__": "datetime", 
+                    "month": 6, 
+                    "second": 25, 
+                    "microsecond": 874000, 
+                    "year": 2017, 
+                    "day": 28, 
+                    "minute": 47
+                }, 
+                "PreferredBackupWindow": "09:45-10:15", 
+                "DBSubnetGroup": "default", 
+                "AllocatedStorage": 1, 
+                "BackupRetentionPeriod": 1, 
+                "PreferredMaintenanceWindow": "sat:05:57-sat:06:27", 
+                "Engine": "aurora", 
+                "Endpoint": "scot-test-cluster.cluster-cz6wy8cci3uf.us-east-1.rds.amazonaws.com", 
+                "EarliestRestorableTime": {
+                    "hour": 16, 
+                    "__class__": "datetime", 
+                    "month": 6, 
+                    "second": 5, 
+                    "microsecond": 774000, 
+                    "year": 2017, 
+                    "day": 27, 
+                    "minute": 14
+                }, 
+                "IAMDatabaseAuthenticationEnabled": false, 
+                "ClusterCreateTime": {
+                    "hour": 16, 
+                    "__class__": "datetime", 
+                    "month": 6, 
+                    "second": 25, 
+                    "microsecond": 714000, 
+                    "year": 2017, 
+                    "day": 27, 
+                    "minute": 13
+                }, 
+                "EngineVersion": "5.6.10a", 
+                "DBClusterIdentifier": "scot-test-cluster", 
+                "DbClusterResourceId": "cluster-AHBKTFUVCNNCCPXFWMQNBEX7GA", 
+                "DBClusterMembers": [
+                    {
+                        "IsClusterWriter": true, 
+                        "DBClusterParameterGroupStatus": "in-sync", 
+                        "PromotionTier": 1, 
+                        "DBInstanceIdentifier": "scot-test"
+                    }
+                ], 
+                "DBClusterArn": "arn:aws:rds:us-east-1:644160558196:cluster:scot-test-cluster", 
+                "StorageEncrypted": false, 
+                "AssociatedRoles": [], 
+                "DBClusterParameterGroup": "default.aurora5.6", 
+                "AvailabilityZones": [
+                    "us-east-1a", 
+                    "us-east-1b", 
+                    "us-east-1e"
+                ], 
+                "Port": 3306
+            }
+        ], 
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "fadda864-5c32-11e7-9765-039fb83f1357", 
+            "HTTPHeaders": {
+                "x-amzn-requestid": "fadda864-5c32-11e7-9765-039fb83f1357", 
+                "vary": "Accept-Encoding", 
+                "content-length": "2711", 
+                "content-type": "text/xml", 
+                "date": "Wed, 28 Jun 2017 18:52:47 GMT"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_rdscluster_tag_augment/rds.ListTagsForResource_1.json b/tests/data/placebo/test_rdscluster_tag_augment/rds.ListTagsForResource_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_rdscluster_tag_augment/rds.ListTagsForResource_1.json
@@ -0,0 +1,22 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "fb11fe91-5c32-11e7-887a-3bb31b4e417f", 
+            "HTTPHeaders": {
+                "x-amzn-requestid": "fb11fe91-5c32-11e7-887a-3bb31b4e417f", 
+                "date": "Wed, 28 Jun 2017 18:52:48 GMT", 
+                "content-length": "384", 
+                "content-type": "text/xml"
+            }
+        }, 
+        "TagList": [
+            {
+                "Value": "cbar", 
+                "Key": "cfoo"
+            }
+        ]
+    }
+}
\ No newline at end of file
diff --git a/tests/test_rdscluster.py b/tests/test_rdscluster.py
--- a/tests/test_rdscluster.py
+++ b/tests/test_rdscluster.py
@@ -13,12 +13,22 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+from c7n.resources.rdscluster import RDSCluster
+
 from .common import BaseTest
 
 
 class RDSClusterTest(BaseTest):
 
+    def remove_augments(self):
+        # This exists because we added tag augmentation after eight other tests
+        # were created and I did not want to re-create the state to re-record
+        # them with the extra API call. If those get re-recorded we can remove
+        # this.
+        self.patch(RDSCluster, 'augment', lambda x, y: y)
+
     def test_rdscluster_security_group(self):
+        self.remove_augments()
         session_factory = self.replay_flight_data('test_rdscluster_sg_filter')
         p = self.load_policy({
             'name': 'rdscluster-sg',
@@ -33,6 +43,7 @@ def test_rdscluster_security_group(self):
         self.assertEqual(resources[0]['DatabaseName'], 'devtest')
 
     def test_rdscluster_subnet(self):
+        self.remove_augments()
         session_factory = self.replay_flight_data('test_rdscluster_subnet')
         p = self.load_policy({
             'name': 'rdscluster-sub',
@@ -47,6 +58,7 @@ def test_rdscluster_subnet(self):
         self.assertEqual(resources[0]['DatabaseName'], 'devtest')
 
     def test_rdscluster_simple(self):
+        self.remove_augments()
         session_factory = self.replay_flight_data('test_rdscluster_simple')
         p = self.load_policy({
             'name': 'rdscluster-simple',
@@ -56,6 +68,7 @@ def test_rdscluster_simple(self):
         self.assertEqual(len(resources), 2)
 
     def test_rdscluster_simple_filter(self):
+        self.remove_augments()
         session_factory = self.replay_flight_data('test_rdscluster_simple')
         p = self.load_policy({
             'name': 'rdscluster-simple-filter',
@@ -69,6 +82,7 @@ def test_rdscluster_simple_filter(self):
         self.assertEqual(len(resources), 1)
 
     def test_rdscluster_delete(self):
+        self.remove_augments()
         session_factory = self.replay_flight_data('test_rdscluster_delete')
         p = self.load_policy({
             'name': 'rdscluster-delete',
@@ -85,6 +99,7 @@ def test_rdscluster_delete(self):
         self.assertEqual(len(resources), 1)
 
     def test_rdscluster_delete_with_instances(self):
+        self.remove_augments()
         session_factory = self.replay_flight_data('test_rdscluster_delete_with_instances')
         p = self.load_policy({
             'name': 'rdscluster-delete',
@@ -101,6 +116,7 @@ def test_rdscluster_delete_with_instances(self):
         self.assertEqual(len(resources), 1)
 
     def test_rdscluster_retention(self):
+        self.remove_augments()
         session_factory = self.replay_flight_data('test_rdscluster_retention')
         p = self.load_policy({
             'name': 'rdscluster-delete',
@@ -115,6 +131,7 @@ def test_rdscluster_retention(self):
         self.assertEqual(len(resources), 1)
 
     def test_rdscluster_snapshot(self):
+        self.remove_augments()
         session_factory = self.replay_flight_data('test_rdscluster_snapshot')
         p = self.load_policy({
             'name': 'rdscluster-snapshot',
@@ -128,6 +145,17 @@ def test_rdscluster_snapshot(self):
         resources = p.run()
         self.assertEqual(len(resources), 1)
 
+    def test_rdscluster_tag_augment(self):
+        session_factory = self.replay_flight_data('test_rdscluster_tag_augment')
+        p = self.load_policy({
+            'name': 'rdscluster-tag-augment',
+            'resource': 'rds-cluster',
+            'filters': [{'tag:cfoo': 'cbar'}]},
+            session_factory=session_factory)
+
+        resources = p.run()
+        self.assertEqual(len(resources), 1)
+
 
 class RDSClusterSnapshotTest(BaseTest):