Repository: cloud-custodian/cloud-custodian

Issue:
Add RDS Cluster 'delete' action to first delete associated RDS DB instances
For the RDS Cluster resource added in https://github.com/capitalone/cloud-custodian/pull/344 it would be good to enhance the **delete** action to add an option (default `false`) to first delete any associated DB instances.



Discussion:

Resolving patch:
diff --git a/c7n/resources/rdscluster.py b/c7n/resources/rdscluster.py
--- a/c7n/resources/rdscluster.py
+++ b/c7n/resources/rdscluster.py
@@ -54,15 +54,26 @@ class Meta(object):
 class Delete(BaseAction):
 
     schema = type_schema(
-        'delete', **{'skip-snapshot': {'type': 'boolean'}})
+        'delete', **{'skip-snapshot': {'type': 'boolean'},
+                     'delete-instances': {'type': 'boolean'}})
 
-    def process(self, resources):
-        self.skip = self.data.get('skip-snapshot', False)
+    def process(self, clusters):
+        skip = self.data.get('skip-snapshot', False)
+        delete_instances = self.data.get('delete-instances', True)
         client = local_session(self.manager.session_factory).client('rds')
 
-        for cluster in resources:
+        for cluster in clusters:
+            if delete_instances:
+                for instance in cluster.get('DBClusterMembers', []):
+                    client.delete_db_instance(
+                        DBInstanceIdentifier=instance['DBInstanceIdentifier'],
+                        SkipFinalSnapshot=True)
+                    self.log.info(
+                        'Deleted RDS instance: %s',
+                        instance['DBInstanceIdentifier'])
+
             params = {'DBClusterIdentifier': cluster['DBClusterIdentifier']}
-            if self.skip:
+            if skip:
                 params['SkipFinalSnapshot'] = True
             else:
                 params['FinalDBSnapshotIdentifier'] = snapshot_identifier(
@@ -70,11 +81,16 @@ def process(self, resources):
             try:
                 client.delete_db_cluster(**params)
             except ClientError as e:
-                if e.response['Error']['Code'] in ['InvalidDBClusterStateFault']:
+                if e.response['Error']['Code'] == 'InvalidDBClusterStateFault':
+                    self.log.info(
+                        'RDS cluster in invalid state: %s',
+                        cluster['DBClusterIdentifier'])
                     continue
                 raise
 
-            self.log.info('Deleted RDS cluster: %s' % cluster['DBClusterIdentifier'])
+            self.log.info(
+                'Deleted RDS cluster: %s',
+                cluster['DBClusterIdentifier'])
 
 
 @actions.register('retention')
@@ -87,36 +103,36 @@ class RetentionWindow(BaseAction):
         'retention',
         **{'days': {'type': 'number'}})
 
-    def process(self, resources):
+    def process(self, clusters):
         with self.executor_factory(max_workers=2) as w:
             futures = []
-            for resource in resources:
+            for cluster in clusters:
                 futures.append(w.submit(
                     self.process_snapshot_retention,
-                    resource))
+                    cluster))
                 for f in as_completed(futures):
                     if f.exception():
                         self.log.error(
-                            "Exception setting RDS cluster retention  \n %s" % (
-                                f.exception()))
+                            "Exception setting RDS cluster retention  \n %s",
+                            f.exception())
 
-    def process_snapshot_retention(self, resource):
-        current_retention = int(resource.get('BackupRetentionPeriod', 0))
+    def process_snapshot_retention(self, cluster):
+        current_retention = int(cluster.get('BackupRetentionPeriod', 0))
         new_retention = self.data['days']
 
         if current_retention < new_retention:
             self.set_retention_window(
-                resource,
+                cluster,
                 max(current_retention, new_retention))
-            return resource
+            return cluster
 
-    def set_retention_window(self, resource, retention):
+    def set_retention_window(self, cluster, retention):
         c = local_session(self.manager.session_factory).client('rds')
         c.modify_db_cluster(
-            DBClusterIdentifier=resource['DBClusterIdentifier'],
+            DBClusterIdentifier=cluster['DBClusterIdentifier'],
             BackupRetentionPeriod=retention,
-            PreferredBackupWindow=resource['PreferredBackupWindow'],
-            PreferredMaintenanceWindow=resource['PreferredMaintenanceWindow'])
+            PreferredBackupWindow=cluster['PreferredBackupWindow'],
+            PreferredMaintenanceWindow=cluster['PreferredMaintenanceWindow'])
 
 
 @actions.register('snapshot')
@@ -124,27 +140,27 @@ class Snapshot(BaseAction):
 
     schema = type_schema('snapshot')
 
-    def process(self, resources):
+    def process(self, clusters):
         with self.executor_factory(max_workers=3) as w:
             futures = []
-            for resource in resources:
+            for cluster in clusters:
                 futures.append(w.submit(
                     self.process_cluster_snapshot,
-                    resource))
+                    cluster))
                 for f in as_completed(futures):
                     if f.exception():
                         self.log.error(
-                            "Exception creating RDS cluster snapshot  \n %s" % (
-                                f.exception()))
-        return resources
+                            "Exception creating RDS cluster snapshot  \n %s",
+                            f.exception())
+        return clusters
 
-    def process_cluster_snapshot(self, resource):
+    def process_cluster_snapshot(self, cluster):
         c = local_session(self.manager.session_factory).client('rds')
         c.create_db_cluster_snapshot(
             DBClusterSnapshotIdentifier=snapshot_identifier(
                 'Backup',
-                resource['DBClusterIdentifier']),
-            DBClusterIdentifier=resource['DBClusterIdentifier'])
+                cluster['DBClusterIdentifier']),
+            DBClusterIdentifier=cluster['DBClusterIdentifier'])
 
 
 @resources.register('rds-cluster-snapshot')
@@ -156,7 +172,8 @@ class Meta(object):
 
         service = 'rds'
         type = 'rds-cluster-snapshot'
-        enum_spec = ('describe_db_cluster_snapshots', 'DBClusterSnapshots', None)
+        enum_spec = (
+            'describe_db_cluster_snapshots', 'DBClusterSnapshots', None)
         name = id = 'DBClusterSnapshotIdentifier'
         filter_name = None
         filter_type = None
@@ -192,15 +209,12 @@ def process(self, snapshots):
                 for f in as_completed(futures):
                     if f.exception():
                         self.log.error(
-                            "Exception deleting snapshot set \n %s" % (
-                                f.exception()))
+                            "Exception deleting snapshot set \n %s",
+                            f.exception())
         return snapshots
 
     def process_snapshot_set(self, snapshots_set):
         c = local_session(self.manager.session_factory).client('rds')
         for s in snapshots_set:
-            try:
-                c.delete_db_cluster_snapshot(
-                    DBClusterSnapshotIdentifier=s['DBClusterSnapshotIdentifier'])
-            except ClientError as e:
-                raise
+            c.delete_db_cluster_snapshot(
+                DBClusterSnapshotIdentifier=s['DBClusterSnapshotIdentifier'])


Test patch:
diff --git a/tests/data/placebo/test_rdscluster_delete_with_instances/rds.DeleteDBCluster_1.json b/tests/data/placebo/test_rdscluster_delete_with_instances/rds.DeleteDBCluster_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_rdscluster_delete_with_instances/rds.DeleteDBCluster_1.json
@@ -0,0 +1,67 @@
+{
+    "status_code": 200,
+    "data": {
+        "DBCluster": {
+            "MasterUsername": "u1",
+            "VpcSecurityGroups": [
+                {
+                    "Status": "active",
+                    "VpcSecurityGroupId": "sg-1"
+                }
+            ],
+            "Status": "deleting",
+            "LatestRestorableTime": {
+                "hour": 14,
+                "__class__": "datetime",
+                "month": 7,
+                "second": 47,
+                "microsecond": 674000,
+                "year": 2016,
+                "day": 27,
+                "minute": 37
+            },
+            "PreferredBackupWindow": "08:28-08:58",
+            "DBSubnetGroup": "dbsubnetgroup1",
+            "AllocatedStorage": 1,
+            "BackupRetentionPeriod": 1,
+            "PreferredMaintenanceWindow": "wed:04:07-wed:04:37",
+            "Engine": "aurora",
+            "Endpoint": "cccc.cluster-abc.us-east-1.rds.amazonaws.com",
+            "EarliestRestorableTime": {
+                "hour": 14,
+                "__class__": "datetime",
+                "month": 7,
+                "second": 47,
+                "microsecond": 674000,
+                "year": 2016,
+                "day": 27,
+                "minute": 37
+            },
+            "EngineVersion": "5.6.10a",
+            "DBClusterIdentifier": "cccc",
+            "DbClusterResourceId": "cluster-ABCD",
+            "DBClusterMembers": [
+                {
+                    "IsClusterWriter": true,
+                    "DBClusterParameterGroupStatus": "in-sync",
+                    "PromotionTier": 0,
+                    "DBInstanceIdentifier": "test-cluster-1"
+                }
+            ],
+            "KmsKeyId": "arn:aws:kms:us-east-1:123:key/k1",
+            "StorageEncrypted": true,
+            "DatabaseName": "mydb",
+            "DBClusterParameterGroup": "default.aurora5.6",
+            "AvailabilityZones": [
+                "us-east-1a",
+                "us-east-1b",
+                "us-east-1e"
+            ],
+            "Port": 3306
+        },
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200,
+            "RequestId": "86256d71-5408-11e6-9e2b-195660273320"
+        }
+    }
+}
diff --git a/tests/data/placebo/test_rdscluster_delete_with_instances/rds.DeleteDBInstance_1.json b/tests/data/placebo/test_rdscluster_delete_with_instances/rds.DeleteDBInstance_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_rdscluster_delete_with_instances/rds.DeleteDBInstance_1.json
@@ -0,0 +1,108 @@
+{
+    "status_code": 200, 
+    "data": {
+        "DBInstance": {
+            "PubliclyAccessible": true, 
+            "MasterUsername": "superduper", 
+            "MonitoringInterval": 0, 
+            "LicenseModel": "postgresql-license", 
+            "VpcSecurityGroups": [
+                {
+                    "Status": "active", 
+                    "VpcSecurityGroupId": "sg-0a08e365"
+                }
+            ], 
+            "InstanceCreateTime": {
+                "hour": 0, 
+                "__class__": "datetime", 
+                "month": 5, 
+                "second": 10, 
+                "microsecond": 155000, 
+                "year": 2016, 
+                "day": 8, 
+                "minute": 22
+            }, 
+            "CopyTagsToSnapshot": false, 
+            "OptionGroupMemberships": [
+                {
+                    "Status": "in-sync", 
+                    "OptionGroupName": "default:postgres-9-4"
+                }
+            ], 
+            "PendingModifiedValues": {}, 
+            "Engine": "postgres", 
+            "MultiAZ": false, 
+            "LatestRestorableTime": {
+                "hour": 0, 
+                "__class__": "datetime", 
+                "month": 5, 
+                "second": 51, 
+                "microsecond": 0, 
+                "year": 2016, 
+                "day": 8, 
+                "minute": 44
+            }, 
+            "DBSecurityGroups": [], 
+            "DBParameterGroups": [
+                {
+                    "DBParameterGroupName": "default.postgres9.4", 
+                    "ParameterApplyStatus": "in-sync"
+                }
+            ], 
+            "AutoMinorVersionUpgrade": true, 
+            "PreferredBackupWindow": "06:38-07:08", 
+            "DBSubnetGroup": {
+                "Subnets": [
+                    {
+                        "SubnetStatus": "Active", 
+                        "SubnetIdentifier": "subnet-389e3d53", 
+                        "SubnetAvailabilityZone": {
+                            "Name": "us-west-2c"
+                        }
+                    }, 
+                    {
+                        "SubnetStatus": "Active", 
+                        "SubnetIdentifier": "subnet-3e9e3d55", 
+                        "SubnetAvailabilityZone": {
+                            "Name": "us-west-2a"
+                        }
+                    }, 
+                    {
+                        "SubnetStatus": "Active", 
+                        "SubnetIdentifier": "subnet-3f9e3d54", 
+                        "SubnetAvailabilityZone": {
+                            "Name": "us-west-2b"
+                        }
+                    }
+                ], 
+                "DBSubnetGroupName": "default", 
+                "VpcId": "vpc-399e3d52", 
+                "DBSubnetGroupDescription": "default", 
+                "SubnetGroupStatus": "Complete"
+            }, 
+            "ReadReplicaDBInstanceIdentifiers": [], 
+            "AllocatedStorage": 5, 
+            "BackupRetentionPeriod": 21, 
+            "PreferredMaintenanceWindow": "mon:09:22-mon:09:52", 
+            "Endpoint": {
+                "Port": 5432, 
+                "Address": "oneness.cj9uic5xuiu0.us-west-2.rds.amazonaws.com"
+            }, 
+            "DBInstanceStatus": "deleting", 
+            "EngineVersion": "9.4.7", 
+            "AvailabilityZone": "us-west-2a", 
+            "DomainMemberships": [], 
+            "StorageType": "gp2", 
+            "DbiResourceId": "db-BHKFSGDR5S2DSUNBF465JT2VRI", 
+            "CACertificateIdentifier": "rds-ca-2015", 
+            "StorageEncrypted": false, 
+            "DBInstanceClass": "db.m3.medium", 
+            "DbInstancePort": 0, 
+            "DBInstanceIdentifier": "oneness"
+        }, 
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RequestId": "cdbbf7c5-14b6-11e6-b743-37a6d3ed4c4c"
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_rdscluster_delete_with_instances/rds.DescribeDBClusters_1.json b/tests/data/placebo/test_rdscluster_delete_with_instances/rds.DescribeDBClusters_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_rdscluster_delete_with_instances/rds.DescribeDBClusters_1.json
@@ -0,0 +1,124 @@
+{
+    "status_code": 200,
+    "data": {
+        "DBClusters": [
+            {
+                "Status": "available",
+                "Engine": "aurora",
+                "Endpoint": "bbb.cluster-xyz.us-east-1.rds.amazonaws.com",
+                "KmsKeyId": "arn:aws:kms:us-east-1:123:key/k1",
+                "DBClusterIdentifier": "bbb",
+                "MasterUsername": "user1",
+                "EarliestRestorableTime": {
+                    "hour": 8,
+                    "__class__": "datetime",
+                    "month": 7,
+                    "second": 23,
+                    "microsecond": 306000,
+                    "year": 2016,
+                    "day": 12,
+                    "minute": 1
+                },
+                "DbClusterResourceId": "cluster-ABCDEF",
+                "DBClusterMembers": [
+                    {
+                        "IsClusterWriter": true,
+                        "DBClusterParameterGroupStatus": "in-sync",
+                        "PromotionTier": 1,
+                        "DBInstanceIdentifier": "bbb-poolmember-1"
+                    },
+                    {
+                        "IsClusterWriter": false,
+                        "DBClusterParameterGroupStatus": "in-sync",
+                        "PromotionTier": 1,
+                        "DBInstanceIdentifier": "bbb-rds-poolmember-2"
+                    }
+                ],
+                "Port": 3306,
+                "PreferredBackupWindow": "08:00-09:00",
+                "VpcSecurityGroups": [
+                    {
+                        "Status": "active",
+                        "VpcSecurityGroupId": "sg-1"
+                    }
+                ],
+                "DBSubnetGroup": "default-vpc-1",
+                "StorageEncrypted": true,
+                "AllocatedStorage": 1,
+                "EngineVersion": "5.6.10a",
+                "DBClusterParameterGroup": "default.aurora5.6",
+                "BackupRetentionPeriod": 14,
+                "AvailabilityZones": [
+                    "us-east-1a",
+                    "us-east-1b",
+                    "us-east-1c"
+                ],
+                "LatestRestorableTime": {
+                    "hour": 0,
+                    "__class__": "datetime",
+                    "month": 7,
+                    "second": 7,
+                    "microsecond": 80000,
+                    "year": 2016,
+                    "day": 27,
+                    "minute": 59
+                },
+                "PreferredMaintenanceWindow": "sun:07:00-sun:08:00"
+            },
+            {
+                "Status": "available",
+                "Engine": "aurora",
+                "Endpoint": "aaa.cluster-xyz.us-east-1.rds.amazonaws.com",
+                "KmsKeyId": "arn:aws:kms:us-east-1:123:key/k1",
+                "DBClusterIdentifier": "aaa",
+                "MasterUsername": "user2",
+                "EarliestRestorableTime": {
+                    "hour": 15,
+                    "__class__": "datetime",
+                    "month": 2,
+                    "second": 31,
+                    "microsecond": 154000,
+                    "year": 2016,
+                    "day": 25,
+                    "minute": 44
+                },
+                "DbClusterResourceId": "cluster-ZYXWVU",
+                "DBClusterMembers": [],
+                "Port": 3306,
+                "PreferredBackupWindow": "08:00-09:00",
+                "VpcSecurityGroups": [
+                    {
+                        "Status": "active",
+                        "VpcSecurityGroupId": "sg-2"
+                    }
+                ],
+                "DBSubnetGroup": "default-vpc-2", 
+                "StorageEncrypted": true,
+                "AllocatedStorage": 1,
+                "EngineVersion": "5.6.10a",
+                "DBClusterParameterGroup": "default.aurora5.6",
+                "BackupRetentionPeriod": 14,
+                "AvailabilityZones": [
+                    "us-east-1a",
+                    "us-east-1b",
+                    "us-east-1c"
+                ],
+                "LatestRestorableTime": {
+                    "hour": 15,
+                    "__class__": "datetime",
+                    "month": 2,
+                    "second": 31,
+                    "microsecond": 154000,
+                    "year": 2016,
+                    "day": 25,
+                    "minute": 44
+                },
+                "PreferredMaintenanceWindow": "sun:07:00-sun:08:00"
+            }
+        ],
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200,
+            "RequestId": "ca5ef80a-5396-11e6-b9e0-ddf0c53d86bf"
+        }
+    }
+}
diff --git a/tests/test_rdscluster.py b/tests/test_rdscluster.py
--- a/tests/test_rdscluster.py
+++ b/tests/test_rdscluster.py
@@ -50,7 +50,25 @@ def test_rdscluster_delete(self):
                 {'type': 'value',
                  'key': 'DBClusterIdentifier',
                  'value': 'bbb'}],
-            'actions': ['delete']},
+            'actions': [
+                {'type': 'delete',
+                 'delete-instances': False}]},
+            session_factory=session_factory)
+        resources = p.run()
+        self.assertEqual(len(resources), 1)
+
+    def test_rdscluster_delete_with_instances(self):
+        session_factory = self.replay_flight_data('test_rdscluster_delete_with_instances')
+        p = self.load_policy({
+            'name': 'rdscluster-delete',
+            'resource': 'rds-cluster',
+            'filters': [
+                {'type': 'value',
+                 'key': 'DBClusterIdentifier',
+                 'value': 'bbb'}],
+            'actions': [
+                {'type': 'delete',
+                 'delete-instances': True}]},
             session_factory=session_factory)
         resources = p.run()
         self.assertEqual(len(resources), 1)