Repository: cloud-custodian/cloud-custodian

Issue:
RDS DB Instance deletion error: "ClientError: An error occurred (InvalidParameterCombination) when calling the DeleteDBInstance operation: FinalDBSnapshotIdentifier can not be specified when deleting a replica instance"
```
2016-09-05 10:10:18,375 - custodian.output - ERROR - Error while executing policy
Traceback (most recent call last):
  File "/usr/local/custodian/lib/python2.7/site-packages/c7n/policy.py", line 190, in run
    results = a.process(resources)
  File "/usr/local/custodian/lib/python2.7/site-packages/c7n/resources/rds.py", line 312, in process
    client.delete_db_instance(**params)
  File "/usr/local/custodian/lib/python2.7/site-packages/botocore/client.py", line 159, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/usr/local/custodian/lib/python2.7/site-packages/botocore/client.py", line 494, in _make_api_call
    raise ClientError(parsed_response, operation_name)
ClientError: An error occurred (InvalidParameterCombination) when calling the DeleteDBInstance operation: FinalDBSnapshotIdentifier can not be specified when deleting a replica instance
```



Discussion:

Resolving patch:
diff --git a/c7n/resources/rds.py b/c7n/resources/rds.py
--- a/c7n/resources/rds.py
+++ b/c7n/resources/rds.py
@@ -50,7 +50,7 @@
 from concurrent.futures import as_completed
 
 from c7n.actions import ActionRegistry, BaseAction, AutoTagUser
-from c7n.filters import FilterRegistry, Filter, AgeFilter, OPERATORS, ValueFilter
+from c7n.filters import FilterRegistry, Filter, AgeFilter, OPERATORS
 from c7n.manager import resources
 from c7n.query import QueryResourceManager
 from c7n import tags
@@ -102,27 +102,27 @@ def generate_arn(self):
                 account_id=self.account_id, resource_type='db', separator=':')
         return self._generate_arn
 
-    def augment(self, resources):
+    def augment(self, dbs):
         filter(None, _rds_tags(
             self.get_model(),
-            resources, self.session_factory, self.executor_factory,
+            dbs, self.session_factory, self.executor_factory,
             self.generate_arn, self.retry))
-        return resources
+        return dbs
 
 
 def _rds_tags(
-        model, dbs, session_factory, executor_factory, generate_arn, retry):
+        model, dbs, session_factory, executor_factory, generator, retry):
     """Augment rds instances with their respective tags."""
 
     def process_tags(db):
         client = local_session(session_factory).client('rds')
-        arn = generate_arn(db[model.id])
+        arn = generator(db[model.id])
         tag_list = None
         try:
             tag_list = retry(client.list_tags_for_resource, ResourceName=arn)['TagList']
         except ClientError as e:
             if e.response['Error']['Code'] not in ['DBInstanceNotFound']:
-                log.warning("Exception getting rds tags  \n %s" % (e))
+                log.warning("Exception getting rds tags  \n %s", e)
             return None
         db['Tags'] = tag_list or []
         return db
@@ -131,40 +131,71 @@ def process_tags(db):
     with executor_factory(max_workers=1) as w:
         return list(w.map(process_tags, dbs))
 
+
 def _db_instance_eligible_for_backup(resource):
     db_instance_id = resource['DBInstanceIdentifier']
 
     # Database instance is not in available state
     if resource.get('DBInstanceStatus', '') != 'available':
-        log.debug("DB instance %s is not in available state" %
-            (db_instance_id))
+        log.debug(
+            "DB instance %s is not in available state",
+            db_instance_id)
         return False
     # The specified DB Instance is a member of a cluster and its backup retention should not be modified directly.
     #   Instead, modify the backup retention of the cluster using the ModifyDbCluster API
     if resource.get('DBClusterIdentifier', ''):
-        log.debug("DB instance %s is a cluster member" %
-            (db_instance_id))
+        log.debug(
+            "DB instance %s is a cluster member",
+            db_instance_id)
         return False
     # DB Backups not supported on a read replica for engine postgres
     if (resource.get('ReadReplicaSourceDBInstanceIdentifier', '') and
             resource.get('Engine', '') == 'postgres'):
-        log.debug("DB instance %s is a postgres read-replica" %
-            (db_instance_id))
+        log.debug(
+            "DB instance %s is a postgres read-replica",
+            db_instance_id)
         return False
-    # DB Backups not supported on a read replica running a mysql version before 5.6.
+    # DB Backups not supported on a read replica running a mysql version before 5.6
     if (resource.get('ReadReplicaSourceDBInstanceIdentifier', '') and
             resource.get('Engine', '') == 'mysql'):
         engine_version = resource.get('EngineVersion', '')
         # Assume "<major>.<minor>.<whatever>"
         match = re.match(r'(?P<major>\d+)\.(?P<minor>\d+)\..*', engine_version)
         if (match and int(match.group('major')) < 5 or
-            (int(match.group('major')) == 5 and int(match.group('minor')) < 6)):
-            log.debug("DB instance %s is a version %s mysql read-replica" %
-                (db_instance_id, engine_version))
+                (int(match.group('major')) == 5 and int(match.group('minor')) < 6)):
+            log.debug(
+                "DB instance %s is a version %s mysql read-replica",
+                db_instance_id,
+                engine_version)
             return False
     return True
 
 
+def _db_instance_eligible_for_final_snapshot(resource):
+    db_instance_id = resource['DBInstanceIdentifier']
+    status = resource.get('DBInstanceStatus', '')
+
+    # If the DB instance you are deleting has a status of "Creating,"
+    # you will not be able to have a final DB snapshot taken
+    # If the DB instance is in a failure state with a status of "failed,"
+    # "incompatible-restore," or "incompatible-network," you can only delete
+    # the instance when the SkipFinalSnapshot parameter is set to "true."
+    if status in ['creating', 'failed',
+                  'incompatible-restore', 'incompatible-network']:
+        log.debug(
+            "DB instance %s is in invalid state",
+            db_instance_id)
+        return False
+
+    # FinalDBSnapshotIdentifier can not be specified when deleting a replica instance
+    if resource.get('ReadReplicaSourceDBInstanceIdentifier', ''):
+        log.debug(
+            "DB instance %s is a read-replica",
+            db_instance_id)
+        return False
+    return True
+
+
 @filters.register('default-vpc')
 class DefaultVpc(Filter):
     """ Matches if an rds database is in the default vpc
@@ -185,7 +216,7 @@ def __call__(self, rdb):
 
         if query_vpc:
             client = local_session(self.manager.session_factory).client('ec2')
-            self.log.debug("querying vpc %s" % vpc_id)
+            self.log.debug("querying vpc %s", vpc_id)
             vpcs = [v['VpcId'] for v
                     in client.describe_vpcs(VpcIds=[vpc_id])['Vpcs']
                     if v['IsDefault']]
@@ -198,8 +229,8 @@ def __call__(self, rdb):
 @filters.register('kms-alias')
 class KmsKeyAlias(ResourceKmsKeyAlias):
 
-    def process(self, resources, event=None):
-        return self.get_matching_aliases(resources)
+    def process(self, dbs, event=None):
+        return self.get_matching_aliases(dbs)
 
 
 @actions.register('mark-for-op')
@@ -211,15 +242,14 @@ class TagDelayedAction(tags.TagDelayedAction):
 
     batch_size = 5
 
-    def process(self, resources):
-        session = local_session(self.manager.session_factory)
-        return super(TagDelayedAction, self).process(resources)
+    def process(self, dbs):
+        return super(TagDelayedAction, self).process(dbs)
 
-    def process_resource_set(self, resources, tags):
+    def process_resource_set(self, dbs, ts):
         client = local_session(self.manager.session_factory).client('rds')
-        for r in resources:
-            arn = self.manager.generate_arn(r['DBInstanceIdentifier'])
-            client.add_tags_to_resource(ResourceName=arn, Tags=tags)
+        for db in dbs:
+            arn = self.manager.generate_arn(db['DBInstanceIdentifier'])
+            client.add_tags_to_resource(ResourceName=arn, Tags=ts)
 
 
 @actions.register('auto-patch')
@@ -229,7 +259,7 @@ class AutoPatch(BaseAction):
         'auto-patch',
         minor={'type': 'boolean'}, window={'type': 'string'})
 
-    def process(self, resources):
+    def process(self, dbs):
         client = local_session(
             self.manager.session_factory).client('rds')
 
@@ -237,9 +267,9 @@ def process(self, resources):
         if self.data.get('window'):
             params['PreferredMaintenanceWindow'] = self.data['minor']
 
-        for r in resources:
+        for db in dbs:
             client.modify_db_instance(
-                DBInstanceIdentifier=r['DBInstanceIdentifier'],
+                DBInstanceIdentifier=db['DBInstanceIdentifier'],
                 **params)
 
 
@@ -250,12 +280,12 @@ class Tag(tags.Tag):
     concurrency = 2
     batch_size = 5
 
-    def process_resource_set(self, resources, tags):
+    def process_resource_set(self, dbs, ts):
         client = local_session(
             self.manager.session_factory).client('rds')
-        for r in resources:
-            arn = self.manager.generate_arn(r['DBInstanceIdentifier'])
-            client.add_tags_to_resource(ResourceName=arn, Tags=tags)
+        for db in dbs:
+            arn = self.manager.generate_arn(db['DBInstanceIdentifier'])
+            client.add_tags_to_resource(ResourceName=arn, Tags=ts)
 
 
 @actions.register('remove-tag')
@@ -265,11 +295,11 @@ class RemoveTag(tags.RemoveTag):
     concurrency = 2
     batch_size = 5
 
-    def process_resource_set(self, resources, tag_keys):
+    def process_resource_set(self, dbs, tag_keys):
         client = local_session(
             self.manager.session_factory).client('rds')
-        for r in resources:
-            arn = self.manager.generate_arn(r['DBInstanceIdentifier'])
+        for db in dbs:
+            arn = self.manager.generate_arn(db['DBInstanceIdentifier'])
             client.remove_tags_from_resource(
                 ResourceName=arn, TagKeys=tag_keys)
 
@@ -295,19 +325,19 @@ class Delete(BaseAction):
             }
         }
 
-    def process(self, resources):
-        self.skip = self.data.get('skip-snapshot', False)
+    def process(self, dbs):
+        skip = self.data.get('skip-snapshot', False)
 
-        # Concurrency feels like over kill here.
+        # Concurrency feels like overkill here.
         client = local_session(self.manager.session_factory).client('rds')
-        for rdb in resources:
+        for db in dbs:
             params = dict(
-                DBInstanceIdentifier=rdb['DBInstanceIdentifier'])
-            if self.skip:
+                DBInstanceIdentifier=db['DBInstanceIdentifier'])
+            if skip or not _db_instance_eligible_for_final_snapshot(db):
                 params['SkipFinalSnapshot'] = True
             else:
                 params['FinalDBSnapshotIdentifier'] = snapshot_identifier(
-                    'Final', rdb['DBInstanceIdentifier'])
+                    'Final', db['DBInstanceIdentifier'])
             try:
                 client.delete_db_instance(**params)
             except ClientError as e:
@@ -315,7 +345,8 @@ def process(self, resources):
                     continue
                 raise
 
-            self.log.info("Deleted rds: %s" % rdb['DBInstanceIdentifier'])
+            self.log.info("Deleted rds: %s", db['DBInstanceIdentifier'])
+        return dbs
 
 
 @actions.register('snapshot')
@@ -325,19 +356,19 @@ class Snapshot(BaseAction):
         'type': {
             'enum': ['snapshot']}}}
 
-    def process(self, resources):
+    def process(self, dbs):
         with self.executor_factory(max_workers=3) as w:
             futures = []
-            for resource in resources:
+            for db in dbs:
                 futures.append(w.submit(
                     self.process_rds_snapshot,
-                    resource))
+                    db))
                 for f in as_completed(futures):
                     if f.exception():
                         self.log.error(
-                            "Exception creating rds snapshot  \n %s" % (
-                                f.exception()))
-        return resources
+                            "Exception creating rds snapshot  \n %s",
+                            f.exception())
+        return dbs
 
     def process_rds_snapshot(self, resource):
         if not _db_instance_eligible_for_backup(resource):
@@ -359,18 +390,19 @@ class RetentionWindow(BaseAction):
         'retention',
         **{'days': {'type': 'number'}, 'copy-tags': {'type': 'boolean'}})
 
-    def process(self, resources):
+    def process(self, dbs):
         with self.executor_factory(max_workers=3) as w:
             futures = []
-            for resource in resources:
+            for db in dbs:
                 futures.append(w.submit(
                     self.process_snapshot_retention,
-                    resource))
+                    db))
                 for f in as_completed(futures):
                     if f.exception():
                         self.log.error(
-                            "Exception setting rds retention  \n %s" % (
-                                f.exception()))
+                            "Exception setting rds retention  \n %s",
+                            f.exception())
+        return dbs
 
     def process_snapshot_retention(self, resource):
         current_retention = int(resource.get('BackupRetentionPeriod', 0))
@@ -379,7 +411,7 @@ def process_snapshot_retention(self, resource):
         new_copy_tags = self.data.get('copy-tags', True)
 
         if ((current_retention < new_retention or
-                current_copy_tags != new_copy_tags) and
+             current_copy_tags != new_copy_tags) and
                 _db_instance_eligible_for_backup(resource)):
             self.set_retention_window(
                 resource,
@@ -440,8 +472,8 @@ def process(self, snapshots):
                 for f in as_completed(futures):
                     if f.exception():
                         self.log.error(
-                            "Exception deleting snapshot set \n %s" % (
-                                f.exception()))
+                            "Exception deleting snapshot set \n %s",
+                            f.exception())
         return snapshots
 
     def process_snapshot_set(self, snapshots_set):
@@ -450,5 +482,5 @@ def process_snapshot_set(self, snapshots_set):
             try:
                 c.delete_db_snapshot(
                     DBSnapshotIdentifier=s['DBSnapshotIdentifier'])
-            except ClientError as e:
+            except ClientError:
                 raise


Test patch:
diff --git a/tests/test_rds.py b/tests/test_rds.py
--- a/tests/test_rds.py
+++ b/tests/test_rds.py
@@ -272,6 +272,60 @@ def test_rds_db_instance_eligible_for_backup(self):
         }
         self.assertTrue(rds._db_instance_eligible_for_backup(resource))
 
+    def test_rds_db_instance_eligible_for_final_snapshot(self):
+        resource = {
+            'DBInstanceIdentifier': 'ABC'
+        }
+        self.assertTrue(rds._db_instance_eligible_for_final_snapshot(resource))
+
+        resource = {
+            'DBInstanceIdentifier': 'ABC',
+            'DBInstanceStatus': 'available'
+        }
+        self.assertTrue(rds._db_instance_eligible_for_final_snapshot(resource))
+
+        resource = {
+            'DBInstanceIdentifier': 'ABC',
+            'DBInstanceStatus': 'creating'
+        }
+        self.assertFalse(rds._db_instance_eligible_for_final_snapshot(resource))
+
+        resource = {
+            'DBInstanceIdentifier': 'ABC',
+            'DBInstanceStatus': 'failed'
+        }
+        self.assertFalse(rds._db_instance_eligible_for_final_snapshot(resource))
+
+        resource = {
+            'DBInstanceIdentifier': 'ABC',
+            'DBInstanceStatus': 'incompatible-restore'
+        }
+        self.assertFalse(rds._db_instance_eligible_for_final_snapshot(resource))
+
+        resource = {
+            'DBInstanceIdentifier': 'ABC',
+            'DBInstanceStatus': 'incompatible-network'
+        }
+        self.assertFalse(rds._db_instance_eligible_for_final_snapshot(resource))
+
+        resource = {
+            'DBInstanceIdentifier': 'ABC',
+            'DBInstanceStatus': 'available',
+            'ReadReplicaSourceDBInstanceIdentifier': 'R1',
+            'Engine': 'mysql',
+            'EngineVersion': '5.7.1'
+        }
+        self.assertFalse(rds._db_instance_eligible_for_final_snapshot(resource))
+
+        resource = {
+            'DBInstanceIdentifier': 'ABC',
+            'DBInstanceStatus': 'available',
+            'ReadReplicaSourceDBInstanceIdentifier': '',
+            'Engine': 'mysql',
+            'EngineVersion': '5.7.1'
+        }
+        self.assertTrue(rds._db_instance_eligible_for_final_snapshot(resource))
+
 
 class RDSSnapshotTest(BaseTest):