Repository: cloud-custodian/cloud-custodian

Issue:
rds retention errors
2016-06-07 19:26:05,278: custodian.policy:INFO policy: rds-retention-window resource:rds has count:27 time:6.50
2016-06-07 19:26:06,784: custodian.actions:ERROR Exception setting rds retention  
 An error occurred (InvalidDBInstanceState) when calling the ModifyDBInstance operation: DB Backups not supported on a read replica for engine postgres



Discussion:
2016-06-07 19:42:16,678: custodian.actions:ERROR Exception setting rds retention  
 An error occurred (InvalidDBInstanceState) when calling the ModifyDBInstance operation: Database instance is not in available state.

re ebs snapshots, this one can safely be ignored, but its polluting some of the logs, potentially move this to a warning and explicitly check error code
2016-06-07 19:37:03,291: custodian.actions:ERROR Exception creating snapshot set 
 An error occurred (IncorrectState) when calling the CreateSnapshot operation: The volume 'vol-771e4bd2' is 'retired'

 An error occurred (InvalidParameterCombination) when calling the ModifyDBInstance operation: The specified DB Instance is a member of a cluster and its backup retention should not be modified directly. Instead, modify the backup retention of the cluster using the ModifyDbCluster API

I'll take this one as we may be seeing some of these errors in our accounts 😃 .


Resolving patch:
diff --git a/c7n/resources/ec2.py b/c7n/resources/ec2.py
--- a/c7n/resources/ec2.py
+++ b/c7n/resources/ec2.py
@@ -381,16 +381,26 @@ def process_volume_set(self, resource):
         for block_device in resource['BlockDeviceMappings']:
             if 'Ebs' not in block_device:
                 continue
+            volume_id = block_device['Ebs']['VolumeId']
             description = "Automated,Backup,%s,%s" % (
                 resource['InstanceId'],
-                block_device['Ebs']['VolumeId'])
-            response = c.create_snapshot(
-                DryRun=self.manager.config.dryrun,
-                VolumeId=block_device['Ebs']['VolumeId'],
-                Description=description)
+                volume_id)
+            try:
+                response = c.create_snapshot(
+                    DryRun=self.manager.config.dryrun,
+                    VolumeId=volume_id,
+                    Description=description)
+            except ClientError as e:
+                if e.response['Error']['Code'] == 'IncorrectState':
+                    log.warning(
+                        "action:%s volume:%s is incorrect state" % (
+                            self.__class__.__name__.lower(),
+                            volume_id))
+                    continue
+                raise
 
             tags = [
-                {'Key': 'Name', 'Value': block_device['Ebs']['VolumeId']},
+                {'Key': 'Name', 'Value': volume_id},
                 {'Key': 'InstanceId', 'Value': resource['InstanceId']},
                 {'Key': 'DeviceName', 'Value': block_device['DeviceName']}
             ]
@@ -406,7 +416,7 @@ def process_volume_set(self, resource):
                 log.warning(
                     "action:%s volume:%s too many tags to copy" % (
                         self.__class__.__name__.lower(),
-                        block_device['Ebs']['VolumeId']))
+                        volume_id))
                 copy_tags = []
 
             tags.extend(copy_tags)
diff --git a/c7n/resources/rds.py b/c7n/resources/rds.py
--- a/c7n/resources/rds.py
+++ b/c7n/resources/rds.py
@@ -60,6 +60,7 @@
 
 """
 import logging
+import re
 
 from botocore.exceptions import ClientError
 from concurrent.futures import as_completed
@@ -290,7 +291,8 @@ def process(self, resources):
 
     def process_snapshot_retention(self, resource):
         v = int(resource.get('BackupRetentionPeriod', 0))
-        if v == 0 or v < self.data['days']:
+        if ((v == 0 or v < self.data['days']) and
+                self._db_instance_eligible_for_backup(resource)):
             self.set_retention_window(resource)
             return resource
 
@@ -300,6 +302,39 @@ def set_retention_window(self, resource):
             DBInstanceIdentifier=resource['DBInstanceIdentifier'],
             BackupRetentionPeriod=self.data['days'])
 
+    def _db_instance_eligible_for_backup(self, resource):
+        db_instance_id = resource['DBInstanceIdentifier']
+
+        # Database instance is not in available state
+        if resource.get('DBInstanceStatus', '') != 'available':
+            log.debug("DB instance %s is not in available state" %
+                (db_instance_id))
+            return False
+        # The specified DB Instance is a member of a cluster and its backup retention should not be modified directly.
+        #   Instead, modify the backup retention of the cluster using the ModifyDbCluster API
+        if resource.get('DBClusterIdentifier', ''):
+            log.debug("DB instance %s is a cluster member" %
+                (db_instance_id))
+            return False
+        # DB Backups not supported on a read replica for engine postgres
+        if (resource.get('ReadReplicaSourceDBInstanceIdentifier', '') and
+                resource.get('Engine', '') == 'postgres'):
+            log.debug("DB instance %s is a postgres read-replica" %
+                (db_instance_id))
+            return False
+        # DB Backups not supported on a read replica running a mysql version before 5.6.
+        if (resource.get('ReadReplicaSourceDBInstanceIdentifier', '') and
+                resource.get('Engine', '') == 'mysql'):
+            engine_version = resource.get('EngineVersion', '')
+            # Assume "<major>.<minor>.<whatever>"
+            match = re.match(r'(?P<major>\d+)\.(?P<minor>\d+)\..*', engine_version)
+            if (match and int(match.group('major')) < 5 or
+                (int(match.group('major')) == 5 and int(match.group('minor')) < 6)):
+                log.debug("DB instance %s is a version %s mysql read-replica" %
+                    (db_instance_id, engine_version))
+                return False
+        return True
+
 
 @resources.register('rds-snapshot')
 class RDSSnapshot(QueryResourceManager):


Test patch:
diff --git a/tests/data/placebo/test_rds_retention/rds.DescribeDBInstances_1.json b/tests/data/placebo/test_rds_retention/rds.DescribeDBInstances_1.json
--- a/tests/data/placebo/test_rds_retention/rds.DescribeDBInstances_1.json
+++ b/tests/data/placebo/test_rds_retention/rds.DescribeDBInstances_1.json
@@ -1,110 +1,617 @@
 {
-    "status_code": 200, 
+    "status_code": 200,
     "data": {
         "DBInstances": [
             {
-                "PubliclyAccessible": true, 
-                "MasterUsername": "superduper", 
-                "MonitoringInterval": 0, 
-                "LicenseModel": "postgresql-license", 
+                "PubliclyAccessible": true,
+                "MasterUsername": "superduper",
+                "MonitoringInterval": 0,
+                "LicenseModel": "postgresql-license",
                 "VpcSecurityGroups": [
                     {
-                        "Status": "active", 
+                        "Status": "active",
                         "VpcSecurityGroupId": "sg-0a08e365"
                     }
-                ], 
+                ],
                 "InstanceCreateTime": {
-                    "hour": 0, 
-                    "__class__": "datetime", 
-                    "month": 5, 
-                    "second": 10, 
-                    "microsecond": 155000, 
-                    "year": 2016, 
-                    "day": 8, 
+                    "hour": 0,
+                    "__class__": "datetime",
+                    "month": 5,
+                    "second": 10,
+                    "microsecond": 155000,
+                    "year": 2016,
+                    "day": 8,
                     "minute": 22
-                }, 
-                "CopyTagsToSnapshot": false, 
+                },
+                "CopyTagsToSnapshot": false,
                 "OptionGroupMemberships": [
                     {
-                        "Status": "in-sync", 
+                        "Status": "in-sync",
                         "OptionGroupName": "default:postgres-9-4"
                     }
-                ], 
-                "PendingModifiedValues": {}, 
-                "Engine": "postgres", 
-                "MultiAZ": false, 
+                ],
+                "PendingModifiedValues": {},
+                "Engine": "postgres",
+                "MultiAZ": false,
                 "LatestRestorableTime": {
-                    "hour": 0, 
-                    "__class__": "datetime", 
-                    "month": 5, 
-                    "second": 50, 
-                    "microsecond": 0, 
-                    "year": 2016, 
-                    "day": 8, 
+                    "hour": 0,
+                    "__class__": "datetime",
+                    "month": 5,
+                    "second": 50,
+                    "microsecond": 0,
+                    "year": 2016,
+                    "day": 8,
                     "minute": 39
-                }, 
-                "DBSecurityGroups": [], 
+                },
+                "DBSecurityGroups": [],
                 "DBParameterGroups": [
                     {
-                        "DBParameterGroupName": "default.postgres9.4", 
+                        "DBParameterGroupName": "default.postgres9.4",
                         "ParameterApplyStatus": "in-sync"
                     }
-                ], 
-                "AutoMinorVersionUpgrade": true, 
-                "PreferredBackupWindow": "06:38-07:08", 
+                ],
+                "AutoMinorVersionUpgrade": true,
+                "PreferredBackupWindow": "06:38-07:08",
                 "DBSubnetGroup": {
                     "Subnets": [
                         {
-                            "SubnetStatus": "Active", 
-                            "SubnetIdentifier": "subnet-389e3d53", 
+                            "SubnetStatus": "Active",
+                            "SubnetIdentifier": "subnet-389e3d53",
                             "SubnetAvailabilityZone": {
                                 "Name": "us-west-2c"
                             }
-                        }, 
+                        },
                         {
-                            "SubnetStatus": "Active", 
-                            "SubnetIdentifier": "subnet-3e9e3d55", 
+                            "SubnetStatus": "Active",
+                            "SubnetIdentifier": "subnet-3e9e3d55",
                             "SubnetAvailabilityZone": {
                                 "Name": "us-west-2a"
                             }
-                        }, 
+                        },
                         {
-                            "SubnetStatus": "Active", 
-                            "SubnetIdentifier": "subnet-3f9e3d54", 
+                            "SubnetStatus": "Active",
+                            "SubnetIdentifier": "subnet-3f9e3d54",
                             "SubnetAvailabilityZone": {
                                 "Name": "us-west-2b"
                             }
                         }
-                    ], 
-                    "DBSubnetGroupName": "default", 
-                    "VpcId": "vpc-399e3d52", 
-                    "DBSubnetGroupDescription": "default", 
+                    ],
+                    "DBSubnetGroupName": "default",
+                    "VpcId": "vpc-399e3d52",
+                    "DBSubnetGroupDescription": "default",
                     "SubnetGroupStatus": "Complete"
-                }, 
-                "ReadReplicaDBInstanceIdentifiers": [], 
-                "AllocatedStorage": 5, 
-                "BackupRetentionPeriod": 7, 
-                "PreferredMaintenanceWindow": "mon:09:22-mon:09:52", 
+                },
+                "ReadReplicaDBInstanceIdentifiers": [],
+                "AllocatedStorage": 5,
+                "BackupRetentionPeriod": 7,
+                "PreferredMaintenanceWindow": "mon:09:22-mon:09:52",
                 "Endpoint": {
-                    "Port": 5432, 
+                    "Port": 5432,
                     "Address": "oneness.cj9uic5xuiu0.us-west-2.rds.amazonaws.com"
-                }, 
-                "DBInstanceStatus": "available", 
-                "EngineVersion": "9.4.7", 
-                "AvailabilityZone": "us-west-2a", 
-                "DomainMemberships": [], 
-                "StorageType": "gp2", 
-                "DbiResourceId": "db-BHKFSGDR5S2DSUNBF465JT2VRI", 
-                "CACertificateIdentifier": "rds-ca-2015", 
-                "StorageEncrypted": false, 
-                "DBInstanceClass": "db.m3.medium", 
-                "DbInstancePort": 0, 
+                },
+                "DBInstanceStatus": "available",
+                "EngineVersion": "9.4.7",
+                "AvailabilityZone": "us-west-2a",
+                "DomainMemberships": [],
+                "StorageType": "gp2",
+                "DbiResourceId": "db-BHKFSGDR5S2DSUNBF465JT2VRI",
+                "CACertificateIdentifier": "rds-ca-2015",
+                "StorageEncrypted": false,
+                "DBInstanceClass": "db.m3.medium",
+                "DbInstancePort": 0,
                 "DBInstanceIdentifier": "oneness"
+            },
+
+            {
+                "PubliclyAccessible": false,
+                "MasterUsername": "dbadmin",
+                "MonitoringInterval": 0,
+                "LicenseModel": "general-public-license",
+                "VpcSecurityGroups": [
+                  {
+                      "Status": "active",
+                      "VpcSecurityGroupId": "sg-0a08e365"
+                  }
+                ],
+                "InstanceCreateTime": {
+                    "hour": 16,
+                    "__class__": "datetime",
+                    "month": 4,
+                    "second": 30,
+                    "microsecond": 589000,
+                    "year": 2016,
+                    "day": 27,
+                    "minute": 4
+                },
+                "CopyTagsToSnapshot": false,
+                "OptionGroupMemberships": [
+                    {
+                        "Status": "in-sync",
+                        "OptionGroupName": "default:mysql-5-6"
+                    }
+                ],
+                "PendingModifiedValues": {},
+                "Engine": "mysql",
+                "MultiAZ": true,
+                "LatestRestorableTime": {
+                    "hour": 1,
+                    "__class__": "datetime",
+                    "month": 6,
+                    "second": 0,
+                    "microsecond": 0,
+                    "year": 2016,
+                    "day": 14,
+                    "minute": 15
+                },
+                "DBSecurityGroups": [],
+                "DBParameterGroups": [
+                    {
+                        "DBParameterGroupName": "default.mysql5.6",
+                        "ParameterApplyStatus": "in-sync"
+                    }
+                ],
+                "AutoMinorVersionUpgrade": true,
+                "PreferredBackupWindow": "02:30-03:30",
+                "DBSubnetGroup": {
+                    "Subnets": [
+                        {
+                            "SubnetStatus": "Active",
+                            "SubnetIdentifier": "subnet-389e3d53",
+                            "SubnetAvailabilityZone": {
+                                "Name": "us-west-2c"
+                            }
+                        },
+                        {
+                            "SubnetStatus": "Active",
+                            "SubnetIdentifier": "subnet-3e9e3d55",
+                            "SubnetAvailabilityZone": {
+                                "Name": "us-west-2a"
+                            }
+                        },
+                        {
+                            "SubnetStatus": "Active",
+                            "SubnetIdentifier": "subnet-3f9e3d54",
+                            "SubnetAvailabilityZone": {
+                                "Name": "us-west-2b"
+                            }
+                        }
+                    ],
+                    "DBSubnetGroupName": "default",
+                    "VpcId": "vpc-399e3d52",
+                    "DBSubnetGroupDescription": "default",
+                    "SubnetGroupStatus": "Complete"
+                },
+                "SecondaryAvailabilityZone": "us-east-1b",
+                "ReadReplicaDBInstanceIdentifiers": [
+                    "sbbdevslave"
+                ],
+                "AllocatedStorage": 100,
+                "BackupRetentionPeriod": 20,
+                "DBName": "SBBDEV",
+                "PreferredMaintenanceWindow": "sun:06:00-sun:08:00",
+                "Endpoint": {
+                    "Port": 3306,
+                    "Address": "sbbdev.cj9uic5xuiu0.us-west-2.rds.amazonaws.com"
+                },
+                "DBInstanceStatus": "failed",
+                "EngineVersion": "5.6.23",
+                "AvailabilityZone": "us-east-1a",
+                "DomainMemberships": [],
+                "StorageType": "io1",
+                "DbiResourceId": "db-TTKJKDV5QDQ3GBGKQYLK5U57NI",
+                "CACertificateIdentifier": "rds-ca-2015",
+                "Iops": 1000,
+                "StorageEncrypted": true,
+                "DBInstanceClass": "db.m3.medium",
+                "DbInstancePort": 0,
+                "DBInstanceIdentifier": "sbbdev"
+            },
+
+            {
+                "PubliclyAccessible": false,
+                "MasterUsername": "gadmin",
+                "MonitoringInterval": 0,
+                "LicenseModel": "general-public-license",
+                "VpcSecurityGroups": [
+                  {
+                      "Status": "active",
+                      "VpcSecurityGroupId": "sg-0a08e365"
+                  }
+                ],
+                "InstanceCreateTime": {
+                    "hour": 19,
+                    "__class__": "datetime",
+                    "month": 2,
+                    "second": 39,
+                    "microsecond": 343000,
+                    "year": 2016,
+                    "day": 17,
+                    "minute": 7
+                },
+                "CopyTagsToSnapshot": false,
+                "OptionGroupMemberships": [
+                    {
+                        "Status": "in-sync",
+                        "OptionGroupName": "default:aurora-5-6"
+                    }
+                ],
+                "PendingModifiedValues": {},
+                "Engine": "aurora",
+                "MultiAZ": false,
+                "DBSecurityGroups": [],
+                "DBParameterGroups": [
+                    {
+                        "DBParameterGroupName": "default.aurora5.6",
+                        "ParameterApplyStatus": "in-sync"
+                    }
+                ],
+                "AutoMinorVersionUpgrade": false,
+                "PreferredBackupWindow": "08:00-09:00",
+                "PromotionTier": 1,
+                "DBSubnetGroup": {
+                    "Subnets": [
+                        {
+                            "SubnetStatus": "Active",
+                            "SubnetIdentifier": "subnet-389e3d53",
+                            "SubnetAvailabilityZone": {
+                                "Name": "us-west-2c"
+                            }
+                        },
+                        {
+                            "SubnetStatus": "Active",
+                            "SubnetIdentifier": "subnet-3e9e3d55",
+                            "SubnetAvailabilityZone": {
+                                "Name": "us-west-2a"
+                            }
+                        },
+                        {
+                            "SubnetStatus": "Active",
+                            "SubnetIdentifier": "subnet-3f9e3d54",
+                            "SubnetAvailabilityZone": {
+                                "Name": "us-west-2b"
+                            }
+                        }
+                    ],
+                    "DBSubnetGroupName": "default",
+                    "VpcId": "vpc-399e3d52",
+                    "DBSubnetGroupDescription": "default",
+                    "SubnetGroupStatus": "Complete"
+                },
+                "ReadReplicaDBInstanceIdentifiers": [],
+                "AllocatedStorage": 1,
+                "BackupRetentionPeriod": 14,
+                "PreferredMaintenanceWindow": "thu:05:01-thu:05:31",
+                "Endpoint": {
+                    "Port": 3306,
+                    "Address": "mobile-e-dev-ms-keshav-rds-poolmember-1.cj9uic5xuiu0.us-west-2.rds.amazonaws.com"
+                },
+                "DBInstanceStatus": "available",
+                "EngineVersion": "5.6.10a",
+                "AvailabilityZone": "us-east-1b",
+                "DomainMemberships": [],
+                "DBClusterIdentifier": "mobile-e-dev-ms-keshav-rds",
+                "StorageType": "aurora",
+                "DbiResourceId": "db-RCRPVEDVXXB5PA36Z7NBZN4KNU",
+                "CACertificateIdentifier": "rds-ca-2015",
+                "StorageEncrypted": true,
+                "DBInstanceClass": "db.r3.large",
+                "DbInstancePort": 0,
+                "DBInstanceIdentifier": "mobile-e-dev-ms-keshav-rds-poolmember-1"
+            },
+
+            {
+                "PubliclyAccessible": false,
+                "MasterUsername": "dbadmin",
+                "MonitoringInterval": 0,
+                "LicenseModel": "postgresql-license",
+                "VpcSecurityGroups": [
+                  {
+                      "Status": "active",
+                      "VpcSecurityGroupId": "sg-0a08e365"
+                  }
+                ],
+                "InstanceCreateTime": {
+                    "hour": 19,
+                    "__class__": "datetime",
+                    "month": 5,
+                    "second": 22,
+                    "microsecond": 52000,
+                    "year": 2016,
+                    "day": 20,
+                    "minute": 28
+                },
+                "CopyTagsToSnapshot": true,
+                "OptionGroupMemberships": [
+                    {
+                        "Status": "in-sync",
+                        "OptionGroupName": "default:postgres-9-4"
+                    }
+                ],
+                "PendingModifiedValues": {},
+                "Engine": "postgres",
+                "MultiAZ": false,
+                "DBSecurityGroups": [],
+                "DBParameterGroups": [
+                    {
+                        "DBParameterGroupName": "default.postgres9.4",
+                        "ParameterApplyStatus": "in-sync"
+                    }
+                ],
+                "ReadReplicaSourceDBInstanceIdentifier": "qapipfit",
+                "AutoMinorVersionUpgrade": true,
+                "PreferredBackupWindow": "02:30-03:30",
+                "DBSubnetGroup": {
+                    "Subnets": [
+                        {
+                            "SubnetStatus": "Active",
+                            "SubnetIdentifier": "subnet-389e3d53",
+                            "SubnetAvailabilityZone": {
+                                "Name": "us-west-2c"
+                            }
+                        },
+                        {
+                            "SubnetStatus": "Active",
+                            "SubnetIdentifier": "subnet-3e9e3d55",
+                            "SubnetAvailabilityZone": {
+                                "Name": "us-west-2a"
+                            }
+                        },
+                        {
+                            "SubnetStatus": "Active",
+                            "SubnetIdentifier": "subnet-3f9e3d54",
+                            "SubnetAvailabilityZone": {
+                                "Name": "us-west-2b"
+                            }
+                        }
+                    ],
+                    "DBSubnetGroupName": "default",
+                    "VpcId": "vpc-399e3d52",
+                    "DBSubnetGroupDescription": "default",
+                    "SubnetGroupStatus": "Complete"
+                },
+                "ReadReplicaDBInstanceIdentifiers": [],
+                "AllocatedStorage": 50,
+                "BackupRetentionPeriod": 0,
+                "DBName": "qapipfit",
+                "PreferredMaintenanceWindow": "sun:06:00-sun:08:00",
+                "Endpoint": {
+                    "Port": 5432,
+                    "Address": "qapipfitread.cj9uic5xuiu0.us-west-2.rds.amazonaws.com"
+                },
+                "DBInstanceStatus": "available",
+                "EngineVersion": "9.4.5",
+                "StatusInfos": [
+                    {
+                        "Status": "replicating",
+                        "StatusType": "read replication",
+                        "Normal": true
+                    }
+                ],
+                "AvailabilityZone": "us-east-1b",
+                "DomainMemberships": [],
+                "StorageType": "gp2",
+                "DbiResourceId": "db-34UUKY7U4AFJLJ2KUIVGBT77QQ",
+                "CACertificateIdentifier": "rds-ca-2015",
+                "StorageEncrypted": true,
+                "DBInstanceClass": "db.m3.medium",
+                "DbInstancePort": 0,
+                "DBInstanceIdentifier": "qapipfitread"
+            },
+
+            {
+                "PubliclyAccessible": false,
+                "MasterUsername": "dbadmin",
+                "MonitoringInterval": 0,
+                "LicenseModel": "general-public-license",
+                "VpcSecurityGroups": [
+                  {
+                      "Status": "active",
+                      "VpcSecurityGroupId": "sg-0a08e365"
+                  }
+                ],
+                "InstanceCreateTime": {
+                    "hour": 19,
+                    "__class__": "datetime",
+                    "month": 4,
+                    "second": 48,
+                    "microsecond": 580000,
+                    "year": 2016,
+                    "day": 29,
+                    "minute": 27
+                },
+                "CopyTagsToSnapshot": false,
+                "OptionGroupMemberships": [
+                    {
+                        "Status": "in-sync",
+                        "OptionGroupName": "default:mysql-5-6"
+                    }
+                ],
+                "PendingModifiedValues": {},
+                "Engine": "mysql",
+                "MultiAZ": false,
+                "LatestRestorableTime": {
+                    "hour": 1,
+                    "__class__": "datetime",
+                    "month": 6,
+                    "second": 0,
+                    "microsecond": 0,
+                    "year": 2016,
+                    "day": 14,
+                    "minute": 50
+                },
+                "DBSecurityGroups": [],
+                "DBParameterGroups": [
+                    {
+                        "DBParameterGroupName": "default.mysql5.6",
+                        "ParameterApplyStatus": "in-sync"
+                    }
+                ],
+                "ReadReplicaSourceDBInstanceIdentifier": "sbbdev",
+                "AutoMinorVersionUpgrade": true,
+                "PreferredBackupWindow": "02:30-03:30",
+                "DBSubnetGroup": {
+                    "Subnets": [
+                        {
+                            "SubnetStatus": "Active",
+                            "SubnetIdentifier": "subnet-389e3d53",
+                            "SubnetAvailabilityZone": {
+                                "Name": "us-west-2c"
+                            }
+                        },
+                        {
+                            "SubnetStatus": "Active",
+                            "SubnetIdentifier": "subnet-3e9e3d55",
+                            "SubnetAvailabilityZone": {
+                                "Name": "us-west-2a"
+                            }
+                        },
+                        {
+                            "SubnetStatus": "Active",
+                            "SubnetIdentifier": "subnet-3f9e3d54",
+                            "SubnetAvailabilityZone": {
+                                "Name": "us-west-2b"
+                            }
+                        }
+                    ],
+                    "DBSubnetGroupName": "default",
+                    "VpcId": "vpc-399e3d52",
+                    "DBSubnetGroupDescription": "default",
+                    "SubnetGroupStatus": "Complete"
+                },
+                "ReadReplicaDBInstanceIdentifiers": [],
+                "AllocatedStorage": 100,
+                "BackupRetentionPeriod": 20,
+                "DBName": "SBBDEV",
+                "PreferredMaintenanceWindow": "sun:06:00-sun:08:00",
+                "Endpoint": {
+                    "Port": 3306,
+                    "Address": "sbbdevslave.cj9uic5xuiu0.us-west-2.rds.amazonaws.com"
+                },
+                "DBInstanceStatus": "available",
+                "EngineVersion": "5.6.23",
+                "StatusInfos": [
+                    {
+                        "Status": "replicating",
+                        "StatusType": "read replication",
+                        "Normal": true
+                    }
+                ],
+                "AvailabilityZone": "us-east-1a",
+                "DomainMemberships": [],
+                "StorageType": "io1",
+                "DbiResourceId": "db-QNSLJKPJCV4VTCLODR4DJ6UIPE",
+                "CACertificateIdentifier": "rds-ca-2015",
+                "Iops": 1000,
+                "StorageEncrypted": true,
+                "DBInstanceClass": "db.m3.medium",
+                "DbInstancePort": 0,
+                "DBInstanceIdentifier": "sbbdevslave"
+            },
+
+            {
+                "PubliclyAccessible": false,
+                "MasterUsername": "dbadmin",
+                "MonitoringInterval": 0,
+                "LicenseModel": "general-public-license",
+                "VpcSecurityGroups": [
+                  {
+                      "Status": "active",
+                      "VpcSecurityGroupId": "sg-0a08e365"
+                  }
+                ],
+                "InstanceCreateTime": {
+                    "hour": 2,
+                    "__class__": "datetime",
+                    "month": 12,
+                    "second": 39,
+                    "microsecond": 673000,
+                    "year": 2015,
+                    "day": 22,
+                    "minute": 37
+                },
+                "CopyTagsToSnapshot": false,
+                "OptionGroupMemberships": [
+                    {
+                        "Status": "in-sync",
+                        "OptionGroupName": "default:mysql-5-1"
+                    }
+                ],
+                "PendingModifiedValues": {},
+                "Engine": "mysql",
+                "MultiAZ": false,
+                "DBSecurityGroups": [],
+                "DBParameterGroups": [
+                    {
+                        "DBParameterGroupName": "eapir3",
+                        "ParameterApplyStatus": "in-sync"
+                    }
+                ],
+                "ReadReplicaSourceDBInstanceIdentifier": "qpsoa",
+                "AutoMinorVersionUpgrade": false,
+                "PreferredBackupWindow": "04:00-04:30",
+                "DBSubnetGroup": {
+                    "Subnets": [
+                        {
+                            "SubnetStatus": "Active",
+                            "SubnetIdentifier": "subnet-389e3d53",
+                            "SubnetAvailabilityZone": {
+                                "Name": "us-west-2c"
+                            }
+                        },
+                        {
+                            "SubnetStatus": "Active",
+                            "SubnetIdentifier": "subnet-3e9e3d55",
+                            "SubnetAvailabilityZone": {
+                                "Name": "us-west-2a"
+                            }
+                        },
+                        {
+                            "SubnetStatus": "Active",
+                            "SubnetIdentifier": "subnet-3f9e3d54",
+                            "SubnetAvailabilityZone": {
+                                "Name": "us-west-2b"
+                            }
+                        }
+                    ],
+                    "DBSubnetGroupName": "default",
+                    "VpcId": "vpc-399e3d52",
+                    "DBSubnetGroupDescription": "default",
+                    "SubnetGroupStatus": "Complete"
+                },
+                "ReadReplicaDBInstanceIdentifiers": [],
+                "AllocatedStorage": 300,
+                "BackupRetentionPeriod": 1,
+                "DBName": "qpsoa",
+                "PreferredMaintenanceWindow": "sun:05:00-sun:05:30",
+                "Endpoint": {
+                    "Port": 3306,
+                    "Address": "qpsoaread.cj9uic5xuiu0.us-west-2.rds.amazonaws.com"
+                },
+                "DBInstanceStatus": "available",
+                "EngineVersion": "5.1.73b",
+                "StatusInfos": [
+                    {
+                        "Status": "replicating",
+                        "StatusType": "read replication",
+                        "Normal": true
+                    }
+                ],
+                "AvailabilityZone": "us-east-1a",
+                "DomainMemberships": [],
+                "StorageType": "io1",
+                "DbiResourceId": "db-3DWNDRYBV7UOBORHWZX775SC6I",
+                "CACertificateIdentifier": "rds-ca-2015",
+                "Iops": 3000,
+                "StorageEncrypted": true,
+                "DBInstanceClass": "db.m3.2xlarge",
+                "DbInstancePort": 0,
+                "DBInstanceIdentifier": "qpsoaread"
             }
-        ], 
+
+        ],
         "ResponseMetadata": {
-            "HTTPStatusCode": 200, 
+            "HTTPStatusCode": 200,
             "RequestId": "dc2cfad1-14b5-11e6-a841-3b6c1aba6927"
         }
     }
-}
\ No newline at end of file
+}
diff --git a/tests/test_rds.py b/tests/test_rds.py
--- a/tests/test_rds.py
+++ b/tests/test_rds.py
@@ -115,7 +115,7 @@ def test_rds_retention(self):
             config={'region': 'us-west-2'},
             session_factory=session_factory)
         resources = p.run()
-        self.assertEqual(len(resources), 1)
+        self.assertEqual(len(resources), 6)
 
     def test_rds_delete(self):
         session_factory = self.replay_flight_data('test_rds_delete')