Repository: cloud-custodian/cloud-custodian

Issue:
Ability to limit which processes are suspended on ASGs
When Custodian suspends an ASG it suspends all processes.  This can have negative repercussions.  For example, this will cause any repetitive scheduled actions to be deleted if they reach their scheduled time and the ScheduledActions process is disabled.  For ASGs that are being suspended for an offhours action we would like to be able to leave this process enabled.  More generally, it might be useful to be able to exclude one or more processes, e.g:
```
    actions:
      - type: suspend
        exclude:
          - AlarmNotifications
          - ScheduledActions
```
So for the default suspend action Custodian could still do the basic suspend_processes, but for exclusions it would need to pass which processes to suspend (the negation of this list).  If you wanted to put the work on the policy creator you could have them instead specify which processes to suspend (use include instead of exclude) then use that list directly.


Discussion:

Resolving patch:
diff --git a/c7n/resources/asg.py b/c7n/resources/asg.py
--- a/c7n/resources/asg.py
+++ b/c7n/resources/asg.py
@@ -1077,6 +1077,8 @@ def process(self, asgs):
 class Suspend(BaseAction):
     """Action to suspend ASG processes and instances
 
+    AWS ASG suspend/resume and process docs https://goo.gl/XYtKQ8
+
     :example:
 
         .. code-block: yaml
@@ -1090,7 +1092,24 @@ class Suspend(BaseAction):
                   - type: suspend
     """
 
-    schema = type_schema('suspend')
+    ASG_PROCESSES = [
+        "Launch",
+        "Terminate",
+        "HealthCheck",
+        "ReplaceUnhealthy",
+        "AZRebalance",
+        "AlarmNotification",
+        "ScheduledActions",
+        "AddToLoadBalancer"]
+
+    schema = type_schema(
+        'suspend',
+        exclude={
+            'type': 'array',
+            'title': 'ASG Processes to not suspend',
+            'items': {'enum': ASG_PROCESSES}})
+
+    ASG_PROCESSES = set(ASG_PROCESSES)
 
     def process(self, asgs):
         original_count = len(asgs)
@@ -1108,9 +1127,13 @@ def process_asg(self, asg):
         """
         session = local_session(self.manager.session_factory)
         asg_client = session.client('autoscaling')
+        processes = list(self.ASG_PROCESSES.difference(
+            self.data.get('exclude', ())))
+
         try:
             self.manager.retry(
                 asg_client.suspend_processes,
+                ScalingProcesses=processes,
                 AutoScalingGroupName=asg['AutoScalingGroupName'])
         except ClientError as e:
             if e.response['Error']['Code'] == 'ValidationError':
diff --git a/c7n/resources/s3.py b/c7n/resources/s3.py
--- a/c7n/resources/s3.py
+++ b/c7n/resources/s3.py
@@ -154,8 +154,8 @@ def assemble_bucket(item):
                 continue
             else:
                 log.warning(
-                    "Bucket:%s unable to invoke method:%s error:%s " % (
-                        b['Name'], m, e.response['Error']['Message']))
+                    "Bucket:%s unable to invoke method:%s error:%s ",
+                        b['Name'], m, e.response['Error']['Message'])
                 return None
         # As soon as we learn location (which generally works)
         if k == 'Location' and v is not None:
@@ -465,7 +465,9 @@ def process_bucket(self, bucket):
 
 @actions.register('toggle-versioning')
 class ToggleVersioning(BucketActionBase):
-    """Action to enable/disable versioning on a S3 bucket
+    """Action to enable/suspend versioning on a S3 bucket
+
+    Note versioning can never be disabled only suspended.
 
     :example:
 
@@ -477,7 +479,7 @@ class ToggleVersioning(BucketActionBase):
                 filter:
                   - type: value
                     key: Versioning
-                    value: Disabled
+                    value: Suspended
                 actions:
                   - type: toggle-versioning
                     enabled: true
@@ -509,6 +511,7 @@ def process(self, resources):
 @actions.register('toggle-logging')
 class ToggleLogging(BucketActionBase):
     """Action to enable/disable logging on a S3 bucket.
+
     Target bucket ACL must allow for WRITE and READ_ACP Permissions
     Not specifying a target_prefix will default to the current bucket name.
     http://goo.gl/PiWWU2
@@ -1466,6 +1469,10 @@ def process_delete_enablement(self, b):
         """
         client = local_session(self.manager.session_factory).client('s3')
 
+        # Stop replication so we can suspend versioning
+        if b.get('Replication') is not None:
+            client.delete_bucket_replication(Bucket=b['Name'])
+
         # Suspend versioning, so we don't get new delete markers
         # as we walk and delete versions
         if (self.get_bucket_style(b) == 'versioned'
@@ -1474,6 +1481,7 @@ def process_delete_enablement(self, b):
             client.put_bucket_versioning(
                 Bucket=b['Name'],
                 VersioningConfiguration={'Status': 'Suspended'})
+
         # Clear our multi-part uploads
         uploads = client.get_paginator('list_multipart_uploads')
         for p in uploads.paginate(Bucket=b['Name']):
@@ -1485,10 +1493,10 @@ def process_delete_enablement(self, b):
 
     def process(self, buckets):
         # might be worth sanity checking all our permissions
-        # on the bucket up front before disabling versioning.
-        with self.executor_factory(max_workers=3) as w:
-            list(w.map(self.process_delete_enablement, buckets))
+        # on the bucket up front before disabling versioning/replication.
         if self.data.get('remove-contents', True):
+            with self.executor_factory(max_workers=3) as w:
+                list(w.map(self.process_delete_enablement, buckets))
             self.empty_buckets(buckets)
         with self.executor_factory(max_workers=3) as w:
             results = w.map(self.delete_bucket, buckets)


Test patch:
diff --git a/tests/data/placebo/test_s3_delete_replicated_bucket/s3.DeleteBucketReplication_1.json b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.DeleteBucketReplication_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.DeleteBucketReplication_1.json
@@ -0,0 +1,17 @@
+{
+    "status_code": 204, 
+    "data": {
+        "ResponseMetadata": {
+            "HTTPStatusCode": 204, 
+            "RetryAttempts": 1, 
+            "HostId": "d59jvs7ZY+qY9czDkun2GML93eV6IeOQssHZKb3IvQtY/uNbakHI6SauJXRAVbWyokUfxYPUg6M=", 
+            "RequestId": "1F1DB1049F2F5877", 
+            "HTTPHeaders": {
+                "x-amz-id-2": "d59jvs7ZY+qY9czDkun2GML93eV6IeOQssHZKb3IvQtY/uNbakHI6SauJXRAVbWyokUfxYPUg6M=", 
+                "date": "Sat, 07 Jan 2017 20:17:19 GMT", 
+                "x-amz-request-id": "1F1DB1049F2F5877", 
+                "server": "AmazonS3"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_delete_replicated_bucket/s3.DeleteBucket_1.json b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.DeleteBucket_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.DeleteBucket_1.json
@@ -0,0 +1,17 @@
+{
+    "status_code": 204, 
+    "data": {
+        "ResponseMetadata": {
+            "HTTPStatusCode": 204, 
+            "RetryAttempts": 0, 
+            "HostId": "sSFCfCgMjgoNWJ48Q/qmGPp7OAa1rmu5plP8d3wIN/6mhQa7t0YRf5iVDAyzGmWXyDzQe4R8Ghc=", 
+            "RequestId": "EFA06ABDD66DA281", 
+            "HTTPHeaders": {
+                "x-amz-id-2": "sSFCfCgMjgoNWJ48Q/qmGPp7OAa1rmu5plP8d3wIN/6mhQa7t0YRf5iVDAyzGmWXyDzQe4R8Ghc=", 
+                "date": "Sat, 07 Jan 2017 20:17:21 GMT", 
+                "x-amz-request-id": "EFA06ABDD66DA281", 
+                "server": "AmazonS3"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_delete_replicated_bucket/s3.DeleteObjects_1.json b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.DeleteObjects_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.DeleteObjects_1.json
@@ -0,0 +1,46 @@
+{
+    "status_code": 200, 
+    "data": {
+        "Deleted": [
+            {
+                "VersionId": "oR8cvbTYzIpUi3qOPDvSIuLUxQGrqmbh", 
+                "Key": "Screen Shot 2017-01-04 at 8.13.36 PM.png"
+            }, 
+            {
+                "VersionId": ".PII_WPyK1JAkmhhT_mV2JmdrKoO7p9X", 
+                "Key": "Screen Shot 2017-01-04 at 8.13.42 PM.png"
+            }, 
+            {
+                "VersionId": "Pxh5frwAbmN9GSSibyt7_6xI5IPUuFEi", 
+                "Key": "Screen Shot 2017-01-04 at 8.13.36 PM.png"
+            }, 
+            {
+                "VersionId": "rjCBi3AfMJdXq.OFXCcHSyQ4yx6GalvG", 
+                "Key": "Screen Shot 2017-01-04 at 8.13.42 PM.png"
+            }, 
+            {
+                "VersionId": "y70jHWgNr.kdCIrzCeJGM7wxgkWcpLfh", 
+                "Key": "Screen Shot 2017-01-04 at 8.18.46 PM.png"
+            }, 
+            {
+                "VersionId": "X2OhbdfpBlhB0Cjb_0p0SL_U0a2ktkSG", 
+                "Key": "Screen Shot 2017-01-04 at 8.18.46 PM.png"
+            }
+        ], 
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RetryAttempts": 0, 
+            "HostId": "bUcWcwqdAgJ+2KyRunvJNym8QfKtt0O3Ik5YewzKPqCUwOsch7/GsYMNupQpAXFLWJ3s6iyOZ3Q=", 
+            "RequestId": "9CA7E558B1AFA628", 
+            "HTTPHeaders": {
+                "x-amz-id-2": "bUcWcwqdAgJ+2KyRunvJNym8QfKtt0O3Ik5YewzKPqCUwOsch7/GsYMNupQpAXFLWJ3s6iyOZ3Q=", 
+                "server": "AmazonS3", 
+                "transfer-encoding": "chunked", 
+                "connection": "close", 
+                "x-amz-request-id": "9CA7E558B1AFA628", 
+                "date": "Sat, 07 Jan 2017 20:17:20 GMT", 
+                "content-type": "application/xml"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketReplication_1.json b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketReplication_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketReplication_1.json
@@ -0,0 +1,24 @@
+{
+    "status_code": 404, 
+    "data": {
+        "ResponseMetadata": {
+            "HTTPStatusCode": 404, 
+            "RetryAttempts": 1, 
+            "HostId": "rIHS+8ojjF0UE9b8C8/K7XPyhUyyh323XqCCkWetpQ6qwr2qqfDUeQktZ0XhZlt8HrF6Vxto3wA=", 
+            "RequestId": "DB13744D52EC4028", 
+            "HTTPHeaders": {
+                "x-amz-id-2": "rIHS+8ojjF0UE9b8C8/K7XPyhUyyh323XqCCkWetpQ6qwr2qqfDUeQktZ0XhZlt8HrF6Vxto3wA=", 
+                "server": "AmazonS3", 
+                "transfer-encoding": "chunked", 
+                "x-amz-request-id": "DB13744D52EC4028", 
+                "date": "Sat, 07 Jan 2017 20:17:12 GMT", 
+                "content-type": "application/xml"
+            }
+        }, 
+        "Error": {
+            "Message": "The replication configuration was not found", 
+            "Code": "ReplicationConfigurationNotFoundError", 
+            "BucketName": "config-bucket-644160558196"
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketReplication_2.json b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketReplication_2.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketReplication_2.json
@@ -0,0 +1,31 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ReplicationConfiguration": {
+            "Rules": [
+                {
+                    "Status": "Enabled", 
+                    "Prefix": "", 
+                    "Destination": {
+                        "Bucket": "arn:aws:s3:::custodian-replicated-west"
+                    }, 
+                    "ID": "custodian-replicated"
+                }
+            ], 
+            "Role": "arn:aws:iam::644160558196:role/custodian-replicated-custodian-replicated-west-s3-repl-role"
+        }, 
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RetryAttempts": 1, 
+            "HostId": "8jCIudGdR5i7XZ4Qc7UlrBkHHoIhpCWYGqMex+XxpYgdmoph/Wnln1o2Cl+zT41G5bLDkzRFBLI=", 
+            "RequestId": "7F8DACCF490C62B0", 
+            "HTTPHeaders": {
+                "x-amz-id-2": "8jCIudGdR5i7XZ4Qc7UlrBkHHoIhpCWYGqMex+XxpYgdmoph/Wnln1o2Cl+zT41G5bLDkzRFBLI=", 
+                "date": "Sat, 07 Jan 2017 20:17:14 GMT", 
+                "transfer-encoding": "chunked", 
+                "x-amz-request-id": "7F8DACCF490C62B0", 
+                "server": "AmazonS3"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketReplication_3.json b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketReplication_3.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketReplication_3.json
@@ -0,0 +1,24 @@
+{
+    "status_code": 404, 
+    "data": {
+        "ResponseMetadata": {
+            "HTTPStatusCode": 404, 
+            "RetryAttempts": 0, 
+            "HostId": "gA8Q6vDfQVJld48/TEP9QqFf7RVIlVKj6IZRFyv9g4OSI9wNywvxfhmGP1kRQTTKymJEDq+rzKQ=", 
+            "RequestId": "25A5CFEE1D662FD9", 
+            "HTTPHeaders": {
+                "x-amz-id-2": "gA8Q6vDfQVJld48/TEP9QqFf7RVIlVKj6IZRFyv9g4OSI9wNywvxfhmGP1kRQTTKymJEDq+rzKQ=", 
+                "server": "AmazonS3", 
+                "transfer-encoding": "chunked", 
+                "x-amz-request-id": "25A5CFEE1D662FD9", 
+                "date": "Sat, 07 Jan 2017 20:17:14 GMT", 
+                "content-type": "application/xml"
+            }
+        }, 
+        "Error": {
+            "Message": "The replication configuration was not found", 
+            "Code": "ReplicationConfigurationNotFoundError", 
+            "BucketName": "custodian-replicated-west"
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketReplication_4.json b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketReplication_4.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketReplication_4.json
@@ -0,0 +1,24 @@
+{
+    "status_code": 404, 
+    "data": {
+        "ResponseMetadata": {
+            "HTTPStatusCode": 404, 
+            "RetryAttempts": 1, 
+            "HostId": "trFqZ9vNduHfeUpLZTbNeQZI4V8HAkk3NQfsYkoZIGE7PgA5RcY5Yh5Nreiv04L3qqrXZjqFQLM=", 
+            "RequestId": "E25399A76EC31107", 
+            "HTTPHeaders": {
+                "x-amz-id-2": "trFqZ9vNduHfeUpLZTbNeQZI4V8HAkk3NQfsYkoZIGE7PgA5RcY5Yh5Nreiv04L3qqrXZjqFQLM=", 
+                "server": "AmazonS3", 
+                "transfer-encoding": "chunked", 
+                "x-amz-request-id": "E25399A76EC31107", 
+                "date": "Sat, 07 Jan 2017 20:17:15 GMT", 
+                "content-type": "application/xml"
+            }
+        }, 
+        "Error": {
+            "Message": "The replication configuration was not found", 
+            "Code": "ReplicationConfigurationNotFoundError", 
+            "BucketName": "custodian-skunk-billing"
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketReplication_5.json b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketReplication_5.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketReplication_5.json
@@ -0,0 +1,24 @@
+{
+    "status_code": 404, 
+    "data": {
+        "ResponseMetadata": {
+            "HTTPStatusCode": 404, 
+            "RetryAttempts": 1, 
+            "HostId": "S040v+G1K2HKeWGGVrI7xqk15P5vflkirDN9wsagpoHZKYaEGrKr6D9lPjqSN1KLPYZP2p25B/Q=", 
+            "RequestId": "D18D267EC8575A7A", 
+            "HTTPHeaders": {
+                "x-amz-id-2": "S040v+G1K2HKeWGGVrI7xqk15P5vflkirDN9wsagpoHZKYaEGrKr6D9lPjqSN1KLPYZP2p25B/Q=", 
+                "server": "AmazonS3", 
+                "transfer-encoding": "chunked", 
+                "x-amz-request-id": "D18D267EC8575A7A", 
+                "date": "Sat, 07 Jan 2017 20:17:16 GMT", 
+                "content-type": "application/xml"
+            }
+        }, 
+        "Error": {
+            "Message": "The replication configuration was not found", 
+            "Code": "ReplicationConfigurationNotFoundError", 
+            "BucketName": "custodian-skunk-trails"
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketVersioning_1.json b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketVersioning_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketVersioning_1.json
@@ -0,0 +1,18 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RetryAttempts": 0, 
+            "HostId": "WhhKDJXuYibouTuTXFWNp6QZ4YvlMM+90yyinuSo99PoyuyA95bruI1Kef7jXWq9gJuJWw4wuTM=", 
+            "RequestId": "939F423850123EDA", 
+            "HTTPHeaders": {
+                "x-amz-id-2": "WhhKDJXuYibouTuTXFWNp6QZ4YvlMM+90yyinuSo99PoyuyA95bruI1Kef7jXWq9gJuJWw4wuTM=", 
+                "date": "Sat, 07 Jan 2017 20:17:13 GMT", 
+                "transfer-encoding": "chunked", 
+                "x-amz-request-id": "939F423850123EDA", 
+                "server": "AmazonS3"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketVersioning_2.json b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketVersioning_2.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketVersioning_2.json
@@ -0,0 +1,19 @@
+{
+    "status_code": 200, 
+    "data": {
+        "Status": "Enabled", 
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RetryAttempts": 0, 
+            "HostId": "p+mi4q/XyPXAe264N71ldZx9f241YQeGnOLIxWNGfiNA3Y+e4iMMkrJ+toB5PHxeBENqpkNUHgM=", 
+            "RequestId": "6BBE4E222FADDF0C", 
+            "HTTPHeaders": {
+                "x-amz-id-2": "p+mi4q/XyPXAe264N71ldZx9f241YQeGnOLIxWNGfiNA3Y+e4iMMkrJ+toB5PHxeBENqpkNUHgM=", 
+                "date": "Sat, 07 Jan 2017 20:17:15 GMT", 
+                "transfer-encoding": "chunked", 
+                "x-amz-request-id": "6BBE4E222FADDF0C", 
+                "server": "AmazonS3"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketVersioning_3.json b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketVersioning_3.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketVersioning_3.json
@@ -0,0 +1,19 @@
+{
+    "status_code": 200, 
+    "data": {
+        "Status": "Enabled", 
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RetryAttempts": 0, 
+            "HostId": "aNYgLd3Lo+TOMC2mXEBu9xe6QO8SSauVeUtjM2miBPpTF/9VI+MxxCWJNtDzMmJ44S3KJ6gerWg=", 
+            "RequestId": "E850AFBC083EC5CA", 
+            "HTTPHeaders": {
+                "x-amz-id-2": "aNYgLd3Lo+TOMC2mXEBu9xe6QO8SSauVeUtjM2miBPpTF/9VI+MxxCWJNtDzMmJ44S3KJ6gerWg=", 
+                "date": "Sat, 07 Jan 2017 20:17:15 GMT", 
+                "transfer-encoding": "chunked", 
+                "x-amz-request-id": "E850AFBC083EC5CA", 
+                "server": "AmazonS3"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketVersioning_4.json b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketVersioning_4.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketVersioning_4.json
@@ -0,0 +1,18 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RetryAttempts": 0, 
+            "HostId": "jJmJ2PcE4EbsGYj7riQmUxET222nr43YnEnojraZqr0ml8Kov2nvZeWuO9WF8sSrm1ih9vilzag=", 
+            "RequestId": "06CC559A04312B2A", 
+            "HTTPHeaders": {
+                "x-amz-id-2": "jJmJ2PcE4EbsGYj7riQmUxET222nr43YnEnojraZqr0ml8Kov2nvZeWuO9WF8sSrm1ih9vilzag=", 
+                "date": "Sat, 07 Jan 2017 20:17:17 GMT", 
+                "transfer-encoding": "chunked", 
+                "x-amz-request-id": "06CC559A04312B2A", 
+                "server": "AmazonS3"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketVersioning_5.json b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketVersioning_5.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.GetBucketVersioning_5.json
@@ -0,0 +1,19 @@
+{
+    "status_code": 200, 
+    "data": {
+        "Status": "Enabled", 
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RetryAttempts": 0, 
+            "HostId": "uoOWuYdKN/7FxIcWft+N/w02Fk2X51nRDr5Tk5BCBSi/YmhYtDIOLWhlldI7TpBezOr3fYTnogs=", 
+            "RequestId": "4ACC948B350AD46C", 
+            "HTTPHeaders": {
+                "x-amz-id-2": "uoOWuYdKN/7FxIcWft+N/w02Fk2X51nRDr5Tk5BCBSi/YmhYtDIOLWhlldI7TpBezOr3fYTnogs=", 
+                "date": "Sat, 07 Jan 2017 20:17:18 GMT", 
+                "transfer-encoding": "chunked", 
+                "x-amz-request-id": "4ACC948B350AD46C", 
+                "server": "AmazonS3"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_delete_replicated_bucket/s3.ListBuckets_1.json b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.ListBuckets_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.ListBuckets_1.json
@@ -0,0 +1,89 @@
+{
+    "status_code": 200, 
+    "data": {
+        "Owner": {
+            "ID": "e7c8bb65a5fc49cf906715eae09de9e4bb7861a96361ba79b833aa45f6833b15"
+        }, 
+        "Buckets": [
+            {
+                "CreationDate": {
+                    "hour": 23, 
+                    "__class__": "datetime", 
+                    "month": 9, 
+                    "second": 53, 
+                    "microsecond": 0, 
+                    "year": 2016, 
+                    "day": 12, 
+                    "minute": 37
+                }, 
+                "Name": "config-bucket-644160558196"
+            }, 
+            {
+                "CreationDate": {
+                    "hour": 20, 
+                    "__class__": "datetime", 
+                    "month": 1, 
+                    "second": 2, 
+                    "microsecond": 0, 
+                    "year": 2017, 
+                    "day": 7, 
+                    "minute": 6
+                }, 
+                "Name": "custodian-replicated"
+            }, 
+            {
+                "CreationDate": {
+                    "hour": 19, 
+                    "__class__": "datetime", 
+                    "month": 1, 
+                    "second": 22, 
+                    "microsecond": 0, 
+                    "year": 2017, 
+                    "day": 7, 
+                    "minute": 57
+                }, 
+                "Name": "custodian-replicated-west"
+            }, 
+            {
+                "CreationDate": {
+                    "hour": 21, 
+                    "__class__": "datetime", 
+                    "month": 11, 
+                    "second": 6, 
+                    "microsecond": 0, 
+                    "year": 2016, 
+                    "day": 6, 
+                    "minute": 19
+                }, 
+                "Name": "custodian-skunk-billing"
+            }, 
+            {
+                "CreationDate": {
+                    "hour": 19, 
+                    "__class__": "datetime", 
+                    "month": 1, 
+                    "second": 6, 
+                    "microsecond": 0, 
+                    "year": 2017, 
+                    "day": 7, 
+                    "minute": 53
+                }, 
+                "Name": "custodian-skunk-trails"
+            }
+        ], 
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RetryAttempts": 0, 
+            "HostId": "wHLpqIYniaDakKj22GvH7bIgZgg+tX9XYwCLVMmRvMgxbxPfato/Qr7X+PW1Nxj+KI+KNvyegPM=", 
+            "RequestId": "BF641869AEE42726", 
+            "HTTPHeaders": {
+                "x-amz-id-2": "wHLpqIYniaDakKj22GvH7bIgZgg+tX9XYwCLVMmRvMgxbxPfato/Qr7X+PW1Nxj+KI+KNvyegPM=", 
+                "server": "AmazonS3", 
+                "transfer-encoding": "chunked", 
+                "x-amz-request-id": "BF641869AEE42726", 
+                "date": "Sat, 07 Jan 2017 20:17:13 GMT", 
+                "content-type": "application/xml"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_delete_replicated_bucket/s3.ListBuckets_2.json b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.ListBuckets_2.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.ListBuckets_2.json
@@ -0,0 +1,76 @@
+{
+    "status_code": 200, 
+    "data": {
+        "Owner": {
+            "ID": "e7c8bb65a5fc49cf906715eae09de9e4bb7861a96361ba79b833aa45f6833b15"
+        }, 
+        "Buckets": [
+            {
+                "CreationDate": {
+                    "hour": 23, 
+                    "__class__": "datetime", 
+                    "month": 9, 
+                    "second": 53, 
+                    "microsecond": 0, 
+                    "year": 2016, 
+                    "day": 12, 
+                    "minute": 37
+                }, 
+                "Name": "config-bucket-644160558196"
+            }, 
+            {
+                "CreationDate": {
+                    "hour": 19, 
+                    "__class__": "datetime", 
+                    "month": 1, 
+                    "second": 22, 
+                    "microsecond": 0, 
+                    "year": 2017, 
+                    "day": 7, 
+                    "minute": 57
+                }, 
+                "Name": "custodian-replicated-west"
+            }, 
+            {
+                "CreationDate": {
+                    "hour": 21, 
+                    "__class__": "datetime", 
+                    "month": 11, 
+                    "second": 6, 
+                    "microsecond": 0, 
+                    "year": 2016, 
+                    "day": 6, 
+                    "minute": 19
+                }, 
+                "Name": "custodian-skunk-billing"
+            }, 
+            {
+                "CreationDate": {
+                    "hour": 19, 
+                    "__class__": "datetime", 
+                    "month": 1, 
+                    "second": 6, 
+                    "microsecond": 0, 
+                    "year": 2017, 
+                    "day": 7, 
+                    "minute": 53
+                }, 
+                "Name": "custodian-skunk-trails"
+            }
+        ], 
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RetryAttempts": 0, 
+            "HostId": "baNNmcBCk/SMEngXEhcTJaqfxp/Sj/1uzqFYpxINryzh8FfniScojeXsupoOhDVcHpvCoQEfztA=", 
+            "RequestId": "2069D883CC82807D", 
+            "HTTPHeaders": {
+                "x-amz-id-2": "baNNmcBCk/SMEngXEhcTJaqfxp/Sj/1uzqFYpxINryzh8FfniScojeXsupoOhDVcHpvCoQEfztA=", 
+                "server": "AmazonS3", 
+                "transfer-encoding": "chunked", 
+                "x-amz-request-id": "2069D883CC82807D", 
+                "date": "Sat, 07 Jan 2017 20:17:22 GMT", 
+                "content-type": "application/xml"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_delete_replicated_bucket/s3.ListMultipartUploads_1.json b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.ListMultipartUploads_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.ListMultipartUploads_1.json
@@ -0,0 +1,26 @@
+{
+    "status_code": 200, 
+    "data": {
+        "UploadIdMarker": "", 
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RetryAttempts": 0, 
+            "HostId": "kX10lVRe/Rx0VOaGDYSEiSCFh9c7hta7KPpjGRXOnMXscKfMjchAe7UmzwJdbD0jGJx3WJAtL0Y=", 
+            "RequestId": "CC597ADB4CA07D69", 
+            "HTTPHeaders": {
+                "x-amz-id-2": "kX10lVRe/Rx0VOaGDYSEiSCFh9c7hta7KPpjGRXOnMXscKfMjchAe7UmzwJdbD0jGJx3WJAtL0Y=", 
+                "server": "AmazonS3", 
+                "transfer-encoding": "chunked", 
+                "x-amz-request-id": "CC597ADB4CA07D69", 
+                "date": "Sat, 07 Jan 2017 20:17:19 GMT", 
+                "content-type": "application/xml"
+            }
+        }, 
+        "NextKeyMarker": "", 
+        "Bucket": "custodian-replicated", 
+        "NextUploadIdMarker": "", 
+        "KeyMarker": "", 
+        "MaxUploads": 1000, 
+        "IsTruncated": false
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_delete_replicated_bucket/s3.ListObjectVersions_1.json b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.ListObjectVersions_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.ListObjectVersions_1.json
@@ -0,0 +1,159 @@
+{
+    "status_code": 200, 
+    "data": {
+        "Name": "custodian-replicated", 
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RetryAttempts": 0, 
+            "HostId": "EFaHLbSaX5BUuyar74o397pmnN/OIAsmOtfLLbbE0U7WIZmR0GhaLlYK6itDADUHixQlfmEM2RA=", 
+            "RequestId": "63F7BCAA23E10B2E", 
+            "HTTPHeaders": {
+                "x-amz-id-2": "EFaHLbSaX5BUuyar74o397pmnN/OIAsmOtfLLbbE0U7WIZmR0GhaLlYK6itDADUHixQlfmEM2RA=", 
+                "server": "AmazonS3", 
+                "transfer-encoding": "chunked", 
+                "x-amz-request-id": "63F7BCAA23E10B2E", 
+                "date": "Sat, 07 Jan 2017 20:17:20 GMT", 
+                "content-type": "application/xml"
+            }
+        }, 
+        "Versions": [
+            {
+                "LastModified": {
+                    "hour": 20, 
+                    "__class__": "datetime", 
+                    "month": 1, 
+                    "second": 38, 
+                    "microsecond": 0, 
+                    "year": 2017, 
+                    "day": 7, 
+                    "minute": 6
+                }, 
+                "VersionId": "oR8cvbTYzIpUi3qOPDvSIuLUxQGrqmbh", 
+                "ETag": "\"9f9c3721240a71beb989b048f0c4ef15\"", 
+                "StorageClass": "STANDARD", 
+                "Key": "Screen Shot 2017-01-04 at 8.13.36 PM.png", 
+                "Owner": {
+                    "DisplayName": "mandeep.bal", 
+                    "ID": "e7c8bb65a5fc49cf906715eae09de9e4bb7861a96361ba79b833aa45f6833b15"
+                }, 
+                "IsLatest": true, 
+                "Size": 9547
+            }, 
+            {
+                "LastModified": {
+                    "hour": 20, 
+                    "__class__": "datetime", 
+                    "month": 1, 
+                    "second": 9, 
+                    "microsecond": 0, 
+                    "year": 2017, 
+                    "day": 7, 
+                    "minute": 5
+                }, 
+                "VersionId": "Pxh5frwAbmN9GSSibyt7_6xI5IPUuFEi", 
+                "ETag": "\"9f9c3721240a71beb989b048f0c4ef15\"", 
+                "StorageClass": "STANDARD", 
+                "Key": "Screen Shot 2017-01-04 at 8.13.36 PM.png", 
+                "Owner": {
+                    "DisplayName": "mandeep.bal", 
+                    "ID": "e7c8bb65a5fc49cf906715eae09de9e4bb7861a96361ba79b833aa45f6833b15"
+                }, 
+                "IsLatest": false, 
+                "Size": 9547
+            }, 
+            {
+                "LastModified": {
+                    "hour": 20, 
+                    "__class__": "datetime", 
+                    "month": 1, 
+                    "second": 38, 
+                    "microsecond": 0, 
+                    "year": 2017, 
+                    "day": 7, 
+                    "minute": 6
+                }, 
+                "VersionId": ".PII_WPyK1JAkmhhT_mV2JmdrKoO7p9X", 
+                "ETag": "\"e478902f393ab701c1d1c810d8ed7f5a\"", 
+                "StorageClass": "STANDARD", 
+                "Key": "Screen Shot 2017-01-04 at 8.13.42 PM.png", 
+                "Owner": {
+                    "DisplayName": "mandeep.bal", 
+                    "ID": "e7c8bb65a5fc49cf906715eae09de9e4bb7861a96361ba79b833aa45f6833b15"
+                }, 
+                "IsLatest": true, 
+                "Size": 109641
+            }, 
+            {
+                "LastModified": {
+                    "hour": 20, 
+                    "__class__": "datetime", 
+                    "month": 1, 
+                    "second": 9, 
+                    "microsecond": 0, 
+                    "year": 2017, 
+                    "day": 7, 
+                    "minute": 5
+                }, 
+                "VersionId": "rjCBi3AfMJdXq.OFXCcHSyQ4yx6GalvG", 
+                "ETag": "\"e478902f393ab701c1d1c810d8ed7f5a\"", 
+                "StorageClass": "STANDARD", 
+                "Key": "Screen Shot 2017-01-04 at 8.13.42 PM.png", 
+                "Owner": {
+                    "DisplayName": "mandeep.bal", 
+                    "ID": "e7c8bb65a5fc49cf906715eae09de9e4bb7861a96361ba79b833aa45f6833b15"
+                }, 
+                "IsLatest": false, 
+                "Size": 109641
+            }, 
+            {
+                "LastModified": {
+                    "hour": 20, 
+                    "__class__": "datetime", 
+                    "month": 1, 
+                    "second": 36, 
+                    "microsecond": 0, 
+                    "year": 2017, 
+                    "day": 7, 
+                    "minute": 6
+                }, 
+                "VersionId": "y70jHWgNr.kdCIrzCeJGM7wxgkWcpLfh", 
+                "ETag": "\"9336789303f0103ebc0a374d1ee88ac0\"", 
+                "StorageClass": "STANDARD", 
+                "Key": "Screen Shot 2017-01-04 at 8.18.46 PM.png", 
+                "Owner": {
+                    "DisplayName": "mandeep.bal", 
+                    "ID": "e7c8bb65a5fc49cf906715eae09de9e4bb7861a96361ba79b833aa45f6833b15"
+                }, 
+                "IsLatest": true, 
+                "Size": 106325
+            }, 
+            {
+                "LastModified": {
+                    "hour": 20, 
+                    "__class__": "datetime", 
+                    "month": 1, 
+                    "second": 7, 
+                    "microsecond": 0, 
+                    "year": 2017, 
+                    "day": 7, 
+                    "minute": 5
+                }, 
+                "VersionId": "X2OhbdfpBlhB0Cjb_0p0SL_U0a2ktkSG", 
+                "ETag": "\"9336789303f0103ebc0a374d1ee88ac0\"", 
+                "StorageClass": "STANDARD", 
+                "Key": "Screen Shot 2017-01-04 at 8.18.46 PM.png", 
+                "Owner": {
+                    "DisplayName": "mandeep.bal", 
+                    "ID": "e7c8bb65a5fc49cf906715eae09de9e4bb7861a96361ba79b833aa45f6833b15"
+                }, 
+                "IsLatest": false, 
+                "Size": 106325
+            }
+        ], 
+        "MaxKeys": 1000, 
+        "Prefix": "", 
+        "KeyMarker": "", 
+        "IsTruncated": false, 
+        "VersionIdMarker": ""
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_delete_replicated_bucket/s3.PutBucketVersioning_1.json b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.PutBucketVersioning_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_delete_replicated_bucket/s3.PutBucketVersioning_1.json
@@ -0,0 +1,18 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "RetryAttempts": 0, 
+            "HostId": "WsWZzcdvlgSjysCzimqqNWEhNbCCqLqsK8mF4e8mMuCtX/tTXnexzZIAi+FTeoqVhAc/yRmEqRM=", 
+            "RequestId": "8474238D705E4C27", 
+            "HTTPHeaders": {
+                "x-amz-id-2": "WsWZzcdvlgSjysCzimqqNWEhNbCCqLqsK8mF4e8mMuCtX/tTXnexzZIAi+FTeoqVhAc/yRmEqRM=", 
+                "date": "Sat, 07 Jan 2017 20:17:19 GMT", 
+                "content-length": "0", 
+                "x-amz-request-id": "8474238D705E4C27", 
+                "server": "AmazonS3"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/test_s3.py b/tests/test_s3.py
--- a/tests/test_s3.py
+++ b/tests/test_s3.py
@@ -115,6 +115,33 @@ def test_metrics(self):
 
 class BucketDelete(BaseTest):
 
+    def test_delete_replicated_bucket(self):
+        # the iam setup is a little for replication to duplicate in a test
+        # preconditions - custodian-replicated and custodian-replicated-west
+        # buckets setup with replication, we're deleting the custodian-replicated
+        # bucket (source).
+        bname = 'custodian-replicated'
+        self.patch(s3.S3, 'executor_factory', MainThreadExecutor)
+        self.patch(
+            s3, 'S3_AUGMENT_TABLE',
+            [('get_bucket_replication', 'Replication', None, None),
+             ('get_bucket_versioning', 'Versioning', None, None)])
+        session_factory = self.replay_flight_data(
+            'test_s3_delete_replicated_bucket')
+        p = self.load_policy({
+            'name': 's3-delete-bucket',
+            'resource': 's3',
+            'filters': [{'Name': bname}],
+            'actions': [{'type': 'delete', 'remove-contents': True}]
+        }, session_factory=session_factory)
+        resources = p.run()
+        self.assertEqual(len(resources), 1)
+
+        session = session_factory()
+        client = session.client('s3')
+        buckets = set([b['Name'] for b in client.list_buckets()['Buckets']])
+        self.assertFalse(bname in buckets)
+
     def test_delete_versioned_bucket(self):
         self.patch(s3.S3, 'executor_factory', MainThreadExecutor)
         self.patch(s3, 'S3_AUGMENT_TABLE',
diff --git a/tests/test_waf.py b/tests/test_waf.py
--- a/tests/test_waf.py
+++ b/tests/test_waf.py
@@ -15,6 +15,3 @@ def test_waf_query(self):
             resources[0]['WebACLId'], '1ebe0b46-0fd2-4e07-a74c-27bf25adc0bf')
         self.assertEqual(
             resources[0]['DefaultAction'], {'Type': 'BLOCK'})
-
-        
-