Repository: cloud-custodian/cloud-custodian

Issue:
err on s3 cloudtrail lambda policy
reported internally, on an s3 custodian policy that was listening to create bucket event, the function fired, but had the following error:

```
string indices must be integers, not str:
TypeError Traceback (most recent call last):
File "/var/task/custodian_policy.py", line 4, in run return handler.dispatch_event(event, context)
File "/var/task/c7n/handler.py", line 72, in dispatch_event p.push(event, context)
File "/var/task/c7n/policy.py", line 134, in push resources = self.resource_manager.get_resources(resource_ids)
File "/var/task/c7n/resources/s3.py", line 91, in get_resources results = filter(None, results)
File "/var/task/concurrent/futures/_base.py", line 581, in result_iterator yield future.result()
File "/var/task/concurrent/futures/_base.py", line 405, in result return self.__get_result()
File "/var/task/concurrent/futures/thread.py", line 55, in run result = self.fn(*self.args, **self.kwargs)
File "/var/task/c7n/resources/s3.py", line 146, in assemble_bucket v = method(Bucket=b['Name'])
TypeError: string indices must be integers, not str
```



Discussion:

Resolving patch:
diff --git a/c7n/cwe.py b/c7n/cwe.py
--- a/c7n/cwe.py
+++ b/c7n/cwe.py
@@ -38,7 +38,7 @@ class CloudWatchEvents(object):
 
         'CreateBucket': {
             'ids': 'requestParameters.bucketName',
-            'source': 'aws.s3'},
+            'source': 's3.amazonaws.com'},
 
         'CreateCluster': {
             'ids': 'requestParameters.clusterIdentifier',
diff --git a/c7n/mu.py b/c7n/mu.py
--- a/c7n/mu.py
+++ b/c7n/mu.py
@@ -173,7 +173,7 @@ def custodian_archive(skip=None):
     """Create a lambda code archive for running custodian."""
 
     # Some aggressive shrinking
-    required = ["concurrent", "yaml", "pkg_resources"]
+    required = ["concurrent", "yaml", "pkg_resources", "skew"]
     host_platform = platform.uname()[0]
 
     def lib_filter(root, dirs, files):
diff --git a/c7n/resources/s3.py b/c7n/resources/s3.py
--- a/c7n/resources/s3.py
+++ b/c7n/resources/s3.py
@@ -84,7 +84,7 @@ def __init__(self, ctx, data):
     def get_resources(self, resource_ids):
         with self.executor_factory(
                 max_workers=min((5, len(resource_ids)))) as w:
-            buckets = {'Name': r for r in resource_ids}
+            buckets = [{'Name': r} for r in resource_ids]
             results = w.map(
                 assemble_bucket,
                 zip(itertools.repeat(self.session_factory), buckets))


Test patch:
diff --git a/tests/data/cwe/event-cloud-trail-create-bucket.json b/tests/data/cwe/event-cloud-trail-create-bucket.json
new file mode 100644
--- /dev/null
+++ b/tests/data/cwe/event-cloud-trail-create-bucket.json
@@ -0,0 +1,42 @@
+{
+    "account": "619193117841",
+    "region": "us-west-2",
+    "detail": {
+        "eventVersion": "1.03",
+        "eventID": "c4fa3db1-2303-40e4-bde0-3b7246bdbf0b",
+        "eventTime": "2016-06-04T12:19:03Z",
+        "requestParameters": {
+            "CreateBucketConfiguration": {
+                "xmlns": "http://s3.amazonaws.com/doc/2006-03-01/",
+                "LocationConstraint": "us-west-2"
+            },
+            "bucketName": "custodian-create-bucket-v4"
+        },
+        "eventType": "AwsApiCall",
+        "responseElements": null,
+        "awsRegion": "us-west-2",
+        "eventName": "CreateBucket",
+        "userIdentity": {
+            "sessionContext": {
+                "attributes": {
+                    "creationDate": "2016-06-04T11:18:26Z",
+                    "mfaAuthenticated": "false"
+                }
+            },
+            "type": "Root",
+            "arn": "arn:aws:iam::619193117841:root",
+            "principalId": "619193117841",
+            "accountId": "619193117841"
+        },
+        "eventSource": "s3.amazonaws.com",
+        "requestID": "A279EE6B879FE5ED",
+        "userAgent": "[S3Console/0.4]",
+        "sourceIPAddress": "204.63.44.148"
+    },
+    "detail-type": "AWS API Call via CloudTrail",
+    "source": "aws.s3",
+    "version": "0",
+    "time": "2016-06-04T12:19:03Z",
+    "id": "075efcc5-cc27-4041-accc-5fe2f65ef80d",
+    "resources": []
+}
diff --git a/tests/data/placebo/test_s3_create/s3.CreateBucket_1.json b/tests/data/placebo/test_s3_create/s3.CreateBucket_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_create/s3.CreateBucket_1.json
@@ -0,0 +1,11 @@
+{
+    "status_code": 200, 
+    "data": {
+        "Location": "/custodian-create-bucket-v4", 
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "HostId": "8Y21GQrN4e0rwxYRR+/GUKRQZH1CDWthMK6koqG5B1yW0VA0VsPc3sjOBHp7eR2pMpTOCRGrvGs=", 
+            "RequestId": "C4A7C25E5ED8FFF0"
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_create/s3.DeleteBucket_1.json b/tests/data/placebo/test_s3_create/s3.DeleteBucket_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_create/s3.DeleteBucket_1.json
@@ -0,0 +1,10 @@
+{
+    "status_code": 204, 
+    "data": {
+        "ResponseMetadata": {
+            "HTTPStatusCode": 204, 
+            "HostId": "iScX6Zpc4P/CvwvdtzoUbI0OHT4mfgbsaUE2G89br2VJJWoj5RZvybOR4G1i4DsC1d0QPZrp9Kg=", 
+            "RequestId": "712AF01654E253A6"
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_create/s3.GetBucketPolicy_1.json b/tests/data/placebo/test_s3_create/s3.GetBucketPolicy_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_create/s3.GetBucketPolicy_1.json
@@ -0,0 +1,15 @@
+{
+    "status_code": 404, 
+    "data": {
+        "ResponseMetadata": {
+            "HTTPStatusCode": 404, 
+            "HostId": "tIDujFrBeEYRHvzjI9W1+DxMQiNucdgkkpQOHrDCeI2kce8dITkInffXW5I7TDgRPTvQTSVxngo=", 
+            "RequestId": "383A59C56334003D"
+        }, 
+        "Error": {
+            "Message": "The bucket policy does not exist", 
+            "Code": "NoSuchBucketPolicy", 
+            "BucketName": "custodian-create-bucket-v4"
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_create/s3.GetBucketPolicy_2.json b/tests/data/placebo/test_s3_create/s3.GetBucketPolicy_2.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_create/s3.GetBucketPolicy_2.json
@@ -0,0 +1,11 @@
+{
+    "status_code": 200, 
+    "data": {
+        "Policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"RequireEncryptedPutObject\",\"Effect\":\"Deny\",\"Principal\":\"*\",\"Action\":\"s3:PutObject\",\"Resource\":\"arn:aws:s3:::custodian-create-bucket-v4/*\",\"Condition\":{\"StringNotEquals\":{\"s3:x-amz-server-side-encryption\":[\"AES256\",\"aws:kms\"]}}}]}", 
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "HostId": "ufxjht6EoJ2pvRJjnbq0FoCReObwCHjwuP5jbobGGj7nA+0t5WTxQ13oQTXekIbC15m49pO6qC0=", 
+            "RequestId": "0EA2913B584F9C4D"
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_create/s3.ListObjects_1.json b/tests/data/placebo/test_s3_create/s3.ListObjects_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_create/s3.ListObjects_1.json
@@ -0,0 +1,16 @@
+{
+    "status_code": 200, 
+    "data": {
+        "MaxKeys": 1000, 
+        "Prefix": "", 
+        "Name": "custodian-create-bucket-v4", 
+        "ResponseMetadata": {
+            "HTTPStatusCode": 200, 
+            "HostId": "2d3PUVY5NsaXf4xxbwPTazzbQa9r2pSBYtiJIXFD9BsKxlM/ESqY3gMUIDQVJ8l7imCmFAubAWk=", 
+            "RequestId": "661100C0789DCFCA"
+        }, 
+        "Marker": "", 
+        "EncodingType": "url", 
+        "IsTruncated": false
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_s3_create/s3.PutBucketPolicy_1.json b/tests/data/placebo/test_s3_create/s3.PutBucketPolicy_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_s3_create/s3.PutBucketPolicy_1.json
@@ -0,0 +1,10 @@
+{
+    "status_code": 204, 
+    "data": {
+        "ResponseMetadata": {
+            "HTTPStatusCode": 204, 
+            "HostId": "alzg3Nwf+Pv4AwjzYqXErWkrbohwdGXlr9UGK8252znO6iye037jfkSVbd3B43Ok3zr9Zbbjuec=", 
+            "RequestId": "80A9A02C4824710A"
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/test_cwe.py b/tests/test_cwe.py
--- a/tests/test_cwe.py
+++ b/tests/test_cwe.py
@@ -65,5 +65,5 @@ def test_cloud_trail_resource(self):
         self.assertEqual(
             CloudWatchEvents.match(
                 event_data('event-cloud-trail-s3.json')),
-            {'source': 'aws.s3',
+            {'source': 's3.amazonaws.com',
              'ids': jmespath.compile('detail.requestParameters.bucketName')})
diff --git a/tests/test_mu.py b/tests/test_mu.py
--- a/tests/test_mu.py
+++ b/tests/test_mu.py
@@ -79,7 +79,7 @@ def test_cwe_trail(self):
         self.assertEqual(
             json.loads(event.render_event_pattern()),
             {u'detail': {u'eventName': [u'CreateBucket'],
-                         u'eventSource': [u'aws.s3']},
+                         u'eventSource': [u's3.amazonaws.com']},
              u'detail-type': ['AWS API Call via CloudTrail']})
 
         self.assert_items(
diff --git a/tests/test_s3.py b/tests/test_s3.py
--- a/tests/test_s3.py
+++ b/tests/test_s3.py
@@ -14,16 +14,17 @@
 import json
 import shutil
 import tempfile
+import time
 from unittest import TestCase
 
 from botocore.exceptions import ClientError
 
 from c7n.executor import MainThreadExecutor
 from c7n.resources import s3
-from c7n.mu import LambdaManager
+from c7n.mu import LambdaManager, PolicyLambda
 from c7n.ufuncs import s3crypt
 
-from common import BaseTest
+from common import BaseTest, event_data
 
 
 class RestoreCompletionTest(TestCase):
@@ -328,6 +329,53 @@ def test_attach_encrypt_requires_role(self):
              'resource': 's3',
              'actions': [{'type': 'attach-encrypt'}]})
 
+    def test_create_bucket_event(self):
+        self.patch(s3, 'S3_AUGMENT_TABLE', [
+            ('get_bucket_policy',  'Policy', None, None),
+        ])
+        self.patch(s3.S3, 'executor_factory', MainThreadExecutor)
+        session_factory = self.replay_flight_data('test_s3_create')
+        bname = 'custodian-create-bucket-v4'
+        session = session_factory()
+        client = session.client('s3')
+
+        client.create_bucket(Bucket=bname)
+        self.addCleanup(destroyBucket, client, bname)
+
+        p = self.load_policy({
+            'name': 'bucket-create-v2',
+            'resource': 's3',
+            'mode': {
+                'type': 'cloudtrail',
+                'role': 'arn:aws:iam::619193117841:role/CustodianDemoRole',
+                'events': ['CreateBucket'],
+                },
+            'actions': [
+                'encryption-policy']}, session_factory=session_factory)
+        p.push(event_data('event-cloud-trail-create-bucket.json'), None)
+
+        try:
+            result = client.get_bucket_policy(Bucket=bname)
+        except:
+            self.fail("Could not get bucket policy")
+
+        self.assertTrue('Policy' in result)
+        policy = json.loads(result['Policy'])
+        self.assertEqual(
+            policy,
+            {u'Statement': [
+                {u'Action': u's3:PutObject',
+                 u'Condition': {
+                     u'StringNotEquals': {
+                         u's3:x-amz-server-side-encryption': [
+                             u'AES256',
+                             u'aws:kms']}},
+                 u'Effect': u'Deny',
+                 u'Principal': u'*',
+                 u'Resource': u'arn:aws:s3:::custodian-create-bucket-v4/*',
+                 u'Sid': u'RequireEncryptedPutObject'}],
+             u'Version': u'2012-10-17'})
+
     def test_attach_encrypt(self):
         self.patch(s3, 'S3_AUGMENT_TABLE', [])
         session_factory = self.replay_flight_data('test_s3_attach_encrypt')
@@ -402,9 +450,9 @@ def test_global_grants_filter_option(self):
         client = session.client('s3')
         client.create_bucket(Bucket=bname)
         self.addCleanup(destroyBucket, client, bname)
-        
+
         public = 'http://acs.amazonaws.com/groups/global/AllUsers'
-    
+
         client.put_bucket_acl(
             Bucket=bname,
             AccessControlPolicy={
@@ -438,8 +486,8 @@ def test_global_grants_filter_option(self):
                   'permissions': ['WRITE']}]},
             session_factory=session_factory)
         resources = p.run()
-        self.assertEqual(len(resources), 1)        
-        
+        self.assertEqual(len(resources), 1)
+
     def test_global_grants_filter_and_remove(self):
         self.patch(s3, 'S3_AUGMENT_TABLE', [
             ('get_bucket_acl', 'Acl', None, None)
@@ -450,7 +498,7 @@ def test_global_grants_filter_and_remove(self):
         session = session_factory()
         client = session.client('s3')
         client.create_bucket(Bucket=bname)
-        
+
         public = 'http://acs.amazonaws.com/groups/global/AllUsers'
         client.put_bucket_acl(
             Bucket=bname,