Repository: cloud-custodian/cloud-custodian

Issue:
handle throttles on log group output



Discussion:

Resolving patch:
diff --git a/c7n/log.py b/c7n/log.py
--- a/c7n/log.py
+++ b/c7n/log.py
@@ -37,6 +37,7 @@
 except ImportError:
     import queue as Queue
 
+from c7n.utils import get_retry
 
 FLUSH_MARKER = object()
 SHUTDOWN_MARKER = object()
@@ -82,9 +83,15 @@ def __init__(self, log_group=__name__, log_stream=None,
         # cleanup atexit, custodian is a bit more explicitly scoping shutdown to
         # each policy, so use a sentinel value to avoid deadlocks.
         self.shutdown = False
+        retry = get_retry(('ThrottlingException',))
         try:
-            self.session_factory().client(
-                'logs').create_log_group(logGroupName=self.log_group)
+            client = self.session_factory().client('logs')
+            logs = retry(
+                client.describe_log_groups,
+                logGroupNamePrefix=self.log_group)['logGroups']
+            if [l for l in logs if l['logGroupName'] == self.log_group]:
+                retry(client.create_log_group,
+                      logGroupName=self.log_group)
         except ClientError as e:
             if Error.code(e) != Error.ResourceExists:
                 raise


Test patch:
diff --git a/tests/data/placebo/test_log_existing_stream/logs.CreateLogGroup_1.json b/tests/data/placebo/test_log_existing_stream/logs.CreateLogGroup_1.json
--- a/tests/data/placebo/test_log_existing_stream/logs.CreateLogGroup_1.json
+++ b/tests/data/placebo/test_log_existing_stream/logs.CreateLogGroup_1.json
@@ -1,13 +1,16 @@
 {
-    "status_code": 400, 
+    "status_code": 200, 
     "data": {
         "ResponseMetadata": {
-            "HTTPStatusCode": 400, 
-            "RequestId": "eac08fe9-cd1a-11e5-a37b-e943bdecb17e"
-        }, 
-        "Error": {
-            "Message": "The specified log group already exists", 
-            "Code": "ResourceAlreadyExistsException"
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "c240e2eb-b4d3-11e6-a157-01f518f82ed7", 
+            "HTTPHeaders": {
+                "x-amzn-requestid": "c240e2eb-b4d3-11e6-a157-01f518f82ed7", 
+                "date": "Sun, 27 Nov 2016 19:00:26 GMT", 
+                "content-length": "0", 
+                "content-type": "application/x-amz-json-1.1"
+            }
         }
     }
 }
\ No newline at end of file
diff --git a/tests/data/placebo/test_log_existing_stream/logs.CreateLogGroup_2.json b/tests/data/placebo/test_log_existing_stream/logs.CreateLogGroup_2.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_log_existing_stream/logs.CreateLogGroup_2.json
@@ -0,0 +1,21 @@
+{
+    "status_code": 400, 
+    "data": {
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 400, 
+            "RequestId": "c2827f46-b4d3-11e6-88f9-4336d5cae6f4", 
+            "HTTPHeaders": {
+                "x-amzn-requestid": "c2827f46-b4d3-11e6-88f9-4336d5cae6f4", 
+                "date": "Sun, 27 Nov 2016 19:00:27 GMT", 
+                "content-length": "94", 
+                "content-type": "application/x-amz-json-1.1", 
+                "cneonction": "close"
+            }
+        }, 
+        "Error": {
+            "Message": "The specified log group already exists", 
+            "Code": "ResourceAlreadyExistsException"
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_log_existing_stream/logs.CreateLogStream_1.json b/tests/data/placebo/test_log_existing_stream/logs.CreateLogStream_1.json
--- a/tests/data/placebo/test_log_existing_stream/logs.CreateLogStream_1.json
+++ b/tests/data/placebo/test_log_existing_stream/logs.CreateLogStream_1.json
@@ -1,13 +1,16 @@
 {
-    "status_code": 400, 
+    "status_code": 200, 
     "data": {
         "ResponseMetadata": {
-            "HTTPStatusCode": 400, 
-            "RequestId": "ead43e10-cd1a-11e5-8be8-cd02a40e563e"
-        }, 
-        "Error": {
-            "Message": "The specified log stream already exists", 
-            "Code": "ResourceAlreadyExistsException"
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "c2b7bfc2-b4d3-11e6-8bc7-43ba34db3530", 
+            "HTTPHeaders": {
+                "x-amzn-requestid": "c2b7bfc2-b4d3-11e6-8bc7-43ba34db3530", 
+                "date": "Sun, 27 Nov 2016 19:00:27 GMT", 
+                "content-length": "0", 
+                "content-type": "application/x-amz-json-1.1"
+            }
         }
     }
 }
\ No newline at end of file
diff --git a/tests/data/placebo/test_log_existing_stream/logs.DescribeLogGroups_1.json b/tests/data/placebo/test_log_existing_stream/logs.DescribeLogGroups_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_log_existing_stream/logs.DescribeLogGroups_1.json
@@ -0,0 +1,25 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "c2742784-b4d3-11e6-98dc-f9e5807ba420", 
+            "HTTPHeaders": {
+                "x-amzn-requestid": "c2742784-b4d3-11e6-98dc-f9e5807ba420", 
+                "date": "Sun, 27 Nov 2016 19:00:27 GMT", 
+                "content-length": "187", 
+                "content-type": "application/x-amz-json-1.1"
+            }
+        }, 
+        "logGroups": [
+            {
+                "arn": "arn:aws:logs:us-east-1:644160558196:log-group:/custodian-dev:*", 
+                "creationTime": 1480273227168, 
+                "metricFilterCount": 0, 
+                "logGroupName": "/custodian-dev", 
+                "storedBytes": 0
+            }
+        ]
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_log_existing_stream/logs.PutLogEvents_1.json b/tests/data/placebo/test_log_existing_stream/logs.PutLogEvents_1.json
--- a/tests/data/placebo/test_log_existing_stream/logs.PutLogEvents_1.json
+++ b/tests/data/placebo/test_log_existing_stream/logs.PutLogEvents_1.json
@@ -1,13 +1,17 @@
 {
-    "status_code": 400, 
+    "status_code": 200, 
     "data": {
+        "nextSequenceToken": "49567511072778230234585680335896857312552072736688602210", 
         "ResponseMetadata": {
-            "HTTPStatusCode": 400, 
-            "RequestId": "eadcf0f3-cd1a-11e5-be2d-e781e8260755"
-        }, 
-        "Error": {
-            "Message": "The given sequenceToken is invalid. The next expected sequenceToken is: 49540751441854908238376988780308869790031137916571370994", 
-            "Code": "InvalidSequenceTokenException"
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "c2c72969-b4d3-11e6-bade-e96ef0364aec", 
+            "HTTPHeaders": {
+                "x-amzn-requestid": "c2c72969-b4d3-11e6-bade-e96ef0364aec", 
+                "date": "Sun, 27 Nov 2016 19:00:28 GMT", 
+                "content-length": "80", 
+                "content-type": "application/x-amz-json-1.1"
+            }
         }
     }
 }
\ No newline at end of file
diff --git a/tests/data/placebo/test_log_existing_stream/logs.PutLogEvents_2.json b/tests/data/placebo/test_log_existing_stream/logs.PutLogEvents_2.json
deleted file mode 100644
--- a/tests/data/placebo/test_log_existing_stream/logs.PutLogEvents_2.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
-    "status_code": 200, 
-    "data": {
-        "nextSequenceToken": "49540751441854908238376988843899576827580298812337440242", 
-        "ResponseMetadata": {
-            "HTTPStatusCode": 200, 
-            "RequestId": "eae41d8e-cd1a-11e5-872d-8bfa8d903e7b"
-        }
-    }
-}
\ No newline at end of file
diff --git a/tests/data/placebo/test_log_time_flush/logs.CreateLogGroup_1.json b/tests/data/placebo/test_log_time_flush/logs.CreateLogGroup_1.json
deleted file mode 100644
--- a/tests/data/placebo/test_log_time_flush/logs.CreateLogGroup_1.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
-    "status_code": 200, 
-    "data": {
-        "ResponseMetadata": {
-            "HTTPStatusCode": 200, 
-            "RequestId": "726a3e7a-cd93-11e5-bdef-0f26886231f0"
-        }
-    }
-}
\ No newline at end of file
diff --git a/tests/data/placebo/test_log_time_flush/logs.CreateLogStream_1.json b/tests/data/placebo/test_log_time_flush/logs.CreateLogStream_1.json
--- a/tests/data/placebo/test_log_time_flush/logs.CreateLogStream_1.json
+++ b/tests/data/placebo/test_log_time_flush/logs.CreateLogStream_1.json
@@ -1,9 +1,21 @@
 {
-    "status_code": 200, 
+    "status_code": 400, 
     "data": {
         "ResponseMetadata": {
-            "HTTPStatusCode": 200, 
-            "RequestId": "734f6e09-cd93-11e5-b7c5-9353fc464fb1"
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 400, 
+            "RequestId": "6d865d1b-b4d3-11e6-8f49-fd2ab443b28f", 
+            "HTTPHeaders": {
+                "x-amzn-requestid": "6d865d1b-b4d3-11e6-8f49-fd2ab443b28f", 
+                "date": "Sun, 27 Nov 2016 18:58:04 GMT", 
+                "content-length": "90", 
+                "content-type": "application/x-amz-json-1.1", 
+                "nncoection": "close"
+            }
+        }, 
+        "Error": {
+            "Message": "The specified log group does not exist.", 
+            "Code": "ResourceNotFoundException"
         }
     }
 }
\ No newline at end of file
diff --git a/tests/data/placebo/test_log_time_flush/logs.DescribeLogGroups_1.json b/tests/data/placebo/test_log_time_flush/logs.DescribeLogGroups_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_log_time_flush/logs.DescribeLogGroups_1.json
@@ -0,0 +1,17 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "6ca17b83-b4d3-11e6-82b9-f5de1afb802b", 
+            "HTTPHeaders": {
+                "x-amzn-requestid": "6ca17b83-b4d3-11e6-82b9-f5de1afb802b", 
+                "date": "Sun, 27 Nov 2016 18:58:02 GMT", 
+                "content-length": "16", 
+                "content-type": "application/x-amz-json-1.1"
+            }
+        }, 
+        "logGroups": []
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_log_time_flush/logs.PutLogEvents_1.json b/tests/data/placebo/test_log_time_flush/logs.PutLogEvents_1.json
deleted file mode 100644
--- a/tests/data/placebo/test_log_time_flush/logs.PutLogEvents_1.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
-    "status_code": 200, 
-    "data": {
-        "nextSequenceToken": "49556335865110179795774509772179688169226661159862555058", 
-        "ResponseMetadata": {
-            "HTTPStatusCode": 200, 
-            "RequestId": "735cb42e-cd93-11e5-bb26-fd51e2c05e26"
-        }
-    }
-}
\ No newline at end of file
diff --git a/tests/data/placebo/test_transport_buffer_flush/logs.CreateLogGroup_1.json b/tests/data/placebo/test_transport_buffer_flush/logs.CreateLogGroup_1.json
deleted file mode 100644
--- a/tests/data/placebo/test_transport_buffer_flush/logs.CreateLogGroup_1.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
-    "status_code": 400, 
-    "data": {
-        "ResponseMetadata": {
-            "HTTPStatusCode": 400, 
-            "RequestId": "8a3ea59e-e2d9-11e5-9102-af4bf900bfaa"
-        }, 
-        "Error": {
-            "Message": "The specified log group already exists", 
-            "Code": "ResourceAlreadyExistsException"
-        }
-    }
-}
\ No newline at end of file
diff --git a/tests/data/placebo/test_transport_buffer_flush/logs.CreateLogStream_1.json b/tests/data/placebo/test_transport_buffer_flush/logs.CreateLogStream_1.json
--- a/tests/data/placebo/test_transport_buffer_flush/logs.CreateLogStream_1.json
+++ b/tests/data/placebo/test_transport_buffer_flush/logs.CreateLogStream_1.json
@@ -2,12 +2,20 @@
     "status_code": 400, 
     "data": {
         "ResponseMetadata": {
+            "RetryAttempts": 0, 
             "HTTPStatusCode": 400, 
-            "RequestId": "8aa7a046-e2d9-11e5-a277-013d8210e78d"
+            "RequestId": "5d401ac1-b4d3-11e6-b1e9-931109f185cd", 
+            "HTTPHeaders": {
+                "x-amzn-requestid": "5d401ac1-b4d3-11e6-b1e9-931109f185cd", 
+                "date": "Sun, 27 Nov 2016 18:57:37 GMT", 
+                "content-length": "90", 
+                "content-type": "application/x-amz-json-1.1", 
+                "nncoection": "close"
+            }
         }, 
         "Error": {
-            "Message": "The specified log stream already exists", 
-            "Code": "ResourceAlreadyExistsException"
+            "Message": "The specified log group does not exist.", 
+            "Code": "ResourceNotFoundException"
         }
     }
 }
\ No newline at end of file
diff --git a/tests/data/placebo/test_transport_buffer_flush/logs.DescribeLogGroups_1.json b/tests/data/placebo/test_transport_buffer_flush/logs.DescribeLogGroups_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_transport_buffer_flush/logs.DescribeLogGroups_1.json
@@ -0,0 +1,17 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "5d107ecf-b4d3-11e6-8ca5-c5e76461387c", 
+            "HTTPHeaders": {
+                "x-amzn-requestid": "5d107ecf-b4d3-11e6-8ca5-c5e76461387c", 
+                "date": "Sun, 27 Nov 2016 18:57:37 GMT", 
+                "content-length": "16", 
+                "content-type": "application/x-amz-json-1.1"
+            }
+        }, 
+        "logGroups": []
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_transport_buffer_flush/logs.PutLogEvents_1.json b/tests/data/placebo/test_transport_buffer_flush/logs.PutLogEvents_1.json
deleted file mode 100644
--- a/tests/data/placebo/test_transport_buffer_flush/logs.PutLogEvents_1.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
-    "status_code": 400, 
-    "data": {
-        "ResponseMetadata": {
-            "HTTPStatusCode": 400, 
-            "RequestId": "8ad36b5d-e2d9-11e5-8deb-e540c8247bdb"
-        }, 
-        "Error": {
-            "Message": "The given batch of log events has already been accepted. The next batch can be sent with sequenceToken: 49547819337610807837224084818942476068710828687556098530", 
-            "Code": "DataAlreadyAcceptedException"
-        }
-    }
-}
\ No newline at end of file
diff --git a/tests/data/placebo/test_transport_buffer_flush/logs.PutLogEvents_2.json b/tests/data/placebo/test_transport_buffer_flush/logs.PutLogEvents_2.json
deleted file mode 100644
--- a/tests/data/placebo/test_transport_buffer_flush/logs.PutLogEvents_2.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
-    "status_code": 200, 
-    "data": {
-        "nextSequenceToken": "49547819337610807837224084845825359519481360597802564066", 
-        "ResponseMetadata": {
-            "HTTPStatusCode": 200, 
-            "RequestId": "8ae6cbfc-e2d9-11e5-bb0b-23e799246a14"
-        }
-    }
-}
\ No newline at end of file
diff --git a/tests/test_log.py b/tests/test_log.py
--- a/tests/test_log.py
+++ b/tests/test_log.py
@@ -23,7 +23,11 @@ class LogTest(BaseTest):
 
     def test_existing_stream(self):
         session_factory = self.replay_flight_data('test_log_existing_stream')
-        handler = CloudWatchLogHandler(session_factory=session_factory)
+        client =  session_factory().client('logs')
+        group_name = "/custodian-dev"
+        client.create_log_group(logGroupName=group_name)
+        handler = CloudWatchLogHandler(
+            group_name, session_factory=session_factory)
         log = logging.getLogger("custodian")
         log.addHandler(handler)
         self.addCleanup(log.removeHandler, handler)
@@ -40,7 +44,7 @@ def test_time_flush(self):
         log = logging.getLogger("test-c7n")
         handler = CloudWatchLogHandler(
             "test-maid-4", "alpha", session_factory=session_factory)
-        handler.batch_interval = 1
+        handler.batch_interval = 0.1
         log.addHandler(handler)
         self.addCleanup(log.removeHandler, handler)
         log.setLevel(logging.DEBUG)
@@ -48,7 +52,7 @@ def test_time_flush(self):
         for i in range(100, 105):
             log.info('hello world %s' % i)
 
-        time.sleep(1.1)
+        time.sleep(0.2)
         log.info('bye world')
         self.assertFalse(handler.buf)
         handler.flush()