Repository: cloud-custodian/cloud-custodian

Issue:
aws - set-vpc-flow log support create log group if not exists
For enabling VPC flow-logs through Custodian, If the LogGroup doesn't exist Custodian should create a LogGroup before enabling VPC flow-logs.


Discussion:
@kapilt should this functionality look something like this?
```
  - name: enable-vpc-flow-logs
    resource: vpc
    filters:
      - type: flow-logs
        enabled: false
    actions:
      - type: set-flow-log
        DeliverLogsPermissionArn: "arn:aws:iam::{account_id}:role/X"
        LogGroupName: /flow-log
        CreateGroup: true
```
Or just:
```
  - name: enable-vpc-flow-logs
    resource: vpc
    filters:
      - type: flow-logs
        enabled: false
    actions:
      - type: set-flow-log
        DeliverLogsPermissionArn: "arn:aws:iam::{account_id}:role/X"
        LogGroupName: /flow-log
```
such that it would create LogGroup: /flow-log automatically if it doesn't exist
@PratMis i think this got addressed?
Don't think so, didn't get a chance to work on this! I'll put something together

Resolving patch:
diff --git a/c7n/resources/vpc.py b/c7n/resources/vpc.py
--- a/c7n/resources/vpc.py
+++ b/c7n/resources/vpc.py
@@ -2200,7 +2200,7 @@ class CreateFlowLogs(BaseAction):
                 DeliverLogsPermissionArn: arn:iam:role
                 LogGroupName: /custodian/vpc/flowlogs/
     """
-    permissions = ('ec2:CreateFlowLogs',)
+    permissions = ('ec2:CreateFlowLogs', 'logs:CreateLogGroup',)
     schema = {
         'type': 'object',
         'additionalProperties': False,
@@ -2297,7 +2297,8 @@ def process(self, resources):
         params['ResourceType'] = self.RESOURCE_ALIAS[model.arn_type]
         params['TrafficType'] = self.data.get('TrafficType', 'ALL').upper()
         params['MaxAggregationInterval'] = self.data.get('MaxAggregationInterval', 600)
-
+        if self.data.get('LogDestinationType', 'cloud-watch-logs') == 'cloud-watch-logs':
+            self.process_log_group(self.data.get('LogGroupName'))
         try:
             results = client.create_flow_logs(**params)
 
@@ -2312,3 +2313,10 @@ def process(self, resources):
                     e.response['Error']['Message'])
             else:
                 raise
+
+    def process_log_group(self, logroup):
+        client = local_session(self.manager.session_factory).client('logs')
+        try:
+            client.create_log_group(logGroupName=logroup)
+        except client.exceptions.ResourceAlreadyExistsException:
+            pass


Test patch:
diff --git a/tests/data/placebo/test_vpc_create_flow_logs/ec2.CreateFlowLogs_1.json b/tests/data/placebo/test_vpc_create_flow_logs/ec2.CreateFlowLogs_1.json
--- a/tests/data/placebo/test_vpc_create_flow_logs/ec2.CreateFlowLogs_1.json
+++ b/tests/data/placebo/test_vpc_create_flow_logs/ec2.CreateFlowLogs_1.json
@@ -1,22 +1,11 @@
 {
-    "status_code": 200, 
+    "status_code": 200,
     "data": {
-        "Unsuccessful": [], 
-        "ResponseMetadata": {
-            "RetryAttempts": 0, 
-            "HTTPStatusCode": 200, 
-            "RequestId": "f1bbb838-5fed-41bc-832f-9ec5b65cc34f", 
-            "HTTPHeaders": {
-                "transfer-encoding": "chunked", 
-                "vary": "Accept-Encoding", 
-                "server": "AmazonEC2", 
-                "content-type": "text/xml;charset=UTF-8", 
-                "date": "Fri, 06 Apr 2018 16:51:45 GMT"
-            }
-        }, 
+        "ClientToken": "6zPNON5jQ9rEuL4pDVwzJlm70HqAQcYDkaOEHQr9gos=",
         "FlowLogIds": [
-            "fl-f076b299"
-        ], 
-        "ClientToken": "zc3APQuNTpR/2VUebosuWy3DOrWJhkyq7pcWIVGeKkc="
+            "fl-041cdc74dc6435446"
+        ],
+        "Unsuccessful": [],
+        "ResponseMetadata": {}
     }
 }
\ No newline at end of file
diff --git a/tests/data/placebo/test_vpc_create_flow_logs/ec2.CreateFlowLogs_2.json b/tests/data/placebo/test_vpc_create_flow_logs/ec2.CreateFlowLogs_2.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_vpc_create_flow_logs/ec2.CreateFlowLogs_2.json
@@ -0,0 +1,11 @@
+{
+    "status_code": 200,
+    "data": {
+        "ClientToken": "JuutuXS0nHXAfsQ1lOWaMAL6+tpqT5mqHMnEGPdD90I=",
+        "FlowLogIds": [
+            "fl-0c28f7d7b5cd29477"
+        ],
+        "Unsuccessful": [],
+        "ResponseMetadata": {}
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeFlowLogs_1.json b/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeFlowLogs_1.json
--- a/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeFlowLogs_1.json
+++ b/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeFlowLogs_1.json
@@ -1,18 +1,7 @@
 {
-    "status_code": 200, 
+    "status_code": 200,
     "data": {
-        "FlowLogs": [], 
-        "ResponseMetadata": {
-            "RetryAttempts": 0, 
-            "HTTPStatusCode": 200, 
-            "RequestId": "a0cd7ea9-229e-4d62-9ab9-fbb6793c95e6", 
-            "HTTPHeaders": {
-                "transfer-encoding": "chunked", 
-                "vary": "Accept-Encoding", 
-                "server": "AmazonEC2", 
-                "content-type": "text/xml;charset=UTF-8", 
-                "date": "Fri, 06 Apr 2018 16:51:44 GMT"
-            }
-        }
+        "FlowLogs": [],
+        "ResponseMetadata": {}
     }
 }
\ No newline at end of file
diff --git a/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeFlowLogs_2.json b/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeFlowLogs_2.json
--- a/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeFlowLogs_2.json
+++ b/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeFlowLogs_2.json
@@ -1,37 +1,31 @@
 {
-    "status_code": 200, 
+    "status_code": 200,
     "data": {
         "FlowLogs": [
             {
-                "ResourceId": "vpc-7af45101", 
                 "CreationTime": {
-                    "hour": 16, 
-                    "__class__": "datetime", 
-                    "month": 4, 
-                    "second": 46, 
-                    "microsecond": 20000, 
-                    "year": 2018, 
-                    "day": 6, 
-                    "minute": 51
-                }, 
-                "LogGroupName": "/custodian/vpc_logs/", 
-                "TrafficType": "ALL", 
-                "FlowLogStatus": "ACTIVE", 
-                "FlowLogId": "fl-f076b299", 
-                "DeliverLogsPermissionArn": "arn:aws:iam::644160558196:role/flowlogsRole"
+                    "__class__": "datetime",
+                    "year": 2020,
+                    "month": 4,
+                    "day": 27,
+                    "hour": 4,
+                    "minute": 33,
+                    "second": 16,
+                    "microsecond": 30000
+                },
+                "DeliverLogsPermissionArn": "arn:aws:iam::644160558196:role/flowlogsRole",
+                "DeliverLogsStatus": "SUCCESS",
+                "FlowLogId": "fl-041cdc74dc6435446",
+                "FlowLogStatus": "ACTIVE",
+                "LogGroupName": "/custodian/vpc_log/",
+                "ResourceId": "vpc-d2d616b5",
+                "TrafficType": "ALL",
+                "LogDestinationType": "cloud-watch-logs",
+                "LogFormat": "${version} ${account-id} ${interface-id} ${srcaddr} ${dstaddr} ${srcport} ${dstport} ${protocol} ${packets} ${bytes} ${start} ${end} ${action} ${log-status}",
+                "Tags": [],
+                "MaxAggregationInterval": 600
             }
-        ], 
-        "ResponseMetadata": {
-            "RetryAttempts": 0, 
-            "HTTPStatusCode": 200, 
-            "RequestId": "844aacb9-04d2-42de-b5db-9cb92b5f2793", 
-            "HTTPHeaders": {
-                "transfer-encoding": "chunked", 
-                "vary": "Accept-Encoding", 
-                "server": "AmazonEC2", 
-                "content-type": "text/xml;charset=UTF-8", 
-                "date": "Fri, 06 Apr 2018 16:51:46 GMT"
-            }
-        }
+        ],
+        "ResponseMetadata": {}
     }
 }
\ No newline at end of file
diff --git a/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeFlowLogs_3.json b/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeFlowLogs_3.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeFlowLogs_3.json
@@ -0,0 +1,31 @@
+{
+    "status_code": 200,
+    "data": {
+        "FlowLogs": [
+            {
+                "CreationTime": {
+                    "__class__": "datetime",
+                    "year": 2020,
+                    "month": 4,
+                    "day": 27,
+                    "hour": 4,
+                    "minute": 33,
+                    "second": 16,
+                    "microsecond": 30000
+                },
+                "DeliverLogsPermissionArn": "arn:aws:iam::644160558196:role/flowlogsRole",
+                "DeliverLogsStatus": "SUCCESS",
+                "FlowLogId": "fl-041cdc74dc6435446",
+                "FlowLogStatus": "ACTIVE",
+                "LogGroupName": "/custodian/vpc_log/",
+                "ResourceId": "vpc-d2d616b5",
+                "TrafficType": "ALL",
+                "LogDestinationType": "cloud-watch-logs",
+                "LogFormat": "${version} ${account-id} ${interface-id} ${srcaddr} ${dstaddr} ${srcport} ${dstport} ${protocol} ${packets} ${bytes} ${start} ${end} ${action} ${log-status}",
+                "Tags": [],
+                "MaxAggregationInterval": 600
+            }
+        ],
+        "ResponseMetadata": {}
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeFlowLogs_4.json b/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeFlowLogs_4.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeFlowLogs_4.json
@@ -0,0 +1,31 @@
+{
+    "status_code": 200,
+    "data": {
+        "FlowLogs": [
+            {
+                "CreationTime": {
+                    "__class__": "datetime",
+                    "year": 2020,
+                    "month": 4,
+                    "day": 27,
+                    "hour": 4,
+                    "minute": 33,
+                    "second": 17,
+                    "microsecond": 121000
+                },
+                "DeliverLogsPermissionArn": "arn:aws:iam::644160558196:role/flowlogsRole",
+                "DeliverLogsStatus": "SUCCESS",
+                "FlowLogId": "fl-0c28f7d7b5cd29477",
+                "FlowLogStatus": "ACTIVE",
+                "LogGroupName": "/custodian/vpc_log/",
+                "ResourceId": "vpc-03005fb9b8740263d",
+                "TrafficType": "ALL",
+                "LogDestinationType": "cloud-watch-logs",
+                "LogFormat": "${version} ${account-id} ${interface-id} ${srcaddr} ${dstaddr} ${srcport} ${dstport} ${protocol} ${packets} ${bytes} ${start} ${end} ${action} ${log-status}",
+                "Tags": [],
+                "MaxAggregationInterval": 600
+            }
+        ],
+        "ResponseMetadata": {}
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeVpcs_1.json b/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeVpcs_1.json
--- a/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeVpcs_1.json
+++ b/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeVpcs_1.json
@@ -1,42 +1,60 @@
 {
-    "status_code": 200, 
+    "status_code": 200,
     "data": {
         "Vpcs": [
             {
-                "VpcId": "vpc-7af45101", 
-                "InstanceTenancy": "default", 
+                "CidrBlock": "10.205.0.0/18",
+                "DhcpOptionsId": "dopt-24ff1940",
+                "State": "available",
+                "VpcId": "vpc-03005fb9b8740263d",
+                "OwnerId": "644160558196",
+                "InstanceTenancy": "default",
+                "CidrBlockAssociationSet": [
+                    {
+                        "AssociationId": "vpc-cidr-assoc-0f6765530d80dc044",
+                        "CidrBlock": "10.205.0.0/18",
+                        "CidrBlockState": {
+                            "State": "associated"
+                        }
+                    }
+                ],
+                "IsDefault": false,
                 "Tags": [
                     {
-                        "Value": "FlowLogTest", 
-                        "Key": "Name"
+                        "Key": "Owner",
+                        "Value": "c7n"
+                    },
+                    {
+                        "Key": "Name",
+                        "Value": "testing-vpc"
                     }
-                ], 
+                ]
+            },
+            {
+                "CidrBlock": "172.31.0.0/16",
+                "DhcpOptionsId": "dopt-24ff1940",
+                "State": "available",
+                "VpcId": "vpc-d2d616b5",
+                "OwnerId": "644160558196",
+                "InstanceTenancy": "default",
                 "CidrBlockAssociationSet": [
                     {
-                        "AssociationId": "vpc-cidr-assoc-7c1eb811", 
-                        "CidrBlock": "10.0.16.0/24", 
+                        "AssociationId": "vpc-cidr-assoc-33669f5a",
+                        "CidrBlock": "172.31.0.0/16",
                         "CidrBlockState": {
                             "State": "associated"
                         }
                     }
-                ], 
-                "State": "available", 
-                "DhcpOptionsId": "dopt-4d4c3035", 
-                "CidrBlock": "10.0.16.0/24", 
-                "IsDefault": false
-            }
-        ], 
-        "ResponseMetadata": {
-            "RetryAttempts": 0, 
-            "HTTPStatusCode": 200, 
-            "RequestId": "e667839a-e438-453c-877a-4c75c0c26add", 
-            "HTTPHeaders": {
-                "transfer-encoding": "chunked", 
-                "vary": "Accept-Encoding", 
-                "server": "AmazonEC2", 
-                "content-type": "text/xml;charset=UTF-8", 
-                "date": "Fri, 06 Apr 2018 16:51:44 GMT"
+                ],
+                "IsDefault": true,
+                "Tags": [
+                    {
+                        "Key": "Name",
+                        "Value": "FlowLogTest"
+                    }
+                ]
             }
-        }
+        ],
+        "ResponseMetadata": {}
     }
 }
\ No newline at end of file
diff --git a/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeVpcs_2.json b/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeVpcs_2.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_vpc_create_flow_logs/ec2.DescribeVpcs_2.json
@@ -0,0 +1,60 @@
+{
+    "status_code": 200,
+    "data": {
+        "Vpcs": [
+            {
+                "CidrBlock": "10.205.0.0/18",
+                "DhcpOptionsId": "dopt-24ff1940",
+                "State": "available",
+                "VpcId": "vpc-03005fb9b8740263d",
+                "OwnerId": "644160558196",
+                "InstanceTenancy": "default",
+                "CidrBlockAssociationSet": [
+                    {
+                        "AssociationId": "vpc-cidr-assoc-0f6765530d80dc044",
+                        "CidrBlock": "10.205.0.0/18",
+                        "CidrBlockState": {
+                            "State": "associated"
+                        }
+                    }
+                ],
+                "IsDefault": false,
+                "Tags": [
+                    {
+                        "Key": "Owner",
+                        "Value": "c7n"
+                    },
+                    {
+                        "Key": "Name",
+                        "Value": "testing-vpc"
+                    }
+                ]
+            },
+            {
+                "CidrBlock": "172.31.0.0/16",
+                "DhcpOptionsId": "dopt-24ff1940",
+                "State": "available",
+                "VpcId": "vpc-d2d616b5",
+                "OwnerId": "644160558196",
+                "InstanceTenancy": "default",
+                "CidrBlockAssociationSet": [
+                    {
+                        "AssociationId": "vpc-cidr-assoc-33669f5a",
+                        "CidrBlock": "172.31.0.0/16",
+                        "CidrBlockState": {
+                            "State": "associated"
+                        }
+                    }
+                ],
+                "IsDefault": true,
+                "Tags": [
+                    {
+                        "Key": "Name",
+                        "Value": "FlowLogTest"
+                    }
+                ]
+            }
+        ],
+        "ResponseMetadata": {}
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_vpc_create_flow_logs/logs.CreateLogGroup_1.json b/tests/data/placebo/test_vpc_create_flow_logs/logs.CreateLogGroup_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_vpc_create_flow_logs/logs.CreateLogGroup_1.json
@@ -0,0 +1,6 @@
+{
+    "status_code": 200,
+    "data": {
+        "ResponseMetadata": {}
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_vpc_create_flow_logs/logs.CreateLogGroup_2.json b/tests/data/placebo/test_vpc_create_flow_logs/logs.CreateLogGroup_2.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_vpc_create_flow_logs/logs.CreateLogGroup_2.json
@@ -0,0 +1,10 @@
+{
+    "status_code": 400,
+    "data": {
+        "Error": {
+            "Message": "The specified log group already exists",
+            "Code": "ResourceAlreadyExistsException"
+        },
+        "ResponseMetadata": {}
+    }
+}
\ No newline at end of file
diff --git a/tests/test_vpc.py b/tests/test_vpc.py
--- a/tests/test_vpc.py
+++ b/tests/test_vpc.py
@@ -2725,6 +2725,7 @@ class FlowLogsTest(BaseTest):
 
     def test_vpc_create_flow_logs(self):
         session_factory = self.replay_flight_data("test_vpc_create_flow_logs")
+        # creates log group
         p = self.load_policy(
             {
                 "name": "c7n-create-vpc-flow-logs",
@@ -2737,7 +2738,7 @@ def test_vpc_create_flow_logs(self):
                     {
                         "type": "set-flow-log",
                         "DeliverLogsPermissionArn": "arn:aws:iam::644160558196:role/flowlogsRole",
-                        "LogGroupName": "/custodian/vpc_logs/",
+                        "LogGroupName": "/custodian/vpc_log/",
                         "TrafficType": "ALL",
                     }
                 ],
@@ -2746,14 +2747,40 @@ def test_vpc_create_flow_logs(self):
         )
         resources = p.run()
         self.assertEqual(len(resources), 1)
-        self.assertEqual(resources[0]["VpcId"], "vpc-7af45101")
+        self.assertEqual(resources[0]["VpcId"], "vpc-d2d616b5")
         client = session_factory(region="us-east-1").client("ec2")
         logs = client.describe_flow_logs(
             Filters=[{"Name": "resource-id", "Values": [resources[0]["VpcId"]]}]
-        )[
-            "FlowLogs"
-        ]
+        )["FlowLogs"]
         self.assertEqual(logs[0]["ResourceId"], resources[0]["VpcId"])
+        # p1 has same log group name
+        p1 = self.load_policy(
+            {
+                "name": "c7n-create-vpc-flow-logs",
+                "resource": "vpc",
+                "filters": [
+                    {"tag:Name": "testing-vpc"},
+                    {"type": "flow-logs", "enabled": False}
+                ],
+                "actions": [
+                    {
+                        "type": "set-flow-log",
+                        "DeliverLogsPermissionArn": "arn:aws:iam::644160558196:role/flowlogsRole",
+                        "LogGroupName": "/custodian/vpc_log/",
+                        "TrafficType": "ALL",
+                    }
+                ],
+            },
+            session_factory=session_factory,
+        )
+        res = p1.run()
+        self.assertEqual(len(res), 1)
+        self.assertEqual(res[0]["VpcId"], "vpc-03005fb9b8740263d")
+        client = session_factory(region="us-east-1").client("ec2")
+        lgs = client.describe_flow_logs(
+            Filters=[{"Name": "resource-id", "Values": [res[0]["VpcId"]]}]
+        )["FlowLogs"]
+        self.assertEqual(lgs[0]["ResourceId"], res[0]["VpcId"])
 
     def test_vpc_flow_log_destination(self):
         session_factory = self.replay_flight_data('test_vpc_flow_filter_destination')