Repository: cloud-custodian/cloud-custodian

Issue:
aws - error for vpc flow-log filter
Actual Command being run 

```shell

-------------------------------------------------------------------------------------------------
(custodian) [ccustodian@bax-global-abc-testing custodian]$ custodian run --verbose enable-vpc-flow.yml  --region us-east-2 --output-dir  /var/tmp/flow-reject2
--------------------------------------------------------------------------------------------------

2019-12-06 19:09:41,769: custodian.commands:DEBUG Loaded file enable-vpc-flow.yml. Contains 1 policies
2019-12-06 19:09:42,066: custodian.output:DEBUG Storing output with <LogFile file:///var/tmp/flow-reject2/vpc-flow-log-check/custodian-run.log>
2019-12-06 19:09:42,080: custodian.policy:DEBUG Running policy:vpc-flow-log-check resource:vpc region:us-east-2 c7n:0.8.45.1
2019-12-06 19:09:42,080: custodian.cache:DEBUG Using cache file /home/ccustodian/.cache/cloud-custodian.cache
2019-12-06 19:09:42,081: custodian.resources.vpc:DEBUG Using cached c7n.resources.vpc.Vpc: 4
2019-12-06 19:09:42,284: custodian.output:DEBUG metric:PolicyException Count:1 policy:vpc-flow-log-check restype:vpc
2019-12-06 19:09:42,284: custodian.output:DEBUG metric:ApiCalls Count:1 policy:vpc-flow-log-check restype:vpc
2019-12-06 19:09:42,284: custodian.output:ERROR Error while executing policy
Traceback (most recent call last):
  File "/home/ccustodian/custodian/lib64/python3.7/site-packages/c7n/policy.py", line 287, in run
    resources = self.policy.resource_manager.resources()
  File "/home/ccustodian/custodian/lib64/python3.7/site-packages/c7n/query.py", line 460, in resources
    resources = self.filter_resources(resources)
  File "/home/ccustodian/custodian/lib64/python3.7/site-packages/c7n/manager.py", line 108, in filter_resources
    resources = f.process(resources, event)
  File "/home/ccustodian/custodian/lib64/python3.7/site-packages/c7n/filters/core.py", line 300, in process
    return self.process_set(resources, event)
  File "/home/ccustodian/custodian/lib64/python3.7/site-packages/c7n/filters/core.py", line 319, in process_set
    resources = f.process(resources, event)
  File "/home/ccustodian/custodian/lib64/python3.7/site-packages/c7n/resources/vpc.py", line 156, in process
    log_group_match = (log_group is None) or op(fl['LogGroupName'], log_group)
KeyError: 'LogGroupName'
2019-12-06 19:09:42,286: custodian.commands:ERROR Error while executing policy vpc-flow-log-check, continuing
Traceback (most recent call last):
  File "/home/ccustodian/custodian/lib64/python3.7/site-packages/c7n/commands.py", line 271, in run
    policy()
  File "/home/ccustodian/custodian/lib64/python3.7/site-packages/c7n/policy.py", line 1028, in __call__
    resources = mode.run()
  File "/home/ccustodian/custodian/lib64/python3.7/site-packages/c7n/policy.py", line 287, in run
    resources = self.policy.resource_manager.resources()
  File "/home/ccustodian/custodian/lib64/python3.7/site-packages/c7n/query.py", line 460, in resources
    resources = self.filter_resources(resources)
  File "/home/ccustodian/custodian/lib64/python3.7/site-packages/c7n/manager.py", line 108, in filter_resources
    resources = f.process(resources, event)
  File "/home/ccustodian/custodian/lib64/python3.7/site-packages/c7n/filters/core.py", line 300, in process
    return self.process_set(resources, event)
  File "/home/ccustodian/custodian/lib64/python3.7/site-packages/c7n/filters/core.py", line 319, in process_set
    resources = f.process(resources, event)
  File "/home/ccustodian/custodian/lib64/python3.7/site-packages/c7n/resources/vpc.py", line 156, in process
    log_group_match = (log_group is None) or op(fl['LogGroupName'], log_group)
KeyError: 'LogGroupName'
```

policy yaml 
```yaml
policies:
  - name: vpc-flow-log-check
    resource: vpc
    filters:
       - not:
            - type: flow-logs
              enabled: true
              set-op: or
              op: equal
              traffic-type: reject
              status: active
              log-group: bax-sand-flow-reject
```


Logging file metadata.json --->

```json
{
  "policy": {
    "name": "vpc-flow-log-check",
    "resource": "vpc",
    "filters": [
      {
        "not": [
          {
            "type": "flow-logs",
            "enabled": true,
            "set-op": "or",
            "op": "equal",
            "traffic-type": "reject",
            "status": "active",
            "log-group": "bax-sand-flow-reject"
          }
        ]
      }
    ]
  },
  "version": "0.8.45.1",
  "execution": {
    "id": "4afbd47c-9b13-4609-9457-ea1ef08755dc",
    "start": 1575659382.0659943,
    "end_time": 1575659382.2835417,
    "duration": 0.21754741668701172
  },
  "config": {
    "region": "us-east-1",
    "regions": [
      "us-east-2"
    ],
    "cache": "~/.cache/cloud-custodian.cache",
    "profile": null,
    "account_id": null,
    "assume_role": null,
    "external_id": null,
    "log_group": null,
    "tracer": null,
    "metrics_enabled": null,
    "metrics": null,
    "output_dir": "/var/tmp/flow-reject2",
    "cache_period": 15,
    "dryrun": false,
    "authorization_file": null,
    "subparser": "run",
    "config": null,
    "configs": [
      "enable-vpc-flow.yml"
    ],
    "policy_filters": [],
    "resource_types": [],
    "verbose": 1,
    "quiet": null,
    "debug": false,
    "skip_validation": false,
    "command": "c7n.commands.run",
    "vars": null
  },
  "sys-stats": {},
  "api-stats": {
    "ec2.DescribeFlowLogs": 1
  },
  "metrics": [
    {
      "MetricName": "PolicyException",
      "Timestamp": "2019-12-06T19:09:42.283523",
      "Value": 1,
      "Unit": "Count"
    }
  ]
}
```



Discussion:

Resolving patch:
diff --git a/c7n/resources/vpc.py b/c7n/resources/vpc.py
--- a/c7n/resources/vpc.py
+++ b/c7n/resources/vpc.py
@@ -98,6 +98,7 @@ class FlowLogFilter(Filter):
            'destination': {'type': 'string'},
            'destination-type': {'enum': ['s3', 'cloud-watch-logs']},
            'traffic-type': {'enum': ['accept', 'reject', 'all']},
+           'log-format': {'type': 'string'},
            'log-group': {'type': 'string'}})
 
     permissions = ('ec2:DescribeFlowLogs',)
@@ -117,6 +118,7 @@ def process(self, resources, event=None):
 
         enabled = self.data.get('enabled', False)
         log_group = self.data.get('log-group')
+        log_format = self.data.get('log-format')
         traffic_type = self.data.get('traffic-type')
         destination_type = self.data.get('destination-type')
         destination = self.data.get('destination')
@@ -153,11 +155,12 @@ def process(self, resources, event=None):
                         traffic_type is None) or op(
                         fl['TrafficType'],
                         traffic_type.upper())
-                    log_group_match = (log_group is None) or op(fl['LogGroupName'], log_group)
-
+                    log_group_match = (log_group is None) or op(fl.get('LogGroupName'), log_group)
+                    log_format_match = (log_format is None) or op(fl.get('LogFormat'), log_format)
                     # combine all conditions to check if flow log matches the spec
                     fl_match = (status_match and traffic_type_match and dest_match and
-                                log_group_match and dest_type_match and delivery_status_match)
+                                log_format_match and log_group_match and
+                                dest_type_match and delivery_status_match)
                     fl_matches.append(fl_match)
 
                 if set_op == 'or':
@@ -1980,6 +1983,7 @@ class CreateFlowLogs(BaseAction):
             'DeliverLogsPermissionArn': {'type': 'string'},
             'LogGroupName': {'type': 'string'},
             'LogDestination': {'type': 'string'},
+            'LogFormat': {'type': 'string'},
             'LogDestinationType': {'enum': ['s3', 'cloud-watch-logs']},
             'TrafficType': {
                 'type': 'string',


Test patch: