Repository: cloud-custodian/cloud-custodian

Issue:
Remove Permissions - Security Group Issue
I am having issues matching for specific security group rules when the there are multiple rules that have the same FromPort.

For Example if the Security Group ruleset looks like this:
Type | Protocol | Port Range | Source
 SSH |     TCP    |       22         | 10.1.23.2/32
 SSH |     TCP    |       22         | 0.0.0.0/0
HTTP|    TCP     |       80        | 10.1.23.2/32

Here is my CC configuration:
    resource: security-group
    filters:
      - type: ingress
        Cidr:
          value: 0.0.0.0/0
          op: eq
          value_type: cidr
    actions:
      - type: remove-permissions

However, when running this configuration, it matches on the 0.0.0.0/0 rule but ALSO any other rules that have the same port.  I only want it to match on 0.0.0.0/0 rule.

DELETES --> SSH |     TCP    |       22         | 10.1.23.2/32
DELETES --> SSH |     TCP    |       22         | 0.0.0.0/0
KEEPS  --> HTTP|    TCP     |       80        | 10.1.23.2/32

Resources File:
    "MatchedIpPermissions": [
      {
        "PrefixListIds": [],
        "FromPort": 22,
        "IpRanges": [
          {
            "CidrIp": "0.0.0.0/0"
          },
          {
            "CidrIp": "10.1.23.2/32"
          }
        ],
        "ToPort": 22,
        "IpProtocol": "tcp",
        "UserIdGroupPairs": [],
        "Ipv6Ranges": []
      }
    ],


Discussion:
thanks for the bug report!  that leads to an interesting find, i did some testing on this, it looks like aws will combine separate rules by port using multiple ip ranges when doing a describe on the rules. i'll need to check around if this rule folding holds for other rule attribute matches. if its just ip range grouping we'll need to ungroup/multiplex by iprange before filter matching or permission removal. afaics this shouldn't affect our diff/patch implementation for security-groups.

![image](https://cloud.githubusercontent.com/assets/21650/21665326/f9379e20-d2b9-11e6-81bf-b3d4426c923c.png)

`aws ec2 describe-security-groups`

![image](https://cloud.githubusercontent.com/assets/21650/21665384/34ea20be-d2ba-11e6-8ee2-6bca8eb063b1.png)

revoking works on an individual iprange/cidr level

`aws ec2 revoke-security-group-ingress --group-id  sg-xxyyzz --cidr 0.0.0.0/0 --port 22 --protocol tcp`

`aws ec2 describe-security-groups`

![image](https://cloud.githubusercontent.com/assets/21650/21665512/e6bab498-d2ba-11e6-8030-a2e8786bf619.png)


looks like we'll need to ungroup/expand on port/protocol by prefixlists, useridgroup pairs, and ipranges.

Resolving patch:
diff --git a/c7n/resources/vpc.py b/c7n/resources/vpc.py
--- a/c7n/resources/vpc.py
+++ b/c7n/resources/vpc.py
@@ -602,10 +602,35 @@ def process_self_reference(self, perm, sg_id):
             found = self_reference & self.data['SelfReference']
         return found
 
+    def expand_permissions(self, permissions):
+        """Expand each list of cidr, prefix list, user id group pair
+        by port/protocol as an individual rule.
+
+        The console ux automatically expands them out as addition/removal is
+        per this expansion, the describe calls automatically group them.
+        """
+        for p in permissions:
+            np = dict(p)
+            values = {}
+            for k in (u'IpRanges',
+                      u'Ipv6Ranges',
+                      u'PrefixListIds',
+                      u'UserIdGroupPairs'):
+                values[k] = np.pop(k, ())
+                np[k] = []
+            for k, v in values.items():
+                if not v:
+                    continue
+                for e in v:
+                    ep = dict(np)
+                    ep[k] = [e]
+                    yield ep
+
     def __call__(self, resource):
         matched = []
         sg_id = resource['GroupId']
-        for perm in resource[self.ip_permissions_key]:
+
+        for perm in self.expand_permissions(resource[self.ip_permissions_key]):
             found = None
             for f in self.vfilters:
                 if f(perm):


Test patch:
diff --git a/tests/data/placebo/test_security_group_perm_expand/ec2.AuthorizeSecurityGroupIngress_1.json b/tests/data/placebo/test_security_group_perm_expand/ec2.AuthorizeSecurityGroupIngress_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_security_group_perm_expand/ec2.AuthorizeSecurityGroupIngress_1.json
@@ -0,0 +1,17 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "978f1be0-1633-4477-861b-1a2cb973e82b", 
+            "HTTPHeaders": {
+                "transfer-encoding": "chunked", 
+                "vary": "Accept-Encoding", 
+                "server": "AmazonEC2", 
+                "content-type": "text/xml;charset=UTF-8", 
+                "date": "Thu, 05 Jan 2017 02:25:38 GMT"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_security_group_perm_expand/ec2.AuthorizeSecurityGroupIngress_2.json b/tests/data/placebo/test_security_group_perm_expand/ec2.AuthorizeSecurityGroupIngress_2.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_security_group_perm_expand/ec2.AuthorizeSecurityGroupIngress_2.json
@@ -0,0 +1,17 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "4af9eee9-8882-47ae-a929-f5e5ceebb3dd", 
+            "HTTPHeaders": {
+                "transfer-encoding": "chunked", 
+                "vary": "Accept-Encoding", 
+                "server": "AmazonEC2", 
+                "content-type": "text/xml;charset=UTF-8", 
+                "date": "Thu, 05 Jan 2017 02:25:39 GMT"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_security_group_perm_expand/ec2.AuthorizeSecurityGroupIngress_3.json b/tests/data/placebo/test_security_group_perm_expand/ec2.AuthorizeSecurityGroupIngress_3.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_security_group_perm_expand/ec2.AuthorizeSecurityGroupIngress_3.json
@@ -0,0 +1,17 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "a01332fe-69c3-4cdb-af5d-53209f9c7803", 
+            "HTTPHeaders": {
+                "transfer-encoding": "chunked", 
+                "vary": "Accept-Encoding", 
+                "server": "AmazonEC2", 
+                "content-type": "text/xml;charset=UTF-8", 
+                "date": "Thu, 05 Jan 2017 02:25:39 GMT"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_security_group_perm_expand/ec2.CreateSecurityGroup_1.json b/tests/data/placebo/test_security_group_perm_expand/ec2.CreateSecurityGroup_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_security_group_perm_expand/ec2.CreateSecurityGroup_1.json
@@ -0,0 +1,18 @@
+{
+    "status_code": 200, 
+    "data": {
+        "GroupId": "sg-e3842c8a", 
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "3d274a94-0590-4aa7-b16e-7a1113918ba3", 
+            "HTTPHeaders": {
+                "transfer-encoding": "chunked", 
+                "vary": "Accept-Encoding", 
+                "server": "AmazonEC2", 
+                "content-type": "text/xml;charset=UTF-8", 
+                "date": "Thu, 05 Jan 2017 02:25:38 GMT"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_security_group_perm_expand/ec2.CreateSecurityGroup_2.json b/tests/data/placebo/test_security_group_perm_expand/ec2.CreateSecurityGroup_2.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_security_group_perm_expand/ec2.CreateSecurityGroup_2.json
@@ -0,0 +1,18 @@
+{
+    "status_code": 200, 
+    "data": {
+        "GroupId": "sg-e2842c8b", 
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "c57c595b-f59e-4971-a793-5beaf7009a86", 
+            "HTTPHeaders": {
+                "transfer-encoding": "chunked", 
+                "vary": "Accept-Encoding", 
+                "server": "AmazonEC2", 
+                "content-type": "text/xml;charset=UTF-8", 
+                "date": "Thu, 05 Jan 2017 02:25:38 GMT"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_security_group_perm_expand/ec2.CreateVpc_1.json b/tests/data/placebo/test_security_group_perm_expand/ec2.CreateVpc_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_security_group_perm_expand/ec2.CreateVpc_1.json
@@ -0,0 +1,27 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "d56b60b1-70a4-4ffc-85ee-3221b2077538", 
+            "HTTPHeaders": {
+                "transfer-encoding": "chunked", 
+                "vary": "Accept-Encoding", 
+                "server": "AmazonEC2", 
+                "content-type": "text/xml;charset=UTF-8", 
+                "date": "Thu, 05 Jan 2017 02:25:38 GMT"
+            }
+        }, 
+        "Vpc": {
+            "VpcId": "vpc-c25285ab", 
+            "InstanceTenancy": "default", 
+            "Tags": [], 
+            "Ipv6CidrBlockAssociationSet": [], 
+            "State": "pending", 
+            "DhcpOptionsId": "dopt-96bc48ff", 
+            "CidrBlock": "10.42.0.0/16", 
+            "IsDefault": false
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_security_group_perm_expand/ec2.DeleteSecurityGroup_1.json b/tests/data/placebo/test_security_group_perm_expand/ec2.DeleteSecurityGroup_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_security_group_perm_expand/ec2.DeleteSecurityGroup_1.json
@@ -0,0 +1,17 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "cf7f6352-4581-4013-8bac-d3ccf78f7c12", 
+            "HTTPHeaders": {
+                "transfer-encoding": "chunked", 
+                "vary": "Accept-Encoding", 
+                "server": "AmazonEC2", 
+                "content-type": "text/xml;charset=UTF-8", 
+                "date": "Thu, 05 Jan 2017 02:25:40 GMT"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_security_group_perm_expand/ec2.DeleteSecurityGroup_2.json b/tests/data/placebo/test_security_group_perm_expand/ec2.DeleteSecurityGroup_2.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_security_group_perm_expand/ec2.DeleteSecurityGroup_2.json
@@ -0,0 +1,17 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "5e994d87-350e-4643-89b1-f69b317ea9c5", 
+            "HTTPHeaders": {
+                "transfer-encoding": "chunked", 
+                "vary": "Accept-Encoding", 
+                "server": "AmazonEC2", 
+                "content-type": "text/xml;charset=UTF-8", 
+                "date": "Thu, 05 Jan 2017 02:25:40 GMT"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_security_group_perm_expand/ec2.DeleteVpc_1.json b/tests/data/placebo/test_security_group_perm_expand/ec2.DeleteVpc_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_security_group_perm_expand/ec2.DeleteVpc_1.json
@@ -0,0 +1,17 @@
+{
+    "status_code": 200, 
+    "data": {
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "b554562e-60f2-4731-840e-dccee836fcff", 
+            "HTTPHeaders": {
+                "transfer-encoding": "chunked", 
+                "vary": "Accept-Encoding", 
+                "server": "AmazonEC2", 
+                "content-type": "text/xml;charset=UTF-8", 
+                "date": "Thu, 05 Jan 2017 02:25:40 GMT"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_security_group_perm_expand/ec2.DescribeSecurityGroups_1.json b/tests/data/placebo/test_security_group_perm_expand/ec2.DescribeSecurityGroups_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_security_group_perm_expand/ec2.DescribeSecurityGroups_1.json
@@ -0,0 +1,157 @@
+{
+    "status_code": 200, 
+    "data": {
+        "SecurityGroups": [
+            {
+                "IpPermissionsEgress": [
+                    {
+                        "IpProtocol": "-1", 
+                        "PrefixListIds": [], 
+                        "IpRanges": [
+                            {
+                                "CidrIp": "0.0.0.0/0"
+                            }
+                        ], 
+                        "UserIdGroupPairs": [], 
+                        "Ipv6Ranges": []
+                    }
+                ], 
+                "Description": "inbound access", 
+                "IpPermissions": [
+                    {
+                        "PrefixListIds": [], 
+                        "FromPort": 443, 
+                        "IpRanges": [
+                            {
+                                "CidrIp": "10.42.1.0/24"
+                            }, 
+                            {
+                                "CidrIp": "10.42.2.0/24"
+                            }
+                        ], 
+                        "ToPort": 443, 
+                        "IpProtocol": "tcp", 
+                        "UserIdGroupPairs": [
+                            {
+                                "UserId": "644160558196", 
+                                "GroupId": "sg-e2842c8b"
+                            }
+                        ], 
+                        "Ipv6Ranges": []
+                    }
+                ], 
+                "GroupName": "allow-some-ingress", 
+                "VpcId": "vpc-c25285ab", 
+                "OwnerId": "644160558196", 
+                "GroupId": "sg-e3842c8a"
+            }, 
+            {
+                "IpPermissionsEgress": [
+                    {
+                        "IpProtocol": "-1", 
+                        "PrefixListIds": [], 
+                        "IpRanges": [
+                            {
+                                "CidrIp": "0.0.0.0/0"
+                            }
+                        ], 
+                        "UserIdGroupPairs": [], 
+                        "Ipv6Ranges": []
+                    }
+                ], 
+                "Description": "inbound ref access", 
+                "IpPermissions": [], 
+                "GroupName": "allowed-reference", 
+                "VpcId": "vpc-c25285ab", 
+                "OwnerId": "644160558196", 
+                "GroupId": "sg-e2842c8b"
+            }, 
+            {
+                "IpPermissionsEgress": [
+                    {
+                        "IpProtocol": "-1", 
+                        "PrefixListIds": [], 
+                        "IpRanges": [
+                            {
+                                "CidrIp": "0.0.0.0/0"
+                            }
+                        ], 
+                        "UserIdGroupPairs": [], 
+                        "Ipv6Ranges": []
+                    }
+                ], 
+                "Description": "default VPC security group", 
+                "Tags": [
+                    {
+                        "Value": "Private", 
+                        "Key": "NetworkLocation"
+                    }
+                ], 
+                "IpPermissions": [
+                    {
+                        "IpProtocol": "-1", 
+                        "PrefixListIds": [], 
+                        "IpRanges": [], 
+                        "UserIdGroupPairs": [
+                            {
+                                "UserId": "644160558196", 
+                                "GroupId": "sg-aa6c90c3"
+                            }
+                        ], 
+                        "Ipv6Ranges": []
+                    }
+                ], 
+                "GroupName": "default", 
+                "VpcId": "vpc-5f6c9936", 
+                "OwnerId": "644160558196", 
+                "GroupId": "sg-aa6c90c3"
+            }, 
+            {
+                "IpPermissionsEgress": [
+                    {
+                        "IpProtocol": "-1", 
+                        "PrefixListIds": [], 
+                        "IpRanges": [
+                            {
+                                "CidrIp": "0.0.0.0/0"
+                            }
+                        ], 
+                        "UserIdGroupPairs": [], 
+                        "Ipv6Ranges": []
+                    }
+                ], 
+                "Description": "default VPC security group", 
+                "IpPermissions": [
+                    {
+                        "IpProtocol": "-1", 
+                        "PrefixListIds": [], 
+                        "IpRanges": [], 
+                        "UserIdGroupPairs": [
+                            {
+                                "UserId": "644160558196", 
+                                "GroupId": "sg-e0842c89"
+                            }
+                        ], 
+                        "Ipv6Ranges": []
+                    }
+                ], 
+                "GroupName": "default", 
+                "VpcId": "vpc-c25285ab", 
+                "OwnerId": "644160558196", 
+                "GroupId": "sg-e0842c89"
+            }
+        ], 
+        "ResponseMetadata": {
+            "RetryAttempts": 0, 
+            "HTTPStatusCode": 200, 
+            "RequestId": "353bba5f-d0ca-4610-82ff-3f374778e796", 
+            "HTTPHeaders": {
+                "transfer-encoding": "chunked", 
+                "vary": "Accept-Encoding", 
+                "server": "AmazonEC2", 
+                "content-type": "text/xml;charset=UTF-8", 
+                "date": "Thu, 05 Jan 2017 02:25:39 GMT"
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/tests/test_vpc.py b/tests/test_vpc.py
--- a/tests/test_vpc.py
+++ b/tests/test_vpc.py
@@ -200,6 +200,7 @@ def test_only_ports(self):
             resources[0]['MatchedIpPermissions'],
             [{u'FromPort': 60000,
               u'IpProtocol': u'tcp',
+              u'Ipv6Ranges': [],
               u'IpRanges': [{u'CidrIp': u'10.2.0.0/16'}],
               u'PrefixListIds': [],
               u'ToPort': 62000,
@@ -369,6 +370,7 @@ def test_port_within_range(self):
             resources[0]['MatchedIpPermissions'],
             [{u'FromPort': 60000,
               u'IpProtocol': u'tcp',
+              u'Ipv6Ranges': [],
               u'IpRanges': [{u'CidrIp': u'10.2.0.0/16'}],
               u'PrefixListIds': [],
               u'ToPort': 62000,
@@ -509,6 +511,75 @@ def test_ports_ingress(self):
         manager = p.get_resource_manager()
         self.assertEqual(len(manager.filter_resources(resources)), 1)
 
+    def test_permission_expansion(self):
+        factory = self.replay_flight_data('test_security_group_perm_expand')
+        client = factory().client('ec2')
+        vpc_id = client.create_vpc(CidrBlock="10.42.0.0/16")['Vpc']['VpcId']
+        self.addCleanup(client.delete_vpc, VpcId=vpc_id)
+        sg_id = client.create_security_group(
+            GroupName="allow-some-ingress",
+            VpcId=vpc_id,
+            Description="inbound access")['GroupId']
+        sg2_id = client.create_security_group(
+            GroupName="allowed-reference",
+            VpcId=vpc_id,
+            Description="inbound ref access")['GroupId']
+        self.addCleanup(client.delete_security_group, GroupId=sg2_id)
+        self.addCleanup(client.delete_security_group, GroupId=sg_id)
+        client.authorize_security_group_ingress(
+            GroupId=sg_id,
+            IpPermissions=[{
+                'IpProtocol': 'tcp',
+                'FromPort': 443,
+                'ToPort': 443,
+                'IpRanges': [
+                    {
+                        'CidrIp': '10.42.1.0/24'
+                    }]
+            }])
+        client.authorize_security_group_ingress(
+            GroupId=sg_id,
+            IpPermissions=[{
+                'IpProtocol': 'tcp',
+                'FromPort': 443,
+                'ToPort': 443,
+                'IpRanges': [
+                    {
+                        'CidrIp': '10.42.2.0/24'
+                    }]
+            }])
+        client.authorize_security_group_ingress(
+            GroupId=sg_id,
+            IpPermissions=[{
+                'IpProtocol': 'tcp',
+                'FromPort': 443,
+                'ToPort': 443,
+                'UserIdGroupPairs': [{'GroupId': sg2_id}]
+                }])
+        p = self.load_policy({
+            'name': 'ingress-access',
+            'resource': 'security-group',
+            'filters': [
+                {'type': 'ingress',
+                 'Cidr': {
+                     'value': '10.42.1.1',
+                     'op': 'in',
+                     'value_type': 'cidr'}}]
+            }, session_factory=factory)
+        resources = p.run()
+        self.assertEqual(len(resources), 1)
+        self.assertEqual(
+            len(resources[0].get('MatchedIpPermissions', [])), 1)
+        self.assertEqual(
+            resources[0].get('MatchedIpPermissions', []),
+            [{u'FromPort': 443,
+              u'IpProtocol': u'tcp',
+              u'Ipv6Ranges': [],
+              u'PrefixListIds': [],
+              u'UserIdGroupPairs': [],
+              u'IpRanges': [{u'CidrIp': u'10.42.1.0/24'}],
+              u'ToPort': 443}])
+
     def test_cidr_ingress(self):
         factory = self.replay_flight_data('test_security_group_cidr_ingress')
         client = factory().client('ec2')
@@ -573,24 +644,25 @@ def test_cidr_size_egress(self):
             'name': 'wide-egress',
             'resource': 'security-group',
             'filters': [
+                {'GroupName': 'wide-egress'},
                 {'type': 'egress',
                  'Cidr': {
                      'value': 24,
                      'op': 'lt',
-                     'value_type': 'cidr_size'}},
-                {'GroupName': 'wide-egress'}]
+                     'value_type': 'cidr_size'}}]
             }, session_factory=factory)
         resources = p.run()
         self.assertEqual(len(resources), 1)
         self.assertEqual(
             len(resources[0].get('MatchedIpPermissionsEgress', [])), 1)
+
         self.assertEqual(
             resources[0]['MatchedIpPermissionsEgress'],
             [{u'FromPort': 443,
               u'IpProtocol': u'tcp',
+              u'Ipv6Ranges': [],
               u'IpRanges': [
-                  {u'CidrIp': u'10.42.0.0/16'},
-                  {u'CidrIp': u'10.42.1.0/24'}],
+                  {u'CidrIp': u'10.42.0.0/16'}],
               u'PrefixListIds': [],
               u'ToPort': 443,
               u'UserIdGroupPairs': []}])