Repository: cloud-custodian/cloud-custodian

Issue:
ci - investigate transient py 2.7 tox failure wrt to iam permissions
https://dev.azure.com/cloud-custodian/cloud-custodian/_build/results?buildId=5534&view=logs&jobId=1fc2e071-3735-5446-375d-4a9ead60c2c3&j=1fc2e071-3735-5446-375d-4a9ead60c2c3&t=46946717-7d1a-5609-1f4f-564b9ccd6573

```
___________________ TestIamGen.test_iam_permissions_validity ___________________
[gw1] linux2 -- Python 2.7.17 /home/vsts/work/1/s/.tox/py27/bin/python
Traceback (most recent call last):
  File "/home/vsts/work/1/s/tests/test_iamgen.py", line 97, in test_iam_permissions_validity
    "invalid permissions \n %s" % ('\n'.join(sorted(map(str, invalid)))))
ValueError: invalid permissions 
 ('asg.filters.invalid', [u'elasticloadbalancing:DescribeTargetGroups'])
('asg.filters.valid', [u'elasticloadbalancing:DescribeTargetGroups'])
```

this looks like an issue in the test itself



Discussion:
seems to be very transient only one occurrence ever afaics
shouldn't py27 support be dropped?

Resolving patch:
diff --git a/c7n/commands.py b/c7n/commands.py
--- a/c7n/commands.py
+++ b/c7n/commands.py
@@ -34,7 +34,7 @@
 from c7n.utils import dumps, load_file, local_session, SafeLoader, yaml_dump
 from c7n.config import Bag, Config
 from c7n import provider
-from c7n.resources import load_resources
+from c7n.resources import load_resources, load_available
 
 
 log = logging.getLogger('custodian.commands')
@@ -348,7 +348,7 @@ def schema_completer(prefix):
     filtering via startswith happens after this list is returned.
     """
     from c7n import schema
-    load_resources()
+    load_available()
     components = prefix.split('.')
 
     if components[0] in provider.clouds.keys():
@@ -394,7 +394,7 @@ def schema_cmd(options):
         schema.json_dump(options.resource)
         return
 
-    load_resources()
+    load_available()
 
     resource_mapping = schema.resource_vocabulary()
     if options.summary:
diff --git a/c7n/resources/__init__.py b/c7n/resources/__init__.py
--- a/c7n/resources/__init__.py
+++ b/c7n/resources/__init__.py
@@ -26,6 +26,9 @@ def load_resources(resource_types=('*',)):
     pmap = {}
     for r in resource_types:
         parts = r.split('.', 1)
+        # support aws.*
+        if parts[-1] == '*':
+            r = '*'
         pmap.setdefault(parts[0], []).append(r)
 
     load_providers(set(pmap))
@@ -48,6 +51,23 @@ def should_load_provider(name, provider_types):
     return False
 
 
+def load_available():
+    """Load available installed providers
+
+    Unlike load_resources() this will catch ImportErrors on uninstalled
+    providers.
+    """
+    found = []
+    for provider in ('aws', 'azure', 'gcp', 'k8s'):
+        try:
+            load_providers((provider,))
+        except ImportError as e: # pragma: no cover
+            continue
+        else:
+            found.append(provider)
+    load_resources(['%s.*' % s for s in found])
+
+
 def load_providers(provider_types):
     global LOADED
 
diff --git a/c7n/schema.py b/c7n/schema.py
--- a/c7n/schema.py
+++ b/c7n/schema.py
@@ -37,7 +37,7 @@
 
 from c7n.policy import execution
 from c7n.provider import clouds
-from c7n.resources import load_resources
+from c7n.resources import load_available
 from c7n.resolver import ValuesFrom
 from c7n.filters.core import ValueFilter, EventFilter, AgeFilter, OPERATORS, VALUE_TYPES
 from c7n.structure import StructureParser # noqa
@@ -574,7 +574,7 @@ def pprint_schema_summary(vocabulary):
 
 
 def json_dump(resource=None):
-    load_resources()
+    load_available()
     print(json.dumps(generate(resource), indent=2))
 
 


Test patch:
diff --git a/tests/test_iamgen.py b/tests/test_iamgen.py
--- a/tests/test_iamgen.py
+++ b/tests/test_iamgen.py
@@ -92,6 +92,6 @@ def test_iam_permissions_validity(self):
             raise ValueError(
                 "resources missing service %s" % ('\n'.join(sorted(missing))))
 
-        if invalid:
+        if all_invalid:
             raise ValueError(
-                "invalid permissions \n %s" % ('\n'.join(sorted(map(str, invalid)))))
+                "invalid permissions \n %s" % ('\n'.join(sorted(map(str, all_invalid)))))