Repository: cloud-custodian/cloud-custodian

Issue:
Azure - Handle AdalError nicely
If user has bad principal/expired token/whatever they get this `AdalError` buried in stack trace. 

This is a very common scenario, so if we can just handle `AdalError` here and print out the error body that would be preferable to dumping the stack.

You should be able to reproduce by just putting in an invalid client secret.

```
Traceback (most recent call last):
  File "/usr/local/lib/python3.7/site-packages/msrestazure/azure_active_directory.py", line 367, in set_token
    self.secret
  File "/usr/local/lib/python3.7/site-packages/adal/authentication_context.py", line 179, in acquire_token_with_client_credentials
    return self._acquire_token(token_func)
  File "/usr/local/lib/python3.7/site-packages/adal/authentication_context.py", line 128, in _acquire_token
    return token_func(self)
  File "/usr/local/lib/python3.7/site-packages/adal/authentication_context.py", line 177, in token_func
    return token_request.get_token_with_client_credentials(client_secret)
  File "/usr/local/lib/python3.7/site-packages/adal/token_request.py", line 315, in get_token_with_client_credentials
    token = self._oauth_get_token(oauth_parameters)
  File "/usr/local/lib/python3.7/site-packages/adal/token_request.py", line 113, in _oauth_get_token
    return client.get_token(oauth_parameters)
  File "/usr/local/lib/python3.7/site-packages/adal/oauth2_client.py", line 289, in get_token
    raise AdalError(return_error_string, error_response)
adal.adal_error.AdalError: Get Token request returned http error: 401 and server response: {"error":"invalid_client","error_description":"AADSTS7000215: Invalid client secret is provided.\r\nTrace ID: e380c8af-e1ad-48c3-a921-eb7f2ac94900\r\nCorrelation ID: 1d69ea7c-37bd-4b1e-80d4-01a3b9768a38\r\nTimestamp: 2019-06-24 19:33:04Z","error_codes":[7000215],"timestamp":"2019-06-24 19:33:04Z","trace_id":"e380c8af-e1ad-48c3-a921-eb7f2ac94900","correlation_id":"1d69ea7c-37bd-4b1e-80d4-01a3b9768a38"}

```


Discussion:
### ACCEPTANCE CRITERIA

1. Identify a good place in the code to "catch" this error.
2. Create a notification that a secret/token has expired.

FYI @stefangordon 
I am targeting two main spots for this fix. Firstly, `session.py` this is when the authentication context gets created. The second is `query.py` after we get an authentication context and try to fetch the resources.

Tested Scenarios (What - How - Where):

- [x] ExpiredToken - set old `AZURE_ACCESS_TOKEN` - query.py
`(ExpiredAuthenticationToken) The access token expiry UTC time...`

- [x] Bad Subscription -  set bad `AZURE_SUBSCRIPTION_ID` - query.py
`(SubscriptionNotFound) The subscription 'abc-123' could not be found.`

- [x] Bad Application -  set bad `AZURE_CLIENT_ID` - session.py
`AADSTS700016: Application with identifier 'abc-123' was not found`

- [x] Bad Tenant -  set bad `AZURE_TENANT_ID` - session.py
`AADSTS90002: Tenant 'abc-123' not found.`

- [x] Bad Client Secret -  set bad `AZURE_CLIENT_SECRET` - session.py
`"AADSTS7000215: Invalid client secret is provided.`

- [x] MSI Bad Auth - don't set perms for MSI - query.py
`(AuthorizationFailed) The client 'abc-123' with object id 'abc-123' does not have authorization to perform action 'Microsoft.Web/serverfarms/read' over scope '/subscriptions/...' or the scope is invalid.`

- [x] MSI UAI Bad Auth - set bad AZURE_CLIENT_ID - session.py
`{"error":"invalid_request","error_description":"Identity not found"}` 
  (Issue where this body cannot be parsed from `HttpResponse` https://github.com/Azure/msrestazure-for-python/issues/134)

- [x] MSI KV Bad Auth - set bad AZURE_KEYVAULT_SECRET or set bad AZURE_KEYVAULT_CLIENT_ID - session.py

- [x] MSI KV Bad KV secret - set bad json in KV - session.py



Resolving patch:
diff --git a/tools/c7n_azure/c7n_azure/query.py b/tools/c7n_azure/c7n_azure/query.py
--- a/tools/c7n_azure/c7n_azure/query.py
+++ b/tools/c7n_azure/c7n_azure/query.py
@@ -42,9 +42,15 @@ def filter(self, resource_manager, **params):
         if extra_args:
             params.update(extra_args)
 
-        op = getattr(getattr(resource_manager.get_client(), enum_op), list_op)
-        data = [r.serialize(True) for r in op(**params)]
-
+        data = []
+        try:
+            op = getattr(getattr(resource_manager.get_client(), enum_op), list_op)
+            data = [r.serialize(True) for r in op(**params)]
+        except Exception as e:
+            log.error("Failed to query resource.\n"
+                      "Type: azure.{0}.\n"
+                      "Error: {1}".format(resource_manager.type, e))
+            six.raise_from(Exception('Failed to query resources.'), e)
         return data
 
     @staticmethod
@@ -114,7 +120,6 @@ def filter(self, resource_manager, **params):
 
 @sources.register('describe-child-azure')
 class ChildDescribeSource(DescribeSource):
-
     resource_query_factory = ChildResourceQuery
 
     def __init__(self, manager):
@@ -163,6 +168,7 @@ def extra_args(cls, parent_resource):
 
 class QueryMeta(type):
     """metaclass to have consistent action/filter registry for new resources."""
+
     def __new__(cls, name, parents, attrs):
         if 'filter_registry' not in attrs:
             attrs['filter_registry'] = FilterRegistry(
@@ -176,7 +182,6 @@ def __new__(cls, name, parents, attrs):
 
 @six.add_metaclass(QueryMeta)
 class QueryResourceManager(ResourceManager):
-
     class resource_type(TypeInfo):
         pass
 
@@ -272,7 +277,6 @@ def register_actions_and_filters(registry, _):
 
 @six.add_metaclass(QueryMeta)
 class ChildResourceManager(QueryResourceManager):
-
     child_source = 'describe-child-azure'
     parent_manager = None
 
diff --git a/tools/c7n_azure/c7n_azure/session.py b/tools/c7n_azure/c7n_azure/session.py
--- a/tools/c7n_azure/c7n_azure/session.py
+++ b/tools/c7n_azure/c7n_azure/session.py
@@ -27,13 +27,18 @@
 from c7n_azure import constants
 from c7n_azure.utils import (ResourceIdParser, StringUtils, custodian_azure_send_override,
                              ManagedGroupHelper, get_keyvault_secret)
+from knack.util import CLIError
+from msrest.exceptions import AuthenticationError
 from msrestazure.azure_active_directory import MSIAuthentication
+from requests import HTTPError
 
 try:
     from azure.cli.core._profile import Profile
 except Exception:
     Profile = None
 
+log = logging.getLogger('custodian.azure.session')
+
 
 class Session(object):
 
@@ -45,7 +50,6 @@ def __init__(self, subscription_id=None, authorization_file=None,
         :param resource: Resource endpoint for OAuth token.
         """
 
-        self.log = logging.getLogger('custodian.azure.session')
         self._provider_cache = {}
         self.subscription_id_override = subscription_id
         self.credentials = None
@@ -63,64 +67,89 @@ def auth_params(self):
         return self._auth_params
 
     def _authenticate(self):
-        keyvault_client_id = self._auth_params.get('keyvault_client_id')
-        keyvault_secret_id = self._auth_params.get('keyvault_secret_id')
-
-        # If user provided KeyVault secret, we will pull auth params information from it
-        if keyvault_secret_id:
-            self._auth_params.update(
-                json.loads(
-                    get_keyvault_secret(keyvault_client_id, keyvault_secret_id)))
-
-        client_id = self._auth_params.get('client_id')
-        client_secret = self._auth_params.get('client_secret')
-        access_token = self._auth_params.get('access_token')
-        tenant_id = self._auth_params.get('tenant_id')
-        use_msi = self._auth_params.get('use_msi')
-        subscription_id = self._auth_params.get('subscription_id')
-
-        if access_token and subscription_id:
-            self.log.info("Creating session with Token Authentication")
-            self.subscription_id = subscription_id
-            self.credentials = BasicTokenAuthentication(
-                token={
-                    'access_token': access_token
-                })
-            self._is_token_auth = True
-
-        elif client_id and client_secret and tenant_id and subscription_id:
-            self.log.info("Creating session with Service Principal Authentication")
-            self.subscription_id = subscription_id
-            self.credentials = ServicePrincipalCredentials(
-                client_id=client_id,
-                secret=client_secret,
-                tenant=tenant_id,
-                resource=self.resource_namespace)
-            self.tenant_id = tenant_id
-
-        elif use_msi and subscription_id:
-            self.log.info("Creating session with MSI Authentication")
-            self.subscription_id = subscription_id
-            if client_id:
-                self.credentials = MSIAuthentication(
+        try:
+            keyvault_client_id = self._auth_params.get('keyvault_client_id')
+            keyvault_secret_id = self._auth_params.get('keyvault_secret_id')
+
+            # If user provided KeyVault secret, we will pull auth params information from it
+            if keyvault_secret_id:
+                self._auth_params.update(
+                    json.loads(
+                        get_keyvault_secret(keyvault_client_id, keyvault_secret_id)))
+
+            client_id = self._auth_params.get('client_id')
+            client_secret = self._auth_params.get('client_secret')
+            access_token = self._auth_params.get('access_token')
+            tenant_id = self._auth_params.get('tenant_id')
+            use_msi = self._auth_params.get('use_msi')
+            subscription_id = self._auth_params.get('subscription_id')
+
+            if access_token and subscription_id:
+                log.info("Creating session with Token Authentication")
+                self.subscription_id = subscription_id
+                self.credentials = BasicTokenAuthentication(
+                    token={
+                        'access_token': access_token
+                    })
+                self._is_token_auth = True
+
+            elif client_id and client_secret and tenant_id and subscription_id:
+                log.info("Creating session with Service Principal Authentication")
+                self.subscription_id = subscription_id
+                self.credentials = ServicePrincipalCredentials(
                     client_id=client_id,
+                    secret=client_secret,
+                    tenant=tenant_id,
                     resource=self.resource_namespace)
-            else:
-                self.credentials = MSIAuthentication(
-                    resource=self.resource_namespace)
-
-        elif self._auth_params.get('enable_cli_auth'):
-            self.log.info("Creating session with Azure CLI Authentication")
-            self._is_cli_auth = True
-            try:
+                self.tenant_id = tenant_id
+
+            elif use_msi and subscription_id:
+                log.info("Creating session with MSI Authentication")
+                self.subscription_id = subscription_id
+                if client_id:
+                    self.credentials = MSIAuthentication(
+                        client_id=client_id,
+                        resource=self.resource_namespace)
+                else:
+                    self.credentials = MSIAuthentication(
+                        resource=self.resource_namespace)
+
+            elif self._auth_params.get('enable_cli_auth'):
+                log.info("Creating session with Azure CLI Authentication")
+                self._is_cli_auth = True
                 (self.credentials,
                  self.subscription_id,
                  self.tenant_id) = Profile().get_login_credentials(
                     resource=self.resource_namespace)
-            except Exception:
-                self.log.error('Unable to authenticate with Azure')
+            log.info("Session using Subscription ID: %s" % self.subscription_id)
 
-        self.log.info("Session using Subscription ID: %s" % self.subscription_id)
+        except AuthenticationError as e:
+            log.error('Azure Authentication Failure\n'
+                      'Error: {0}'
+                      .format(json.dumps(e.inner_exception.error_response, indent=2)))
+            sys.exit(1)
+        except HTTPError as e:
+            if keyvault_client_id and keyvault_secret_id:
+                log.error('Azure Authentication Failure\n'
+                          'Error: Cannot retrieve SP credentials from the Key Vault '
+                          '(KV uses MSI to access) with client id: {0}'
+                          .format(keyvault_client_id))
+            elif use_msi:
+                log.error('Azure Authentication Failure\n'
+                          'Error: Could not authenticate using managed service identity {0}'
+                          .format(client_id if client_id else '(system identity)'))
+            else:
+                log.error('Azure Authentication Failure: %s' % e.response)
+            sys.exit(1)
+        except CLIError as e:
+            log.error('Azure Authentication Failure\n'
+                      'Error: Could not authenticate with Azure CLI credentials: {0}'
+                      .format(e))
+            sys.exit(1)
+        except Exception as e:
+            log.error('Azure Authentication Failure\n'
+                      'Error: {0}'.format(e))
+            sys.exit(1)
 
     def _initialize_session(self):
         """
@@ -138,11 +167,11 @@ def _initialize_session(self):
             return
 
         if self.authorization_file:
-            self.log.info("Using file for authentication parameters")
+            log.info("Using file for authentication parameters")
             with open(self.authorization_file) as json_file:
                 self._auth_params = json.load(json_file)
         else:
-            self.log.info("Using environment variables for authentication parameters")
+            log.info("Using environment variables for authentication parameters")
             self._auth_params = {
                 'client_id': os.environ.get(constants.ENV_CLIENT_ID),
                 'client_secret': os.environ.get(constants.ENV_CLIENT_SECRET),
@@ -162,7 +191,7 @@ def _initialize_session(self):
         self._authenticate()
 
         if self.credentials is None:
-            self.log.error('Unable to authenticate with Azure.')
+            log.error('Unable to authenticate with Azure.')
             sys.exit(1)
 
         # TODO: cleanup this workaround when issue resolved.
@@ -249,7 +278,7 @@ def resource_api_version(self, resource_id):
             return resource_client.providers.api_version
 
         rt = next((t for t in provider.resource_types
-            if StringUtils.equal(t.resource_type, resource_type)), None)
+                   if StringUtils.equal(t.resource_type, resource_type)), None)
 
         if rt and rt.api_versions:
             versions = [v for v in rt.api_versions if 'preview' not in v.lower()]


Test patch:
diff --git a/tools/c7n_azure/tests/test_azure_query.py b/tools/c7n_azure/tests/test_azure_query.py
--- a/tools/c7n_azure/tests/test_azure_query.py
+++ b/tools/c7n_azure/tests/test_azure_query.py
@@ -17,9 +17,11 @@
 
 from azure_common import BaseTest, arm_template
 from azure_common import cassette_name
-from mock import mock
+from c7n_azure.session import Session
+from mock import mock, patch
 
 from c7n.exceptions import ResourceLimitExceeded
+from c7n.utils import local_session
 
 
 class QueryResourceManagerTest(BaseTest):
@@ -66,3 +68,31 @@ def test_policy_resource_limits_count(self):
                         in output.getvalue())
         self.assertEqual(
             p.ctx.metrics.buf[0]['MetricName'], 'ResourceLimitExceeded')
+
+    @patch('c7n_azure.query.log.error')
+    def test_query_filter_generic_exception(self, mock_log):
+        with self.assertRaises(Exception):
+            with patch('azure.mgmt.resource.resources.v%s.operations.'
+                       'resource_groups_operations.ResourceGroupsOperations.list'
+                       % QueryResourceManagerTest._get_resource_group_client_api_string()) \
+                    as mock_list:
+                mock_list.side_effect = Exception('test query exception')
+
+                p = self.load_policy(
+                    {
+                        "name": "limits",
+                        "resource": "azure.resourcegroup",
+                    },
+                    validate=True)
+
+                p.run()
+
+                mock_log.assert_called_once_with(
+                    'Failed to query resource.'
+                    '\nType: azure.resourcegroup.\nError: test query exception')
+
+    @staticmethod
+    def _get_resource_group_client_api_string():
+        return local_session(Session) \
+            .client('azure.mgmt.resource.ResourceManagementClient') \
+            .DEFAULT_API_VERSION.replace("-", "_")
diff --git a/tools/c7n_azure/tests/test_session.py b/tools/c7n_azure/tests/test_session.py
--- a/tools/c7n_azure/tests/test_session.py
+++ b/tools/c7n_azure/tests/test_session.py
@@ -13,6 +13,11 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+from adal import AdalError
+from knack.util import CLIError
+from msrest.exceptions import AuthenticationError
+from requests import HTTPError
+
 try:
     from importlib import reload
 except Exception:
@@ -34,7 +39,6 @@
 
 
 class SessionTest(BaseTest):
-
     authorization_file = os.path.join(os.path.dirname(__file__), 'data', 'test_auth_file.json')
     authorization_file_kv = os.path.join(os.path.dirname(__file__), 'data',
                                          'test_auth_file_kv.json')
@@ -84,13 +88,35 @@ def test_initialize_session_principal(self):
                                 constants.ENV_CLIENT_ID: 'client',
                                 constants.ENV_CLIENT_SECRET: 'secret'
                             }, clear=True):
-
                 s = Session()
 
                 self.assertIs(type(s.get_credentials()), ServicePrincipalCredentials)
                 self.assertEqual(s.get_subscription_id(), DEFAULT_SUBSCRIPTION_ID)
                 self.assertEqual(s.get_tenant_id(), DEFAULT_TENANT_ID)
 
+    @patch('azure.common.credentials.ServicePrincipalCredentials.__init__')
+    @patch('c7n_azure.session.log.error')
+    def test_initialize_session_authentication_error(self, mock_log, mock_cred):
+        with self.assertRaises(SystemExit):
+            adal_err = AdalError("test")
+            adal_err.error_response = {'error': 'test'}
+            err = AuthenticationError('test')
+            err.inner_exception = adal_err
+            mock_cred.side_effect = err
+
+            with patch.dict(os.environ,
+                            {
+                                constants.ENV_TENANT_ID: DEFAULT_TENANT_ID,
+                                constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID,
+                                constants.ENV_CLIENT_ID: 'client',
+                                constants.ENV_CLIENT_SECRET: 'secret'
+                            }, clear=True):
+                s = Session()
+                s.get_subscription_id()
+
+        mock_log.assert_called_once_with(
+            'Azure Authentication Failure\nError: {\n  "error": "test"\n}')
+
     def test_initialize_msi_auth_system(self):
         with patch('msrestazure.azure_active_directory.MSIAuthentication.__init__',
                    autospec=True, return_value=None):
@@ -118,13 +144,30 @@ def test_initialize_msi_auth_user(self):
                 self.assertIs(type(s.get_credentials()), MSIAuthentication)
                 self.assertEqual(s.get_subscription_id(), DEFAULT_SUBSCRIPTION_ID)
 
+    @patch('msrestazure.azure_active_directory.MSIAuthentication.__init__')
+    @patch('c7n_azure.session.log.error')
+    def test_initialize_session_msi_authentication_error(self, mock_log, mock_cred):
+        with self.assertRaises(SystemExit):
+            mock_cred.side_effect = HTTPError()
+
+            with patch.dict(os.environ,
+                            {
+                                constants.ENV_USE_MSI: 'true',
+                                constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID
+                            }, clear=True):
+                s = Session()
+                s.get_subscription_id()
+
+        mock_log.assert_called_once_with(
+            'Azure Authentication Failure\n'
+            'Error: Could not authenticate using managed service identity (system identity)')
+
     def test_initialize_session_token(self):
         with patch.dict(os.environ,
                         {
                             constants.ENV_ACCESS_TOKEN: 'token',
                             constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID
                         }, clear=True):
-
             s = Session()
 
             self.assertIs(type(s.get_credentials()), BasicTokenAuthentication)
@@ -287,3 +330,45 @@ def test_kv_patch(self, _1):
             self.assertEqual(auth_params.get('keyvault_secret_id'), 'kv_secret')
             self.assertEqual(auth_params.get('client_id'), 'client')
             self.assertEqual(auth_params.get('client_secret'), 'secret')
+
+    @patch('c7n_azure.utils.get_keyvault_secret')
+    @patch('c7n_azure.session.log.error')
+    def test_initialize_session_kv_authentication_error(self, mock_log, mock_get_kv_secret):
+        reload(sys.modules['c7n_azure.session'])
+
+        with self.assertRaises(SystemExit):
+            mock_get_kv_secret.side_effect = HTTPError()
+
+            with patch.dict(os.environ,
+                            {
+                                constants.ENV_TENANT_ID: 'tenant',
+                                constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID,
+                                constants.ENV_KEYVAULT_CLIENT_ID: 'kv_client',
+                                constants.ENV_KEYVAULT_SECRET_ID: 'kv_secret'
+                            }, clear=True):
+                s = Session()
+                s.get_subscription_id()
+
+        mock_log.assert_called_once_with(
+            'Azure Authentication Failure\nError: '
+            'Cannot retrieve SP credentials from the Key Vault '
+            '(KV uses MSI to access) with client id: kv_client')
+
+    @patch('azure.cli.core._profile.Profile.get_login_credentials')
+    @patch('c7n_azure.session.log.error')
+    def test_initialize_session_cli_error(self, mock_log, mock_cli_creds):
+        with self.assertRaises(SystemExit):
+            mock_cli_creds.side_effect = CLIError("Bad CLI credentials")
+
+            with patch.dict(os.environ,
+                            {
+                                constants.ENV_TENANT_ID: 'tenant',
+                                constants.ENV_SUB_ID: DEFAULT_SUBSCRIPTION_ID,
+                            }, clear=True):
+                s = Session()
+                s.get_subscription_id()
+
+        mock_log.assert_called_once_with(
+            'Azure Authentication Failure\nError: '
+            'Could not authenticate with Azure CLI '
+            'credentials: Bad CLI credentials')