Repository: cloud-custodian/cloud-custodian

Issue:
azure - delete locked resource group should be logged and skipped
## Action
Try to delete a resource group that is locked

## Actual
Custodian is throwing an exception.

## Expected
Custodian should skip and log the error

```
(cloud-custodian) ➜  cloud-custodian git:(master) ✗ custodian-cask run -s=. policies/policy.yml
Custodian Cask (cloudcustodian/c7n:latest)
Skipped image pull - Last checked 23 minutes ago.

2019-07-08 20:57:46,374: custodian.azure.session:INFO Creating session with Azure CLI Authentication
2019-07-08 20:57:46,374: custodian.azure.session:INFO Session using Subscription ID: <my sub id>
2019-07-08 20:57:46,505: custodian.cache:WARNING Could not save cache /home/custodian/.cache/cloud-custodian.cache err: [Errno 2] No such file or directory: '/home/custodian/.cache/cloud-custodian.cache'
2019-07-08 20:57:46,506: custodian.cache:INFO Generating Cache directory: /home/custodian/.cache.
2019-07-08 20:57:49,283: custodian.policy:INFO policy:delete-empty-resource-groups resource:azure.resourcegroup region: count:1 time:2.90
2019-07-08 20:57:49,290: custodian.resources.resourcegroup:INFO Removing resource group test1
2019-07-08 20:57:49,383: custodian.output:ERROR Error while executing policy
Traceback (most recent call last):
  File "/usr/local/lib/python3.7/site-packages/c7n/policy.py", line 261, in run
    results = a.process(resources)
  File "/usr/local/lib/python3.7/site-packages/c7n_azure/resources/resourcegroup.py", line 93, in process
    self.manager.get_client().resource_groups.delete(group['name'])
  File "/usr/local/lib/python3.7/site-packages/azure/mgmt/resource/resources/v2018_05_01/operations/resource_groups_operations.py", line 231, in delete
    **operation_config
  File "/usr/local/lib/python3.7/site-packages/azure/mgmt/resource/resources/v2018_05_01/operations/resource_groups_operations.py", line 199, in _delete_initial
    raise exp
msrestazure.azure_exceptions.CloudError: Azure Error: ScopeLocked
Message: The scope '/subscriptions/<my sub id>/resourcegroups/test1' cannot perform delete operation because following scope(s) are locked: '/subscriptions/<my sub id>/resourceGroups/test1'. Please remove the lock and try again.
2019-07-08 20:57:49,384: custodian.commands:ERROR Error while executing policy delete-empty-resource-groups, continuing
Traceback (most recent call last):
  File "/usr/local/lib/python3.7/site-packages/c7n/commands.py", line 269, in run
    policy()
  File "/usr/local/lib/python3.7/site-packages/c7n/policy.py", line 945, in __call__
    resources = mode.run()
  File "/usr/local/lib/python3.7/site-packages/c7n/policy.py", line 261, in run
    results = a.process(resources)
  File "/usr/local/lib/python3.7/site-packages/c7n_azure/resources/resourcegroup.py", line 93, in process
    self.manager.get_client().resource_groups.delete(group['name'])
  File "/usr/local/lib/python3.7/site-packages/azure/mgmt/resource/resources/v2018_05_01/operations/resource_groups_operations.py", line 231, in delete
    **operation_config
  File "/usr/local/lib/python3.7/site-packages/azure/mgmt/resource/resources/v2018_05_01/operations/resource_groups_operations.py", line 199, in _delete_initial
    raise exp
msrestazure.azure_exceptions.CloudError: Azure Error: ScopeLocked
Message: The scope '/subscriptions/<my sub id>/resourcegroups/test1' cannot perform delete operation because following scope(s) are locked: '/subscriptions/<my sub id>/resourceGroups/test1'. Please remove the lock and try again.
```


Discussion:
Resource group has custom `delete` operation. It needs to be updated to inherit from `AzureBaseOperation` to get consistent error handling.

https://github.com/cloud-custodian/cloud-custodian/blob/master/tools/c7n_azure/c7n_azure/resources/resourcegroup.py#L90

Resolving patch:
diff --git a/tools/c7n_azure/c7n_azure/actions/notify.py b/tools/c7n_azure/c7n_azure/actions/notify.py
--- a/tools/c7n_azure/c7n_azure/actions/notify.py
+++ b/tools/c7n_azure/c7n_azure/actions/notify.py
@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from azure.common import AzureHttpError
+
 from c7n_azure.storage_utils import StorageUtilities
 
 from c7n import utils
@@ -103,5 +105,16 @@ def send_data_message(self, message, session):
             return self.send_to_azure_queue(queue_uri, message, session)
 
     def send_to_azure_queue(self, queue_uri, message, session):
-        queue_service, queue_name = StorageUtilities.get_queue_client_by_uri(queue_uri, session)
-        return StorageUtilities.put_queue_message(queue_service, queue_name, self.pack(message)).id
+        try:
+            queue_service, queue_name = StorageUtilities.get_queue_client_by_uri(queue_uri, session)
+            return StorageUtilities.put_queue_message(
+                queue_service,
+                queue_name,
+                self.pack(message)).id
+        except AzureHttpError as e:
+            if e.status_code == 403:
+                self.log.error("Access Error - Storage Queue Data Contributor Role is required "
+                               "to enqueue messages to the Azure Queue Storage.")
+            else:
+                self.log.error("Error putting message to the queue.\n" +
+                               str(e))
diff --git a/tools/c7n_azure/c7n_azure/output.py b/tools/c7n_azure/c7n_azure/output.py
--- a/tools/c7n_azure/c7n_azure/output.py
+++ b/tools/c7n_azure/c7n_azure/output.py
@@ -89,8 +89,12 @@ def upload(self):
                         blob_name,
                         os.path.join(root, f))
                 except AzureHttpError as e:
-                    self.log.error("Error writing output. Confirm output storage URL is correct "
-                                   "and that 'Storage Blob Contributor' role is assigned. \n" +
+                    if e.status_code == 403:
+                        self.log.error("Access Error: Storage Blob Data Contributor Role "
+                                       "is required to write to Azure Blob Storage.")
+                    else:
+                        self.log.error("Error writing output. "
+                                       "Confirm output storage URL is correct. \n" +
                                    str(e))
 
                 self.log.debug("%s uploaded" % blob_name)
diff --git a/tools/c7n_azure/c7n_azure/resources/resourcegroup.py b/tools/c7n_azure/c7n_azure/resources/resourcegroup.py
--- a/tools/c7n_azure/c7n_azure/resources/resourcegroup.py
+++ b/tools/c7n_azure/c7n_azure/resources/resourcegroup.py
@@ -11,12 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+from c7n_azure.actions.base import AzureBaseAction
 from c7n_azure.provider import resources
 from c7n_azure.resources.arm import ArmResourceManager
 from c7n_azure.utils import ResourceIdParser
 
-from c7n.actions import BaseAction
 from c7n.filters import Filter
 from c7n.utils import type_schema
 
@@ -78,7 +77,7 @@ def __call__(self, group):
 
 
 @ResourceGroup.action_registry.register('delete')
-class DeleteResourceGroup(BaseAction):
+class DeleteResourceGroup(AzureBaseAction):
     # policies:
     #   - name: test - azure
     #   resource: azure.resourcegroup
@@ -87,7 +86,6 @@ class DeleteResourceGroup(BaseAction):
 
     schema = type_schema('delete')
 
-    def process(self, groups):
-        for group in groups:
-            self.manager.log.info('Removing resource group ' + group['name'])
-            self.manager.get_client().resource_groups.delete(group['name'])
+    def _process_resource(self, group):
+        self.manager.log.info('Removing resource group ' + group['name'])
+        self.manager.get_client().resource_groups.delete(group['name'])


Test patch:
diff --git a/tools/c7n_azure/tests/test_notify.py b/tools/c7n_azure/tests/test_notify.py
--- a/tools/c7n_azure/tests/test_notify.py
+++ b/tools/c7n_azure/tests/test_notify.py
@@ -13,9 +13,14 @@
 # limitations under the License.
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+from azure.common import AzureHttpError
 from azure_common import BaseTest, arm_template
 from c7n_azure.session import Session
 from c7n_azure.storage_utils import StorageUtilities
+from mock import patch
+from c7n_azure.actions.notify import Notify
+import re
+from c7n.utils import local_session
 
 
 class NotifyTest(BaseTest):
@@ -77,3 +82,40 @@ def test_notify_though_storage_queue(self):
         # Pull messages, should be 1
         messages = StorageUtilities.get_queue_messages(queue, name)
         self.assertEqual(len(messages), 1)
+
+    @patch('c7n_azure.storage_utils.StorageUtilities.put_queue_message')
+    @patch('c7n_azure.storage_utils.StorageUtilities.get_queue_client_by_uri')
+    @patch('logging.Logger.error')
+    def test_access_error(self,
+                          logger_mock,
+                          get_queue_client_by_uri,
+                          put_queue_message):
+        put_queue_message.side_effect = AzureHttpError('forbidden', 403)
+        get_queue_client_by_uri.return_value = 'service', 'name'
+
+        action = Notify()
+
+        action.send_to_azure_queue("url", "message", local_session(Session))
+
+        args, _ = logger_mock.call_args
+
+        self.assertIsNotNone(re.match("Access Error*", args[0]))
+
+    @patch('c7n_azure.storage_utils.StorageUtilities.put_queue_message')
+    @patch('c7n_azure.storage_utils.StorageUtilities.get_queue_client_by_uri')
+    @patch('logging.Logger.error')
+    def test_error_putting_to_queue(self,
+                                    logger_mock,
+                                    get_queue_client_by_uri,
+                                    put_queue_message):
+        put_queue_message.side_effect = AzureHttpError('not found', 404)
+        get_queue_client_by_uri.return_value = 'service', 'name'
+
+        action = Notify()
+
+        action.send_to_azure_queue("url", "message", local_session(Session))
+
+        args, _ = logger_mock.call_args
+
+        self.assertIsNone(re.match("Access Error*", args[0]))
+        self.assertIsNotNone(re.match("Error*", args[0]))
diff --git a/tools/c7n_azure/tests/test_output.py b/tools/c7n_azure/tests/test_output.py
--- a/tools/c7n_azure/tests/test_output.py
+++ b/tools/c7n_azure/tests/test_output.py
@@ -19,7 +19,9 @@
 from datetime import date
 
 import mock
+from azure.common import AzureHttpError
 from azure_common import BaseTest
+import re
 from c7n_azure.output import AzureStorageOutput
 
 from c7n.config import Bag, Config
@@ -28,6 +30,7 @@
 
 from c7n_azure.output import MetricsOutput, AppInsightsLogOutput
 from c7n.output import log_outputs, metrics_outputs
+from mock import patch
 
 
 class OutputTest(BaseTest):
@@ -120,3 +123,47 @@ def test_app_insights_metrics(self):
         self.assertTrue(isinstance(sink, MetricsOutput))
         sink.put_metric('ResourceCount', 101, 'Count')
         sink.flush()
+
+    @patch('logging.Logger.error')
+    def test_access_error(self, logger_mock):
+
+        AzureStorageOutput.get_blob_client_wrapper = gm = mock.MagicMock()
+        gm.return_value = None, "logs", 'xyz'
+
+        output = self.get_azure_output()
+
+        output.blob_service = mock.MagicMock()
+        output.blob_service.create_blob_from_path = mock.MagicMock()
+        output.blob_service.create_blob_from_path.side_effect = AzureHttpError('forbidden', 403)
+
+        # Generate fake output file
+        with open(os.path.join(output.root_dir, "foo.txt"), "w") as fh:
+            fh.write("abc")
+
+        output.upload()
+
+        args, _ = logger_mock.call_args
+
+        self.assertIsNotNone(re.match("Access Error*", args[0]))
+
+    @patch('logging.Logger.error')
+    def test_error_writing_to_blob(self, logger_mock):
+        AzureStorageOutput.get_blob_client_wrapper = gm = mock.MagicMock()
+        gm.return_value = None, "logs", 'xyz'
+
+        output = self.get_azure_output()
+
+        output.blob_service = mock.MagicMock()
+        output.blob_service.create_blob_from_path = mock.MagicMock()
+        output.blob_service.create_blob_from_path.side_effect = AzureHttpError('not found', 404)
+
+        # Generate fake output file
+        with open(os.path.join(output.root_dir, "foo.txt"), "w") as fh:
+            fh.write("abc")
+
+        output.upload()
+
+        args, _ = logger_mock.call_args
+
+        self.assertIsNone(re.match("Access Error*", args[0]))
+        self.assertIsNotNone(re.match("Error*", args[0]))