Repository: cloud-custodian/cloud-custodian

Issue:
aws - cache-cluster - error occurred (InvalidParameterValue)
### Describe the bug

When trying to delete a cache-cluster resource type, the operation fails when the cache-cluster's replication group is part of a global cluster. Traceback


### What did you expect to happen?

Log a warning and proceed?

### Cloud Provider

Amazon Web Services (AWS)

### Cloud Custodian version and dependency information

```shell
Custodian:   0.9.28
Python:      3.11.4 (main, Jul 25 2023, 17:36:13) [Clang 14.0.3 (clang-1403.0.22.14.1)]
```


### Policy

```shell
- name: cache-cluster-minor-version-existing-delete
  resource: aws.cache-cluster
  filters:
  - type: value
    key: AutoMinorVersionUpgrade
    op: not-equal
    value: true
  actions:
  - type: delete
```


### Relevant log/traceback output

```shell
Traceback (most recent call last):
  File "/usr/local/lib/python3.9/dist-packages/c7n/policy.py", line 364, in run
    results = a.process(resources)
  File "/usr/local/lib/python3.9/dist-packages/c7n/resources/elasticache.py", line 170, in process
    client.delete_replication_group(**params)
  File "/usr/local/lib/python3.9/dist-packages/botocore/client.py", line 530, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/usr/local/lib/python3.9/dist-packages/botocore/client.py", line 964, in _make_api_call
    raise error_class(parsed_response, operation_name)
botocore.errorfactory.InvalidParameterValueException: An error occurred (InvalidParameterValue) when calling the DeleteReplicationGroup operation: Cluster [xyz] is part of a global cluster [xyz]. Request rejected.
```


### Extra information or context

_No response_


Discussion:

Resolving patch:
diff --git a/c7n/resources/elasticache.py b/c7n/resources/elasticache.py
--- a/c7n/resources/elasticache.py
+++ b/c7n/resources/elasticache.py
@@ -16,7 +16,7 @@
 from c7n.query import QueryResourceManager, TypeInfo
 from c7n.tags import universal_augment
 from c7n.utils import (
-    local_session, chunks, snapshot_identifier, type_schema)
+    local_session, chunks, snapshot_identifier, type_schema, jmespath_search)
 
 filters = FilterRegistry('elasticache.filters')
 actions = ActionRegistry('elasticache.actions')
@@ -129,6 +129,10 @@ def process(self, clusters):
         clusters_to_delete = []
         replication_groups_to_delete = set()
         for cluster in clusters:
+            if cluster.get('ReplicationGroupId') in self.fetch_global_ds_rpgs():
+                self.log.info(
+                  f"Skipping {cluster['CacheClusterId']}: associated with a global datastore")
+                continue
             if cluster.get('ReplicationGroupId', ''):
                 replication_groups_to_delete.add(cluster['ReplicationGroupId'])
             else:
@@ -173,6 +177,11 @@ def process(self, clusters):
                 'Deleted ElastiCache replication group: %s',
                 replication_group)
 
+    def fetch_global_ds_rpgs(self):
+        rpgs = self.manager.get_resource_manager('elasticache-group').resources(augment=False)
+        global_rpgs = get_global_datastore_association(rpgs)
+        return global_rpgs
+
 
 @actions.register('snapshot')
 class SnapshotElastiCacheCluster(BaseAction):
@@ -485,6 +494,16 @@ class KmsFilter(KmsRelatedFilter):
     RelatedIdsExpression = 'KmsKeyId'
 
 
+def get_global_datastore_association(resources):
+    global_ds_rpgs = set()
+    for r in resources:
+        global_ds_association = jmespath_search(
+            'GlobalReplicationGroupInfo.GlobalReplicationGroupId', r)
+        if global_ds_association:
+            global_ds_rpgs.add(r['ReplicationGroupId'])
+    return global_ds_rpgs
+
+
 @ElastiCacheReplicationGroup.action_registry.register('delete')
 class DeleteReplicationGroup(BaseAction):
     """Action to delete a cache replication group
@@ -514,7 +533,12 @@ class DeleteReplicationGroup(BaseAction):
     def process(self, resources):
         resources = self.filter_resources(resources, 'Status', self.valid_origin_states)
         client = local_session(self.manager.session_factory).client('elasticache')
+        global_datastore_association_map = get_global_datastore_association(resources)
         for r in resources:
+            if r['ReplicationGroupId'] in global_datastore_association_map:
+                self.log.info(
+                  f"Skipping {r['ReplicationGroupId']}: associated with a global datastore")
+                continue
             params = {'ReplicationGroupId': r['ReplicationGroupId']}
             if self.data.get('snapshot', False):
                 params.update({'FinalSnapshotIdentifier': r['ReplicationGroupId'] + '-snapshot'})


Test patch:
diff --git a/tests/data/placebo/test_elasticache_global_ds_cluster_delete/elasticache.DescribeCacheClusters_1.json b/tests/data/placebo/test_elasticache_global_ds_cluster_delete/elasticache.DescribeCacheClusters_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_elasticache_global_ds_cluster_delete/elasticache.DescribeCacheClusters_1.json
@@ -0,0 +1,50 @@
+{
+    "status_code": 200,
+    "data": {
+        "CacheClusters": [
+            {
+                "CacheClusterId": "c7n-test-global-001",
+                "ClientDownloadLandingPage": "https://console.aws.amazon.com/elasticache/home#client-download:",
+                "CacheNodeType": "cache.m5.large",
+                "Engine": "redis",
+                "EngineVersion": "7.0.7",
+                "CacheClusterStatus": "available",
+                "NumCacheNodes": 1,
+                "PreferredAvailabilityZone": "us-east-1b",
+                "CacheClusterCreateTime": {
+                    "__class__": "datetime",
+                    "year": 2023,
+                    "month": 8,
+                    "day": 25,
+                    "hour": 22,
+                    "minute": 6,
+                    "second": 38,
+                    "microsecond": 192000
+                },
+                "PreferredMaintenanceWindow": "tue:06:00-tue:07:00",
+                "PendingModifiedValues": {},
+                "CacheSecurityGroups": [],
+                "CacheParameterGroup": {
+                    "CacheParameterGroupName": "global-datastore-c7n-test-globalesyd",
+                    "ParameterApplyStatus": "in-sync",
+                    "CacheNodeIdsToReboot": []
+                },
+                "CacheSubnetGroupName": "sandbox-dev-east-vpc-elasticache-subnet-group",
+                "AutoMinorVersionUpgrade": false,
+                "ReplicationGroupId": "c7n-test-global",
+                "SnapshotRetentionLimit": 0,
+                "SnapshotWindow": "09:30-10:30",
+                "AuthTokenEnabled": false,
+                "TransitEncryptionEnabled": true,
+                "AtRestEncryptionEnabled": true,
+                "ARN": "arn:aws:elasticache:us-east-1:644160558196:cluster:c7n-test-global-001",
+                "ReplicationGroupLogDeliveryEnabled": false,
+                "LogDeliveryConfigurations": [],
+                "NetworkType": "ipv4",
+                "IpDiscovery": "ipv4",
+                "TransitEncryptionMode": "required"
+            }
+        ],
+        "ResponseMetadata": {}
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_elasticache_global_ds_cluster_delete/elasticache.DescribeReplicationGroups_1.json b/tests/data/placebo/test_elasticache_global_ds_cluster_delete/elasticache.DescribeReplicationGroups_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_elasticache_global_ds_cluster_delete/elasticache.DescribeReplicationGroups_1.json
@@ -0,0 +1,74 @@
+{
+    "status_code": 200,
+    "data": {
+        "ReplicationGroups": [
+            {
+                "ReplicationGroupId": "c7n-test-global",
+                "Description": " ",
+                "GlobalReplicationGroupInfo": {
+                    "GlobalReplicationGroupId": "ldgnf-c7n-test-global",
+                    "GlobalReplicationGroupMemberRole": "PRIMARY"
+                },
+                "Status": "available",
+                "PendingModifiedValues": {},
+                "MemberClusters": [
+                    "c7n-test-global-001"
+                ],
+                "NodeGroups": [
+                    {
+                        "NodeGroupId": "0001",
+                        "Status": "available",
+                        "PrimaryEndpoint": {
+                            "Address": "master.c7n-test-global.tgeny2.use1.cache.amazonaws.com",
+                            "Port": 6379
+                        },
+                        "ReaderEndpoint": {
+                            "Address": "replica.c7n-test-global.tgeny2.use1.cache.amazonaws.com",
+                            "Port": 6379
+                        },
+                        "NodeGroupMembers": [
+                            {
+                                "CacheClusterId": "c7n-test-global-001",
+                                "CacheNodeId": "0001",
+                                "ReadEndpoint": {
+                                    "Address": "c7n-test-global-001.c7n-test-global.tgeny2.use1.cache.amazonaws.com",
+                                    "Port": 6379
+                                },
+                                "PreferredAvailabilityZone": "us-east-1b",
+                                "CurrentRole": "primary"
+                            }
+                        ]
+                    }
+                ],
+                "AutomaticFailover": "disabled",
+                "MultiAZ": "disabled",
+                "SnapshotRetentionLimit": 0,
+                "SnapshotWindow": "09:30-10:30",
+                "ClusterEnabled": false,
+                "CacheNodeType": "cache.m5.large",
+                "AuthTokenEnabled": false,
+                "TransitEncryptionEnabled": true,
+                "AtRestEncryptionEnabled": true,
+                "ARN": "arn:aws:elasticache:us-east-1:644160558196:replicationgroup:c7n-test-global",
+                "LogDeliveryConfigurations": [],
+                "ReplicationGroupCreateTime": {
+                    "__class__": "datetime",
+                    "year": 2023,
+                    "month": 8,
+                    "day": 25,
+                    "hour": 21,
+                    "minute": 59,
+                    "second": 9,
+                    "microsecond": 247000
+                },
+                "DataTiering": "disabled",
+                "AutoMinorVersionUpgrade": false,
+                "NetworkType": "ipv4",
+                "IpDiscovery": "ipv4",
+                "TransitEncryptionMode": "required",
+                "ClusterMode": "disabled"
+            }
+        ],
+        "ResponseMetadata": {}
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_elasticache_global_ds_cluster_delete/tagging.GetResources_1.json b/tests/data/placebo/test_elasticache_global_ds_cluster_delete/tagging.GetResources_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_elasticache_global_ds_cluster_delete/tagging.GetResources_1.json
@@ -0,0 +1,8 @@
+{
+    "status_code": 200,
+    "data": {
+        "PaginationToken": "",
+        "ResourceTagMappingList": [],
+        "ResponseMetadata": {}
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_elasticache_global_ds_cluster_delete/tagging.GetResources_2.json b/tests/data/placebo/test_elasticache_global_ds_cluster_delete/tagging.GetResources_2.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_elasticache_global_ds_cluster_delete/tagging.GetResources_2.json
@@ -0,0 +1,8 @@
+{
+    "status_code": 200,
+    "data": {
+        "PaginationToken": "",
+        "ResourceTagMappingList": [],
+        "ResponseMetadata": {}
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_replication_group_global_ds_cluster_delete/elasticache.DescribeReplicationGroups_1.json b/tests/data/placebo/test_replication_group_global_ds_cluster_delete/elasticache.DescribeReplicationGroups_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_replication_group_global_ds_cluster_delete/elasticache.DescribeReplicationGroups_1.json
@@ -0,0 +1,74 @@
+{
+    "status_code": 200,
+    "data": {
+        "ReplicationGroups": [
+            {
+                "ReplicationGroupId": "c7n-test-global",
+                "Description": " ",
+                "GlobalReplicationGroupInfo": {
+                    "GlobalReplicationGroupId": "ldgnf-c7n-test-global",
+                    "GlobalReplicationGroupMemberRole": "PRIMARY"
+                },
+                "Status": "available",
+                "PendingModifiedValues": {},
+                "MemberClusters": [
+                    "c7n-test-global-001"
+                ],
+                "NodeGroups": [
+                    {
+                        "NodeGroupId": "0001",
+                        "Status": "available",
+                        "PrimaryEndpoint": {
+                            "Address": "master.c7n-test-global.tgeny2.use1.cache.amazonaws.com",
+                            "Port": 6379
+                        },
+                        "ReaderEndpoint": {
+                            "Address": "replica.c7n-test-global.tgeny2.use1.cache.amazonaws.com",
+                            "Port": 6379
+                        },
+                        "NodeGroupMembers": [
+                            {
+                                "CacheClusterId": "c7n-test-global-001",
+                                "CacheNodeId": "0001",
+                                "ReadEndpoint": {
+                                    "Address": "c7n-test-global-001.c7n-test-global.tgeny2.use1.cache.amazonaws.com",
+                                    "Port": 6379
+                                },
+                                "PreferredAvailabilityZone": "us-east-1b",
+                                "CurrentRole": "primary"
+                            }
+                        ]
+                    }
+                ],
+                "AutomaticFailover": "disabled",
+                "MultiAZ": "disabled",
+                "SnapshotRetentionLimit": 0,
+                "SnapshotWindow": "09:30-10:30",
+                "ClusterEnabled": false,
+                "CacheNodeType": "cache.m5.large",
+                "AuthTokenEnabled": false,
+                "TransitEncryptionEnabled": true,
+                "AtRestEncryptionEnabled": true,
+                "ARN": "arn:aws:elasticache:us-east-1:644160558196:replicationgroup:c7n-test-global",
+                "LogDeliveryConfigurations": [],
+                "ReplicationGroupCreateTime": {
+                    "__class__": "datetime",
+                    "year": 2023,
+                    "month": 8,
+                    "day": 25,
+                    "hour": 21,
+                    "minute": 59,
+                    "second": 9,
+                    "microsecond": 247000
+                },
+                "DataTiering": "disabled",
+                "AutoMinorVersionUpgrade": false,
+                "NetworkType": "ipv4",
+                "IpDiscovery": "ipv4",
+                "TransitEncryptionMode": "required",
+                "ClusterMode": "disabled"
+            }
+        ],
+        "ResponseMetadata": {}
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_replication_group_global_ds_cluster_delete/tagging.GetResources_1.json b/tests/data/placebo/test_replication_group_global_ds_cluster_delete/tagging.GetResources_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_replication_group_global_ds_cluster_delete/tagging.GetResources_1.json
@@ -0,0 +1,8 @@
+{
+    "status_code": 200,
+    "data": {
+        "PaginationToken": "",
+        "ResourceTagMappingList": [],
+        "ResponseMetadata": {}
+    }
+}
\ No newline at end of file
diff --git a/tests/test_elasticache.py b/tests/test_elasticache.py
--- a/tests/test_elasticache.py
+++ b/tests/test_elasticache.py
@@ -247,6 +247,27 @@ def test_elasticache_cluster_snapshot(self):
         resources = p.run()
         self.assertEqual(len(resources), 3)
 
+    def test_elasticache_global_ds_cluster_delete(self):
+        session_factory = self.replay_flight_data("test_elasticache_global_ds_cluster_delete")
+        log_output = self.capture_logging('custodian.actions')
+        p = self.load_policy(
+            {
+                "name": "elasticache-cluster-delete",
+                "resource": "cache-cluster",
+                "filters": [{
+                    "type": "value",
+                    "key": "CacheParameterGroup.CacheParameterGroupName",
+                    "value": "global-datastore",
+                    "op": "contains"}],
+                "actions": [{"type": "delete"}],
+            },
+            session_factory=session_factory,
+        )
+        resources = p.run()
+        self.assertEqual(len(resources), 1)
+        self.assertEqual(resources[0]['CacheClusterId'], 'c7n-test-global-001')
+        assert "Skipping c7n-test-global-001" in log_output.getvalue()
+
 
 class TestElastiCacheSubnetGroup(BaseTest):
 
@@ -518,3 +539,24 @@ def test_elasticache_replication_group_tag(self):
         arn = p.resource_manager.get_arns(resources)[0]
         tags = client.list_tags_for_resource(ResourceName=arn)["TagList"]
         self.assertEqual(tags[0]["Value"], "added")
+
+    def test_replication_group_global_ds_cluster_delete(self):
+        session_factory = self.replay_flight_data("test_replication_group_global_ds_cluster_delete")
+        log_output = self.capture_logging('custodian.actions')
+        p = self.load_policy(
+            {
+                "name": "elasticache-cluster-delete",
+                "resource": "aws.elasticache-group",
+                "filters": [{
+                    "type": "value",
+                    "key": "GlobalReplicationGroupInfo.GlobalReplicationGroupId",
+                    "value": "ldgnf-c7n-test-global",
+                    "op": "eq"}],
+                "actions": [{"type": "delete"}],
+            },
+            session_factory=session_factory,
+        )
+        resources = p.run()
+        self.assertEqual(len(resources), 1)
+        self.assertEqual(resources[0]['ReplicationGroupId'], 'c7n-test-global')
+        assert "Skipping c7n-test-global" in log_output.getvalue()