Repository: cloud-custodian/cloud-custodian

Issue:
Support percentile statistics in CloudWatch metrics filter
### Describe the feature

I would like to be able to use percentile statistics (like p99) in the [AWS metrics filter](https://cloudcustodian.io/docs/aws/resources/aws-common-filters.html?highlight=metric#metrics).

For example, I want to be able to write a rule like

```yaml
policies:
  - name: ec2-underutilized
    resource: ec2
    filters:
    - type: metrics
      name: CPUUtilization
      statistics: p99
      period: 86400
      days: 7
      value: 1
      op: less-than
```

to find EC2 instances where the 99th percentile CPU usage is less than 1 percent. 

The [CloudWatch:GetMetricStatistics API](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricStatistics.html) supports an `ExtendedStatistics` option for requesting percentile statistics, so I think it should be technically possible to support this.

### Extra information or context

_No response_


Discussion:

Resolving patch:
diff --git a/c7n/filters/metrics.py b/c7n/filters/metrics.py
--- a/c7n/filters/metrics.py
+++ b/c7n/filters/metrics.py
@@ -3,6 +3,8 @@
 """
 CloudWatch Metrics suppport for resources
 """
+import re
+
 from collections import namedtuple
 from concurrent.futures import as_completed
 from datetime import datetime, timedelta
@@ -76,8 +78,7 @@ class MetricsFilter(Filter):
                'patternProperties': {
                    '^.*$': {'type': 'string'}}},
            # Type choices
-           'statistics': {'type': 'string', 'enum': [
-               'Average', 'Sum', 'Maximum', 'Minimum', 'SampleCount']},
+           'statistics': {'type': 'string'},
            'days': {'type': 'number'},
            'op': {'type': 'string', 'enum': list(OPERATORS.keys())},
            'value': {'type': 'number'},
@@ -125,11 +126,19 @@ class MetricsFilter(Filter):
         'workspaces': 'AWS/WorkSpaces',
     }
 
+    standard_stats = {'Average', 'Sum', 'Maximum', 'Minimum', 'SampleCount'}
+    extended_stats_re = re.compile(r'^p\d{1,3}\.{0,1}\d{0,1}$')
+
     def __init__(self, data, manager=None):
         super(MetricsFilter, self).__init__(data, manager)
         self.days = self.data.get('days', 14)
 
     def validate(self):
+        stats = self.data.get('statistics', 'Average')
+        if stats not in self.standard_stats and not self.extended_stats_re.match(stats):
+            raise PolicyValidationError(
+                "metrics filter statistics method %s not supported" % stats)
+
         if self.days > 455:
             raise PolicyValidationError(
                 "metrics filter days value (%s) cannot exceed 455" % self.days)
@@ -230,15 +239,23 @@ def process_resource_set(self, resource_set):
             # means multiple filters within a policy using the same metric
             # across different periods or dimensions would be problematic.
             key = "%s.%s.%s.%s" % (self.namespace, self.metric, self.statistics, str(self.days))
+
+            params = dict(
+                Namespace=self.namespace,
+                MetricName=self.metric,
+                StartTime=self.start,
+                EndTime=self.end,
+                Period=self.period,
+                Dimensions=dimensions
+            )
+
+            stats_key = (self.statistics in self.standard_stats
+                         and 'Statistics' or 'ExtendedStatistics')
+            params[stats_key] = [self.statistics]
+
             if key not in collected_metrics:
                 collected_metrics[key] = client.get_metric_statistics(
-                    Namespace=self.namespace,
-                    MetricName=self.metric,
-                    Statistics=[self.statistics],
-                    StartTime=self.start,
-                    EndTime=self.end,
-                    Period=self.period,
-                    Dimensions=dimensions)['Datapoints']
+                    **params)['Datapoints']
 
             # In certain cases CloudWatch reports no data for a metric.
             # If the policy specifies a fill value for missing data, add


Test patch:
diff --git a/tests/test_ebs.py b/tests/test_ebs.py
--- a/tests/test_ebs.py
+++ b/tests/test_ebs.py
@@ -833,6 +833,29 @@ def test_ebs_non_fault_tolerant(self):
 
 class PiopsMetricsFilterTest(BaseTest):
 
+    def test_metrics_validation(self):
+        policy = self.load_policy(
+            {
+                "name": "ebs-metrics-test",
+                "resource": "ebs",
+                "filters": [{
+                    "type": "metrics",
+                    "name": "VOlumeConsumedReadWriteOps",
+                    "value": 50,
+                    "op": "gt"}]})
+        metrics = policy.resource_manager.filters[0]
+        metrics.data['statistics'] = 'p99'
+        metrics.validate()
+        metrics.data['statistics'] = 'p99.5'
+        metrics.validate()
+        metrics.data['statistics'] = 'pabc'
+        try:
+            metrics.validate()
+        except PolicyValidationError:
+            pass
+        else:
+            self.fail()
+
     def test_ebs_metrics_percent_filter(self):
         session = self.replay_flight_data("test_ebs_metrics_percent_filter")
         policy = self.load_policy(