Repository: cloud-custodian/cloud-custodian

Issue:
aws - notify action batch_size flexibility
### Describe the feature

Currently, notify action batch_size is set to 250. Some of our policies are hitting `SNS:MessagetooLongError` error even after lowering the batch size to 100 especially on ecs-task-definition/ecs-services policies.
Does is make sense to have this as a parameter in the policy within the notify action? That way we could stick to the batch_size of 250 in general and modify policies to have lower values as needed. The lower batch_size then would apply to a fewer set of policies instead of having a lower threshold across the board


### Extra information or context

_No response_


Discussion:
ideally we would dynamically size the batch, by serializing each resource, track size, and then manually assemble them the json from the set of serialized resources, we would need to assume a default compression ratio for gzip and measure the b64 encoding on top of that.
Thanks @kapilt. Sounds good, will try to give it a shot

Resolving patch:
diff --git a/c7n/actions/notify.py b/c7n/actions/notify.py
--- a/c7n/actions/notify.py
+++ b/c7n/actions/notify.py
@@ -14,9 +14,109 @@
 from c7n.version import version
 
 
+class ResourceMessageBuffer:
+
+    # conservative ratio calculated over all extant json test data
+    # files, most resources have many common repeated keys and values
+    # re compress down well.
+    #
+    # base64 increases size, but the compression still reduces total size versus raw.
+    # https://lemire.me/blog/2019/01/30/what-is-the-space-overhead-of-base64-encoding/
+    #
+    # script to caculate ratio
+    # https://gist.github.com/kapilt/8c3558a7db0d178cb1c4e91d47dacc77
+    #
+    # we use this conservative value as a seed and adapt based on observed data
+    seed_b64_zlib_ratio = 0.5
+
+    def __init__(self, envelope, buffer_max_size):
+        self.buffer_max_size = buffer_max_size
+        self.resource_parts = []
+
+        envelope['resources'] = []
+        self.envelope = utils.dumps(envelope)
+        self.raw_size = float(len(self.envelope))
+        self.observed_ratio = 0
+        self.fill_sizes = []
+
+    def add(self, resource):
+        self.resource_parts.append(utils.dumps(resource))
+        self.raw_size += len(self.resource_parts[-1])
+
+    def __len__(self):
+        return len(self.resource_parts)
+
+    def __repr__(self):
+        return (f"<ResourceBuffer count:{len(self)} esize:{self.estimated_size:.1f}"
+                f" ratio:{self.compress_ratio:.2f} avg_rsize:{self.average_rsize:.1f}"
+                f" fill:{self.fill_ratio:.2f}>")
+
+    @property
+    def fill_ratio(self):
+        cardinality = float(len(self.fill_sizes) or 1)
+        return sum(self.fill_sizes) / (self.buffer_max_size * cardinality)
+
+    @property
+    def estimated_size(self):
+        return self.raw_size * self.compress_ratio
+
+    @property
+    def compress_ratio(self):
+        return self.observed_ratio or self.seed_b64_zlib_ratio
+
+    @property
+    def average_rsize(self):
+        rcount = len(self)
+        if not rcount:
+            return 0
+        return (self.raw_size - len(self.envelope)) / float(rcount)
+
+    @property
+    def full(self):
+        """ heuristic to calculate size of payload
+        """
+        if (self.raw_size + self.average_rsize * 2) * self.compress_ratio > self.buffer_max_size:
+            return True
+        return False
+
+    def consume(self):
+        rbegin_idx = self.envelope.rfind('[')
+        rend_idx = self.envelope.rfind(']')
+
+        payload = self.envelope
+        payload = "%s%s%s" % (
+            payload[:rbegin_idx+1],
+            ",".join(self.resource_parts),
+            payload[rend_idx:]
+        )
+
+        serialized_payload = base64.b64encode(
+            zlib.compress(
+                payload.encode('utf8')
+            )
+        ).decode('ascii')
+
+        if len(serialized_payload) > self.buffer_max_size:
+            raise AssertionError(
+                f"{self} payload over max size:{len(serialized_payload)}"
+            )
+
+        self.fill_sizes.append(len(serialized_payload))
+        self.resource_parts = []
+        # adapative ratio based on payload contents, with a static
+        # increment for headroom on resource variance.
+        self.observed_ratio = min(
+            (len(serialized_payload) / float(self.raw_size)) + 0.1,
+            self.seed_b64_zlib_ratio
+        )
+        self.raw_size = float(len(self.envelope))
+        return serialized_payload
+
+
 class BaseNotify(EventAction):
 
-    batch_size = 250
+    message_buffer_class = ResourceMessageBuffer
+    buffer_max_size = 262144
 
     def expand_variables(self, message):
         """expand any variables in the action to_from/cc_from fields.
@@ -177,12 +277,22 @@ def process(self, resources, event=None):
             'policy': self.manager.data}
         message['action'] = self.expand_variables(message)
 
-        for batch in utils.chunks(resources, self.batch_size):
-            message['resources'] = self.prepare_resources(batch)
-            receipt = self.send_data_message(message)
-            self.log.info("sent message:%s policy:%s template:%s count:%s" % (
-                receipt, self.manager.data['name'],
-                self.data.get('template', 'default'), len(batch)))
+        rbuffer = self.message_buffer_class(message, self.buffer_max_size)
+        for r in self.prepare_resources(resources):
+            rbuffer.add(r)
+            if rbuffer.full:
+                self.consume_buffer(message, rbuffer)
+
+        if len(rbuffer):
+            self.consume_buffer(message, rbuffer)
+
+    def consume_buffer(self, message, rbuffer):
+        rcount = len(rbuffer)
+        payload = rbuffer.consume()
+        receipt = self.send_data_message(message, payload)
+        self.log.info("sent message:%s policy:%s template:%s count:%s" % (
+            receipt, self.manager.data['name'],
+            self.data.get('template', 'default'), rcount))
 
     def prepare_resources(self, resources):
         """Resources preparation for transport.
@@ -203,6 +313,11 @@ def prepare_resources(self, resources):
             return resources
         return handler(resources)
 
+    def prepare_ecs_service(self, resources):
+        for r in resources:
+            r.pop('events', None)
+        return resources
+
     def prepare_launch_config(self, resources):
         for r in resources:
             r.pop('UserData', None)
@@ -228,13 +343,13 @@ def prepare_iam_saml_provider(self, resources):
                 r.pop('IDPSSODescriptor')
         return resources
 
-    def send_data_message(self, message):
+    def send_data_message(self, message, payload):
         if self.data['transport']['type'] == 'sqs':
-            return self.send_sqs(message)
+            return self.send_sqs(message, payload)
         elif self.data['transport']['type'] == 'sns':
-            return self.send_sns(message)
+            return self.send_sns(message, payload)
 
-    def send_sns(self, message):
+    def send_sns(self, message, payload):
         topic = self.data['transport']['topic'].format(**message)
         user_attributes = self.data['transport'].get('attributes')
         if topic.startswith('arn:'):
@@ -260,12 +375,12 @@ def send_sns(self, message):
                     attrs[k] = {'DataType': 'String', 'StringValue': v}
         result = client.publish(
             TopicArn=topic_arn,
-            Message=self.pack(message),
+            Message=payload,
             MessageAttributes=attrs
         )
         return result['MessageId']
 
-    def send_sqs(self, message):
+    def send_sqs(self, message, payload):
         queue = self.data['transport']['queue'].format(**message)
         if queue.startswith('https://queue.amazonaws.com'):
             region = 'us-east-1'
@@ -299,7 +414,7 @@ def send_sqs(self, message):
         }
         result = client.send_message(
             QueueUrl=queue_url,
-            MessageBody=self.pack(message),
+            MessageBody=payload,
             MessageAttributes=attrs)
         return result['MessageId']
 


Test patch:
diff --git a/tests/test_notify.py b/tests/test_notify.py
--- a/tests/test_notify.py
+++ b/tests/test_notify.py
@@ -10,6 +10,50 @@
 import zlib
 
 from c7n.exceptions import PolicyValidationError
+from c7n.actions.notify import ResourceMessageBuffer
+
+import pytest
+
+
+def test_msg_buffer():
+    buf_size = 1024
+    mbuffer = ResourceMessageBuffer({'env': 'dev', 'region': 'us-east-2'}, buf_size)
+    assert mbuffer.full is False
+
+    for i in range(0, 50):
+        mbuffer.add({'id': 'x%s' % i, 'a': 1, 'b': 2 + i, 'c': 5 * i})
+        if mbuffer.full:
+            break
+
+    assert len(mbuffer) == 47
+    assert int(mbuffer.estimated_size) == 995
+    payload = mbuffer.consume()
+    assert len(payload) == 532
+    assert mbuffer.observed_ratio > 0.36 and mbuffer.observed_ratio < 0.5
+    assert 'resources' in json.loads(zlib.decompress(base64.b64decode(payload)))
+
+    assert len(mbuffer) == 0
+    # raw size reverts back to envelope
+    assert mbuffer.raw_size == 56
+
+    # repeat, but with observed dynamic ratio now
+    for i in range(0, 100):
+        mbuffer.add({'id': 'x%s' % i, 'a': 1, 'b': 2 + i, 'c': 5 * i})
+        if mbuffer.full:
+            break
+
+    assert len(mbuffer) == 65
+    payload = mbuffer.consume()
+    assert len(payload) == 680
+
+
+def test_msg_buffer_exceed():
+    mbuffer = ResourceMessageBuffer({'env': 'dev', 'region': 'us-west-2'}, 100)
+    assert mbuffer.full is False
+    mbuffer.add({'id': 'x', 'values': list(range(100))})
+    with pytest.raises(AssertionError) as e_info:
+        mbuffer.consume()
+    assert str(mbuffer) in str(e_info.value)
 
 
 class NotifyTest(BaseTest):