Repository: cloud-custodian/cloud-custodian

Issue:
Add config support for ECS
**Is your feature request related to a problem? Please describe.**
We have a lot of ecs task definitions(100K+) in our environment and are running into api throttling issue for controls around task definitions.

**Describe the solution you'd like**
AWS recently rolled out Config support for ECS resources(https://aws.amazon.com/about-aws/whats-new/2021/02/aws-config-supports-amazon-container-services/)

**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.

**Additional context**
Add any other context or screenshots about the feature request here.



Discussion:
what percent of those ecs task definitions are just garbage (not used) lying around? is that something that would be helpful wrt to cleanup. re config support sure.
Based off of some preliminary testing, about 90% are unused. It's hard to gauge though because while we can scan services, task sets and scheduled tasks to extract the definition arns and extrapolate which ones are in use, the list_tasks api (when scanning for stopped status) will only spit out that last hour or so of runs. That list_tasks scan limitation is an issue because of the potential for one off tasks, or things created from external processes like lambda. 

We have a PFR out for adding a lifecycle rule on the definitions to track last used date and help with cleanup but there is no ETA yet. 
intriguing challenge ;-) so basically due to the possibility of adhoc one off tasks, there's no clear mechanism to apriori safely clean out tasks defs. i think there's still some utility in a cleanup here, with guidance to app teams to reuse ref'd defs for those tasks (aka command override). 

anyways... regardless of that we should move forward with config support.

Resolving patch:
diff --git a/c7n/resources/ecs.py b/c7n/resources/ecs.py
--- a/c7n/resources/ecs.py
+++ b/c7n/resources/ecs.py
@@ -8,6 +8,7 @@
 from c7n.manager import resources
 from c7n.utils import local_session, chunks, get_retry, type_schema, group_by
 from c7n import query
+from c7n.query import DescribeSource, ConfigSource
 import jmespath
 from c7n.tags import Tag, TagDelayedAction, RemoveTag, TagActionFilter
 from c7n.actions import AutoTagUser
@@ -476,33 +477,25 @@ def process(self, resources):
                     raise
 
 
-@resources.register('ecs-task-definition')
-class TaskDefinition(query.QueryResourceManager):
-
-    class resource_type(query.TypeInfo):
-        service = 'ecs'
-        arn = id = name = 'taskDefinitionArn'
-        enum_spec = ('list_task_definitions', 'taskDefinitionArns', None)
-        cfn_type = 'AWS::ECS::TaskDefinition'
-        arn_type = 'task-definition'
+class DescribeTaskDefinition(DescribeSource):
 
     def get_resources(self, ids, cache=True):
         if cache:
-            resources = self._get_cached_resources(ids)
+            resources = self.manager._get_cached_resources(ids)
             if resources is not None:
                 return resources
         try:
             resources = self.augment(ids)
             return resources
         except ClientError as e:
-            self.log.warning("event ids not resolved: %s error:%s" % (ids, e))
+            self.manager.log.warning("event ids not resolved: %s error:%s" % (ids, e))
             return []
 
     def augment(self, resources):
         results = []
-        client = local_session(self.session_factory).client('ecs')
+        client = local_session(self.manager.session_factory).client('ecs')
         for task_def_set in resources:
-            response = self.retry(
+            response = self.manager.retry(
                 client.describe_task_definition,
                 taskDefinition=task_def_set,
                 include=['TAGS'])
@@ -513,6 +506,48 @@ def augment(self, resources):
         return results
 
 
+class ConfigECSTaskDefinition(ConfigSource):
+
+    preserve_empty = {'mountPoints', 'portMappings', 'volumesFrom'}
+    preserve_case = {'Tags'}
+
+    @classmethod
+    def lowerKeys(cls, data):
+        if isinstance(data, dict):
+            for k, v in list(data.items()):
+                if k in cls.preserve_case:
+                    continue
+                lk = k[0].lower() + k[1:]
+                data[lk] = data.pop(k)
+                # describe doesn't return empty list/dict by default
+                if isinstance(v, (list, dict)) and not v and lk not in cls.preserve_empty:
+                    data.pop(lk)
+                elif isinstance(v, (dict, list)):
+                    data[lk] = cls.lowerKeys(v)
+        elif isinstance(data, list):
+            return list(map(cls.lowerKeys, data))
+        return data
+
+    def load_resource(self, item):
+        return self.lowerKeys(super().load_resource(item))
+
+
+@resources.register('ecs-task-definition')
+class TaskDefinition(query.QueryResourceManager):
+
+    class resource_type(query.TypeInfo):
+        service = 'ecs'
+        arn = id = name = 'taskDefinitionArn'
+        enum_spec = ('list_task_definitions', 'taskDefinitionArns', None)
+        cfn_type = config_type = 'AWS::ECS::TaskDefinition'
+        arn_type = 'task-definition'
+
+    source_mapping = {
+        'config': ConfigECSTaskDefinition,
+        'describe': DescribeTaskDefinition
+    }
+
+
 @TaskDefinition.action_registry.register('delete')
 class DeleteTaskDefinition(BaseAction):
     """Delete/DeRegister a task definition.


Test patch:
diff --git a/tests/data/placebo/test_ecs_task_def_config/config.GetResourceConfigHistory_1.json b/tests/data/placebo/test_ecs_task_def_config/config.GetResourceConfigHistory_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_ecs_task_def_config/config.GetResourceConfigHistory_1.json
@@ -0,0 +1,37 @@
+{
+    "status_code": 200,
+    "data": {
+        "configurationItems": [
+            {
+                "version": "1.3",
+                "accountId": "644160558196",
+                "configurationItemCaptureTime": {
+                    "__class__": "datetime",
+                    "year": 2021,
+                    "month": 3,
+                    "day": 8,
+                    "hour": 0,
+                    "minute": 57,
+                    "second": 12,
+                    "microsecond": 720000
+                },
+                "configurationItemStatus": "OK",
+                "configurationStateId": "1615183032720",
+                "configurationItemMD5Hash": "",
+                "arn": "arn:aws:ecs:us-east-1:644160558196:task-definition/TEST:1",
+                "resourceType": "AWS::ECS::TaskDefinition",
+                "resourceId": "TEST:1",
+                "resourceName": "TEST:1",
+                "awsRegion": "us-east-1",
+                "availabilityZone": "Regional",
+                "tags": {},
+                "relatedEvents": [],
+                "relationships": [],
+                "configuration": "{\"ContainerDefinitions\":[{\"Name\":\"dwcqwc\",\"Image\":\"qwcqwc.comwqe\",\"Cpu\":0,\"Links\":[],\"PortMappings\":[],\"Essential\":true,\"EntryPoint\":[],\"Command\":[],\"Environment\":[],\"EnvironmentFiles\":[],\"MountPoints\":[],\"VolumesFrom\":[],\"Secrets\":[],\"DependsOn\":[],\"DnsServers\":[],\"DnsSearchDomains\":[],\"ExtraHosts\":[],\"DockerSecurityOptions\":[],\"DockerLabels\":{},\"Ulimits\":[],\"LogConfiguration\":{\"LogDriver\":\"awslogs\",\"Options\":{\"awslogs-group\":\"/ecs/TEST\",\"awslogs-region\":\"us-east-1\",\"awslogs-stream-prefix\":\"ecs\"},\"SecretOptions\":[]},\"SystemControls\":[],\"ResourceRequirements\":[]}],\"Cpu\":\"256\",\"ExecutionRoleArn\":\"arn:aws:iam::644160558196:role/ecsTaskExecutionRole\",\"Family\":\"TEST\",\"InferenceAccelerators\":[],\"Memory\":\"512\",\"NetworkMode\":\"awsvpc\",\"PlacementConstraints\":[],\"RequiresCompatibilities\":[\"FARGATE\"],\"Status\":\"INACTIVE\",\"Tags\":[],\"TaskDefinitionArn\":\"arn:aws:ecs:us-east-1:644160558196:task-definition/TEST:1\",\"TaskRoleArn\":\"arn:aws:iam::644160558196:role/ecsTaskExecutionRole\",\"Volumes\":[]}",
+                "supplementaryConfiguration": {}
+            }
+        ],
+        "nextToken": "eyJlbmNyeXB0ZWREYXRhIjpbMzYsLTgsNzYsLTY0LC05NCwtNzYsMTIzLDQ0LDE2LC04OCwtMTYsMTI2LDI2LC0xMDgsNzUsNzAsNDYsMTAyLC0xLC00NywtODQsLTM3LC00Nyw2MiwxMSw0NywtNCw0OCw3MCwtMTA1LDU5LDEyNiw5MCwxMjAsLTkzLC00MSwtMTIsNDQsMjEsMTEsMjEsLTU0LC0xNywtMTIyLDMyLC03NSw4MywyMCwyNiwtMTI1LDEwNSwxOSw5MiwtNjAsLTksMTE2LC0xMDUsLTM5LDExMSwtMTQsLTI1LDY1LC00NiwtMzEsLTIyLC01MywtOTEsNjQsLTcxLC0xMDUsMjYsMTAxLC0yNCwzNSwtODQsMTIyLC0xNSw1MCwtMTIyLDUsLTM5LC00LC0xOCwtNjUsMTE3LDExNCwxMDcsLTk5LC0xMDgsLTcwLC00NiwtNzksLTc5LDEzLC0zNyw4OSwzNCwtNDIsLTk5LDIxLC0xMDMsMzksMzAsLTczLC0xMTUsLTk2LDcsODUsLTgzLC0xMjcsLTE4LC0yNSwzMSwzMywxMTQsMTA4LDMxLDUyLDAsLTEyMCw4Nyw0MiwtNzMsLTgxLDUyLDQ3LDYxLDkzLDc1LDQ3LDksMiwtMSwzMyw5MywxNSwtMTI0LC0xMTAsLTkwLC00OCwtODEsODgsMTEsMjQsLTMzLDU5LC05OCwtNjIsLTgyLC03MCwtNDgsNDcsLTQwLC0xMDAsMjEsLTExMCwtMjMsNDksLTQxLDExNyw4OCwtODcsLTc4LC0xOSwyMiwtMTE0XSwibWF0ZXJpYWxTZXRTZXJpYWxOdW1iZXIiOjEsIml2UGFyYW1ldGVyU3BlYyI6eyJpdiI6WzYsLTI5LDMsLTUsLTEwMiwtMTIxLC0zOCwyNiwxMDcsLTEwMCwtNjEsOTIsOCwtNzEsNzcsLTc3XX19",
+        "ResponseMetadata": {}
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_ecs_task_def_config/config.GetResourceConfigHistory_2.json b/tests/data/placebo/test_ecs_task_def_config/config.GetResourceConfigHistory_2.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_ecs_task_def_config/config.GetResourceConfigHistory_2.json
@@ -0,0 +1,39 @@
+{
+    "status_code": 200,
+    "data": {
+        "configurationItems": [
+            {
+                "version": "1.3",
+                "accountId": "644160558196",
+                "configurationItemCaptureTime": {
+                    "__class__": "datetime",
+                    "year": 2021,
+                    "month": 3,
+                    "day": 9,
+                    "hour": 6,
+                    "minute": 33,
+                    "second": 56,
+                    "microsecond": 616000
+                },
+                "configurationItemStatus": "OK",
+                "configurationStateId": "1615289636616",
+                "configurationItemMD5Hash": "",
+                "arn": "arn:aws:ecs:us-east-1:644160558196:task-definition/app-fargate-task:2",
+                "resourceType": "AWS::ECS::TaskDefinition",
+                "resourceId": "app-fargate-task:2",
+                "resourceName": "app-fargate-task:2",
+                "awsRegion": "us-east-1",
+                "availabilityZone": "Regional",
+                "tags": {
+                    "test": "name"
+                },
+                "relatedEvents": [],
+                "relationships": [],
+                "configuration": "{\"ContainerDefinitions\":[{\"Name\":\"fargate-app-2\",\"Image\":\"httpd:2.4\",\"Cpu\":0,\"Links\":[],\"PortMappings\":[{\"ContainerPort\":80,\"HostPort\":80,\"Protocol\":\"tcp\"}],\"Essential\":true,\"EntryPoint\":[\"sh\",\"-c\"],\"Command\":[\"/bin/sh -c \\\"echo \\u0027\\u003chtml\\u003e \\u003chead\\u003e \\u003ctitle\\u003eAmazon ECS Sample App\\u003c/title\\u003e \\u003cstyle\\u003ebody {margin-top: 40px; background-color: #333;} \\u003c/style\\u003e \\u003c/head\\u003e\\u003cbody\\u003e \\u003cdiv style\\u003dcolor:white;text-align:center\\u003e \\u003ch1\\u003eAmazon ECS Sample App\\u003c/h1\\u003e \\u003ch2\\u003eCongratulations!\\u003c/h2\\u003e \\u003cp\\u003eYour application is now running on a container in Amazon ECS.\\u003c/p\\u003e \\u003c/div\\u003e\\u003c/body\\u003e\\u003c/html\\u003e\\u0027 \\u003e  /usr/local/apache2/htdocs/index.html \\u0026\\u0026 httpd-foreground\\\"\"],\"Environment\":[],\"EnvironmentFiles\":[],\"MountPoints\":[],\"VolumesFrom\":[],\"Secrets\":[],\"DependsOn\":[],\"DnsServers\":[],\"DnsSearchDomains\":[],\"ExtraHosts\":[],\"DockerSecurityOptions\":[],\"DockerLabels\":{},\"Ulimits\":[],\"SystemControls\":[],\"ResourceRequirements\":[]}],\"Cpu\":\"256\",\"Family\":\"app-fargate-task\",\"InferenceAccelerators\":[],\"Memory\":\"512\",\"NetworkMode\":\"awsvpc\",\"PlacementConstraints\":[],\"RequiresCompatibilities\":[\"FARGATE\"],\"Status\":\"ACTIVE\",\"Tags\":[{\"Key\":\"test\",\"Value\":\"name\"}],\"TaskDefinitionArn\":\"arn:aws:ecs:us-east-1:644160558196:task-definition/app-fargate-task:2\",\"Volumes\":[]}",
+                "supplementaryConfiguration": {}
+            }
+        ],
+        "nextToken": "eyJlbmNyeXB0ZWREYXRhIjpbMTAzLC03MiwxMjEsOTMsOTEsNjMsLTc5LC0zOSwtNDUsLTE4LDExMCwtMTA0LC02NSw5MiwtNDcsNzMsMjksMTksLTEsMCw1OCw5MywyMCwxOCwtOTYsLTc3LDI5LC05NCwxMTMsNzksLTEyMywxMjUsLTkxLC01MSwtODYsOTEsOTUsMzcsLTE0LC05LDU5LC0xMTAsLTk1LDM1LDEyLC02Miw2OCwyNywtMTE3LC0zNiwtMTAyLC0yNywzNSw1MSw4MCw4MCw5MCw0NCwxMTUsLTYzLC0xMjAsLTEyNCwtOTYsLTkxLDg5LDEyNiwtNzgsMTI1LC01OSw5OSwtMzcsLTcwLC0zMyw0MCwtOTAsLTg1LC0yNCwxMjIsLTEyMiw1MywtMTksLTgxLDM2LC01LC05MiwxMjIsLTkwLC00MCwxMjIsNzYsMTcsLTUsNDgsNTMsOTcsNDAsLTQsNzgsMTEzLC0xMDUsMTIwLC0xOCwxMDgsODgsMTEyLC0xMDYsLTU4LDI3LC0zNiwzLDEwMSwxMTgsMjksLTExNCwzOCwxMTYsMzUsLTIwLDk4LC0xMTcsNTEsODQsLTkzLDcxLDEzLC0xLDg3LDQ0LC0xMDAsOTIsLTQ3LDY1LC03OCwtMTUsLTc5LC05NSw0MiwtNTYsLTgsNiw3OCwtMTA4LC0zNCwtOSw3NCwtMTE4LC0yLDEyNSwtMTcsMjIsLTQsMjUsLTYwLC0xMDEsOTgsLTUxLDI2LC0xMTgsNTYsOTgsLTE4LC0xMjAsNzUsNDgsLTQsLTJdLCJtYXRlcmlhbFNldFNlcmlhbE51bWJlciI6MSwiaXZQYXJhbWV0ZXJTcGVjIjp7Iml2IjpbLTYzLDEyNywyNywtOTIsLTQxLC0xMTMsMTE5LDI2LC00NCwxNywyOCwtNzIsLTQzLC0xMTQsLTEwMiw1OF19fQ==",
+        "ResponseMetadata": {}
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_ecs_task_def_config/config.ListDiscoveredResources_1.json b/tests/data/placebo/test_ecs_task_def_config/config.ListDiscoveredResources_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_ecs_task_def_config/config.ListDiscoveredResources_1.json
@@ -0,0 +1,18 @@
+{
+    "status_code": 200,
+    "data": {
+        "resourceIdentifiers": [
+            {
+                "resourceType": "AWS::ECS::TaskDefinition",
+                "resourceId": "TEST:1",
+                "resourceName": "TEST:1"
+            },
+            {
+                "resourceType": "AWS::ECS::TaskDefinition",
+                "resourceId": "app-fargate-task:2",
+                "resourceName": "app-fargate-task:2"
+            }
+        ],
+        "ResponseMetadata": {}
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_ecs_task_def_config/config.SelectResourceConfig_1.json b/tests/data/placebo/test_ecs_task_def_config/config.SelectResourceConfig_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_ecs_task_def_config/config.SelectResourceConfig_1.json
@@ -0,0 +1,20 @@
+{
+    "status_code": 200,
+    "data": {
+        "Results": [],
+        "QueryInfo": {
+            "SelectFields": [
+                {
+                    "Name": "resourceId"
+                },
+                {
+                    "Name": "configuration"
+                },
+                {
+                    "Name": "supplementaryConfiguration"
+                }
+            ]
+        },
+        "ResponseMetadata": {}
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_ecs_task_def_config/ecs.ListTagsForResource_1.json b/tests/data/placebo/test_ecs_task_def_config/ecs.ListTagsForResource_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_ecs_task_def_config/ecs.ListTagsForResource_1.json
@@ -0,0 +1,7 @@
+{
+    "status_code": 200,
+    "data": {
+        "tags": [],
+        "ResponseMetadata": {}
+    }
+}
\ No newline at end of file
diff --git a/tests/data/placebo/test_ecs_task_def_config/ecs.UntagResource_1.json b/tests/data/placebo/test_ecs_task_def_config/ecs.UntagResource_1.json
new file mode 100644
--- /dev/null
+++ b/tests/data/placebo/test_ecs_task_def_config/ecs.UntagResource_1.json
@@ -0,0 +1,6 @@
+{
+    "status_code": 200,
+    "data": {
+        "ResponseMetadata": {}
+    }
+}
\ No newline at end of file
diff --git a/tests/test_ecs.py b/tests/test_ecs.py
--- a/tests/test_ecs.py
+++ b/tests/test_ecs.py
@@ -262,7 +262,7 @@ def test_task_definition_get_resources(self):
             session_factory=session_factory,
         )
         arn = "arn:aws:ecs:us-east-1:644160558196:task-definition/ecs-read-only-root:1"
-        resources = p.resource_manager.get_resources([arn])
+        resources = p.resource_manager.get_source('describe').get_resources([arn])
         self.assertEqual(len(resources), 1)
         self.assertEqual(resources[0]["taskDefinitionArn"], arn)
         self.assertEqual(
@@ -307,6 +307,53 @@ def test_ecs_task_def_tags(self):
                     resourceArn=resources[0]["taskDefinitionArn"]).get("tags")}
         self.assertEqual(tags, {"TestKey": "TestValue", "c7n-tag": "present"})
 
+    def test_ecs_task_def_config(self):
+        session_factory = self.replay_flight_data("test_ecs_task_def_config")
+        p = self.load_policy(
+            {
+                "name": "ecs-task-def-config-tag",
+                "resource": "ecs-task-definition",
+                "source": "config",
+                "filters": [
+                    {"tag:test": "name"}
+                ],
+                "actions": [
+                    {"type": "remove-tag", "tags": ["test"]}
+                ],
+            },
+            session_factory=session_factory,
+        )
+        resources = p.run()
+        self.assertEqual(len(resources), 1)
+        assert resources[0]['containerDefinitions'] == [
+            {'command': ['/bin/sh -c "echo \'<html> <head> '
+                         '<title>Amazon ECS Sample App</title> '
+                         '<style>body {margin-top: 40px; '
+                         'background-color: #333;} </style> '
+                         '</head><body> <div '
+                         'style=color:white;text-align:center> '
+                         '<h1>Amazon ECS Sample App</h1> '
+                         '<h2>Congratulations!</h2> <p>Your '
+                         'application is now running on a '
+                         'container in Amazon ECS.</p> '
+                         "</div></body></html>' >  "
+                         '/usr/local/apache2/htdocs/index.html '
+                         '&& httpd-foreground"'],
+             'cpu': 0,
+             'entryPoint': ['sh', '-c'],
+             'essential': True,
+             'image': 'httpd:2.4',
+             'mountPoints': [],
+             'name': 'fargate-app-2',
+             'portMappings': [{'containerPort': 80,
+                               'hostPort': 80,
+                               'protocol': 'tcp'}],
+             'volumesFrom': []}]
+        assert resources[0]['Tags'] == [{'Key': 'test', 'Value': 'name'}]
+        client = session_factory().client("ecs")
+        self.assertEqual(len(client.list_tags_for_resource(
+            resourceArn=resources[0]["taskDefinitionArn"]).get("tags")), 0)
+
 
 class TestEcsTask(BaseTest):
 
diff --git a/tests/test_policy.py b/tests/test_policy.py
--- a/tests/test_policy.py
+++ b/tests/test_policy.py
@@ -274,6 +274,7 @@ def test_config_resource_support(self):
             'AWS::EC2::EgressOnlyInternetGateway',
             'AWS::EC2::VPCEndpointService',
             'AWS::EC2::FlowLog',
+            'AWS::ECS::TaskDefinition',
             'AWS::RDS::DBSecurityGroup',
             'AWS::RDS::EventSubscription',
             'AWS::S3::AccountPublicAccessBlock',
@@ -323,7 +324,10 @@ def test_config_resource_support(self):
             raise AssertionError(
                 "Missing config types \n %s" % ('\n'.join(missing)))
 
+        # config service can't be bothered to update their sdk correctly
+        invalid_ignore = {'AWS::ECS::TaskDefinition'}
         bad_types = resource_config_types.difference(config_types)
+        bad_types = bad_types.difference(invalid_ignore)
         if bad_types:
             raise AssertionError(
                 "Invalid config types \n %s" % ('\n'.join(bad_types)))