Repository: cloudtools/troposphere

Issue:
EMR: AutoScalingPolicy property missing from InstanceGroupConfigProperty
InstanceGroupConfigProperty is missing AutoScalingPolicy


Discussion:
This is already included as part of boto3 (as well as CloudFormation, AWS CLI, etc., but hasn't been implemented within Troposphere.

https://boto3.readthedocs.io/en/latest/reference/services/emr.html#EMR.Client.put_auto_scaling_policy

Any assistance for implementing this is appreciated.

Resolving patch:
diff --git a/examples/EMR_Cluster.py b/examples/EMR_Cluster.py
--- a/examples/EMR_Cluster.py
+++ b/examples/EMR_Cluster.py
@@ -4,6 +4,47 @@
 import troposphere.iam as iam
 
 
+scaling_policy = emr.SimpleScalingPolicyConfiguration(
+                    AdjustmentType="EXACT_CAPACITY",
+                    ScalingAdjustment="1",
+                    CoolDown="300"
+                )
+
+
+def generate_rules(rules_name):
+    global emr, scaling_policy
+
+    rules = [
+        emr.ScalingRule(
+            Name=rules_name,
+            Description="%s rules" % rules_name,
+            Action=emr.ScalingAction(
+                Market="ON_DEMAND",
+                SimpleScalingPolicyConfiguration=scaling_policy
+            ),
+            Trigger=emr.ScalingTrigger(
+                CloudWatchAlarmDefinition=emr.CloudWatchAlarmDefinition(
+                    ComparisonOperator="GREATER_THAN",
+                    EvaluationPeriods="120",
+                    MetricName="TestMetric",
+                    Namespace="AWS/ElasticMapReduce",
+                    Period="300",
+                    Statistic="AVERAGE",
+                    Threshold="50",
+                    Unit="PERCENT",
+                    Dimensions=[
+                        emr.KeyValue(
+                            'my.custom.master.property',
+                            'my.custom.master.value'
+                        )
+                    ]
+                )
+            )
+        )
+    ]
+    return rules
+
+
 template = Template()
 template.add_description(
     "Sample CloudFormation template for creating an EMR cluster"
@@ -59,6 +100,8 @@
     ]
 ))
 
+emr_autoscaling_role = "EMR_AutoScaling_DefaultRole"
+
 emr_job_flow_role = template.add_resource(iam.Role(
     "EMRJobFlowRole",
     AssumeRolePolicyDocument={
@@ -126,6 +169,7 @@
     ],
     JobFlowRole=Ref(emr_instance_profile),
     ServiceRole=Ref(emr_service_role),
+    AutoScalingRole=Ref(emr_autoscaling_role),
     Instances=emr.JobFlowInstancesConfig(
         Ec2KeyName=Ref(keyname),
         Ec2SubnetId=Ref(subnet),
@@ -133,12 +177,26 @@
             Name="Master Instance",
             InstanceCount="1",
             InstanceType=M4_LARGE,
-            Market="ON_DEMAND"
+            Market="ON_DEMAND",
+            AutoScalingPolicy=emr.AutoScalingPolicy(
+                Constraints=emr.ScalingConstraints(
+                    MinCapacity="1",
+                    MaxCapacity="3"
+                ),
+                Rules=generate_rules("MasterAutoScalingPolicy")
+            )
         ),
         CoreInstanceGroup=emr.InstanceGroupConfigProperty(
             Name="Core Instance",
             BidPrice=If(withSpotPrice, Ref(spot), Ref("AWS::NoValue")),
             Market=If(withSpotPrice, "SPOT", "ON_DEMAND"),
+            AutoScalingPolicy=emr.AutoScalingPolicy(
+                Constraints=emr.ScalingConstraints(
+                    MinCapacity="1",
+                    MaxCapacity="3"
+                ),
+                Rules=generate_rules("CoreAutoScalingPolicy"),
+            ),
             EbsConfiguration=emr.EbsConfiguration(
                 EbsBlockDeviceConfigs=[
                     emr.EbsBlockDeviceConfigs(
diff --git a/troposphere/emr.py b/troposphere/emr.py
--- a/troposphere/emr.py
+++ b/troposphere/emr.py
@@ -122,6 +122,66 @@ class EbsConfiguration(AWSProperty):
     }
 
 
+class ScalingConstraints(AWSProperty):
+    props = {
+        'MinCapacity': (integer, True),
+        'MaxCapacity': (integer, True)
+    }
+
+
+class CloudWatchAlarmDefinition(AWSProperty):
+    props = {
+        'ComparisonOperator': (basestring, True),
+        'EvaluationPeriods': (positive_integer, False),
+        'MetricName': (basestring, True),
+        'Namespace': (basestring, False),
+        'Period': (positive_integer, True),
+        'Statistic': (basestring, False),
+        'Threshold': (positive_integer, True),
+        'Unit': (basestring, False),
+        'Dimensions': ([KeyValue], False)
+    }
+
+
+class ScalingTrigger(AWSProperty):
+    props = {
+        'CloudWatchAlarmDefinition': (CloudWatchAlarmDefinition, True),
+    }
+
+
+class SimpleScalingPolicyConfiguration(AWSProperty):
+    props = {
+        'AdjustmentType': (basestring, False),
+        'ScalingAdjustment': (positive_integer, True),
+        'CoolDown': (positive_integer, False)
+    }
+
+
+class ScalingAction(AWSProperty):
+    props = {
+        'Market': (market_validator, False),
+        'SimpleScalingPolicyConfiguration': (
+            SimpleScalingPolicyConfiguration, True
+        )
+    }
+
+
+class ScalingRule(AWSProperty):
+    props = {
+        'Name': (basestring, True),
+        'Description': (basestring, False),
+        'Action': (ScalingAction, True),
+        'Trigger': (ScalingTrigger, True)
+    }
+
+
+class AutoScalingPolicy(AWSProperty):
+    props = {
+        'Constraints': (ScalingConstraints, True),
+        'Rules': ([ScalingRule], False)
+    }
+
+
 class InstanceGroupConfigProperty(AWSProperty):
     props = {
         'BidPrice': (basestring, False),
@@ -130,7 +190,8 @@ class InstanceGroupConfigProperty(AWSProperty):
         'InstanceCount': (positive_integer, True),
         'InstanceType': (basestring, True),
         'Market': (market_validator, False),
-        'Name': (basestring, False)
+        'Name': (basestring, False),
+        'AutoScalingPolicy': (AutoScalingPolicy, False)
     }
 
 
@@ -171,6 +232,7 @@ class Cluster(AWSObject):
         'Name': (basestring, True),
         'ReleaseLabel': (basestring, False),
         'ServiceRole': (basestring, True),
+        'AutoScalingRole': (basestring, False),
         'Tags': (list, False),
         'VisibleToAllUsers': (boolean, False)
     }


Test patch:
diff --git a/tests/examples_output/EMR_Cluster.template b/tests/examples_output/EMR_Cluster.template
--- a/tests/examples_output/EMR_Cluster.template
+++ b/tests/examples_output/EMR_Cluster.template
@@ -87,6 +87,9 @@
                         "Name": "Spark"
                     }
                 ],
+                "AutoScalingRole": {
+                    "Ref": "EMR_AutoScaling_DefaultRole"
+                },
                 "BootstrapActions": [
                     {
                         "Name": "Dummy bootstrap action",
@@ -139,6 +142,44 @@
                 ],
                 "Instances": {
                     "CoreInstanceGroup": {
+                        "AutoScalingPolicy": {
+                            "Constraints": {
+                                "MaxCapacity": "3",
+                                "MinCapacity": "1"
+                            },
+                            "Rules": [
+                                {
+                                    "Action": {
+                                        "Market": "ON_DEMAND",
+                                        "SimpleScalingPolicyConfiguration": {
+                                            "AdjustmentType": "EXACT_CAPACITY",
+                                            "CoolDown": "300",
+                                            "ScalingAdjustment": "1"
+                                        }
+                                    },
+                                    "Description": "CoreAutoScalingPolicy rules",
+                                    "Name": "CoreAutoScalingPolicy",
+                                    "Trigger": {
+                                        "CloudWatchAlarmDefinition": {
+                                            "ComparisonOperator": "GREATER_THAN",
+                                            "Dimensions": [
+                                                {
+                                                    "Key": "my.custom.master.property",
+                                                    "Value": "my.custom.master.value"
+                                                }
+                                            ],
+                                            "EvaluationPeriods": "120",
+                                            "MetricName": "TestMetric",
+                                            "Namespace": "AWS/ElasticMapReduce",
+                                            "Period": "300",
+                                            "Statistic": "AVERAGE",
+                                            "Threshold": "50",
+                                            "Unit": "PERCENT"
+                                        }
+                                    }
+                                }
+                            ]
+                        },
                         "BidPrice": {
                             "Fn::If": [
                                 "WithSpotPrice",
@@ -180,6 +221,44 @@
                         "Ref": "Subnet"
                     },
                     "MasterInstanceGroup": {
+                        "AutoScalingPolicy": {
+                            "Constraints": {
+                                "MaxCapacity": "3",
+                                "MinCapacity": "1"
+                            },
+                            "Rules": [
+                                {
+                                    "Action": {
+                                        "Market": "ON_DEMAND",
+                                        "SimpleScalingPolicyConfiguration": {
+                                            "AdjustmentType": "EXACT_CAPACITY",
+                                            "CoolDown": "300",
+                                            "ScalingAdjustment": "1"
+                                        }
+                                    },
+                                    "Description": "MasterAutoScalingPolicy rules",
+                                    "Name": "MasterAutoScalingPolicy",
+                                    "Trigger": {
+                                        "CloudWatchAlarmDefinition": {
+                                            "ComparisonOperator": "GREATER_THAN",
+                                            "Dimensions": [
+                                                {
+                                                    "Key": "my.custom.master.property",
+                                                    "Value": "my.custom.master.value"
+                                                }
+                                            ],
+                                            "EvaluationPeriods": "120",
+                                            "MetricName": "TestMetric",
+                                            "Namespace": "AWS/ElasticMapReduce",
+                                            "Period": "300",
+                                            "Statistic": "AVERAGE",
+                                            "Threshold": "50",
+                                            "Unit": "PERCENT"
+                                        }
+                                    }
+                                }
+                            ]
+                        },
                         "InstanceCount": "1",
                         "InstanceType": "m4.large",
                         "Market": "ON_DEMAND",
diff --git a/tests/test_emr.py b/tests/test_emr.py
new file mode 100644
--- /dev/null
+++ b/tests/test_emr.py
@@ -0,0 +1,119 @@
+from troposphere import If, Ref, Tags
+from troposphere.constants import M4_LARGE
+
+import unittest
+import troposphere.emr as emr
+
+
+scaling_policy = emr.SimpleScalingPolicyConfiguration(
+                    AdjustmentType="EXACT_CAPACITY",
+                    ScalingAdjustment="1",
+                    CoolDown="300"
+                )
+
+
+class TestEMR(unittest.TestCase):
+
+    def generate_rules(self, rules_name):
+        rules = [
+            emr.ScalingRule(
+                Name=rules_name,
+                Description="%s rules" % rules_name,
+                Action=emr.ScalingAction(
+                    Market="ON_DEMAND",
+                    SimpleScalingPolicyConfiguration=scaling_policy
+                ),
+                Trigger=emr.ScalingTrigger(
+                    CloudWatchAlarmDefinition=emr.CloudWatchAlarmDefinition(
+                        ComparisonOperator="GREATER_THAN",
+                        EvaluationPeriods="120",
+                        MetricName="TestMetric",
+                        Namespace="AWS/ElasticMapReduce",
+                        Period="300",
+                        Statistic="AVERAGE",
+                        Threshold="50",
+                        Unit="PERCENT",
+                        Dimensions=[
+                            emr.KeyValue(
+                                'my.custom.master.property',
+                                'my.custom.master.value'
+                            )
+                        ]
+                    )
+                )
+            )
+        ]
+        return rules
+
+    def test_allow_string_cluster(self):
+        spot = "2"
+        withSpotPrice = "WithSpotPrice"
+        cluster = emr.Cluster(
+            'Cluster',
+            # AdditionalInfo="Additional Info",
+            Applications=[
+                emr.Application(Name="Hadoop"),
+                emr.Application(Name="Hive"),
+                emr.Application(Name="Mahout"),
+                emr.Application(Name="Pig"),
+                emr.Application(Name="Spark")
+            ],
+            BootstrapActions=[
+                emr.BootstrapActionConfig(
+                    Name='Dummy bootstrap action',
+                    ScriptBootstrapAction=emr.ScriptBootstrapActionConfig(
+                        Path='file:/usr/share/aws/emr/scripts/install-hue',
+                        Args=["dummy", "parameter"]
+                    )
+                )
+            ],
+            Configurations=[
+                emr.Configuration(
+                    Classification="core-site",
+                    ConfigurationProperties={
+                        'hadoop.security.groups.cache.secs': '250'
+                    }
+                )
+            ],
+            Instances=emr.JobFlowInstancesConfig(
+                Ec2KeyName="KeyName",
+                Ec2SubnetId="SubnetId",
+                MasterInstanceGroup=emr.InstanceGroupConfigProperty(
+                    InstanceCount="1",
+                    InstanceType=M4_LARGE,
+                    AutoScalingPolicy=emr.AutoScalingPolicy(
+                      Constraints=emr.ScalingConstraints(
+                        MinCapacity="1",
+                        MaxCapacity="3"
+                      ),
+                      Rules=self.generate_rules("MasterAutoScalingPolicy")
+                    ),
+                ),
+                CoreInstanceGroup=emr.InstanceGroupConfigProperty(
+                    Name="Core Instance",
+                    BidPrice=If(withSpotPrice, Ref(spot), Ref("AWS::NoValue")),
+                    Market=If(withSpotPrice, "SPOT", "ON_DEMAND"),
+                    InstanceCount="1",
+                    InstanceType=M4_LARGE,
+                    AutoScalingPolicy=emr.AutoScalingPolicy(
+                        Constraints=emr.ScalingConstraints(
+                            MinCapacity="1",
+                            MaxCapacity="3"
+                        ),
+                        Rules=self.generate_rules("CoreAutoScalingPolicy"),
+                    )
+                ),
+            ),
+            JobFlowRole="EMRJobFlowRole",
+            LogUri="s3://cluster-logs",
+            Name="EMR Cluster",
+            ReleaseLabel="emr-5.5.0",
+            ServiceRole="EMRServiceRole",
+            AutoScalingRole="EMR_AutoScaling_DefaultRole",
+            VisibleToAllUsers="true",
+            Tags=Tags(
+                Name="EMR Sample Cluster"
+            )
+        )
+
+        cluster.to_dict()