Repository: aws-quickstart/taskcat
Pull request #585 (2020-07-21 22:09:24)

Adding support for fetching SSM via pseudo-parameters
## Overview

This PR adds support for fetching data from SSM via psuedo-parameters.

Usage:
`$[taskcat_ssm_/path/to/my/parameter]`

## Testing/Steps taken to ensure quality
unit tests. launched a stack. 



Patch:
diff --git a/examples/sample-taskcat-project/ci/debug-input.json b/examples/sample-taskcat-project/ci/debug-input.json
deleted file mode 100644
index c26a0e6..0000000
--- a/examples/sample-taskcat-project/ci/debug-input.json
+++ /dev/null
@@ -1,54 +0,0 @@
-[
-    {
-        "ParameterKey": "AvailabilityZones",
-        "ParameterValue": "$[taskcat_genaz_3]"
-    },
-    {
-        "ParameterKey": "StackName",
-        "ParameterValue": "TestStack"
-    },
-    {
-        "ParameterKey": "ByteValue",
-        "ParameterValue": 1
-    },
-    {
-        "ParameterKey": "UUID",
-        "ParameterValue": "$[taskcat_genuuid]"
-    },
-    {
-        "ParameterKey": "RandomNumber",
-        "ParameterValue": "$[taskcat_random-numbers]"
-    },
-    {
-        "ParameterKey": "RandomString",
-        "ParameterValue": "$[taskcat_random-string]"
-    },
-    {
-        "ParameterKey": "UUID",
-        "ParameterValue": "$[taskcat_genuuid]"
-    },
-    {
-        "ParameterKey": "PasswordA",
-        "ParameterValue": "$[taskcat_genpass_8A]"
-    },
-    {
-        "ParameterKey": "PasswordAConfirm",
-        "ParameterValue": "$[taskcat_getval_PasswordA]"
-    },
-    {
-        "ParameterKey": "PasswordB",
-        "ParameterValue": "$[taskcat_genpass_32S]"
-    },
-    {
-        "ParameterKey": "LocalOverrideTest",
-        "ParameterValue": "override"
-    },
-    {
-        "ParameterKey": "GlobalOverrideTest",
-        "ParameterValue": "override"
-    },
-    {
-        "ParameterKey": "LicenseContentTest",
-        "ParameterValue": "$[taskcat_getlicensecontent]/aws-quickstart-datasets/taskcat-license-param-test/example.txt"
-    }
-]
diff --git a/examples/sample-taskcat-project/ci/taskcat-autobucket-json.yml b/examples/sample-taskcat-project/ci/taskcat-autobucket-json.yml
deleted file mode 100644
index d4286b3..0000000
--- a/examples/sample-taskcat-project/ci/taskcat-autobucket-json.yml
+++ /dev/null
@@ -1,21 +0,0 @@
----
-global:
-  owner: tonynv@amazon.com
-  qsname: sample-taskcat-project
-  regions:
-    - ap-northeast-1
-    - ap-northeast-2
-    - ap-southeast-1
-    - ap-southeast-2
-    - eu-central-1
-    - eu-west-1
-    - sa-east-1
-    - us-east-1
-    - us-west-1
-    - us-west-2
-tests:
-  taskcat-json:
-    parameter_input: debug-input.json
-    regions:
-      - us-east-1
-    template_file: debug.template
diff --git a/examples/sample-taskcat-project/ci/taskcat-autobucket.yml b/examples/sample-taskcat-project/ci/taskcat-autobucket.yml
deleted file mode 100644
index 7cfdc86..0000000
--- a/examples/sample-taskcat-project/ci/taskcat-autobucket.yml
+++ /dev/null
@@ -1,26 +0,0 @@
----
-global:
-  owner: tonynv@amazon.com
-  qsname: sample-taskcat-project
-  regions:
-    - ap-northeast-1
-    - ap-northeast-2
-    - ap-southeast-1
-    - ap-southeast-2
-    - eu-central-1
-    - eu-west-1
-    - sa-east-1
-    - us-east-1
-    - us-west-1
-    - us-west-2
-tests:
-  taskcat-json:
-    parameter_input: debug-input.json
-    template_file: debug.template
-    regions:
-      - us-east-1
-  taskcat-yaml:
-    parameter_input: debug-input.json
-    template_file: debug-yaml.template
-    regions:
-      - us-east-1
diff --git a/examples/sample-taskcat-project/ci/taskcat.yml b/examples/sample-taskcat-project/ci/taskcat.yml
deleted file mode 100644
index 878637c..0000000
--- a/examples/sample-taskcat-project/ci/taskcat.yml
+++ /dev/null
@@ -1,21 +0,0 @@
----
-global:
-  owner: owner@company.com
-  qsname: sample-taskcat-project
-  regions:
-    - ap-northeast-1
-    - ap-northeast-2
-    - ap-southeast-1
-    - ap-southeast-2
-    - eu-central-1
-    - eu-west-1
-    - sa-east-1
-    - us-east-1
-    - us-west-1
-    - us-west-2
-tests:
-  taskcat-json:
-    parameter_input: debug-input.json
-    regions:
-      - us-east-1
-    template_file: debug.template
diff --git a/examples/sample-taskcat-project/ci/taskcat_global_override.json b/examples/sample-taskcat-project/ci/taskcat_global_override.json
deleted file mode 100644
index 17983fd..0000000
--- a/examples/sample-taskcat-project/ci/taskcat_global_override.json
+++ /dev/null
@@ -1,6 +0,0 @@
-[
-    {
-        "ParameterKey": "GlobalOverrideTest",
-        "ParameterValue": "success"
-    }
-]
diff --git a/examples/sample-taskcat-project/ci/taskcat_project_override.json b/examples/sample-taskcat-project/ci/taskcat_project_override.json
deleted file mode 100644
index 38879fe..0000000
--- a/examples/sample-taskcat-project/ci/taskcat_project_override.json
+++ /dev/null
@@ -1,6 +0,0 @@
-[
-    {
-        "ParameterKey": "LocalOverrideTest",
-        "ParameterValue": "foobar"
-    }
-]
diff --git a/examples/sample-taskcat-project/templates/debug-yaml.template b/examples/sample-taskcat-project/templates/debug-yaml.template
deleted file mode 100644
index 7928a16..0000000
--- a/examples/sample-taskcat-project/templates/debug-yaml.template
+++ /dev/null
@@ -1,144 +0,0 @@
-AWSTemplateFormatVersion: '2010-09-09'
-Description: Lambda  Cluster Id generator
-Parameters:
-  StackName:
-    Description: StackName
-    Type: String
-  RandomString:
-    Description: RandomString
-    Type: String
-  RandomNumber:
-    Description: RandomNumber
-    Type: String
-  UUID:
-    Description: UUID
-    Type: String
-  PasswordA:
-    Description: Password (Alpha)
-    Type: String
-  PasswordAConfirm:
-    Description: Password (Alpha)
-    Type: String
-  PasswordB:
-    Description: Password (Special)
-    Type: String
-  ByteValue:
-    Description: NumberValueInput
-    Type: Number
-  AvailabilityZones:
-    Description: 'List of 3 Availability Zones '
-    Type: List<AWS::EC2::AvailabilityZone::Name>
-  LocalOverrideTest:
-    Description: Test of the local parameter override functionality
-    Type: String
-    Default: override
-  GlobalOverrideTest:
-    Description: Test of the global parameter override functionality
-    Type: String
-    Default: override
-  LicenseContentTest:
-    Description: Test the Get License Content functionality.
-    Type: String
-    Default: AWS::NoValue
-Rules:
-  VerifyOverrides:
-    Assertions:
-      - Assert: !Not
-          - !Equals
-            - !Ref 'LocalOverrideTest'
-            - override
-        AssertDescription: Param key 'LocalOverrideTest' cannot equal 'override'
-      - Assert: !Not
-          - !Equals
-            - !Ref 'GlobalOverrideTest'
-            - override
-        AssertDescription: Param key 'GlobalOverrideTest' cannot equal 'override'
-  VerifyLicenseContent:
-    Assertions:
-      - Assert: !Equals
-        - !Ref "LicenseContentTest"
-        - "pass"
-        AssertDescription: Param key 'LicenseContentTest' did not pass.
-  VerifyPasswordsMatch:
-    Assertions:
-      - Assert: !Equals
-          - !Ref "PasswordA"
-          - !Ref "PasswordAConfirm"
-        AssertionDescription: Params PasswordA and PasswordAConfirm must match
-Resources:
-  LambdaExecutionRole:
-    Type: AWS::IAM::Role
-    Properties:
-      AssumeRolePolicyDocument:
-        Version: '2012-10-17'
-        Statement:
-          - Effect: Allow
-            Principal:
-              Service:
-                - lambda.amazonaws.com
-            Action:
-              - sts:AssumeRole
-      Path: /
-      Policies:
-        - PolicyName: lambda_policy
-          PolicyDocument:
-            Version: '2012-10-17'
-            Statement:
-              - Effect: Allow
-                Action:
-                  - logs:CreateLogGroup
-                  - logs:CreateLogStream
-                  - logs:PutLogEvents
-                Resource: arn:aws:logs:*:*:*
-              - Effect: Allow
-                Action:
-                  - cloudformation:DescribeStacks
-                Resource: '*'
-  GenID:
-    Type: AWS::Lambda::Function
-    Properties:
-      Code:
-        ZipFile: !Join
-          - "\n"
-          - - import random
-            - import json
-            - import cfnresponse
-            - from cfnresponse import send, SUCCESS
-            - 'def handler(event, context):'
-            - '   if event[''RequestType''] == ''Delete'':'
-            - '       send(event, context, ''SUCCESS'', {})'
-            - '       return'
-            - '   if event[''RequestType''] == ''Create'':'
-            - '       token= "%0x.%0x" % (random.SystemRandom().getrandbits(3*8),
-              random.SystemRandom().getrandbits(8*8))'
-            - '       responseData = {}'
-            - '       responseData[''Data''] = token'
-            - '       send(event, context, ''SUCCESS'', responseData)'
-            - '       return token'
-      Handler: index.handler
-      Runtime: python2.7
-      Timeout: '5'
-      Role: !GetAtt 'LambdaExecutionRole.Arn'
-  GetID:
-    Type: Custom::GenerateID
-    Version: '1.0'
-    Properties:
-      ServiceToken: !GetAtt 'GenID.Arn'
-      ResponseURL: !Join
-        - ''
-        - - http://ResponseURL
-          - !Ref 'AWS::StackId'
-          - RequestId
-      StackId: !Ref 'AWS::StackId'
-      ResourceProperties:
-        RequestType: Create
-        RequestId: !Join
-          - ''
-          - - !Ref 'AWS::StackId'
-            - RequestId
-        LogicalResourceId: GenIDLogicalResourceId
-    DependsOn:
-      - GenID
-Outputs:
-  ClusterID:
-    Value: !GetAtt 'GetID.Data'
diff --git a/examples/sample-taskcat-project/templates/debug.template b/examples/sample-taskcat-project/templates/debug.template
deleted file mode 100644
index 8e30f9e..0000000
--- a/examples/sample-taskcat-project/templates/debug.template
+++ /dev/null
@@ -1,267 +0,0 @@
-{
-    "AWSTemplateFormatVersion": "2010-09-09",
-    "Description": "Lambda  Cluster Id generator",
-    "Parameters": {
-        "StackName": {
-            "Description": "StackName",
-            "Type": "String"
-        },
-        "RandomString": {
-            "Description": "RandomString",
-            "Type": "String"
-        },
-        "RandomNumber": {
-            "Description": "RandomNumber",
-            "Type": "String"
-        },
-        "UUID": {
-            "Description": "UUID",
-            "Type": "String"
-        },
-        "PasswordA": {
-            "Description": "Password (Alpha)",
-            "Type": "String"
-        },
-        "PasswordAConfirm": {
-            "Description": "Password (Alpha)",
-            "Type": "String"
-        },
-        "PasswordB": {
-            "Description": "Password (Special)",
-            "Type": "String"
-        },
-        "ByteValue": {
-            "Description": "NumberValueInput",
-            "Type": "Number"
-        },
-        "AvailabilityZones": {
-            "Description": "List of 3 Availability Zones ",
-            "Type": "List<AWS::EC2::AvailabilityZone::Name>"
-        },
-        "LocalOverrideTest": {
-            "Description": "Test of the local parameter override functionality",
-            "Type": "String",
-            "Default": "override"
-        },
-        "GlobalOverrideTest": {
-            "Description": "Test of the global parameter override functionality",
-            "Type": "String",
-            "Default": "override"
-        },
-        "LicenseContentTest": {
-            "Description": "Test the Get License Content functionality.",
-            "Type": "String",
-            "Default": "AWS::NoValue"
-        }
-    },
-    "Rules": {
-        "VerifyOverrides": {
-            "Assertions": [
-                {
-                    "Assert": {
-                        "Fn::Not": [
-                            {
-                                "Fn::Equals": [
-                                    {
-                                        "Ref": "LocalOverrideTest"
-                                    },
-                                    "override"
-                                ]
-                            }
-                        ]
-                    },
-                    "AssertDescription": "Param key 'LocalOverrideTest' cannot equal 'override'"
-                },
-                {
-                    "Assert": {
-                        "Fn::Not": [
-                            {
-                                "Fn::Equals": [
-                                    {
-                                        "Ref": "GlobalOverrideTest"
-                                    },
-                                    "override"
-                                ]
-                            }
-                        ]
-                    },
-                    "AssertDescription": "Param key 'GlobalOverrideTest' cannot equal 'override'"
-                }
-            ]
-        },
-        "VerifyLicenseContent": {
-            "Assertions": [
-                {
-                    "Assert": {
-                        "Fn::Equals": [
-                            {
-                                "Ref": "LicenseContentTest"
-                            },
-                            "pass"
-                        ]
-                    },
-                    "AssertDescription": "Param key 'LicenseContentTest' did not pass."
-                }
-            ]
-        },
-        "VerifyPasswordsMatch": {
-            "Assertions": [
-                {
-                    "Assert": {
-                        "Fn::Equals": [
-                            {
-                                "Ref": "PasswordA"
-                            },
-                            {
-                                "Ref": "PasswordAConfirm"
-                            }
-                        ]
-                    },
-                    "AssertionDescription": "Params PasswordA and PasswordAConfirm must match"
-                }
-            ]
-        }
-    },
-    "Resources": {
-        "LambdaExecutionRole": {
-            "Type": "AWS::IAM::Role",
-            "Properties": {
-                "AssumeRolePolicyDocument": {
-                    "Version": "2012-10-17",
-                    "Statement": [
-                        {
-                            "Effect": "Allow",
-                            "Principal": {
-                                "Service": [
-                                    "lambda.amazonaws.com"
-                                ]
-                            },
-                            "Action": [
-                                "sts:AssumeRole"
-                            ]
-                        }
-                    ]
-                },
-                "Path": "/",
-                "Policies": [
-                    {
-                        "PolicyName": "lambda_policy",
-                        "PolicyDocument": {
-                            "Version": "2012-10-17",
-                            "Statement": [
-                                {
-                                    "Effect": "Allow",
-                                    "Action": [
-                                        "logs:CreateLogGroup",
-                                        "logs:CreateLogStream",
-                                        "logs:PutLogEvents"
-                                    ],
-                                    "Resource": "arn:aws:logs:*:*:*"
-                                },
-                                {
-                                    "Effect": "Allow",
-                                    "Action": [
-                                        "cloudformation:DescribeStacks"
-                                    ],
-                                    "Resource": "*"
-                                }
-                            ]
-                        }
-                    }
-                ]
-            }
-        },
-        "GenID": {
-            "Type": "AWS::Lambda::Function",
-            "Properties": {
-                "Code": {
-                    "ZipFile": {
-                        "Fn::Join": [
-                            "\n",
-                            [
-                                "import random",
-                                "import json",
-                                "import cfnresponse",
-                                "from cfnresponse import send, SUCCESS",
-                                "def handler(event, context):",
-                                "   if event['RequestType'] == 'Delete':",
-                                "       send(event, context, 'SUCCESS', {})",
-                                "       return",
-                                "   if event['RequestType'] == 'Create':",
-                                "       token= \"%0x.%0x\" % (random.SystemRandom().getrandbits(3*8), random.SystemRandom().getrandbits(8*8))",
-                                "       responseData = {}",
-                                "       responseData['Data'] = token",
-                                "       send(event, context, 'SUCCESS', responseData)",
-                                "       return token"
-                            ]
-                        ]
-                    }
-                },
-                "Handler": "index.handler",
-                "Runtime": "python2.7",
-                "Timeout": 5,
-                "Role": {
-                    "Fn::GetAtt": [
-                        "LambdaExecutionRole",
-                        "Arn"
-                    ]
-                }
-            }
-        },
-        "GetID": {
-            "Type": "Custom::GenerateID",
-            "Version": "1.0",
-            "Properties": {
-                "ServiceToken": {
-                    "Fn::GetAtt": [
-                        "GenID",
-                        "Arn"
-                    ]
-                },
-                "ResponseURL": {
-                    "Fn::Join": [
-                        "",
-                        [
-                            "http://ResponseURL",
-                            {
-                                "Ref": "AWS::StackId"
-                            },
-                            "RequestId"
-                        ]
-                    ]
-                },
-                "StackId": {
-                    "Ref": "AWS::StackId"
-                },
-                "ResourceProperties": {
-                    "RequestType": "Create",
-                    "RequestId": {
-                        "Fn::Join": [
-                            "",
-                            [
-                                {
-                                    "Ref": "AWS::StackId"
-                                },
-                                "RequestId"
-                            ]
-                        ]
-                    },
-                    "LogicalResourceId": "GenIDLogicalResourceId"
-                }
-            },
-            "DependsOn": [
-                "GenID"
-            ]
-        }
-    },
-    "Outputs": {
-        "ClusterID": {
-            "Value": {
-                "Fn::GetAtt": [
-                    "GetID",
-                    "Data"
-                ]
-            }
-        }
-    }
-}
diff --git a/taskcat/_common_utils.py b/taskcat/_common_utils.py
index db17451..caf80d2 100644
--- a/taskcat/_common_utils.py
+++ b/taskcat/_common_utils.py
@@ -215,3 +215,9 @@ def neglect_submodule_templates(project_root, template_list):
 def determine_profile_for_region(auth_dict, region):
     profile = auth_dict.get(region, auth_dict.get("default", "default"))
     return profile
+
+
+def fetch_ssm_parameter_value(boto_client, parameter_path):
+    ssm = boto_client("ssm")
+    response = ssm.get_parameter(Name=parameter_path)
+    return response["Parameter"]["Value"]
diff --git a/taskcat/_template_params.py b/taskcat/_template_params.py
index b88a024..be48647 100644
--- a/taskcat/_template_params.py
+++ b/taskcat/_template_params.py
@@ -4,7 +4,7 @@ import re
 import uuid
 from typing import Set
 
-from taskcat._common_utils import CommonTools
+from taskcat._common_utils import CommonTools, fetch_ssm_parameter_value
 from taskcat.exceptions import TaskCatException
 
 LOG = logging.getLogger(__name__)
@@ -32,6 +32,7 @@ class ParamGen:
     )
     RE_GETVAL = re.compile(r"(?<=._getval_)(\w+)(?=]$)", re.IGNORECASE)
     RE_CURRENT_REGION = re.compile(r"\$\[taskcat_current_region]", re.IGNORECASE)
+    RE_SSM_PARAMETER = re.compile(r"\$\[taskcat_ssm_.*]$", re.IGNORECASE)
 
     def __init__(self, param_dict, bucket_name, region, boto_client, az_excludes=None):
         self.regxfind = CommonTools.regxfind
@@ -133,6 +134,8 @@ class ParamGen:
             # $[taskcat_genuuid]
             self._regex_replace_param_value(self.RE_GENUUID, self._gen_uuid())
 
+            # $[taskcat_ssm_X]
+            self._get_ssm_param_value_wrapper(self.RE_SSM_PARAMETER)
             # $[taskcat_current_region]
             self._regex_replace_param_value(
                 self.RE_CURRENT_REGION, self._gen_current_region()
@@ -365,6 +368,13 @@ class ParamGen:
             self._regex_replace_param_value(re.compile("^.*$"), param_value)
             self._regex_replace_param_value(re.compile("^.*$"), param_value)
 
+    def _get_ssm_param_value_wrapper(self, ssm_param_value_regex):
+        if ssm_param_value_regex.search(self.param_value):
+            ssm_value_str = self.regxfind(ssm_param_value_regex, self.param_value)
+            param_path = "_".join(ssm_value_str[:-1].split("_")[2:])
+            param_value = fetch_ssm_parameter_value(self._boto_client, param_path)
+            self._regex_replace_param_value(re.compile("^.*"), param_value)
+
     def _getval_wrapper(self, getval_regex):
         if getval_regex.search(self.param_value):
             requested_key = self.regxfind(getval_regex, self.param_value)


Test patch:
diff --git a/tests/test_common_utils.py b/tests/test_common_utils.py
index b1335d2..ff497f4 100644
--- a/tests/test_common_utils.py
+++ b/tests/test_common_utils.py
@@ -5,6 +5,7 @@ import unittest
 import mock
 from taskcat._common_utils import (
     exit_with_code,
+    fetch_ssm_parameter_value,
     get_s3_domain,
     make_dir,
     merge_dicts,
@@ -105,3 +106,29 @@ class TestCommonUtils(unittest.TestCase):
     def test_s3_bucket_name_from_url(self):
         bucket = s3_bucket_name_from_url("https://buk.s3.amazonaws.com/obj.yaml")
         self.assertEqual("buk", bucket)
+
+    def test_fetch_ssm_parameter_value(self):
+        # String, no explicit version.
+        m_boto_client = mock.Mock()
+        m_ssm = mock.Mock()
+        m_boto_client.return_value = m_ssm
+        m_ssm.get_parameter.return_value = {
+            "Parameter": {"Name": "foo", "Type": "String", "Value": "bar", "Version": 1}
+        }
+
+        expected = "bar"
+        actual = fetch_ssm_parameter_value(m_boto_client, "foo")
+        self.assertEqual(expected, actual)
+
+        m_ssm.get_parameter.return_value = {
+            "Parameter": {
+                "Name": "foo",
+                "Type": "StringList",
+                "Value": "bar,baz,11",
+                "Version": 1,
+            }
+        }
+
+        expected = "bar,baz,11"
+        actual = fetch_ssm_parameter_value(m_boto_client, "foo")
+        self.assertEqual(expected, actual)
diff --git a/tests/test_template_params.py b/tests/test_template_params.py
index a762e43..a4710bb 100644
--- a/tests/test_template_params.py
+++ b/tests/test_template_params.py
@@ -237,6 +237,10 @@ class TestParamGen(unittest.TestCase):
             test_string="$[taskcat_url_http://example.com]",
             test_pattern_attribute="RE_GETURL",
         ),
+        rp_namedtup(
+            test_string="$[taskcat_ssm_/aws/foo/bar/blah]",
+            test_pattern_attribute="RE_SSM_PARAMETER",
+        ),
     ]
 
     def test_regxfind(self):
@@ -485,3 +489,21 @@ class TestParamGen(unittest.TestCase):
             "ExampleString": "us-east-1a",
         }
         self.assertEqual(pg.results, expected_result)
+
+    @mock.patch("taskcat._template_params.fetch_ssm_parameter_value")
+    def test_ssm_parameter_wrapper(self, m_fetch_ssm):
+        m_fetch_ssm.return_value = "blah"
+        input_params = {
+            "Example": "$[taskcat_ssm_/path/to/my/example/ssm_param/nested_with_underscores]"
+        }
+        bclient = MockClient
+        bclient.logger = logger
+        class_kwargs = self.class_kwargs
+        class_kwargs["param_dict"] = input_params
+        class_kwargs["boto_client"] = bclient
+        pg = ParamGen(**class_kwargs)
+        pg._get_ssm_param_value_wrapper(pg.RE_SSM_PARAMETER)
+        m_fetch_ssm.assert_called_once_with(
+            bclient, "/path/to/my/example/ssm_param/nested_with_underscores"
+        )
+        self.assertEqual(pg.param_value, "blah")