Repository: StackStorm/st2

Issue:
Dynamic Configuration values don't work with nested objects
I use the following cmd to set a key in the datastore.
`$ st2 key set -e smtp_account.me.password "I'm Encrypted"`

and here are my email pack config /opt/stackstorm/configs/email.yaml

```yaml
---
smtp_accounts:
  - name: name
    password: "{{ st2kv.system.smtp_account.me.password }}"
```
(followed by the rest of the account configuration).

This is based on the email pack's `config.schema.yaml`, which contains this:

```yaml
imap_accounts:
  description: "IMAP accounts"
  type: "array"
  required: true
  items:
    type: "object"
    required: true
    properties:
      name:
        description: "Name of the account"
        type: "string"
        secret: false
        required: true
      server:
        description: "Email server name - e.g. imap.gmail.com"
        type: "string"
        secret: false
        required: true
      username:
        description: "Mailbox username"
        type: "string"
        required: true
      password:
        description: "Mailbox password."
        type: "string"
        secret: true
        required: true
```

Running the email action fails:
```
Traceback (most recent call last):
  File "/opt/stackstorm/st2/lib/python2.7/site-packages/st2common/runners/python_action_wrapper.py", line 278, in <module>
    obj.run()
  File "/opt/stackstorm/st2/lib/python2.7/site-packages/st2common/runners/python_action_wrapper.py", line 171, in run
    output = action.run(**self._parameters)
  File "/opt/stackstorm/packs/email/actions/send_email.py", line 49, in run
    s.login(account_data['username'], account_data['password'])
  File "/usr/lib64/python2.7/smtplib.py", line 621, in login
    raise SMTPAuthenticationError(code, resp)
smtplib.SMTPAuthenticationError: (460, 'ERR.LOGIN.PASSERR')
```

I added some log to see the password value in File "/opt/stackstorm/packs/email/actions/send_email.py", line 49.
It seems that we get
`{{ st2kv.system.smtp_account.me.password }}` as the password.

FYI
If I run the send email action and use `{{ st2kv.system.smtp_account.me.password }}` as the message
I can see the message's value is correct.


Discussion:
From Slack - probably related to https://github.com/StackStorm/st2/blob/master/st2common/st2common/util/config_loader.py#L102-L138

> ...conditional for nested objects, but not nested lists

Resolving patch:
diff --git a/st2common/st2common/util/config_loader.py b/st2common/st2common/util/config_loader.py
--- a/st2common/st2common/util/config_loader.py
+++ b/st2common/st2common/util/config_loader.py
@@ -110,22 +110,42 @@ def _assign_dynamic_config_values(self, schema, config, parent_keys=None):
         """
         parent_keys = parent_keys or []
 
-        for config_item_key, config_item_value in six.iteritems(config):
-            schema_item = schema.get(config_item_key, {})
+        config_is_dict = isinstance(config, dict)
+        config_is_list = isinstance(config, list)
+        iterator = six.iteritems(config) if config_is_dict else enumerate(config)
+
+        # config_item_key - if config_is_dict then this is the key in the dictionary
+        #                   if config_is_list then this is the index of them item
+        # config_item_value - the value of the key/index for the current item
+        for config_item_key, config_item_value in iterator:
+            if config_is_dict:
+                # different schema for each key/value pair
+                schema_item = schema.get(config_item_key, {})
+            if config_is_list:
+                # same schema is shared between every item in the list
+                schema_item = schema
+
             is_dictionary = isinstance(config_item_value, dict)
+            is_list = isinstance(config_item_value, list)
 
             # Inspect nested object properties
             if is_dictionary:
-                parent_keys += [config_item_key]
+                parent_keys += [str(config_item_key)]
                 self._assign_dynamic_config_values(schema=schema_item.get('properties', {}),
                                                    config=config[config_item_key],
                                                    parent_keys=parent_keys)
+            # Inspect nested list items
+            elif is_list:
+                parent_keys += [str(config_item_key)]
+                self._assign_dynamic_config_values(schema=schema_item.get('items', {}),
+                                                   config=config[config_item_key],
+                                                   parent_keys=parent_keys)
             else:
                 is_jinja_expression = jinja_utils.is_jinja_expression(value=config_item_value)
 
                 if is_jinja_expression:
                     # Resolve / render the Jinja template expression
-                    full_config_item_key = '.'.join(parent_keys + [config_item_key])
+                    full_config_item_key = '.'.join(parent_keys + [str(config_item_key)])
                     value = self._get_datastore_value_for_expression(key=full_config_item_key,
                         value=config_item_value,
                         config_schema_item=schema_item)
@@ -188,8 +208,9 @@ def _get_datastore_value_for_expression(self, key, value, config_schema_item=Non
             exc_class = type(e)
             original_msg = str(e)
             msg = ('Failed to render dynamic configuration value for key "%s" with value '
-                   '"%s" for pack "%s" config: %s ' % (key, value, self.pack_name, original_msg))
-            raise exc_class(msg)
+                   '"%s" for pack "%s" config: %s %s ' % (key, value, self.pack_name,
+                                                          exc_class, original_msg))
+            raise RuntimeError(msg)
 
         if value:
             # Deserialize the value


Test patch:
diff --git a/st2common/tests/unit/test_config_loader.py b/st2common/tests/unit/test_config_loader.py
--- a/st2common/tests/unit/test_config_loader.py
+++ b/st2common/tests/unit/test_config_loader.py
@@ -16,7 +16,9 @@
 from __future__ import absolute_import
 from st2common.persistence.pack import Config
 from st2common.models.db.pack import ConfigDB
+from st2common.models.db.keyvalue import KeyValuePairDB
 from st2common.exceptions.db import StackStormDBObjectNotFoundError
+from st2common.persistence.keyvalue import KeyValuePair
 from st2common.services.config import set_datastore_value_for_config_key
 from st2common.util.config_loader import ContentPackConfigLoader
 
@@ -296,8 +298,9 @@ def test_get_config_dynamic_config_item_render_fails_user_friendly_exception_is_
 
         expected_msg = ('Failed to render dynamic configuration value for key "level0_key" with '
                         'value "{{st2kvXX.invalid}}" for pack ".*?" config: '
+                        '<class \'jinja2.exceptions.UndefinedError\'> '
                         '\'st2kvXX\' is undefined')
-        self.assertRaisesRegexp(Exception, expected_msg, loader.get_config)
+        self.assertRaisesRegexp(RuntimeError, expected_msg, loader.get_config)
         config_db.delete()
 
         # Renders fails on fist level item
@@ -311,8 +314,9 @@ def test_get_config_dynamic_config_item_render_fails_user_friendly_exception_is_
 
         expected_msg = ('Failed to render dynamic configuration value for key '
                         '"level0_object.level1_key" with value "{{st2kvXX.invalid}}"'
-                        ' for pack ".*?" config: \'st2kvXX\' is undefined')
-        self.assertRaisesRegexp(Exception, expected_msg, loader.get_config)
+                        ' for pack ".*?" config: <class \'jinja2.exceptions.UndefinedError\'>'
+                        ' \'st2kvXX\' is undefined')
+        self.assertRaisesRegexp(RuntimeError, expected_msg, loader.get_config)
         config_db.delete()
 
         # Renders fails on second level item
@@ -328,6 +332,199 @@ def test_get_config_dynamic_config_item_render_fails_user_friendly_exception_is_
 
         expected_msg = ('Failed to render dynamic configuration value for key '
                         '"level0_object.level1_object.level2_key" with value "{{st2kvXX.invalid}}"'
-                        ' for pack ".*?" config: \'st2kvXX\' is undefined')
-        self.assertRaisesRegexp(Exception, expected_msg, loader.get_config)
+                        ' for pack ".*?" config: <class \'jinja2.exceptions.UndefinedError\'>'
+                        ' \'st2kvXX\' is undefined')
+        self.assertRaisesRegexp(RuntimeError, expected_msg, loader.get_config)
+        config_db.delete()
+
+        # Renders fails on list item
+        values = {
+            'level0_object': [
+                'abc',
+                '{{st2kvXX.invalid}}'
+            ]
+        }
+        config_db = ConfigDB(pack=pack_name, values=values)
+        Config.add_or_update(config_db)
+
+        expected_msg = ('Failed to render dynamic configuration value for key '
+                        '"level0_object.1" with value "{{st2kvXX.invalid}}"'
+                        ' for pack ".*?" config: <class \'jinja2.exceptions.UndefinedError\'>'
+                        ' \'st2kvXX\' is undefined')
+        self.assertRaisesRegexp(RuntimeError, expected_msg, loader.get_config)
+        config_db.delete()
+
+        # Renders fails on nested object in list item
+        values = {
+            'level0_object': [
+                {'level2_key': '{{st2kvXX.invalid}}'}
+            ]
+        }
+        config_db = ConfigDB(pack=pack_name, values=values)
+        Config.add_or_update(config_db)
+
+        expected_msg = ('Failed to render dynamic configuration value for key '
+                        '"level0_object.0.level2_key" with value "{{st2kvXX.invalid}}"'
+                        ' for pack ".*?" config: <class \'jinja2.exceptions.UndefinedError\'>'
+                        ' \'st2kvXX\' is undefined')
+        self.assertRaisesRegexp(RuntimeError, expected_msg, loader.get_config)
+        config_db.delete()
+
+        # Renders fails on invalid syntax
+        values = {
+            'level0_key': '{{ this is some invalid Jinja }}'
+        }
+        config_db = ConfigDB(pack=pack_name, values=values)
+        Config.add_or_update(config_db)
+
+        expected_msg = ('Failed to render dynamic configuration value for key '
+                        '"level0_key" with value "{{ this is some invalid Jinja }}"'
+                        ' for pack ".*?" config: <class \'jinja2.exceptions.TemplateSyntaxError\'>'
+                        ' expected token \'end of print statement\', got \'Jinja\'')
+        self.assertRaisesRegexp(RuntimeError, expected_msg, loader.get_config)
+        config_db.delete()
+
+    def test_get_config_dynamic_config_item(self):
+        pack_name = 'dummy_pack_schema_with_nested_object_6'
+        loader = ContentPackConfigLoader(pack_name=pack_name)
+
+        ####################
+        # value in top level item
+        KeyValuePair.add_or_update(KeyValuePairDB(name='k1', value='v1'))
+        values = {
+            'level0_key': '{{st2kv.system.k1}}'
+        }
+        config_db = ConfigDB(pack=pack_name, values=values)
+        config_db = Config.add_or_update(config_db)
+
+        config_rendered = loader.get_config()
+
+        self.assertEquals(config_rendered, {'level0_key': 'v1'})
+
+        config_db.delete()
+
+    def test_get_config_dynamic_config_item_nested_dict(self):
+        pack_name = 'dummy_pack_schema_with_nested_object_7'
+        loader = ContentPackConfigLoader(pack_name=pack_name)
+
+        KeyValuePair.add_or_update(KeyValuePairDB(name='k0', value='v0'))
+        KeyValuePair.add_or_update(KeyValuePairDB(name='k1', value='v1'))
+        KeyValuePair.add_or_update(KeyValuePairDB(name='k2', value='v2'))
+
+        ####################
+        # values nested dictionaries
+        values = {
+            'level0_key': '{{st2kv.system.k0}}',
+            'level0_object': {
+                'level1_key': '{{st2kv.system.k1}}',
+                'level1_object': {
+                    'level2_key': '{{st2kv.system.k2}}'
+                }
+            }
+        }
+        config_db = ConfigDB(pack=pack_name, values=values)
+        config_db = Config.add_or_update(config_db)
+
+        config_rendered = loader.get_config()
+
+        self.assertEquals(config_rendered,
+                          {
+                              'level0_key': 'v0',
+                              'level0_object': {
+                                  'level1_key': 'v1',
+                                  'level1_object': {
+                                      'level2_key': 'v2'
+                                  }
+                              }
+                          })
+
+        config_db.delete()
+
+    def test_get_config_dynamic_config_item_list(self):
+        pack_name = 'dummy_pack_schema_with_nested_object_7'
+        loader = ContentPackConfigLoader(pack_name=pack_name)
+
+        KeyValuePair.add_or_update(KeyValuePairDB(name='k0', value='v0'))
+        KeyValuePair.add_or_update(KeyValuePairDB(name='k1', value='v1'))
+
+        ####################
+        # values in list
+        values = {
+            'level0_key': [
+                'a',
+                '{{st2kv.system.k0}}',
+                'b',
+                '{{st2kv.system.k1}}',
+            ]
+        }
+        config_db = ConfigDB(pack=pack_name, values=values)
+        config_db = Config.add_or_update(config_db)
+
+        config_rendered = loader.get_config()
+
+        self.assertEquals(config_rendered,
+                          {
+                              'level0_key': [
+                                  'a',
+                                  'v0',
+                                  'b',
+                                  'v1'
+                              ]
+                          })
+
+        config_db.delete()
+
+    def test_get_config_dynamic_config_item_nested_list(self):
+        pack_name = 'dummy_pack_schema_with_nested_object_8'
+        loader = ContentPackConfigLoader(pack_name=pack_name)
+
+        KeyValuePair.add_or_update(KeyValuePairDB(name='k0', value='v0'))
+        KeyValuePair.add_or_update(KeyValuePairDB(name='k1', value='v1'))
+        KeyValuePair.add_or_update(KeyValuePairDB(name='k2', value='v2'))
+
+        ####################
+        # values in objects embedded in lists and nested lists
+        values = {
+            'level0_key': [
+                {
+                    'level1_key0': '{{st2kv.system.k0}}'
+                },
+                '{{st2kv.system.k1}}',
+                [
+                    '{{st2kv.system.k0}}',
+                    '{{st2kv.system.k1}}',
+                    '{{st2kv.system.k2}}',
+                ],
+                {
+                    'level1_key2': [
+                        '{{st2kv.system.k2}}',
+                    ]
+                }
+            ]
+        }
+        config_db = ConfigDB(pack=pack_name, values=values)
+        config_db = Config.add_or_update(config_db)
+
+        config_rendered = loader.get_config()
+
+        self.assertEquals(config_rendered,
+                          {
+                              'level0_key': [
+                                  {
+                                      'level1_key0': 'v0'
+                                  },
+                                  'v1',
+                                  [
+                                      'v0',
+                                      'v1',
+                                      'v2',
+                                  ],
+                                  {
+                                      'level1_key2': [
+                                          'v2',
+                                      ]
+                                  }
+                              ]
+                          })
+
         config_db.delete()