Repository: ManageIQ/integration_tests

Issue:
Better YAML overriding
Now it does not take just the root element into the account, but it crawls throught the dictionary and only updates the values that are present in the new dictionary. It converts all dicts to Configs, other values than specified in override dict are not touched.

It also improves the `__getattribute__` behaviour - now it propagates the interface to the child nodes by converting all `dict` to `Config` before returning the value, so the dot operator can be used everywhere.



Discussion:

Resolving patch:
diff --git a/utils/conf_loader.py b/utils/conf_loader.py
--- a/utils/conf_loader.py
+++ b/utils/conf_loader.py
@@ -1,17 +1,19 @@
 import os
-from collections import OrderedDict
 
 import py.path
 import yaml
 from yaml.loader import Loader
 
 
-class OrderedYamlLoader(Loader):
+class YamlConfigLoader(Loader):
+    # Override the root yaml node to be a RecursiveUpdateDict
     def construct_yaml_map(self, node):
-        data = OrderedDict()
+        data = RecursiveUpdateDict()
         yield data
         value = self.construct_mapping(node)
         data.update(value)
+# Do the same for child nodes of the yaml mapping type
+YamlConfigLoader.add_constructor('tag:yaml.org,2002:map', YamlConfigLoader.construct_yaml_map)
 
 
 class ConfigNotFoundException(Exception):
@@ -62,6 +64,43 @@ def __getitem__(self, key):
             return self[key]
 
 
+class RecursiveUpdateDict(dict):
+    def update(self, new_data):
+        """ More intelligent dictionary update.
+
+        This method changes just data that have been changed. How does it work?
+        Imagine you want to change just VM name, other things should stay the same.
+
+        Original config:
+        something:
+            somewhere:
+                VM:
+                    a: 1
+                    b: 2
+                    name: qwer
+                    c: 3
+
+        Instead of copying the whole part from original to the override with just 'name' changed,
+        you will write this:
+
+        something:
+            somewhere:
+                VM:
+                    name: tzui
+
+        This digging deeper affects only dictionary values. Lists are unaffected! And so do other
+        types.
+
+        Args:
+            new_data: Update data.
+        """
+        for key, value in new_data.iteritems():
+            if isinstance(value, type(self)) and key in self:
+                type(self).update(self[key], value)
+            else:
+                self[key] = new_data[key]
+
+
 def load_yaml(filename=None):
     # Find the requested yaml in the config dir, relative to this file's location
     # (aiming for cfme_tests/config)
@@ -70,7 +109,7 @@ def load_yaml(filename=None):
 
     if path.check():
         with path.open() as config_fh:
-            return yaml.load(config_fh, Loader=OrderedYamlLoader)
+            return yaml.load(config_fh, Loader=YamlConfigLoader)
     else:
         msg = 'Unable to load configuration file at %s' % path
         raise ConfigNotFoundException(msg)


Test patch:
diff --git a/utils/tests/test_config.py b/utils/tests/test_config.py
--- a/utils/tests/test_config.py
+++ b/utils/tests/test_config.py
@@ -3,13 +3,19 @@
 import pytest
 
 from utils import conf
+from utils.conf_loader import Config, RecursiveUpdateDict
 
 test_yaml_contents = '''
 test_key: test_value
+nested_test_root:
+    nested_test_key_1: nested_test_value_1
+    nested_test_key_2: nested_test_value_2
 '''
 
 local_test_yaml_contents = '''
 test_key: test_overridden_value
+nested_test_root:
+    nested_test_key_1: overridden_nested_test_value
 '''
 
 
@@ -46,6 +52,9 @@ def create_test_yaml(request, contents, filename, local=False):
 def test_conf_yamls_dict(test_yaml):
     # Dict lookup method works
     assert conf[test_yaml]['test_key'] == 'test_value'
+    assert isinstance(conf, Config)
+    assert isinstance(conf[test_yaml], RecursiveUpdateDict)
+    assert isinstance(conf[test_yaml]['nested_test_root'], RecursiveUpdateDict)
 
 
 def test_conf_yamls_attr(test_yaml):
@@ -55,10 +64,15 @@ def test_conf_yamls_attr(test_yaml):
 
 def test_conf_yamls_override(request, test_yaml):
     # Make a .local.yaml file with the same root name as test_yaml,
-    # Check that the local override works.
     with create_test_yaml(request, local_test_yaml_contents, test_yaml, local=True):
+        # Check that the simple local override works.
         assert conf[test_yaml]['test_key'] == 'test_overridden_value'
 
+        # Check that the local override of specific nested keys works.
+        nested_root = conf[test_yaml]['nested_test_root']
+        assert nested_root['nested_test_key_1'] == 'overridden_nested_test_value'
+        assert nested_root['nested_test_key_2'] == 'nested_test_value_2'
+
 
 def test_conf_yamls_import(test_yaml):
     # Emulate from utils.conf import $test_yaml