Repository: boto/botocore
Pull request #592 (2015-06-25 22:09:17)

Add append_lists option to merge_dicts()
Allows user to append to lists as opposed to replace lists completely when
merging dictionaries.

This is needed for a subsequent PR to boto3 that fixes an issue in collections.

cc @jamesls @mtdowling 


Patch:
diff --git a/botocore/utils.py b/botocore/utils.py
index 2e128c7ea..075e59d6f 100644
--- a/botocore/utils.py
+++ b/botocore/utils.py
@@ -200,11 +200,13 @@ class InstanceMetadataFetcher(object):
         return final_data
 
 
-def merge_dicts(dict1, dict2):
+def merge_dicts(dict1, dict2, append_lists=False):
     """Given two dict, merge the second dict into the first.
 
     The dicts can have arbitrary nesting.
 
+    :param append_lists: If true, instead of clobbering a list with the new
+        value, append all of the new values onto the original list.
     """
     for key in dict2:
         if isinstance(dict2[key], dict):
@@ -212,6 +214,15 @@ def merge_dicts(dict1, dict2):
                 merge_dicts(dict1[key], dict2[key])
             else:
                 dict1[key] = dict2[key]
+        # If the value is a list and the ``append_lists`` flag is set,
+        # append the new values onto the original list
+        elif isinstance(dict2[key], list) and append_lists:
+            # The value in dict1 must be a list in order to append new
+            # values onto it.
+            if key in dict1 and isinstance(dict1[key], list):
+                dict1[key].extend(dict2[key])
+            else:
+                dict1[key] = dict2[key]
         else:
             # At scalar types, we iterate and merge the
             # current dict that we're on.


Test patch:
diff --git a/tests/unit/test_translate.py b/tests/unit/test_translate.py
index 95b152773..5fc44c295 100644
--- a/tests/unit/test_translate.py
+++ b/tests/unit/test_translate.py
@@ -687,52 +687,6 @@ class TestTranslateModel(unittest.TestCase):
         self.assertEqual(new_model['pagination'], extra['pagination'])
 
 
-class TestDictMerge(unittest.TestCase):
-    def test_merge_dicts_overrides(self):
-        first = {'foo': {'bar': {'baz': {'one': 'ORIGINAL', 'two': 'ORIGINAL'}}}}
-        second = {'foo': {'bar': {'baz': {'one': 'UPDATE'}}}}
-
-        merge_dicts(first, second)
-        # The value from the second dict wins.
-        self.assertEqual(first['foo']['bar']['baz']['one'], 'UPDATE')
-        # And we still preserve the other attributes.
-        self.assertEqual(first['foo']['bar']['baz']['two'], 'ORIGINAL')
-
-    def test_merge_dicts_new_keys(self):
-        first = {'foo': {'bar': {'baz': {'one': 'ORIGINAL', 'two': 'ORIGINAL'}}}}
-        second = {'foo': {'bar': {'baz': {'three': 'UPDATE'}}}}
-
-        merge_dicts(first, second)
-        self.assertEqual(first['foo']['bar']['baz']['one'], 'ORIGINAL')
-        self.assertEqual(first['foo']['bar']['baz']['two'], 'ORIGINAL')
-        self.assertEqual(first['foo']['bar']['baz']['three'], 'UPDATE')
-
-    def test_merge_empty_dict_does_nothing(self):
-        first = {'foo': {'bar': 'baz'}}
-        merge_dicts(first, {})
-        self.assertEqual(first, {'foo': {'bar': 'baz'}})
-
-    def test_more_than_one_sub_dict(self):
-        first = {'one': {'inner': 'ORIGINAL', 'inner2': 'ORIGINAL'},
-                 'two': {'inner': 'ORIGINAL', 'inner2': 'ORIGINAL'}}
-        second = {'one': {'inner': 'UPDATE'}, 'two': {'inner': 'UPDATE'}}
-
-        merge_dicts(first, second)
-        self.assertEqual(first['one']['inner'], 'UPDATE')
-        self.assertEqual(first['one']['inner2'], 'ORIGINAL')
-
-        self.assertEqual(first['two']['inner'], 'UPDATE')
-        self.assertEqual(first['two']['inner2'], 'ORIGINAL')
-
-    def test_new_keys(self):
-        first = {'one': {'inner': 'ORIGINAL'}, 'two': {'inner': 'ORIGINAL'}}
-        second = {'three': {'foo': {'bar': 'baz'}}}
-        # In this case, second has no keys in common, but we'd still expect
-        # this to get merged.
-        merge_dicts(first, second)
-        self.assertEqual(first['three']['foo']['bar'], 'baz')
-
-
 class TestBuildRetryConfig(unittest.TestCase):
     def setUp(self):
         self.retry = {
diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py
index 6e8ed436b..4b7e9aef0 100644
--- a/tests/unit/test_utils.py
+++ b/tests/unit/test_utils.py
@@ -36,6 +36,7 @@ from botocore.utils import calculate_sha256
 from botocore.utils import is_valid_endpoint_url
 from botocore.utils import fix_s3_host
 from botocore.utils import instance_cache
+from botocore.utils import merge_dicts
 from botocore.model import DenormalizedStructureBuilder
 from botocore.model import ShapeResolver
 
@@ -597,5 +598,81 @@ class TestInstanceCache(unittest.TestCase):
         self.assertEqual(len(self.cache), 1)
 
 
+class TestMergeDicts(unittest.TestCase):
+    def test_merge_dicts_overrides(self):
+        first = {
+            'foo': {'bar': {'baz': {'one': 'ORIGINAL', 'two': 'ORIGINAL'}}}}
+        second = {'foo': {'bar': {'baz': {'one': 'UPDATE'}}}}
+
+        merge_dicts(first, second)
+        # The value from the second dict wins.
+        self.assertEqual(first['foo']['bar']['baz']['one'], 'UPDATE')
+        # And we still preserve the other attributes.
+        self.assertEqual(first['foo']['bar']['baz']['two'], 'ORIGINAL')
+
+    def test_merge_dicts_new_keys(self):
+        first = {
+            'foo': {'bar': {'baz': {'one': 'ORIGINAL', 'two': 'ORIGINAL'}}}}
+        second = {'foo': {'bar': {'baz': {'three': 'UPDATE'}}}}
+
+        merge_dicts(first, second)
+        self.assertEqual(first['foo']['bar']['baz']['one'], 'ORIGINAL')
+        self.assertEqual(first['foo']['bar']['baz']['two'], 'ORIGINAL')
+        self.assertEqual(first['foo']['bar']['baz']['three'], 'UPDATE')
+
+    def test_merge_empty_dict_does_nothing(self):
+        first = {'foo': {'bar': 'baz'}}
+        merge_dicts(first, {})
+        self.assertEqual(first, {'foo': {'bar': 'baz'}})
+
+    def test_more_than_one_sub_dict(self):
+        first = {'one': {'inner': 'ORIGINAL', 'inner2': 'ORIGINAL'},
+                 'two': {'inner': 'ORIGINAL', 'inner2': 'ORIGINAL'}}
+        second = {'one': {'inner': 'UPDATE'}, 'two': {'inner': 'UPDATE'}}
+
+        merge_dicts(first, second)
+        self.assertEqual(first['one']['inner'], 'UPDATE')
+        self.assertEqual(first['one']['inner2'], 'ORIGINAL')
+
+        self.assertEqual(first['two']['inner'], 'UPDATE')
+        self.assertEqual(first['two']['inner2'], 'ORIGINAL')
+
+    def test_new_keys(self):
+        first = {'one': {'inner': 'ORIGINAL'}, 'two': {'inner': 'ORIGINAL'}}
+        second = {'three': {'foo': {'bar': 'baz'}}}
+        # In this case, second has no keys in common, but we'd still expect
+        # this to get merged.
+        merge_dicts(first, second)
+        self.assertEqual(first['three']['foo']['bar'], 'baz')
+
+    def test_list_values_no_append(self):
+        dict1 = {'Foo': ['old_foo_value']}
+        dict2 = {'Foo': ['new_foo_value']}
+        merge_dicts(dict1, dict2)
+        self.assertEqual(
+            dict1, {'Foo': ['new_foo_value']})
+
+    def test_list_values_append(self):
+        dict1 = {'Foo': ['old_foo_value']}
+        dict2 = {'Foo': ['new_foo_value']}
+        merge_dicts(dict1, dict2, append_lists=True)
+        self.assertEqual(
+            dict1, {'Foo': ['old_foo_value', 'new_foo_value']})
+
+    def test_list_values_mismatching_types(self):
+        dict1 = {'Foo': 'old_foo_value'}
+        dict2 = {'Foo': ['new_foo_value']}
+        merge_dicts(dict1, dict2, append_lists=True)
+        self.assertEqual(
+            dict1, {'Foo': ['new_foo_value']})
+
+    def test_list_values_missing_key(self):
+        dict1 = {}
+        dict2 = {'Foo': ['foo_value']}
+        merge_dicts(dict1, dict2, append_lists=True)
+        self.assertEqual(
+            dict1, {'Foo': ['foo_value']})
+
+
 if __name__ == '__main__':
     unittest.main()