Repository: spack/spack

Issue:
_merge_yaml() doesn't work
See `lib/spack/spack/config.py`.  I've added some debugging printout:
```
    print('----------------- BEGIN')
    for scope in scopes:
        # read potentially cached data from the scope.

        data = scope.get_section(section)

        # Skip empty configs
        if not data or not isinstance(data, dict):
            continue

        if section not in data:
            tty.warn("Skipping bad configuration file: '%s'" % scope.path)
            continue

        print('********* data', data)
        merged_section = _merge_yaml(merged_section, data)



    print('********* result', merged_section)
    print('----------------- END')
```
See below for what I get.  In my case, 4 YAML files are being merged into a final result.  The result is obviously wrong.  Note that the result has `all: version: [develop]`, which doesn't appear anywhere in the files being merged.  And I'm not a happy camper with Spack trying the `develop` version on everything it can.
```
----------------- BEGIN
('********* data', {'packages': {'all': {'compiler': ['gcc', 'intel', 'pgi', 'clang', 'xl', 'nag'], 'providers': {'awk': ['gawk'], 'blas': ['openblas'], 'daal': ['intel-daal'], 'elf': ['elfutils'], 'gl': ['mesa', 'opengl'], 'glu': ['mesa-glu', 'openglu'], 'golang': ['gcc'], 'ipp': ['intel-ipp'], 'java': ['jdk'], 'lapack': ['openblas'], 'mkl': ['intel-mkl'], 'mpe': ['mpe2'], 'mpi': ['openmpi', 'mpich'], 'opencl': ['pocl'], 'openfoam': ['openfoam-com', 'openfoam-org', 'foam-extend'], 'pil': ['py-pillow'], 'pkgconfig': ['pkgconf', 'pkg-config'], 'scalapack': ['netlib-scalapack'], 'szip': ['libszip', 'libaec'], 'tbb': ['intel-tbb'], 'jpeg': ['libjpeg-turbo', 'libjpeg']}, 'paths': {}, 'modules': {}, 'buildable': True, 'version': []}}})
('********* data', {'packages': {'foo-a': {'version': ['develop'], 'paths': {}, 'providers': {}, 'modules': {}, 'buildable': True, 'compiler': []}}})
('********* data', {'packages': {'icebin': {'variants': ['+coupler', '+modele', '+pism'], 'paths': {}, 'providers': {}, 'modules': {}, 'buildable': True, 'version': [], 'compiler': []}}})
('********* data', {'packages': {'icebin': {'version': ['develop'], 'paths': {}, 'providers': {}, 'modules': {}, 'buildable': True, 'compiler': []}}})
('********* result', {'packages': {'all': {'compiler': ['gcc', 'intel', 'pgi', 'clang', 'xl', 'nag'], 'providers': {'awk': ['gawk'], 'blas': ['openblas'], 'daal': ['intel-daal'], 'elf': ['elfutils'], 'gl': ['mesa', 'opengl'], 'glu': ['mesa-glu', 'openglu'], 'golang': ['gcc'], 'ipp': ['intel-ipp'], 'java': ['jdk'], 'lapack': ['openblas'], 'mkl': ['intel-mkl'], 'mpe': ['mpe2'], 'mpi': ['openmpi', 'mpich'], 'opencl': ['pocl'], 'openfoam': ['openfoam-com', 'openfoam-org', 'foam-extend'], 'pil': ['py-pillow'], 'pkgconfig': ['pkgconf', 'pkg-config'], 'scalapack': ['netlib-scalapack'], 'szip': ['libszip', 'libaec'], 'tbb': ['intel-tbb'], 'jpeg': ['libjpeg-turbo', 'libjpeg']}, 'paths': {}, 'modules': {}, 'buildable': True, 'version': ['develop']}, 'foo-a': {'version': ['develop'], 'paths': {}, 'providers': {}, 'modules': {}, 'buildable': True, 'compiler': []}, 'icebin': {'variants': ['+coupler', '+modele', '+pism'], 'paths': {}, 'providers': {}, 'modules': {}, 'buildable': True, 'version': ['develop'], 'compiler': []}}})
----------------- END
```



Discussion:
Fixed by #7925 now we just have to merge.

Resolving patch:
diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -116,7 +116,8 @@ def extend_with_default(validator_class):
     def set_defaults(validator, properties, instance, schema):
         for property, subschema in iteritems(properties):
             if "default" in subschema:
-                instance.setdefault(property, subschema["default"])
+                instance.setdefault(
+                    property, copy.deepcopy(subschema["default"]))
         for err in validate_properties(
                 validator, properties, instance, schema):
             yield err
@@ -127,7 +128,7 @@ def set_pp_defaults(validator, properties, instance, schema):
                 if isinstance(instance, dict):
                     for key, val in iteritems(instance):
                         if re.match(property, key) and val is None:
-                            instance[key] = subschema["default"]
+                            instance[key] = copy.deepcopy(subschema["default"])
 
         for err in validate_pattern_properties(
                 validator, properties, instance, schema):


Test patch:
diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py
--- a/lib/spack/spack/test/config.py
+++ b/lib/spack/spack/test/config.py
@@ -167,6 +167,50 @@
         'build_stage:': ['patha', 'pathb']}}
 
 
+packages_merge_low = {
+    'packages': {
+        'foo': {
+            'variants': ['+v1']
+        },
+        'bar': {
+            'variants': ['+v2']
+        }
+    }
+}
+
+packages_merge_high = {
+    'packages': {
+        'foo': {
+            'version': ['a']
+        },
+        'bar': {
+            'version': ['b'],
+            'variants': ['+v3']
+        },
+        'baz': {
+            'version': ['c']
+        }
+    }
+}
+
+
+@pytest.mark.regression('7924')
+def test_merge_with_defaults(config, write_config_file):
+    """This ensures that specified preferences merge with defaults as
+       expected. Originally all defaults were initialized with the
+       exact same object, which led to aliasing problems. Therefore
+       the test configs used here leave 'version' blank for multiple
+       packages in 'packages_merge_low'.
+    """
+    write_config_file('packages', packages_merge_low, 'low')
+    write_config_file('packages', packages_merge_high, 'high')
+    cfg = spack.config.get_config('packages')
+
+    assert cfg['foo']['version'] == ['a']
+    assert cfg['bar']['version'] == ['b']
+    assert cfg['baz']['version'] == ['c']
+
+
 def check_compiler_config(comps, *compiler_names):
     """Check that named compilers in comps match Spack's config."""
     config = spack.config.get_config('compilers')