Repository: spack/spack

Issue:
KeyError in Spack Environments
I have an environment already installed. I want to update that environment by running `spack concretize --force`, but I'm seeing a KeyError when I try to.

### Steps to reproduce the issue

```console
$ spack concretize
$ spack install
# update one of the packages in the environment
$ spack concretize --force
```

### Error Message

```console
$ spack -d concretize -f
Traceback (most recent call last):
  File "/Users/Adam/spack/bin/spack", line 64, in <module>
    sys.exit(spack.main.main())
  File "/Users/Adam/spack/lib/spack/spack/main.py", line 653, in main
    ev.activate(env, args.use_env_repo)
  File "/Users/Adam/spack/lib/spack/spack/environment.py", line 163, in activate
    cmds += env.add_default_view_to_shell(shell)
  File "/Users/Adam/spack/lib/spack/spack/environment.py", line 1090, in add_default_view_to_shell
    for _, spec in self.concretized_specs():
  File "/Users/Adam/spack/lib/spack/spack/environment.py", line 1215, in concretized_specs
    yield (s, self.specs_by_hash[h])
KeyError: 'pxw6s2yxrbws3sg5jxzosiqc3xiisane'
```

### Information on your system

I've been able to reproduce this on both macOS 10.15.1 and Cray CNL5. The only workaround I've found so far is to `rm spack.lock` and start from scratch in the environment.

@alalazo This may be related to past problems we've encountered where the `package.py` used to install a package and generate module files is different from the current `package.py`. In this case, the only major change I know of is that `sqlite` had a new version added, so that may be the problem.


Discussion:
I have the same issue with clean spack installation and clean software repo:

>     git clone https://github.com/spack/spack spack-0.13.1
>     cd spack-0.13.1
>     git checkout releases/v0.13
>     spack install zlib@1.2.8
>     spack env create userenv-0131
>     spack env activate userenv-0131
>     spack add zlib@1.2.8
>     spack add libxml2
>     spack install

```Traceback (most recent call last):
  File "/cvmfs/software.myrepo/tests/software_basics/spack-0.13.1/bin/spack", line 64, in <module>
    sys.exit(spack.main.main())
  File "/cvmfs/software.myrepo/tests/software_basics/spack-0.13.1/lib/spack/spack/main.py", line 653, in main
    ev.activate(env, args.use_env_repo)
  File "/cvmfs/software.myrepo/tests/software_basics/spack-0.13.1/lib/spack/spack/environment.py", line 162, in activate
    cmds += env.add_default_view_to_shell(shell)
  File "/cvmfs/software.myrepo/tests/software_basics/spack-0.13.1/lib/spack/spack/environment.py", line 1119, in add_default_view_to_shell
    for _, spec in self.concretized_specs():
  File "/cvmfs/software.myrepo/tests/software_basics/spack-0.13.1/lib/spack/spack/environment.py", line 1244, in concretized_specs
    yield (s, self.specs_by_hash[h])
KeyError: '3yl7xdhjft4obodsxdavnem42q57q624'
```
@phanousk could you report this in a separate issue? I don't think your error is related to this. Note that if you are invoking `spack` without a full path, I would expect that you have to invoke `setup-env.sh` after cloning the Spack repo.
> @phanousk could you report this in a separate issue? I don't think your error is related to this. Note that if you are invoking `spack` without a full path, I would expect that you have to invoke `setup-env.sh` after cloning the Spack repo.

Hi, I've tried today the same test with prior 'git pull' and can not simulate the problem again. So perhaps it's not necessary to file another issue.
Petr

Resolving patch:
diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py
--- a/lib/spack/spack/architecture.py
+++ b/lib/spack/spack/architecture.py
@@ -356,10 +356,10 @@ def _cmp_key(self):
         return (self.name, self.version)
 
     def to_dict(self):
-        return {
-            'name': self.name,
-            'version': self.version
-        }
+        return syaml_dict([
+            ('name', self.name),
+            ('version', self.version)
+        ])
 
 
 @key_ordering
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -1492,10 +1492,10 @@ def to_node_dict(self, hash=ht.dag_hash):
             d['parameters'] = params
 
         if self.external:
-            d['external'] = {
-                'path': self.external_path,
-                'module': self.external_module
-            }
+            d['external'] = syaml.syaml_dict([
+                ('path', self.external_path),
+                ('module', self.external_module),
+            ])
 
         if not self._concrete:
             d['concrete'] = False


Test patch:
diff --git a/lib/spack/spack/test/spec_yaml.py b/lib/spack/spack/test/spec_yaml.py
--- a/lib/spack/spack/test/spec_yaml.py
+++ b/lib/spack/spack/test/spec_yaml.py
@@ -8,13 +8,20 @@
 YAML format preserves DAG information in the spec.
 
 """
+import ast
+import inspect
 import os
 
 from collections import Iterable, Mapping
 
+import pytest
+
+import spack.architecture
 import spack.hash_types as ht
+import spack.spec
 import spack.util.spack_json as sjson
 import spack.util.spack_yaml as syaml
+import spack.version
 
 from spack import repo
 from spack.spec import Spec, save_dependency_spec_yamls
@@ -204,6 +211,72 @@ def test_ordered_read_not_required_for_consistent_dag_hash(
         assert spec.full_hash() == round_trip_reversed_json_spec.full_hash()
 
 
+@pytest.mark.parametrize("module", [
+    spack.spec,
+    spack.architecture,
+    spack.version,
+])
+def test_hashes_use_no_python_dicts(module):
+    """Coarse check to make sure we don't use dicts in Spec.to_node_dict().
+
+    Python dicts are not guaranteed to iterate in a deterministic order
+    (at least not in all python versions) so we need to use lists and
+    syaml_dicts.  syaml_dicts are ordered and ensure that hashes in Spack
+    are deterministic.
+
+    This test is intended to handle cases that are not covered by the
+    consistency checks above, or that would be missed by a dynamic check.
+    This test traverses the ASTs of functions that are used in our hash
+    algorithms, finds instances of dictionaries being constructed, and
+    prints out the line numbers where they occur.
+
+    """
+    class FindFunctions(ast.NodeVisitor):
+        """Find a function definition called to_node_dict."""
+        def __init__(self):
+            self.nodes = []
+
+        def visit_FunctionDef(self, node):  # noqa
+            if node.name in ("to_node_dict", "to_dict", "to_dict_or_value"):
+                self.nodes.append(node)
+
+    class FindDicts(ast.NodeVisitor):
+        """Find source locations of dicts in an AST."""
+        def __init__(self, filename):
+            self.nodes = []
+            self.filename = filename
+
+        def add_error(self, node):
+            self.nodes.append(
+                "Use syaml_dict instead of dict at %s:%s:%s"
+                % (self.filename, node.lineno, node.col_offset)
+            )
+
+        def visit_Dict(self, node):  # noqa
+            self.add_error(node)
+
+        def visit_Call(self, node):  # noqa
+            name = None
+            if isinstance(node.func, ast.Name):
+                name = node.func.id
+            elif isinstance(node.func, ast.Attribute):
+                name = node.func.attr
+
+            if name == 'dict':
+                self.add_error(node)
+
+    find_functions = FindFunctions()
+    module_ast = ast.parse(inspect.getsource(module))
+    find_functions.visit(module_ast)
+
+    find_dicts = FindDicts(module.__file__)
+    for node in find_functions.nodes:
+        find_dicts.visit(node)
+
+    # fail with offending lines if we found some dicts.
+    assert [] == find_dicts.nodes
+
+
 def reverse_all_dicts(data):
     """Descend into data and reverse all the dictionaries"""
     if isinstance(data, dict):