Repository: spack/spack

Issue:
Traversing specs in the root direction doesn't work if an environment is not loaded from spack.lock
### Steps to reproduce

Using this environment:
```yaml
spack:
  specs:
  - hdf5~mpi
  - hdf5+mpi
```
and this simple `script.py`:
```python
import spack.ci
import spack.environment

e = spack.environment.Environment('/tmp/ci')
e.concretize()
# e.write()

for item in spack.ci.get_spec_filter_list(e, ['zlib']):
    print(item)
```
shows a lot less specs than expected if a `spack.lock` is not present. With a `spack.lock` the result is correct.

### Error message

This is the result we get:
```console
$ spack python script.py 
==> Starting concretization pool with 2 processes
==> Environment concretized in 8.00 seconds.
zlib@1.2.12%gcc@9.4.0+optimize+pic+shared patches=0d38234 arch=linux-ubuntu20.04-icelake
hdf5@1.12.1%gcc@9.4.0~cxx~fortran~hl~ipo~java~mpi+shared~szip~threadsafe+tools api=default build_type=RelWithDebInfo patches=ee351eb arch=linux-ubuntu20.04-icelake ^cmake@3.23.0%gcc@9.4.0~doc+ncurses+ownlibs~qt build_type=Release arch=linux-ubuntu20.04-icelake ^ncurses@6.2%gcc@9.4.0~symlinks+termlib abi=none arch=linux-ubuntu20.04-icelake ^openssl@1.1.1n%gcc@9.4.0~docs~shared certs=system arch=linux-ubuntu20.04-icelake ^pkgconf@1.8.0%gcc@9.4.0 arch=linux-ubuntu20.04-icelake ^zlib@1.2.12%gcc@9.4.0+optimize+pic+shared patches=0d38234 arch=linux-ubuntu20.04-icelake
```
The error is that, when we concretize the specs separately in memory we have separate Python objects for specs which share the same hash. That means each of the objects with a given hash has only a part of the references towards the root specs.

With a `spack.lock` we have instead a single Python object per spec hash, and so traversing towards roots works correctly.

### Information on your system

* **Spack:** 0.18.0.dev0 (b2eda32e55601dd80bd0602527e5736e24ef8544)
* **Python:** 3.8.10
* **Platform:** linux-ubuntu20.04-icelake
* **Concretizer:** clingo

*This error occurs when pruning specs in Gitlab Pipelines*

### General information

- [X] I have run `spack debug report` and reported the version of Spack/Python/Platform
- [X] I have searched the issues of this repo and believe this is not a duplicate
- [X] I have run the failing commands in debug mode and reported the output


Discussion:

Resolving patch:
diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py
--- a/lib/spack/spack/environment/environment.py
+++ b/lib/spack/spack/environment/environment.py
@@ -1271,11 +1271,33 @@ def _concretize_separately(self, tests=False):
 
         finish = time.time()
         tty.msg('Environment concretized in %.2f seconds.' % (finish - start))
-        results = []
+        by_hash = {}
         for abstract, concrete in zip(root_specs, concretized_root_specs):
             self._add_concrete_spec(abstract, concrete)
-            results.append((abstract, concrete))
+            by_hash[concrete.build_hash()] = concrete
+
+        # Unify the specs objects, so we get correct references to all parents
+        self._read_lockfile_dict(self._to_lockfile_dict())
+
+        # Re-attach information on test dependencies
+        if tests:
+            # This is slow, but the information on test dependency is lost
+            # after unification or when reading from a lockfile.
+            for h in self.specs_by_hash:
+                current_spec, computed_spec = self.specs_by_hash[h], by_hash[h]
+                for node in computed_spec.traverse():
+                    test_deps = node.dependencies(deptype='test')
+                    for test_dependency in test_deps:
+                        if test_dependency in current_spec[node.name]:
+                            continue
+                        current_spec[node.name].add_dependency_edge(
+                            test_dependency.copy(), deptype='test'
+                        )
 
+        results = [
+            (abstract, self.specs_by_hash[h]) for abstract, h in
+            zip(self.concretized_user_specs, self.concretized_order)
+        ]
         return results
 
     def concretize_and_add(self, user_spec, concrete_spec=None, tests=False):
diff --git a/var/spack/repos/builtin/packages/mpich/package.py b/var/spack/repos/builtin/packages/mpich/package.py
--- a/var/spack/repos/builtin/packages/mpich/package.py
+++ b/var/spack/repos/builtin/packages/mpich/package.py
@@ -174,10 +174,10 @@ class Mpich(AutotoolsPackage):
     depends_on('argobots', when='+argobots')
 
     # building from git requires regenerating autotools files
-    depends_on('automake@1.15:', when='@develop', type=("build"))
-    depends_on('libtool@2.4.4:', when='@develop', type=("build"))
-    depends_on("m4", when="@develop", type=("build")),
-    depends_on("autoconf@2.67:", when='@develop', type=("build"))
+    depends_on('automake@1.15:', when='@develop', type='build')
+    depends_on('libtool@2.4.4:', when='@develop', type='build')
+    depends_on("m4", when="@develop", type='build'),
+    depends_on("autoconf@2.67:", when='@develop', type='build')
 
     # building with "+hwloc' also requires regenerating autotools files
     depends_on('automake@1.15:', when='@3.3:3.3.99 +hwloc', type="build")


Test patch:
diff --git a/lib/spack/spack/test/ci.py b/lib/spack/spack/test/ci.py
--- a/lib/spack/spack/test/ci.py
+++ b/lib/spack/spack/test/ci.py
@@ -579,3 +579,15 @@ def test_get_spec_filter_list(mutable_mock_env_path, config, mutable_mock_repo):
                                        'libelf'])
 
     assert affected_pkg_names == expected_affected_pkg_names
+
+
+@pytest.mark.regression('29947')
+def test_affected_specs_on_first_concretization(mutable_mock_env_path, config):
+    e = ev.create('first_concretization')
+    e.add('hdf5~mpi~szip')
+    e.add('hdf5~mpi+szip')
+    e.concretize()
+
+    affected_specs = spack.ci.get_spec_filter_list(e, ['zlib'])
+    hdf5_specs = [s for s in affected_specs if s.name == 'hdf5']
+    assert len(hdf5_specs) == 2
diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py
--- a/lib/spack/spack/test/cmd/ci.py
+++ b/lib/spack/spack/test/cmd/ci.py
@@ -1731,12 +1731,12 @@ def test_ci_reproduce(tmpdir, mutable_mock_env_path,
             job_spec_yaml_path = os.path.join(
                 working_dir.strpath, 'archivefiles.yaml')
             with open(job_spec_yaml_path, 'w') as fd:
-                fd.write(job_spec.to_yaml(hash=ht.full_hash))
+                fd.write(job_spec.to_yaml(hash=ht.build_hash))
 
             root_spec_yaml_path = os.path.join(
                 working_dir.strpath, 'root.yaml')
             with open(root_spec_yaml_path, 'w') as fd:
-                fd.write(root_spec.to_yaml(hash=ht.full_hash))
+                fd.write(root_spec.to_yaml(hash=ht.build_hash))
 
             artifacts_root = os.path.join(working_dir.strpath, 'scratch_dir')
             pipeline_path = os.path.join(artifacts_root, 'pipeline.yml')
diff --git a/lib/spack/spack/test/cmd/install.py b/lib/spack/spack/test/cmd/install.py
--- a/lib/spack/spack/test/cmd/install.py
+++ b/lib/spack/spack/test/cmd/install.py
@@ -851,7 +851,7 @@ def test_install_no_add_in_env(tmpdir, mock_fetch, install_mockery,
         # but not added as a root
         mpi_spec_yaml_path = tmpdir.join('{0}.yaml'.format(mpi_spec.name))
         with open(mpi_spec_yaml_path.strpath, 'w') as fd:
-            fd.write(mpi_spec.to_yaml(hash=ht.full_hash))
+            fd.write(mpi_spec.to_yaml(hash=ht.build_hash))
 
         install('--no-add', '-f', mpi_spec_yaml_path.strpath)
         assert(mpi_spec not in e.roots())