Repository: spack/spack

Issue:
Reusing specs doesn't respect variant constraints
This issue has been reported by @haampie on Slack

### Steps to reproduce

Apply the following diff:
```diff
diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py
index 48c05308cc..cfbfca6db2 100644
--- a/var/spack/repos/builtin/packages/hdf5/package.py
+++ b/var/spack/repos/builtin/packages/hdf5/package.py
@@ -90,6 +90,8 @@ class Hdf5(CMakePackage):
     depends_on('szip', when='+szip')
     depends_on('zlib@1.1.2:')
 
+
+    depends_on('zlib tls=mbedtls +shared +optimize')
     # The compiler wrappers (h5cc, h5fc, etc.) run 'pkg-config'.
     depends_on('pkgconfig', type='run')
 
diff --git a/var/spack/repos/builtin/packages/zlib/package.py b/var/spack/repos/builtin/packages/zlib/package.py
index 7ca45d4215..0dd246dd75 100644
--- a/var/spack/repos/builtin/packages/zlib/package.py
+++ b/var/spack/repos/builtin/packages/zlib/package.py
@@ -2,7 +2,7 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
+import sys
 
 # Although zlib comes with a configure script, it does not use Autotools
 # The AutotoolsPackage causes zlib to fail to build with PGI
@@ -27,6 +27,26 @@ class Zlib(Package):
             description='Enables the build of shared libraries.')
     variant('optimize', default=True,
             description='Enable -O2 for a more optimized lib')
+    default_tls = 'openssl'
+    if sys.platform == 'darwin':
+        default_tls = 'secure_transport'
+
+    # TODO: add dependencies for other possible TLS backends
+    values_tls = [
+        # 'amissl',
+        # 'bearssl',
+        'gnutls',
+        'mbedtls',
diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py
index 48c05308cc..cfbfca6db2 100644
--- a/var/spack/repos/builtin/packages/hdf5/package.py
+++ b/var/spack/repos/builtin/packages/hdf5/package.py
@@ -90,6 +90,8 @@ class Hdf5(CMakePackage):
     depends_on('szip', when='+szip')
     depends_on('zlib@1.1.2:')
 
+
+    depends_on('zlib tls=mbedtls +shared +optimize')
     # The compiler wrappers (h5cc, h5fc, etc.) run 'pkg-config'.
     depends_on('pkgconfig', type='run')
 
diff --git a/var/spack/repos/builtin/packages/zlib/package.py b/var/spack/repos/builtin/packages/zlib/package.py
index 7ca45d4215..0dd246dd75 100644
--- a/var/spack/repos/builtin/packages/zlib/package.py
+++ b/var/spack/repos/builtin/packages/zlib/package.py
@@ -2,7 +2,7 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
+import sys
 
 # Although zlib comes with a configure script, it does not use Autotools
 # The AutotoolsPackage causes zlib to fail to build with PGI
@@ -27,6 +27,26 @@ class Zlib(Package):
             description='Enables the build of shared libraries.')
     variant('optimize', default=True,
             description='Enable -O2 for a more optimized lib')
+    default_tls = 'openssl'
+    if sys.platform == 'darwin':
+        default_tls = 'secure_transport'
+
+    # TODO: add dependencies for other possible TLS backends
+    values_tls = [
+        # 'amissl',
+        # 'bearssl',
+        'gnutls',
+        'mbedtls',
+        # 'mesalink',
+        'nss',
+        'openssl',
+        # 'rustls',
+        # 'schannel',
+        'secure_transport',
+        # 'wolfssl',
+    ]
+
+    variant('tls', default=default_tls, description='TLS backend', values=values_tls, multi=True)
 
     patch('w_patch.patch', when="@1.2.11%cce")
 
```
to e13e697067d004107655bc206ac18e9191abf26c and then:
```
 spack install zlib tls=openssl ~shared ~optimize
==> Installing zlib-1.2.11-yxtja5d7dmt25hgt7vd4bxu7diiqc6ee
==> No binary for zlib-1.2.11-yxtja5d7dmt25hgt7vd4bxu7diiqc6ee found: installing from source
==> Using cached archive: /home/culpo/PycharmProjects/spack/var/spack/cache/_source-cache/archive/c3/c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1.tar.gz
==> No patches needed for zlib
==> zlib: Executing phase: 'install'
==> zlib: Successfully installed zlib-1.2.11-yxtja5d7dmt25hgt7vd4bxu7diiqc6ee
  Fetch: 0.00s.  Build: 0.77s.  Total: 0.78s.
[+] /home/culpo/PycharmProjects/spack/opt/spack/linux-ubuntu20.04-icelake/gcc-9.3.0/zlib-1.2.11-yxtja5d7dmt25hgt7vd4bxu7diiqc6ee
```

### Error message

There's no error message, but:
```
$ spack solve -I --reuse hdf5~mpi
==> Best of 10 considered solutions.
==> Optimization Criteria:
  Priority  Criterion                                            Installed  ToBuild
  1         number of packages to build (vs. reuse)                      -        3
  2         deprecated versions used                                     0        0
  3         version weight                                               0        0
  4         number of non-default variants (roots)                       0        0
  5         preferred providers for roots                                0        0
  6         default values of variants not being used (roots)            0        0
  7         number of non-default variants (non-roots)                   2        0
  8         preferred providers (non-roots)                              0        0
  9         compiler mismatches                                          0        0
  10        OS mismatches                                                0        0
  11        non-preferred OS's                                           0        0
  12        version badness                                              0        0
  13        default values of variants not being used (non-roots)        0        0
  14        non-preferred compilers                                      0        0
  15        target mismatches                                            0        0
  16        non-preferred targets                                        0        0

 -   hdf5@1.10.7%gcc@9.3.0~cxx~fortran~hl~ipo~java~mpi+shared~szip~threadsafe+tools api=default build_type=RelWithDebInfo arch=linux-ubuntu20.04-icelake
 -       ^cmake@3.21.2%gcc@9.3.0~doc+ncurses+openssl+ownlibs~qt build_type=Release arch=linux-ubuntu20.04-icelake
 -       ^pkgconf@1.8.0%gcc@9.3.0 arch=linux-ubuntu20.04-icelake
[+]      ^zlib@1.2.11%gcc@9.3.0~optimize+pic~shared tls=openssl arch=linux-ubuntu20.04-icelake
```
is reusing a spec that doesn't meet the specifications.

### Information on your system

* **Spack:** 0.16.3-5278-e13e697067
* **Python:** 3.8.10
* **Platform:** linux-ubuntu20.04-icelake
* **Concretizer:** clingo


### General information

- [X] I have run `spack debug report` and reported the version of Spack/Python/Platform
- [X] I have searched the issues of this repo and believe this is not a duplicate
- [X] I have run the failing commands in debug mode and reported the output


Discussion:
Simpler reproducer:

```
spack uninstall --all zlib~shared
spack install zlib+shared
```
```
spack spec --reuse zlib~shared
```
concretizes to `+shared`.
It seems we need some additional integrity constraint for reused specs. This is the answer set for the `hdf5` case:
```
node("hdf5") variant_value("hdf5","mpi","False") node("zlib") node("cmake") variant_value("zlib","tls","mbedtls") variant_value("zlib","shared","True") variant_value("zlib","optimize","True") variant_value("hdf5","tools","True") variant_value("hdf5","threadsafe","False") variant_value("hdf5","szip","False") variant_value("hdf5","shared","True") variant_value("hdf5","java","False") variant_value("hdf5","ipo","False") variant_value("hdf5","hl","False") variant_value("hdf5","fortran","False") variant_value("hdf5","cxx","False") variant_value("hdf5","build_type","RelWithDebInfo") variant_value("hdf5","api","default") node("pkgconf") variant_value("zlib","tls","openssl") variant_value("zlib","shared","False") variant_value("zlib","pic","True") variant_value("zlib","optimize","False") variant_value("cmake","qt","False") variant_value("cmake","ownlibs","True") variant_value("cmake","openssl","True") variant_value("cmake","ncurses","True") variant_value("cmake","doc","False") variant_value("cmake","build_type","Release")
```
and zlib has both:
```
variant_value("zlib","shared","True")
variant_value("zlib","shared","False")
```
Going through this:
```
% if a hash is selected, we impose all the constraints that implies
impose(Hash) :- hash(Package, Hash).
```
it seems to me we miss rules to say that we cannot have additional constraints which are not imposed by the spec.
This diff:
```diff
diff --git a/lib/spack/spack/solver/concretize.lp b/lib/spack/spack/solver/concretize.lp
index d0a8a8a308..3291c0fb4e 100644
--- a/lib/spack/spack/solver/concretize.lp
+++ b/lib/spack/spack/solver/concretize.lp
diff --git a/lib/spack/spack/solver/concretize.lp b/lib/spack/spack/solver/concretize.lp
index d0a8a8a308..3291c0fb4e 100644
--- a/lib/spack/spack/solver/concretize.lp
+++ b/lib/spack/spack/solver/concretize.lp
@@ -79,6 +79,11 @@ attr(Name, A1)         :- impose(ID), imposed_constraint(ID, Name, A1).
 attr(Name, A1, A2)     :- impose(ID), imposed_constraint(ID, Name, A1, A2).
 attr(Name, A1, A2, A3) :- impose(ID), imposed_constraint(ID, Name, A1, A2, A3).
 
+% we cannot have additional constraints when we are working with concrete specs
+:- impose(ID), hash(_, ID), not imposed_constraint(ID, Name, A1),         attr(Name, A1).
+:- impose(ID), hash(_, ID), not imposed_constraint(ID, Name, A1, A2),     attr(Name, A1, A2).
+:- impose(ID), hash(_, ID), not imposed_constraint(ID, Name, A1, A2, A3), attr(Name, A1, A2, A3).
+
 #defined condition/1.
 #defined condition_requirement/3.
 #defined condition_requirement/4.
```
seems to fix the reproducers for me. I'll be testing this locally to see if any unit-test fails and try to write a unit-test to avoid regressions.
I can't test my original example anymore cause i just installed new stuff. But seems like this works for the trivial examples.

One issue, with --reuse:

real    1m57.911s

without --reuse:

real    0m7.970s

it's 15x slower

And this is a result of the patch above :( 
This commit https://github.com/spack/spack/pull/25310/commits/d3eb1ca4b8ad3b9a1f65d275c7114ee9c4182f55 is related to the issue  

We didn't see this error before the commit above since we were relying on certain rules being imposed on both reused and built specs. Before that commit though we were implicitly relying on the current directives to hold true also for reused specs, which is certainly not the case.

Resolving patch:
diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py
--- a/lib/spack/spack/solver/asp.py
+++ b/lib/spack/spack/solver/asp.py
@@ -1617,7 +1617,7 @@ def define_installed_packages(self, specs, possible):
             index = spack.binary_distribution.update_cache_and_get_specs()
             for spec in index:
                 self._facts_from_concrete_spec(spec, possible)
-        except spack.binary_distribution.FetchCacheError:
+        except (spack.binary_distribution.FetchCacheError, IndexError):
             # this is raised when no mirrors had indices.
             # TODO: update mirror configuration so it can indicate that the source cache
             # TODO: (or any mirror really) doesn't have binaries.


Test patch:
diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py
--- a/lib/spack/spack/test/concretize.py
+++ b/lib/spack/spack/test/concretize.py
@@ -1324,3 +1324,23 @@ def test_non_default_provider_of_multiple_virtuals(self):
                 if pkg.name == 'low-priority-provider':
                     continue
                 assert pkg not in s
+
+    @pytest.mark.regression('27237')
+    @pytest.mark.parametrize('spec_str,expect_installed', [
+        ('mpich', True),
+        ('mpich+debug', False),
+        ('mpich~debug', True)
+    ])
+    def test_concrete_specs_are_not_modified_on_reuse(
+            self, mutable_database, spec_str, expect_installed
+    ):
+        if spack.config.get('config:concretizer') == 'original':
+            pytest.skip('Original concretizer cannot reuse specs')
+
+        # Test the internal consistency of solve + DAG reconstruction
+        # when reused specs are added to the mix. This prevents things
+        # like additional constraints being added to concrete specs in
+        # the answer set produced by clingo.
+        s = spack.spec.Spec(spec_str).concretized(reuse=True)
+        assert s.package.installed is expect_installed
+        assert s.satisfies(spec_str, strict=True)