Repository: spack/spack

Issue:
Weight selection when reusing specs is wrong
### Steps to reproduce

From a fresh clone, install two versions of `zlib`:
```console
$ spack install zlib@1.2.11
$ spack install zlib@1.2.12
```
Then try to check the optimization scores:
```console
$ spack solve zlib
```
and compare them to the version weights:
```console
$ spack solve --show=asp zlib
```
There will be:
1. Errors in how the weights are emitted by the Python code
2. Errors in how the weights are assigned to each version

In more complicated cases, e.g. involving also external packages, this might lead Spack to select the wrong version.

### Error message

With my setup:
```console
$ spack find
==> 2 installed packages
-- linux-ubuntu20.04-icelake / gcc@9.4.0 ------------------------
zlib@1.2.11  zlib@1.2.12
```
we have:

<details>

<summary>$ spack solve zlib</summary>

```console
==> Best of 2 considered solutions.
==> Optimization Criteria:
  Priority  Criterion                                            Installed  ToBuild
  1         number of input specs not concretized                        -        0
  2         number of packages to build (vs. reuse)                      -        0
  3         deprecated versions used                                     0        0
  4         version weight                                               0        0
  5         number of non-default variants (roots)                       1        0
  6         preferred providers for roots                                0        0
  7         default values of variants not being used (roots)            0        0
  8         number of non-default variants (non-roots)                   0        0
  9         preferred providers (non-roots)                              0        0
  10        compiler mismatches                                          0        0
  11        OS mismatches                                                0        0
  12        non-preferred OS's                                           0        0
  13        version badness                                              0        0
  14        default values of variants not being used (non-roots)        0        0
  15        non-preferred compilers                                      0        0
  16        target mismatches                                            0        0
  17        non-preferred targets                                       30        0

zlib@1.2.12%gcc@9.4.0+optimize+pic+shared patches=0d38234 arch=linux-ubuntu20.04-icelake
```

</details>

<details>

<summary>$ spack solve --show=asp zlib</summary>

```
%============================================================================
% Concrete input spec definitions
%============================================================================

%============================================================================
% Reusable specs
%============================================================================
optimize_for_reuse.
installed_hash("zlib","mbyn7mqdh2gj67dgdmwhkabpkzl3532x").
imposed_constraint("mbyn7mqdh2gj67dgdmwhkabpkzl3532x","node","zlib").
imposed_constraint("mbyn7mqdh2gj67dgdmwhkabpkzl3532x","version","zlib","1.2.11").
imposed_constraint("mbyn7mqdh2gj67dgdmwhkabpkzl3532x","node_platform","zlib","linux").
imposed_constraint("mbyn7mqdh2gj67dgdmwhkabpkzl3532x","node_os","zlib","ubuntu20.04").
imposed_constraint("mbyn7mqdh2gj67dgdmwhkabpkzl3532x","node_target","zlib","icelake").
imposed_constraint("mbyn7mqdh2gj67dgdmwhkabpkzl3532x","variant_value","zlib","optimize","True").
imposed_constraint("mbyn7mqdh2gj67dgdmwhkabpkzl3532x","variant_value","zlib","pic","True").
imposed_constraint("mbyn7mqdh2gj67dgdmwhkabpkzl3532x","variant_value","zlib","shared","True").
imposed_constraint("mbyn7mqdh2gj67dgdmwhkabpkzl3532x","node_compiler","zlib","gcc").
imposed_constraint("mbyn7mqdh2gj67dgdmwhkabpkzl3532x","node_compiler_version","zlib","gcc","9.4.0").
imposed_constraint("mbyn7mqdh2gj67dgdmwhkabpkzl3532x","hash","zlib","mbyn7mqdh2gj67dgdmwhkabpkzl3532x").

installed_hash("zlib","6jwud6k4455g42yxfqsmjnco2olbupjg").
imposed_constraint("6jwud6k4455g42yxfqsmjnco2olbupjg","node","zlib").
imposed_constraint("6jwud6k4455g42yxfqsmjnco2olbupjg","version","zlib","1.2.12").
imposed_constraint("6jwud6k4455g42yxfqsmjnco2olbupjg","node_platform","zlib","linux").
imposed_constraint("6jwud6k4455g42yxfqsmjnco2olbupjg","node_os","zlib","ubuntu20.04").
imposed_constraint("6jwud6k4455g42yxfqsmjnco2olbupjg","node_target","zlib","icelake").
imposed_constraint("6jwud6k4455g42yxfqsmjnco2olbupjg","variant_value","zlib","optimize","True").
imposed_constraint("6jwud6k4455g42yxfqsmjnco2olbupjg","variant_value","zlib","patches","0d38234384870bfd34dfcb738a9083952656f0c766a0f5990b1893076b084b76").
imposed_constraint("6jwud6k4455g42yxfqsmjnco2olbupjg","variant_value","zlib","pic","True").
imposed_constraint("6jwud6k4455g42yxfqsmjnco2olbupjg","variant_value","zlib","shared","True").
imposed_constraint("6jwud6k4455g42yxfqsmjnco2olbupjg","node_compiler","zlib","gcc").
imposed_constraint("6jwud6k4455g42yxfqsmjnco2olbupjg","node_compiler_version","zlib","gcc","9.4.0").
imposed_constraint("6jwud6k4455g42yxfqsmjnco2olbupjg","hash","zlib","6jwud6k4455g42yxfqsmjnco2olbupjg").


%============================================================================
% General Constraints
%============================================================================

%----------------------------------------------------------------------------
% Available compilers
%----------------------------------------------------------------------------
compiler_version("gcc","9.4.0").


%----------------------------------------------------------------------------
% Default compiler preferences
%----------------------------------------------------------------------------
default_compiler_preference("gcc","9.4.0",0).
compiler_supports_target("gcc","9.4.0","x86_64").
compiler_supports_os("gcc","9.4.0","ubuntu20.04").

%----------------------------------------------------------------------------
% Default platform
%----------------------------------------------------------------------------
node_platform_default("linux").

%----------------------------------------------------------------------------
% Possible operating systems
%----------------------------------------------------------------------------
buildable_os("ubuntu20.04").
os("ubuntu20.04",0).

%----------------------------------------------------------------------------
% Default target
%----------------------------------------------------------------------------

%----------------------------------------------------------------------------
% Target compatibility
%----------------------------------------------------------------------------
compiler_supports_target("gcc","9.4.0","icelake").
compiler_supports_target("gcc","9.4.0","cascadelake").
compiler_supports_target("gcc","9.4.0","cannonlake").
compiler_supports_target("gcc","9.4.0","skylake_avx512").
compiler_supports_target("gcc","9.4.0","skylake").
compiler_supports_target("gcc","9.4.0","x86_64_v4").
compiler_supports_target("gcc","9.4.0","broadwell").
compiler_supports_target("gcc","9.4.0","haswell").
compiler_supports_target("gcc","9.4.0","ivybridge").
compiler_supports_target("gcc","9.4.0","x86_64_v3").
compiler_supports_target("gcc","9.4.0","sandybridge").
compiler_supports_target("gcc","9.4.0","westmere").
compiler_supports_target("gcc","9.4.0","nehalem").
compiler_supports_target("gcc","9.4.0","core2").
compiler_supports_target("gcc","9.4.0","x86_64_v2").
compiler_supports_target("gcc","9.4.0","nocona").
compiler_supports_target("gcc","9.4.0","x86_64").
compiler_supports_target("gcc","9.4.0","x86_64").
target("icelake").
target_family("icelake","x86_64").
target_parent("icelake","cascadelake").
target_parent("icelake","cannonlake").

target("cascadelake").
target_family("cascadelake","x86_64").
target_parent("cascadelake","skylake_avx512").

target("cannonlake").
target_family("cannonlake","x86_64").
target_parent("cannonlake","skylake").

target("skylake_avx512").
target_family("skylake_avx512","x86_64").
target_parent("skylake_avx512","skylake").
target_parent("skylake_avx512","x86_64_v4").

target("skylake").
target_family("skylake","x86_64").
target_parent("skylake","broadwell").

target("x86_64_v4").
target_family("x86_64_v4","x86_64").
target_parent("x86_64_v4","x86_64_v3").

target("broadwell").
target_family("broadwell","x86_64").
target_parent("broadwell","haswell").

target("haswell").
target_family("haswell","x86_64").
target_parent("haswell","ivybridge").
target_parent("haswell","x86_64_v3").

target("ivybridge").
target_family("ivybridge","x86_64").
target_parent("ivybridge","sandybridge").

target("x86_64_v3").
target_family("x86_64_v3","x86_64").
target_parent("x86_64_v3","x86_64_v2").

target("sandybridge").
target_family("sandybridge","x86_64").
target_parent("sandybridge","westmere").

target("westmere").
target_family("westmere","x86_64").
target_parent("westmere","nehalem").

target("nehalem").
target_family("nehalem","x86_64").
target_parent("nehalem","core2").
target_parent("nehalem","x86_64_v2").

target("core2").
target_family("core2","x86_64").
target_parent("core2","nocona").

target("x86_64_v2").
target_family("x86_64_v2","x86_64").
target_parent("x86_64_v2","x86_64").

target("nocona").
target_family("nocona","x86_64").
target_parent("nocona","x86_64").

target("x86_64").
target_family("x86_64","x86_64").


%----------------------------------------------------------------------------
% Virtual providers
%----------------------------------------------------------------------------


%----------------------------------------------------------------------------
% Default virtual providers
%----------------------------------------------------------------------------

%============================================================================
% External packages
%============================================================================

%----------------------------------------------------------------------------
% Compiler flag defaults
%----------------------------------------------------------------------------
flag_type("cflags").
flag_type("cxxflags").
flag_type("fflags").
flag_type("ldflags").
flag_type("ldlibs").
flag_type("cppflags").


%============================================================================
% Package Constraints
%============================================================================

%----------------------------------------------------------------------------
% Package rules: zlib
%----------------------------------------------------------------------------
version_declared("zlib","1.2.12",0,"package_py").
version_declared("zlib","1.2.11",1,"package_py").
version_declared("zlib","1.2.8",2,"package_py").
version_declared("zlib","1.2.3",3,"package_py").
version_declared("zlib","1.2.11",4,"installed").
version_declared("zlib","1.2.12",5,"installed").
deprecated_version("zlib","1.2.3").
deprecated_version("zlib","1.2.8").

variant("zlib","optimize").
variant_single_value("zlib","optimize").
variant_default_value_from_package_py("zlib","optimize","True").
variant_possible_value("zlib","optimize","False").
variant_possible_value("zlib","optimize","True").

variant("zlib","pic").
variant_single_value("zlib","pic").
variant_default_value_from_package_py("zlib","pic","True").
variant_possible_value("zlib","pic","False").
variant_possible_value("zlib","pic","True").

variant("zlib","shared").
variant_single_value("zlib","shared").
variant_default_value_from_package_py("zlib","shared","True").
variant_possible_value("zlib","shared","False").
variant_possible_value("zlib","shared","True").


%----------------------------------------------------------------------------
% Package preferences: zlib
%----------------------------------------------------------------------------
default_target_weight("icelake","zlib",30).
default_target_weight("cascadelake","zlib",31).
default_target_weight("cannonlake","zlib",32).
default_target_weight("skylake_avx512","zlib",33).
default_target_weight("skylake","zlib",34).
default_target_weight("x86_64_v4","zlib",35).
default_target_weight("broadwell","zlib",36).
default_target_weight("haswell","zlib",37).
default_target_weight("ivybridge","zlib",38).
default_target_weight("x86_64_v3","zlib",39).
default_target_weight("sandybridge","zlib",40).
default_target_weight("westmere","zlib",41).
default_target_weight("nehalem","zlib",42).
default_target_weight("core2","zlib",43).
default_target_weight("x86_64_v2","zlib",44).
default_target_weight("nocona","zlib",45).
default_target_weight("x86_64","zlib",46).

%============================================================================
% Spec Constraints
%============================================================================

%----------------------------------------------------------------------------
% Spec: zlib
%----------------------------------------------------------------------------
literal(0).
literal(0,"root","zlib").
literal(0,"node","zlib").
concretize_everything.

%============================================================================
% Variant Values defined in specs
%============================================================================
variant_possible_value("zlib","shared","True").
variant_possible_value("zlib","optimize","True").
variant_possible_value("zlib","pic","True").
variant_possible_value("zlib","patches","0d38234384870bfd34dfcb738a9083952656f0c766a0f5990b1893076b084b76").

%============================================================================
% Virtual Constraints
%============================================================================

%============================================================================
% Version Constraints
%============================================================================

%============================================================================
% Compiler Version Constraints
%============================================================================


%============================================================================
% Target Constraints
%============================================================================

%============================================================================
% Error messages
%============================================================================
internal_error("Two versions with identical weights").
internal_error("Virtual when provides not respected").
internal_error("Package provider weights must be unique").
internal_error("External weight used for internal spec").
internal_error("Mismatch between selected compiler and compiler version").
internal_error("Package must resolve to at most one hash").


```

</details>

From the two excerpt above we can see that we are using the installed `zlib@1.2.12` and we are giving it a weight of 0, while the facts are:
```
version_declared("zlib","1.2.12",0,"package_py").
version_declared("zlib","1.2.11",1,"package_py").
version_declared("zlib","1.2.8",2,"package_py").
version_declared("zlib","1.2.3",3,"package_py").
version_declared("zlib","1.2.11",4,"installed").
version_declared("zlib","1.2.12",5,"installed").
```
There are two errors:
1. The installed `zlib@1.2.12` should have a lower weight than the installed `zlib@1.2.11`
2. We should be using weight 4 and 5 for installed versions

### Information on your system

* **Spack:** 0.19.0.dev0 (8c58c14c3d77420c86b0ae852bc4a680c2aca472)
* **Python:** 3.8.10
* **Platform:** linux-ubuntu20.04-icelake
* **Concretizer:** clingo



### General information

- [X] I have run `spack debug report` and reported the version of Spack/Python/Platform
- [X] I have searched the issues of this repo and believe this is not a duplicate
- [X] I have run the failing commands in debug mode and reported the output


Discussion:
Diff to solve the error during emission of the facts:
```diff
diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py
index bcdb929fdc..5f5a128c44 100644
--- a/lib/spack/spack/solver/asp.py
+++ b/lib/spack/spack/solver/asp.py
@@ -748,7 +748,13 @@ def key_fn(version):
 
         pkg = packagize(pkg)
         declared_versions = self.declared_versions[pkg.name]
-        most_to_least_preferred = sorted(declared_versions, key=key_fn)
+        partially_sorted_versions = sorted(set(declared_versions), key=key_fn)
+
+        most_to_least_preferred = []
+        for _, group in itertools.groupby(partially_sorted_versions, key=key_fn):
+            most_to_least_preferred.extend(
+                list(sorted(group, reverse=True, key=lambda x: spack.version.ver(x)))
+            )
 
         for weight, declared_version in enumerate(most_to_least_preferred):
             self.gen.fact(fn.version_declared(
```
The other bug is in:

https://github.com/spack/spack/blob/0205fefe0c80dc1fe03b9ec5da93bf24b1c9103f/lib/spack/spack/solver/concretize.lp#L110-L113

There we cannot just select the minimum, without checking if the provenance of the version is the correct one. For instance, if we have an external `zlib@1.2.11` that will have weight 0, and be preferred to `zlib@1.2.12` from `package.py` at weight 1. With the logic above we might end up installing `zlib@1.2.11` from `package.py` using the weight from the external spec.

Resolving patch:
diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py
--- a/lib/spack/spack/solver/asp.py
+++ b/lib/spack/spack/solver/asp.py
@@ -748,7 +748,13 @@ def key_fn(version):
 
         pkg = packagize(pkg)
         declared_versions = self.declared_versions[pkg.name]
-        most_to_least_preferred = sorted(declared_versions, key=key_fn)
+        partially_sorted_versions = sorted(set(declared_versions), key=key_fn)
+
+        most_to_least_preferred = []
+        for _, group in itertools.groupby(partially_sorted_versions, key=key_fn):
+            most_to_least_preferred.extend(list(sorted(
+                group, reverse=True, key=lambda x: spack.version.ver(x.version)
+            )))
 
         for weight, declared_version in enumerate(most_to_least_preferred):
             self.gen.fact(fn.version_declared(


Test patch:
diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py
--- a/lib/spack/spack/test/concretize.py
+++ b/lib/spack/spack/test/concretize.py
@@ -1753,3 +1753,36 @@ def test_misleading_error_message_on_version(self, mutable_database):
             with pytest.raises(spack.solver.asp.UnsatisfiableSpecError,
                                match="'dep-with-variants' satisfies '@999'"):
                 solver.driver.solve(setup, [root_spec], reuse=reusable_specs)
+
+    @pytest.mark.regression('31148')
+    def test_version_weight_and_provenance(self):
+        """Test package preferences during coconcretization."""
+        import spack.solver.asp
+        if spack.config.get('config:concretizer') == 'original':
+            pytest.skip('Original concretizer cannot reuse')
+
+        reusable_specs = [
+            spack.spec.Spec(spec_str).concretized()
+            for spec_str in ('b@0.9', 'b@1.0')
+        ]
+        root_spec = spack.spec.Spec('a foobar=bar')
+
+        with spack.config.override("concretizer:reuse", True):
+            solver = spack.solver.asp.Solver()
+            setup = spack.solver.asp.SpackSolverSetup()
+            result = solver.driver.solve(
+                setup, [root_spec], reuse=reusable_specs, out=sys.stdout
+            )
+            # The result here should have a single spec to build ('a')
+            # and it should be using b@1.0 with a version badness of 2
+            # The provenance is:
+            # version_declared("b","1.0",0,"package_py").
+            # version_declared("b","0.9",1,"package_py").
+            # version_declared("b","1.0",2,"installed").
+            # version_declared("b","0.9",3,"installed").
+            for criterion in [
+                (1, None, 'number of packages to build (vs. reuse)'),
+                (2, 0, 'version badness')
+            ]:
+                assert criterion in result.criteria
+            assert result.specs[0].satisfies('^b@1.0')