Repository: spack/spack

Issue:
Spack Environment Concretization Duplicating Dependencies
We are investigating spack environments as a way to replicate build environments and noticed top level dependencies being built as separate variants.

Is this expected behavior or should the concretization process be spread across all specs in the spack.yaml?

### Steps to reproduce the issue
Reproduced on multiple platforms and verified in the spack/ubuntu:18.04 container with commit 7a28a61a59b1233a82ea21c1b09dd40e4628183f 


```
[ubuntu] tmp2: cat spack.yaml
spack:
  specs:
  - boost@1.68.0
  - trilinos@12.12.1
```

### Error Message

```
[ubuntu] tmp2: spack concretize -f
==> Concretizing boost@1.68.0
 -   7znc464  boost@1.68.0%gcc@7.3.0+atomic+chrono~clanglibcpp cxxstd=default +date_time~debug+exception+filesystem+graph~icu+iostreams+locale+log+math~mpi+multithreaded~numpy patches=2ab6c72d03dec6a4ae20220a9dfd5c8c572c5294252155b85c6874d97c323199 ~pic+program_options~python+random+regex+serialization+shared+signals~singlethreaded+system~taggedlayout+test+thread+timer~versionedlayout+wave arch=linux-ubuntu18.04-x86_64
...
==> Concretizing trilinos@12.12.1
 -   kmmhk7k  trilinos@12.12.1%gcc@7.3.0~alloptpkgs+amesos+amesos2+anasazi+aztec+belos+boost build_type=RelWithDebInfo ~cgns~complex~debug~dtk+epetra+epetraext+exodus+explicit_template_instantiation~float+fortran~fortrilinos+gtest+hdf5+hypre+ifpack+ifpack2~intrepid~intrepid2~isorropia+kokkos+metis~minitensor+ml+muelu+mumps~nox~openmp~phalanx~piro~pnetcdf~python~rol~rythmos+sacado~shards+shared~stk+suite-sparse~superlu~superlu-dist~teko~tempus+teuchos+tpetra~x11~xsdkflags~zlib+zoltan+zoltan2 arch=linux-ubuntu18.04-x86_64
 -   tumf2uc      ^boost@1.69.0%gcc@7.3.0+atomic+chrono~clanglibcpp cxxstd=default +date_time~debug+exception+filesystem+graph~icu+iostreams+locale+log+math~mpi+multithreaded~numpy patches=2ab6c72d03dec6a4ae20220a9dfd5c8c572c5294252155b85c6874d97c323199 ~pic+program_options~python+random+regex+serialization+shared+signals~singlethreaded+system~taggedlayout+test+thread+timer~versionedlayout+wave arch=linux-ubuntu18.04-x86_64
```



Discussion:
i think this is a duplicate of https://github.com/spack/spack/issues/9902
@rspavel Having all the abstract requirements concretized together is definitely in the road-map and as far as I know it will become an option in `spack.yaml`.
Thanks. I guess I just got really lucky with the first few spackage combinations I tested.

Resolving patch:
diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py
--- a/lib/spack/spack/environment.py
+++ b/lib/spack/spack/environment.py
@@ -3,6 +3,7 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import collections
 import os
 import re
 import sys
@@ -19,6 +20,7 @@
 import llnl.util.tty as tty
 from llnl.util.tty.color import colorize
 
+import spack.concretize
 import spack.error
 import spack.hash_types as ht
 import spack.repo
@@ -514,6 +516,9 @@ def __init__(self, path, init_file=None, with_view=None):
                 path to the view.
         """
         self.path = os.path.abspath(path)
+        # This attribute will be set properly from configuration
+        # during concretization
+        self.concretization = None
         self.clear()
 
         if init_file:
@@ -591,6 +596,9 @@ def _read_manifest(self, f):
                               for name, values in enable_view.items())
         else:
             self.views = {}
+        # Retrieve the current concretization strategy
+        configuration = config_dict(self.yaml)
+        self.concretization = configuration.get('concretization')
 
     @property
     def user_specs(self):
@@ -845,6 +853,59 @@ def concretize(self, force=False):
             self.concretized_order = []
             self.specs_by_hash = {}
 
+        # Pick the right concretization strategy
+        if self.concretization == 'together':
+            return self._concretize_together()
+        if self.concretization == 'separately':
+            return self._concretize_separately()
+
+        msg = 'concretization strategy not implemented [{0}]'
+        raise SpackEnvironmentError(msg.format(self.concretization))
+
+    def _concretize_together(self):
+        """Concretization strategy that concretizes all the specs
+        in the same DAG.
+        """
+        # Exit early if the set of concretized specs is the set of user specs
+        user_specs_did_not_change = not bool(
+            set(self.user_specs) - set(self.concretized_user_specs)
+        )
+        if user_specs_did_not_change:
+            return []
+
+        # Check that user specs don't have duplicate packages
+        counter = collections.defaultdict(int)
+        for user_spec in self.user_specs:
+            counter[user_spec.name] += 1
+
+        duplicates = []
+        for name, count in counter.items():
+            if count > 1:
+                duplicates.append(name)
+
+        if duplicates:
+            msg = ('environment that are configured to concretize specs'
+                   ' together cannot contain more than one spec for each'
+                   ' package [{0}]'.format(', '.join(duplicates)))
+            raise SpackEnvironmentError(msg)
+
+        # Proceed with concretization
+        self.concretized_user_specs = []
+        self.concretized_order = []
+        self.specs_by_hash = {}
+
+        concrete_specs = spack.concretize.concretize_specs_together(
+            *self.user_specs
+        )
+        concretized_specs = [x for x in zip(self.user_specs, concrete_specs)]
+        for abstract, concrete in concretized_specs:
+            self._add_concrete_spec(abstract, concrete)
+        return concretized_specs
+
+    def _concretize_separately(self):
+        """Concretization strategy that concretizes separately one
+        user spec after the other.
+        """
         # keep any concretized specs whose user specs are still in the manifest
         old_concretized_user_specs = self.concretized_user_specs
         old_concretized_order = self.concretized_order
@@ -875,6 +936,13 @@ def install(self, user_spec, concrete_spec=None, **install_args):
         This will automatically concretize the single spec, but it won't
         affect other as-yet unconcretized specs.
         """
+        if self.concretization == 'together':
+            msg = 'cannot install a single spec in an environment that is ' \
+                  'configured to be concretized together. Run instead:\n\n' \
+                  '    $ spack add <spec>\n' \
+                  '    $ spack install\n'
+            raise SpackEnvironmentError(msg)
+
         spec = Spec(user_spec)
 
         if self.add(spec):
@@ -1291,7 +1359,7 @@ def write(self):
     def __enter__(self):
         self._previous_active = _active_environment
         activate(self)
-        return
+        return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         deactivate()
diff --git a/lib/spack/spack/schema/env.py b/lib/spack/spack/schema/env.py
--- a/lib/spack/spack/schema/env.py
+++ b/lib/spack/spack/schema/env.py
@@ -119,6 +119,11 @@
                                 }
                             }
                         ]
+                    },
+                    'concretization': {
+                        'type': 'string',
+                        'enum': ['together', 'separately'],
+                        'default': 'separately'
                     }
                 }
             )


Test patch:
diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py
--- a/lib/spack/spack/test/cmd/env.py
+++ b/lib/spack/spack/test/cmd/env.py
@@ -1694,3 +1694,51 @@ def test_env_activate_csh_prints_shell_output(
     assert "setenv SPACK_ENV" in out
     assert "set prompt=" in out
     assert "alias despacktivate" in out
+
+
+def test_concretize_user_specs_together():
+    e = ev.create('coconcretization')
+    e.concretization = 'together'
+
+    # Concretize a first time using 'mpich' as the MPI provider
+    e.add('mpileaks')
+    e.add('mpich')
+    e.concretize()
+
+    assert all('mpich' in spec for _, spec in e.concretized_specs())
+    assert all('mpich2' not in spec for _, spec in e.concretized_specs())
+
+    # Concretize a second time using 'mpich2' as the MPI provider
+    e.remove('mpich')
+    e.add('mpich2')
+    e.concretize()
+
+    assert all('mpich2' in spec for _, spec in e.concretized_specs())
+    assert all('mpich' not in spec for _, spec in e.concretized_specs())
+
+    # Concretize again without changing anything, check everything
+    # stays the same
+    e.concretize()
+
+    assert all('mpich2' in spec for _, spec in e.concretized_specs())
+    assert all('mpich' not in spec for _, spec in e.concretized_specs())
+
+
+def test_cant_install_single_spec_when_concretizing_together():
+    e = ev.create('coconcretization')
+    e.concretization = 'together'
+
+    with pytest.raises(ev.SpackEnvironmentError, match=r'cannot install'):
+        e.install('zlib')
+
+
+def test_duplicate_packages_raise_when_concretizing_together():
+    e = ev.create('coconcretization')
+    e.concretization = 'together'
+
+    e.add('mpileaks+opt')
+    e.add('mpileaks~opt')
+    e.add('mpich')
+
+    with pytest.raises(ev.SpackEnvironmentError, match=r'cannot contain more'):
+        e.concretize()