Repository: spack/spack

Issue:
Bootstrap disregards command line scopes
### Steps to reproduce

When running `spack -C my_config_scope spec zlib` Spack disregards `my_config_scope` during bootstrapping.

I haven't checked what the behavior of `spack -c x:y:z spec zlib` is during bootstrapping.

I have to be able to specify the staging dir on the command line in some way, also the one used during bootstrapping.



Discussion:

Resolving patch:
diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py
--- a/lib/spack/spack/bootstrap.py
+++ b/lib/spack/spack/bootstrap.py
@@ -26,6 +26,7 @@
 import spack.architecture
 import spack.binary_distribution
 import spack.config
+import spack.detection
 import spack.environment
 import spack.main
 import spack.modules
@@ -209,7 +210,7 @@ def try_import(self, module, abstract_spec_str):
         buildcache = spack.main.SpackCommand('buildcache')
         # Ensure we see only the buildcache being used to bootstrap
         mirror_scope = spack.config.InternalConfigScope(
-            'bootstrap', {'mirrors:': {self.name: self.url}}
+            'bootstrap_buildcache', {'mirrors:': {self.name: self.url}}
         )
         with spack.config.override(mirror_scope):
             # This index is currently needed to get the compiler used to build some
@@ -218,7 +219,7 @@ def try_import(self, module, abstract_spec_str):
             index = spack.binary_distribution.update_cache_and_get_specs()
 
             if not index:
-                raise RuntimeError("Could not populate the binary index")
+                raise RuntimeError("could not populate the binary index")
 
             for item in data['verified']:
                 candidate_spec = item['spec']
@@ -279,6 +280,10 @@ def try_import(module, abstract_spec_str):
 
         tty.info("Bootstrapping {0} from sources".format(module))
 
+        # If we compile code from sources detecting a few build tools
+        # might reduce compilation time by a fair amount
+        _add_externals_if_missing()
+
         # Try to build and install from sources
         with spack_python_interpreter():
             # Add hint to use frontend operating system on Cray
@@ -492,7 +497,11 @@ def _bootstrap_config_scopes():
     config_scopes = [
         spack.config.InternalConfigScope('_builtin', spack.config.config_defaults)
     ]
-    for name, path in spack.config.configuration_paths:
+    configuration_paths = (
+        spack.config.configuration_defaults_path,
+        ('bootstrap', _config_path())
+    )
+    for name, path in configuration_paths:
         platform = spack.architecture.platform().name
         platform_scope = spack.config.ConfigScope(
             '/'.join([name, platform]), os.path.join(path, platform)
@@ -517,9 +526,19 @@ def _add_compilers_if_missing():
             spack.compilers.add_compilers_to_config(new_compilers, init_config=False)
 
 
+def _add_externals_if_missing():
+    search_list = [
+        spack.repo.path.get('cmake'),
+        spack.repo.path.get('bison')
+    ]
+    detected_packages = spack.detection.by_executable(search_list)
+    spack.detection.update_configuration(detected_packages, scope='bootstrap')
+
+
 @contextlib.contextmanager
 def ensure_bootstrap_configuration():
     bootstrap_store_path = store_path()
+    user_configuration = _read_and_sanitize_configuration()
     with spack.environment.deactivate_environment():
         with spack.architecture.use_platform(spack.architecture.real_platform()):
             with spack.repo.use_repositories(spack.paths.packages_path):
@@ -531,11 +550,29 @@ def ensure_bootstrap_configuration():
                         # We may need to compile code from sources, so ensure we have
                         # compilers for the current platform before switching parts.
                         _add_compilers_if_missing()
+                        spack.config.set('bootstrap', user_configuration['bootstrap'])
+                        spack.config.set('config', user_configuration['config'])
                         with spack.modules.disable_modules():
                             with spack_python_interpreter():
                                 yield
 
 
+def _read_and_sanitize_configuration():
+    """Read the user configuration that needs to be reused for bootstrapping
+    and remove the entries that should not be copied over.
+    """
+    # Read the "config" section but pop the install tree (the entry will not be
+    # considered due to the use_store context manager, so it will be confusing
+    # to have it in the configuration).
+    config_yaml = spack.config.get('config')
+    config_yaml.pop('install_tree', None)
+    user_configuration = {
+        'bootstrap': spack.config.get('bootstrap'),
+        'config': config_yaml
+    }
+    return user_configuration
+
+
 def store_path():
     """Path to the store used for bootstrapped software"""
     enabled = spack.config.get('bootstrap:enable', True)
@@ -544,13 +581,28 @@ def store_path():
                'Use "spack bootstrap enable" to enable it')
         raise RuntimeError(msg)
 
-    bootstrap_root_path = spack.config.get(
+    return _store_path()
+
+
+def _root_path():
+    """Root of all the bootstrap related folders"""
+    return spack.config.get(
         'bootstrap:root', spack.paths.user_bootstrap_path
     )
-    bootstrap_store_path = spack.util.path.canonicalize_path(
+
+
+def _store_path():
+    bootstrap_root_path = _root_path()
+    return spack.util.path.canonicalize_path(
         os.path.join(bootstrap_root_path, 'store')
     )
-    return bootstrap_store_path
+
+
+def _config_path():
+    bootstrap_root_path = _root_path()
+    return spack.util.path.canonicalize_path(
+        os.path.join(bootstrap_root_path, 'config')
+    )
 
 
 def clingo_root_spec():
diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py
--- a/lib/spack/spack/cmd/clean.py
+++ b/lib/spack/spack/cmd/clean.py
@@ -7,6 +7,7 @@
 import os
 import shutil
 
+import llnl.util.filesystem
 import llnl.util.tty as tty
 
 import spack.bootstrap
@@ -14,9 +15,9 @@
 import spack.cmd.common.arguments as arguments
 import spack.cmd.test
 import spack.config
-import spack.main
 import spack.repo
 import spack.stage
+import spack.util.path
 from spack.paths import lib_path, var_path
 
 description = "remove temporary build files and/or downloaded archives"
@@ -27,7 +28,7 @@
 class AllClean(argparse.Action):
     """Activates flags -s -d -f -m and -p simultaneously"""
     def __call__(self, parser, namespace, values, option_string=None):
-        parser.parse_args(['-sdfmpb'], namespace=namespace)
+        parser.parse_args(['-sdfmp'], namespace=namespace)
 
 
 def setup_parser(subparser):
@@ -48,9 +49,11 @@ def setup_parser(subparser):
         help="remove .pyc, .pyo files and __pycache__ folders")
     subparser.add_argument(
         '-b', '--bootstrap', action='store_true',
-        help="remove software needed to bootstrap Spack")
+        help="remove software and configuration needed to bootstrap Spack")
     subparser.add_argument(
-        '-a', '--all', action=AllClean, help="equivalent to -sdfmpb", nargs=0
+        '-a', '--all', action=AllClean,
+        help="equivalent to -sdfmp (does not include --bootstrap)",
+        nargs=0
     )
     arguments.add_common_arguments(subparser, ['specs'])
 
@@ -102,8 +105,9 @@ def clean(parser, args):
                         shutil.rmtree(dname)
 
     if args.bootstrap:
-        msg = 'Removing software in "{0}"'
-        tty.msg(msg.format(spack.bootstrap.store_path()))
-        with spack.bootstrap.ensure_bootstrap_configuration():
-            uninstall = spack.main.SpackCommand('uninstall')
-            uninstall('-a', '-y')
+        bootstrap_prefix = spack.util.path.canonicalize_path(
+            spack.config.get('bootstrap:root')
+        )
+        msg = 'Removing bootstrapped software and configuration in "{0}"'
+        tty.msg(msg.format(bootstrap_prefix))
+        llnl.util.filesystem.remove_directory_contents(bootstrap_prefix)
diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py
--- a/lib/spack/spack/config.py
+++ b/lib/spack/spack/config.py
@@ -84,11 +84,16 @@
 all_schemas.update(dict((key, spack.schema.env.schema)
                         for key in spack.schema.env.keys))
 
+#: Path to the default configuration
+configuration_defaults_path = (
+    'defaults', os.path.join(spack.paths.etc_path, 'spack', 'defaults')
+)
+
 #: Builtin paths to configuration files in Spack
 configuration_paths = (
     # Default configuration scope is the lowest-level scope. These are
     # versioned with Spack and can be overridden by systems, sites or users
-    ('defaults', os.path.join(spack.paths.etc_path, 'spack', 'defaults')),
+    configuration_defaults_path,
 
     # System configuration is per machine.
     # No system-level configs should be checked into spack by default


Test patch:
diff --git a/lib/spack/spack/test/bootstrap.py b/lib/spack/spack/test/bootstrap.py
--- a/lib/spack/spack/test/bootstrap.py
+++ b/lib/spack/spack/test/bootstrap.py
@@ -99,3 +99,22 @@ def test_bootstrap_search_for_compilers_with_environment_active(
     with spack.bootstrap.ensure_bootstrap_configuration():
         assert spack.compilers.all_compiler_specs(init_config=False)
     assert not spack.compilers.all_compiler_specs(init_config=False)
+
+
+@pytest.mark.regression('26189')
+def test_config_yaml_is_preserved_during_bootstrap(mutable_config):
+    # Mock the command line scope
+    expected_dir = '/tmp/test'
+    internal_scope = spack.config.InternalConfigScope(
+        name='command_line', data={
+            'config': {
+                'test_stage': expected_dir
+            }
+        }
+    )
+    spack.config.config.push_scope(internal_scope)
+
+    assert spack.config.get('config:test_stage') == expected_dir
+    with spack.bootstrap.ensure_bootstrap_configuration():
+        assert spack.config.get('config:test_stage') == expected_dir
+    assert spack.config.get('config:test_stage') == expected_dir