Repository: spack/spack

Issue:
"Error: KeyError: 'LIBDIR'" when an external python is used
### Steps to reproduce

1. Create a packages.yaml that defines an external python (that doesn't have dev libraries installed?)
2. `spack install exodusii`

### Error message

```==> Installing exodusii-2021-04-05-7z43vabdqjnmbdrgwpkv6qjrwgai2hwt
==> No binary for exodusii-2021-04-05-7z43vabdqjnmbdrgwpkv6qjrwgai2hwt found: installing from source
==> Error: KeyError: 'LIBDIR'

/home/user/spack/var/spack/repos/builtin/packages/python/package.py:795, in libs:
        792        # installs them into lib64. If the user is using an externally
        793        # installed package, it may be in either lib or lib64, so we need
        794        # to ask Python where its LIBDIR is.
        795        libdir = self.config_vars['LIBDIR']
        796
        797        # In Ubuntu 16.04.6 and python 2.7.12 from the system, lib could be
        798        # in LBPL```

### Information on your system

```
* **Spack:** 0.17.1
* **Python:** 3.9.5
* **Platform:** linux-centos8-haswell
* **Concretizer:** clingo
```

### General information

- [X] I have run `spack debug report` and reported the version of Spack/Python/Platform
- [X] I have searched the issues of this repo and believe this is not a duplicate
- [X] I have run the failing commands in debug mode and reported the output


Discussion:
@adamjstewart requested to be assigned to this
On the line before the `LIBDIR` line in the Python package, can you add `print(self.config_vars)` and run again with `spack --debug install exodusii`? I want to see if `config_vars` is completely empty or just missing the `LIBDIR` key. Also, what directory is Python installed in? Can you run `find <prefix> -name 'libpython*.so'` to see if there are any library files there?

Resolving patch:
diff --git a/lib/spack/spack/bootstrap.py b/lib/spack/spack/bootstrap.py
--- a/lib/spack/spack/bootstrap.py
+++ b/lib/spack/spack/bootstrap.py
@@ -77,17 +77,13 @@ def _try_import_from_store(module, query_spec, query_info=None):
     installed_specs = spack.store.db.query(query_spec, installed=True)
 
     for candidate_spec in installed_specs:
-        python_spec = candidate_spec['python']
-        lib_spd = python_spec.package.default_site_packages_dir
-        lib64_spd = lib_spd.replace('lib/', 'lib64/')
-        lib_debian_derivative = os.path.join(
-            'lib', 'python{0}'.format(python_spec.version.up_to(1)), 'dist-packages'
-        )
+        pkg = candidate_spec['python'].package
+        purelib = pkg.config_vars['python_lib']['false']['false']
+        platlib = pkg.config_vars['python_lib']['true']['false']
 
         module_paths = [
-            os.path.join(candidate_spec.prefix, lib_debian_derivative),
-            os.path.join(candidate_spec.prefix, lib_spd),
-            os.path.join(candidate_spec.prefix, lib64_spd)
+            os.path.join(candidate_spec.prefix, purelib),
+            os.path.join(candidate_spec.prefix, platlib),
         ]
         sys.path.extend(module_paths)
 
diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py
--- a/var/spack/repos/builtin/packages/python/package.py
+++ b/var/spack/repos/builtin/packages/python/package.py
@@ -7,12 +7,15 @@
 import os
 import platform
 import re
+import sys
 
 import llnl.util.tty as tty
 from llnl.util.filesystem import get_filetype, path_contains_subdirectory
 from llnl.util.lang import match_predicate
 
 from spack import *
+from spack.build_environment import dso_suffix
+from spack.compilers import NoCompilerForSpecError
 from spack.util.environment import is_system_path
 from spack.util.prefix import Prefix
 
@@ -752,11 +755,61 @@ def config_vars(self):
 """ % self.print_string("json.dumps(config)")
 
         dag_hash = self.spec.dag_hash()
+
         if dag_hash not in self._config_vars:
+            # Default config vars
+            version = self.version.up_to(2)
+            try:
+                cc = self.compiler.cc
+                cxx = self.compiler.cxx
+            except (TypeError, NoCompilerForSpecError):
+                cc = 'cc'
+                cxx = 'c++'
+
+            config = {
+                'CC': cc,
+                'CXX': cxx,
+                'INCLUDEPY': self.prefix.include.join('python{}').format(version),
+                'LIBDEST': self.prefix.lib.join('python{}').format(version),
+                'LIBDIR': self.prefix.lib,
+                'LIBPL': self.prefix.lib.join('python{0}').join(
+                    'config-{0}-{1}').format(version, sys.platform),
+                'LDLIBRARY': 'libpython{}.{}'.format(version, dso_suffix),
+                'LIBRARY': 'libpython{}.a'.format(version),
+                'LDSHARED': cc,
+                'LDCXXSHARED': cxx,
+                'PYTHONFRAMEWORKPREFIX': '/System/Library/Frameworks',
+                'prefix': self.prefix,
+                'config_h_filename': self.prefix.include.join('python{}').join(
+                    'pyconfig.h').format(version),
+                'makefile_filename': self.prefix.lib.join('python{0}').join(
+                    'config-{0}-{1}').Makefile.format(version, sys.platform),
+                'python_inc': {
+                    # plat_specific
+                    'true': os.path.join('include64', 'python{}'.format(version)),
+                    'false': os.path.join('include', 'python{}'.format(version)),
+                },
+                'python_lib': {
+                    # plat_specific
+                    'true': {
+                        # standard_lib
+                        'true': os.path.join('lib64', 'python{}'.format(version)),
+                        'false': os.path.join(
+                            'lib64', 'python{}'.format(version), 'site-packages'),
+                    },
+                    'false': {
+                        # standard_lib
+                        'true': os.path.join('lib', 'python{}'.format(version)),
+                        'false': os.path.join(
+                            'lib', 'python{}'.format(version), 'site-packages'),
+                    },
+                },
+            }
+
             try:
-                config = json.loads(self.command('-c', cmd, output=str))
+                config.update(json.loads(self.command('-c', cmd, output=str)))
             except (ProcessError, RuntimeError):
-                config = {}
+                pass
             self._config_vars[dag_hash] = config
         return self._config_vars[dag_hash]
 
@@ -784,14 +837,9 @@ def home(self):
         and symlinks it to ``/usr/local``. Users may not know the actual
         installation directory and add ``/usr/local`` to their
         ``packages.yaml`` unknowingly. Query the python executable to
-        determine exactly where it is installed. Fall back on
-        ``spec['python'].prefix`` if that doesn't work."""
-
-        if 'prefix' in self.config_vars:
-            prefix = self.config_vars['prefix']
-        else:
-            prefix = self.prefix
-        return Prefix(prefix)
+        determine exactly where it is installed.
+        """
+        return Prefix(self.config_vars['prefix'])
 
     @property
     def libs(self):
@@ -846,14 +894,9 @@ def libs(self):
 
     @property
     def headers(self):
-        if 'config_h_filename' in self.config_vars:
-            config_h = self.config_vars['config_h_filename']
-
-            if not os.path.exists(config_h):
-                includepy = self.config_vars['INCLUDEPY']
-                msg = 'Unable to locate {0} headers in {1}'
-                raise RuntimeError(msg.format(self.name, includepy))
+        config_h = self.config_vars['config_h_filename']
 
+        if os.path.exists(config_h):
             headers = HeaderList(config_h)
         else:
             headers = find_headers(
@@ -876,10 +919,7 @@ def python_include_dir(self):
         Returns:
             str: include files directory
         """
-        try:
-            return self.config_vars['python_inc']['false']
-        except KeyError:
-            return os.path.join('include', 'python{0}'.format(self.version.up_to(2)))
+        return self.config_vars['python_inc']['false']
 
     @property
     def python_lib_dir(self):
@@ -900,10 +940,7 @@ def python_lib_dir(self):
         Returns:
             str: standard library directory
         """
-        try:
-            return self.config_vars['python_lib']['false']['true']
-        except KeyError:
-            return os.path.join('lib', 'python{0}'.format(self.version.up_to(2)))
+        return self.config_vars['python_lib']['false']['true']
 
     @property
     def site_packages_dir(self):
@@ -924,15 +961,7 @@ def site_packages_dir(self):
         Returns:
             str: site-packages directory
         """
-        try:
-            return self.config_vars['python_lib']['true']['false']
-        except KeyError:
-            return self.default_site_packages_dir
-
-    @property
-    def default_site_packages_dir(self):
-        python_dir = 'python{0}'.format(self.version.up_to(2))
-        return os.path.join('lib', python_dir, 'site-packages')
+        return self.config_vars['python_lib']['false']['false']
 
     @property
     def easy_install_file(self):


Test patch: