Repository: spack/spack

Issue:
External packages appear installed even if they are not
If Spack is configured with a `packages.yaml` that includes external packages, these packages may appear installed even if they are not. The cause of that is how we compute the `installed` property:

https://github.com/spack/spack/blob/1307ad3979498916011df47121bba1020da9f2e8/lib/spack/spack/package.py#L957-L959

which returns `True` if the prefix is there (without any check of the DB).

### Expected Result
Starting with this `packages.yaml`:
```yaml
packages:
  openssl:
    buildable: False
    paths:
      openssl@system: /usr/lib
```
and Spack at 231664e22cfcc6f72fcd9d67b718d6723fdb29b2:
```
$ spack find
==> 0 installed packages.
```
I would expect the following:
```console
$ spack-python
Spack version 0.11.2
Python 2.7.15rc1, Linux x86_64
>>> import spack.spec
>>> s = spack.spec.Spec('openssl')
>>> s.concretize()
>>> s.package.installed
False
```

### Actual Result

What we obtain instead is that `openssl` appears to be installed:
```console
$ spack-python
Spack version 0.11.2
Python 2.7.15rc1, Linux x86_64
>>> import spack.spec
>>> s = spack.spec.Spec('openssl')
>>> s.concretize()
>>> s.package.installed
True
```
### Information on your system

The only relevant part should be the `packages.yaml` above



Discussion:
I'll try to see if doing both:
- A check on the prefix
- A check on `spack.store.db`

does not compromise performance much. If so I'll submit a PR which does both checks unconditionally. Otherwise I'll try to check the DB only for external specs.

Resolving patch:
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -995,7 +995,23 @@ def provides(self, vpkg_name):
 
     @property
     def installed(self):
-        return os.path.isdir(self.prefix)
+        """Installation status of a package.
+
+        Returns:
+            True if the package has been installed, False otherwise.
+        """
+        has_prefix = os.path.isdir(self.prefix)
+        try:
+            # If the spec is in the DB, check the installed
+            # attribute of the record
+            rec = spack.store.db.get_record(self.spec)
+            db_says_installed = rec.installed
+        except KeyError:
+            # If the spec is not in the DB, the method
+            #  above raises a Key error
+            db_says_installed = False
+
+        return has_prefix and db_says_installed
 
     @property
     def prefix(self):
@@ -1650,11 +1666,18 @@ def unit_test_check(self):
     def check_for_unfinished_installation(
             self, keep_prefix=False, restage=False):
         """Check for leftover files from partially-completed prior install to
-           prepare for a new install attempt. Options control whether these
-           files are reused (vs. destroyed). This function considers a package
-           fully-installed if there is a DB entry for it (in that way, it is
-           more strict than Package.installed). The return value is used to
-           indicate when the prefix exists but the install is not complete.
+        prepare for a new install attempt.
+
+        Options control whether these files are reused (vs. destroyed).
+
+        Args:
+            keep_prefix (bool): True if the installation prefix needs to be
+                kept, False otherwise
+            restage (bool): False if the stage has to be kept, True otherwise
+
+        Returns:
+            True if the prefix exists but the install is not complete, False
+            otherwise.
         """
         if self.spec.external:
             raise ExternalPackageError("Attempted to repair external spec %s" %
diff --git a/var/spack/repos/builtin.mock/packages/externaltool/package.py b/var/spack/repos/builtin.mock/packages/externaltool/package.py
--- a/var/spack/repos/builtin.mock/packages/externaltool/package.py
+++ b/var/spack/repos/builtin.mock/packages/externaltool/package.py
@@ -30,6 +30,7 @@ class Externaltool(Package):
     url      = "http://somewhere.com/tool-1.0.tar.gz"
 
     version('1.0', '1234567890abcdef1234567890abcdef')
+    version('0.9', '1234567890abcdef1234567890abcdef')
 
     depends_on('externalprereq')
 


Test patch:
diff --git a/lib/spack/spack/test/data/packages.yaml b/lib/spack/spack/test/data/packages.yaml
--- a/lib/spack/spack/test/data/packages.yaml
+++ b/lib/spack/spack/test/data/packages.yaml
@@ -3,6 +3,7 @@ packages:
     buildable: False
     paths:
       externaltool@1.0%gcc@4.5.0: /path/to/external_tool
+      externaltool@0.9%gcc@4.5.0: /usr
   externalvirtual:
     buildable: False
     paths:
diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py
--- a/lib/spack/spack/test/database.py
+++ b/lib/spack/spack/test/database.py
@@ -27,6 +27,7 @@
 both in memory and in its file
 """
 import datetime
+import functools
 import multiprocessing
 import os
 import pytest
@@ -42,6 +43,23 @@
 pytestmark = pytest.mark.db
 
 
+@pytest.fixture()
+def usr_folder_exists(monkeypatch):
+    """The ``/usr`` folder is assumed to be existing in some tests. This
+    fixture makes it such that its existence is mocked, so we have no
+    requirements on the system running tests.
+    """
+    isdir = os.path.isdir
+
+    @functools.wraps(os.path.isdir)
+    def mock_isdir(path):
+        if path == '/usr':
+            return True
+        return isdir(path)
+
+    monkeypatch.setattr(os.path, 'isdir', mock_isdir)
+
+
 def _print_ref_counts():
     """Print out all ref counts for the graph used here, for debugging"""
     recs = []
@@ -436,3 +454,18 @@ def test_external_entries_in_db(database):
     assert rec.spec.external_path == '/path/to/external_tool'
     assert rec.spec.external_module is None
     assert rec.explicit is True
+
+
+@pytest.mark.regression('8036')
+def test_regression_issue_8036(mutable_database, usr_folder_exists):
+    # The test ensures that the external package prefix is treated as
+    # existing. Even when the package prefix exists, the package should
+    # not be considered installed until it is added to the database with
+    # do_install.
+    s = spack.spec.Spec('externaltool@0.9')
+    s.concretize()
+    assert not s.package.installed
+
+    # Now install the external package and check again the `installed` property
+    s.package.do_install(fake=True)
+    assert s.package.installed