Repository: spack/spack

Issue:
"spack reindex" fails with "invalid ref count"
A previous install of `git` failed, leaving a half-installed `git` ghost. This is usually remedied via `spack reindex`, but this fails here.

My original command is:

```
$ spack install -v julia
==> Installing julia
==> git is already installed in /home/eschnett/src/spack/opt/spack/linux-centos7-x86_64/gcc-6.1.0-spack/git-2.9.3-kgah4b5gurqx2rswlt44xoinshfw7doi
Traceback (most recent call last):
  File "/home/eschnett/src/spack/bin/spack", line 192, in <module>
    main()
  File "/home/eschnett/src/spack/bin/spack", line 169, in main
    return_val = command(parser, args)
  File "/home/eschnett/src/spack/lib/spack/spack/cmd/install.py", line 92, in install
    explicit=True)
  File "/home/eschnett/src/spack/lib/spack/spack/package.py", line 919, in do_install
    run_tests=run_tests)
  File "/home/eschnett/src/spack/lib/spack/spack/package.py", line 1069, in do_install_dependencies
    dep.package.do_install(**kwargs)
  File "/home/eschnett/src/spack/lib/spack/spack/package.py", line 901, in do_install
    rec = spack.installed_db.get_record(self.spec)
  File "/home/eschnett/src/spack/lib/spack/spack/database.py", line 75, in converter
    return function(self, spec_like, *args, **kwargs)
  File "/home/eschnett/src/spack/lib/spack/spack/database.py", line 521, in get_record
    key = self._get_matching_spec_key(spec, **kwargs)
  File "/home/eschnett/src/spack/lib/spack/spack/database.py", line 516, in _get_matching_spec_key
    raise KeyError("No such spec in database! %s" % spec)
KeyError: 'No such spec in database! git@2.9.3%gcc@6.1.0-spack arch=linux-centos7-x86_64^autoconf@2.69%gcc@6.1.0-spack arch=linux-centos7-x86_64^bzip2@1.0.6%gcc@6.1.0-spack arch=linux-centos7-x86_64^curl@7.50.1%gcc@6.1.0-spack arch=linux-centos7-x86_64^expat@2.2.0%gcc@6.1.0-spack arch=linux-centos7-x86_64^gettext@0.19.8.1%gcc@6.1.0-spack+bzip2+curses+git~libunistring+libxml2+tar+xz arch=linux-centos7-x86_64^libsigsegv@2.10%gcc@6.1.0-spack arch=linux-centos7-x86_64^libxml2@2.9.4%gcc@6.1.0-spack~python arch=linux-centos7-x86_64^m4@1.4.17%gcc@6.1.0-spack+sigsegv arch=linux-centos7-x86_64^ncurses@6.0%gcc@6.1.0-spack arch=linux-centos7-x86_64^openssl@1.0.2h%gcc@6.1.0-spack arch=linux-centos7-x86_64^pcre@8.39%gcc@6.1.0-spack+utf arch=linux-centos7-x86_64^perl@5.24.0%gcc@6.1.0-spack+cpanm arch=linux-centos7-x86_64^tar@1.29%gcc@6.1.0-spack arch=linux-centos7-x86_64^xz@5.2.2%gcc@6.1.0-spack arch=linux-centos7-x86_64^zlib@1.2.8%gcc@6.1.0-spack arch=linux-centos7-x86_64'
```

To remedy, I try to reindex:

```
$ spack reindex
Traceback (most recent call last):
  File "/home/eschnett/src/spack/bin/spack", line 192, in <module>
    main()
  File "/home/eschnett/src/spack/bin/spack", line 169, in main
    return_val = command(parser, args)
  File "/home/eschnett/src/spack/lib/spack/spack/cmd/reindex.py", line 31, in reindex
    spack.installed_db.reindex(spack.install_layout)
  File "/home/eschnett/src/spack/lib/spack/spack/database.py", line 375, in reindex
    self._check_ref_counts()
  File "/home/eschnett/src/spack/lib/spack/spack/database.py", line 407, in _check_ref_counts
    (key, found, expected, self._index_path))
AssertionError: Invalid ref_count: 3f2b2nmz2brdouohnoxiik3zch7d4v7p: 31 (expected 30), in DB /home/eschnett/src/spack/opt/spack/.spack-db/index.yaml
```



Discussion:
@eschnett: I can't reproduce this one.  Can you pull the latest `develop` and attach the results of `spack debug create-db-tarball` when this happens again?

This error happens reliably. I attached the tarball. The two previous (failing) commands were

```
$ spack reindex                                                                 Traceback (most recent call last):
  File "/home/eschnett/src/spack/bin/spack", line 192, in <module>
    main()
  File "/home/eschnett/src/spack/bin/spack", line 169, in main
    return_val = command(parser, args)
  File "/home/eschnett/src/spack/lib/spack/spack/cmd/reindex.py", line 31, in reindex
    spack.installed_db.reindex(spack.install_layout)
  File "/home/eschnett/src/spack/lib/spack/spack/database.py", line 375, in reindex
    self._check_ref_counts()
  File "/home/eschnett/src/spack/lib/spack/spack/database.py", line 407, in _check_ref_counts
    (key, found, expected, self._index_path))
AssertionError: Invalid ref_count: 3f2b2nmz2brdouohnoxiik3zch7d4v7p: 31 (expected 30), in DB /home/eschnett/src/spack/opt/spack/.spack-db/index.yaml
eschnett@bethe:~/src/spack (08:37:11)
$ spack install -v julia                                                        ==> Installing julia
==> git is already installed in /home/eschnett/src/spack/opt/spack/linux-centos7-x86_64/gcc-6.1.0-spack/git-2.9.3-kgah4b5gurqx2rswlt44xoinshfw7doi
Traceback (most recent call last):
  File "/home/eschnett/src/spack/bin/spack", line 192, in <module>
    main()
  File "/home/eschnett/src/spack/bin/spack", line 169, in main
    return_val = command(parser, args)
  File "/home/eschnett/src/spack/lib/spack/spack/cmd/install.py", line 92, in install
    explicit=True)
  File "/home/eschnett/src/spack/lib/spack/spack/package.py", line 919, in do_install
    run_tests=run_tests)
  File "/home/eschnett/src/spack/lib/spack/spack/package.py", line 1069, in do_install_dependencies
    dep.package.do_install(**kwargs)
  File "/home/eschnett/src/spack/lib/spack/spack/package.py", line 901, in do_install
    rec = spack.installed_db.get_record(self.spec)
  File "/home/eschnett/src/spack/lib/spack/spack/database.py", line 75, in converter
    return function(self, spec_like, *args, **kwargs)
  File "/home/eschnett/src/spack/lib/spack/spack/database.py", line 521, in get_record
    key = self._get_matching_spec_key(spec, **kwargs)
  File "/home/eschnett/src/spack/lib/spack/spack/database.py", line 516, in _get_matching_spec_key
    raise KeyError("No such spec in database! %s" % spec)
KeyError: 'No such spec in database! git@2.9.3%gcc@6.1.0-spack arch=linux-centos7-x86_64^autoconf@2.69%gcc@6.1.0-spack arch=linux-centos7-x86_64^bzip2@1.0.6%gcc@6.1.0-spack arch=linux-centos7-x86_64^curl@7.50.1%gcc@6.1.0-spack arch=linux-centos7-x86_64^expat@2.2.0%gcc@6.1.0-spack arch=linux-centos7-x86_64^gettext@0.19.8.1%gcc@6.1.0-spack+bzip2+curses+git~libunistring+libxml2+tar+xz arch=linux-centos7-x86_64^libsigsegv@2.10%gcc@6.1.0-spack arch=linux-centos7-x86_64^libxml2@2.9.4%gcc@6.1.0-spack~python arch=linux-centos7-x86_64^m4@1.4.17%gcc@6.1.0-spack+sigsegv arch=linux-centos7-x86_64^ncurses@6.0%gcc@6.1.0-spack arch=linux-centos7-x86_64^openssl@1.0.2h%gcc@6.1.0-spack arch=linux-centos7-x86_64^pcre@8.39%gcc@6.1.0-spack+utf arch=linux-centos7-x86_64^perl@5.24.0%gcc@6.1.0-spack+cpanm arch=linux-centos7-x86_64^tar@1.29%gcc@6.1.0-spack arch=linux-centos7-x86_64^xz@5.2.2%gcc@6.1.0-spack arch=linux-centos7-x86_64^zlib@1.2.8%gcc@6.1.0-spack arch=linux-centos7-x86_64'
```

[spack-db.develop.f34dd94.2016-09-06-084822.tar.gz](https://github.com/LLNL/spack/files/457125/spack-db.develop.f34dd94.2016-09-06-084822.tar.gz)

Do these four directories have successful git installs in them?  They appear to have `spec.yaml` files, but I'm curious whether anything else was installed.

```
opt/spack/linux-centos7-x86_64/gcc-6.1.0-spack/git-2.8.1-jkzywad7z7nagmgmaqooa3ae4fmjubns/
opt/spack/linux-centos7-x86_64/gcc-6.1.0-spack/git-2.8.1-nqkxuakqdb5odnoksi6yvolvvkk7iowd/
opt/spack/linux-centos7-x86_64/gcc-6.1.0-spack/git-2.8.1-v6rx5mebb32iuky57xzyi6degit26fcm/
opt/spack/linux-centos7-x86_64/gcc-6.1.0-spack/git-2.9.3-kgah4b5gurqx2rswlt44xoinshfw7doi/
```

If they _do_ have partial installs, are you ever using `spack install --keep-prefix`?

I'm running into this issue as well, but not after a failed install that I'm aware of.

I've never used `spack install --keep-prefix`

```
Traceback (most recent call last):
  File "/cluster/rcss-spack/bin/spack", line 192, in <module>
    main()
  File "/cluster/rcss-spack/bin/spack", line 169, in main
    return_val = command(parser, args)
  File "/cluster/rcss-spack/lib/spack/spack/cmd/reindex.py", line 31, in reindex
    spack.installed_db.reindex(spack.install_layout)
  File "/cluster/rcss-spack/lib/spack/spack/database.py", line 375, in reindex
    self._check_ref_counts()
  File "/cluster/rcss-spack/lib/spack/spack/database.py", line 407, in _check_ref_counts
    (key, found, expected, self._index_path))
AssertionError: Invalid ref_count: x3atd7mkaebyjjs52sfhhtjrfy5l2pnz: 1 (expected 0), in DB /cluster/rcss-spack/opt/spack/.spack-db/index.yaml
```

If it helps, I can attach a db tarbal as well.

@tgamblin Yes, these four `git` directories seem to contain complete installations.

No, I haven't used `--keep-prefix` with git recently, and not on this system. I don't think these are partial installs. (If that is an issue, then Spack could leave a big human-readable marker in the directory, marking it as incomplete?)

Is this fixed by #1688?

It is not fixed. The current head of the development branch still has this issue:

```
$ spack reindex
Traceback (most recent call last):
  File "/home/eschnett/src/spack/bin/spack", line 192, in <module>
    main()
  File "/home/eschnett/src/spack/bin/spack", line 169, in main
    return_val = command(parser, args)
  File "/home/eschnett/src/spack/lib/spack/spack/cmd/reindex.py", line 31, in reindex
    spack.installed_db.reindex(spack.install_layout)
  File "/home/eschnett/src/spack/lib/spack/spack/database.py", line 375, in reindex
    self._check_ref_counts()
  File "/home/eschnett/src/spack/lib/spack/spack/database.py", line 407, in _check_ref_counts
    (key, found, expected, self._index_path))
AssertionError: Invalid ref_count: 3f2b2nmz2brdouohnoxiik3zch7d4v7p: 31 (expected 30), in DB /home/eschnett/src/spack/opt/spack/.spack-db/index.yaml
```

I think we need a procedure to rebuild this database from the installed packages in `opt/spack`.  Unless we use an ACID database (we don't), there will always be ways to break it.

... isn't that what `reindex` does?

@tgamblin I'm not sure.  But clearly, `reindex` shouldn't be confused by
some erroneous refcount if it's supposed to re-build those refcounts.

On Wed, Sep 14, 2016 at 9:37 AM, Erik Schnetter notifications@github.com
wrote:

> ... isn't that what reindex does?
> 
> —
> You are receiving this because you commented.
> Reply to this email directly, view it on GitHub
> https://github.com/LLNL/spack/issues/1720#issuecomment-247014879, or mute
> the thread
> https://github.com/notifications/unsubscribe-auth/AB1cd05hou3pGzt-EKPIILvisQrqaHAWks5qp_h_gaJpZM4J0KAl
> .

I am also running into this. @tgamblin would another db tarball be helpful?

I think this should be a high-priority bug to fix: it messes up the entire repo, and we have no workaround; once a Spack installation gets into this state, we don't know how to get it out.  That prevents uninstalling packages (even after they've been uninstalled), and really breaks everything.  We need, first and foremost, a way to get Spack out of this state and back into "clean" working order.

I am getting this same issue as well.

Other folks with the invalid refcount bug: is the hash in your error message referencing something in your packages.yaml file? 

I somehow ended up with two different hashes that both referenced my SLES-provided ncurses, because I was toying around with packages.yaml and just decided to throw something basic in there. 

I was able to salvage this spack install by changing all of the `opt/spack/linux-SuSE11-x86_64/gcc-5.3.0/*/.spack/spec.yaml` files to refer to ncurses with the same hash. 

Unfortunately, I do not know how I got in that state to begin with.


Resolving patch:
diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py
--- a/lib/spack/spack/database.py
+++ b/lib/spack/spack/database.py
@@ -50,7 +50,7 @@
 
 import spack.spec
 from spack.version import Version
-from spack.spec import Spec
+from spack.spec import *
 from spack.error import SpackError
 from spack.repository import UnknownPackageError
 import spack.util.spack_yaml as syaml
@@ -64,6 +64,9 @@
 # Default timeout for spack database locks is 5 min.
 _db_lock_timeout = 60
 
+# Types of dependencies tracked by the database
+_tracked_deps = nobuild
+
 
 def _autospec(function):
     """Decorator that automatically converts the argument of a single-arg
@@ -232,8 +235,6 @@ def _assign_dependencies(self, hash_key, installs, data):
                                  spec.format('$_$#'), dname, dhash[:7]))
                     continue
 
-                # defensive copy (not sure everything handles extra
-                # parent links yet)
                 child = data[dhash].spec
                 spec._add_dependency(child, dtypes)
 
@@ -328,7 +329,7 @@ def invalid_record(hash_key, error):
         self._data = data
 
     def reindex(self, directory_layout):
-        """Build database index from scratch based from a directory layout.
+        """Build database index from scratch based on a directory layout.
 
         Locks the DB if it isn't locked already.
 
@@ -359,9 +360,6 @@ def _read_suppress_error():
 
                 # Ask the directory layout to traverse the filesystem.
                 for spec in directory_layout.all_specs():
-                    # Create a spec for each known package and add it.
-                    path = directory_layout.path_for_spec(spec)
-
                     # Try to recover explicit value from old DB, but
                     # default it to False if DB was corrupt.
                     explicit = False
@@ -370,7 +368,7 @@ def _read_suppress_error():
                         if old_info is not None:
                             explicit = old_info.explicit
 
-                    self._add(spec, path, directory_layout, explicit=explicit)
+                    self._add(spec, directory_layout, explicit=explicit)
 
                 self._check_ref_counts()
 
@@ -389,10 +387,7 @@ def _check_ref_counts(self):
         counts = {}
         for key, rec in self._data.items():
             counts.setdefault(key, 0)
-            # XXX(deptype): This checks all dependencies, but build
-            #               dependencies might be able to be dropped in the
-            #               future.
-            for dep in rec.spec.dependencies():
+            for dep in rec.spec.dependencies(_tracked_deps):
                 dep_key = dep.dag_hash()
                 counts.setdefault(dep_key, 0)
                 counts[dep_key] += 1
@@ -450,52 +445,62 @@ def _read(self):
             # reindex() takes its own write lock, so no lock here.
             self.reindex(spack.install_layout)
 
-    def _add(self, spec, path, directory_layout=None, explicit=False):
-        """Add an install record for spec at path to the database.
+    def _add(self, spec, directory_layout=None, explicit=False):
+        """Add an install record for this spec to the database.
 
-        This assumes that the spec is not already installed. It
-        updates the ref counts on dependencies of the spec in the DB.
+        Assumes spec is installed in ``layout.path_for_spec(spec)``.
 
-        This operation is in-memory, and does not lock the DB.
+        Also ensures dependencies are present and updated in the DB as
+        either intsalled or missing.
 
         """
-        key = spec.dag_hash()
-        if key in self._data:
-            rec = self._data[key]
-            rec.installed = True
+        if not spec.concrete:
+            raise NonConcreteSpecAddError(
+                "Specs added to DB must be concrete.")
 
-            # TODO: this overwrites a previous install path (when path !=
-            # self._data[key].path), and the old path still has a
-            # dependent in the DB. We could consider re-RPATH-ing the
-            # dependents.  This case is probably infrequent and may not be
-            # worth fixing, but this is where we can discover it.
-            rec.path = path
-
-        else:
-            self._data[key] = InstallRecord(spec, path, True,
-                                            explicit=explicit)
-            for dep in spec.dependencies(('link', 'run')):
-                self._increment_ref_count(dep, directory_layout)
+        for dep in spec.dependencies(_tracked_deps):
+            dkey = dep.dag_hash()
+            if dkey not in self._data:
+                self._add(dep, directory_layout, explicit=False)
 
-    def _increment_ref_count(self, spec, directory_layout=None):
-        """Recursively examine dependencies and update their DB entries."""
         key = spec.dag_hash()
         if key not in self._data:
             installed = False
             path = None
             if directory_layout:
                 path = directory_layout.path_for_spec(spec)
-                installed = os.path.isdir(path)
+                try:
+                    directory_layout.check_installed(spec)
+                    installed = True
+                except DirectoryLayoutError as e:
+                    tty.warn(
+                        'Dependency missing due to corrupt install directory:',
+                        path, str(e))
+
+            # Create a new install record with no deps initially.
+            new_spec = spec.copy(deps=False)
+            self._data[key] = InstallRecord(
+                new_spec, path, installed, ref_count=0, explicit=explicit)
+
+            # Connect dependencies from the DB to the new copy.
+            for name, dep in spec.dependencies_dict(_tracked_deps).iteritems():
+                dkey = dep.spec.dag_hash()
+                new_spec._add_dependency(self._data[dkey].spec, dep.deptypes)
+                self._data[dkey].ref_count += 1
+
+            # Mark concrete once everything is built, and preserve
+            # the original hash of concrete specs.
+            new_spec._mark_concrete()
+            new_spec._hash = key
 
-            self._data[key] = InstallRecord(spec.copy(), path, installed)
-
-            for dep in spec.dependencies(('link', 'run')):
-                self._increment_ref_count(dep)
+        else:
+            # If it is already there, mark it as installed.
+            self._data[key].installed = True
 
-        self._data[key].ref_count += 1
+        self._data[key].explicit = explicit
 
     @_autospec
-    def add(self, spec, path, explicit=False):
+    def add(self, spec, directory_layout, explicit=False):
         """Add spec at path to database, locking and reading DB to sync.
 
         ``add()`` will lock and read from the DB on disk.
@@ -504,7 +509,7 @@ def add(self, spec, path, explicit=False):
         # TODO: ensure that spec is concrete?
         # Entire add is transactional.
         with self.write_transaction():
-            self._add(spec, path, explicit=explicit)
+            self._add(spec, directory_layout, explicit=explicit)
 
     def _get_matching_spec_key(self, spec, **kwargs):
         """Get the exact spec OR get a single spec that matches."""
@@ -534,7 +539,7 @@ def _decrement_ref_count(self, spec):
 
         if rec.ref_count == 0 and not rec.installed:
             del self._data[key]
-            for dep in spec.dependencies('link'):
+            for dep in spec.dependencies(_tracked_deps):
                 self._decrement_ref_count(dep)
 
     def _remove(self, spec):
@@ -548,7 +553,7 @@ def _remove(self, spec):
             return rec.spec
 
         del self._data[key]
-        for dep in rec.spec.dependencies('link'):
+        for dep in rec.spec.dependencies(_tracked_deps):
             self._decrement_ref_count(dep)
 
         # Returns the concrete spec so we know it in the case where a
@@ -657,6 +662,10 @@ class CorruptDatabaseError(SpackError):
     """Raised when errors are found while reading the database."""
 
 
+class NonConcreteSpecAddError(SpackError):
+    """Raised when attemptint to add non-concrete spec to DB."""
+
+
 class InvalidDatabaseVersionError(SpackError):
 
     def __init__(self, expected, found):
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -1053,7 +1053,8 @@ def build_process():
 
         # note: PARENT of the build process adds the new package to
         # the database, so that we don't need to re-read from file.
-        spack.installed_db.add(self.spec, self.prefix, explicit=explicit)
+        spack.installed_db.add(
+            self.spec, spack.install_layout, explicit=explicit)
 
     def sanity_check_prefix(self):
         """This function checks whether install succeeded."""


Test patch:
diff --git a/lib/spack/spack/test/database.py b/lib/spack/spack/test/database.py
--- a/lib/spack/spack/test/database.py
+++ b/lib/spack/spack/test/database.py
@@ -124,6 +124,19 @@ def test_015_write_and_read(self):
             self.assertEqual(new_rec.path,      rec.path)
             self.assertEqual(new_rec.installed, rec.installed)
 
+    def _check_merkleiness(self):
+        """Ensure the spack database is a valid merkle graph."""
+        all_specs = spack.installed_db.query(installed=any)
+
+        seen = {}
+        for spec in all_specs:
+            for dep in spec.dependencies():
+                hash_key = dep.dag_hash()
+                if hash_key not in seen:
+                    seen[hash_key] = id(dep)
+                else:
+                    self.assertEqual(seen[hash_key], id(dep))
+
     def _check_db_sanity(self):
         """Utiilty function to check db against install layout."""
         expected = sorted(spack.install_layout.all_specs())
@@ -133,10 +146,17 @@ def _check_db_sanity(self):
         for e, a in zip(expected, actual):
             self.assertEqual(e, a)
 
+        self._check_merkleiness()
+
     def test_020_db_sanity(self):
         """Make sure query() returns what's actually in the db."""
         self._check_db_sanity()
 
+    def test_025_reindex(self):
+        """Make sure reindex works and ref counts are valid."""
+        spack.installed_db.reindex(spack.install_layout)
+        self._check_db_sanity()
+
     def test_030_db_sanity_from_another_process(self):
         def read_and_modify():
             self._check_db_sanity()  # check that other process can read DB
@@ -203,9 +223,10 @@ def _check_remove_and_add_package(self, spec):
         self.assertTrue(concrete_spec not in remaining)
 
         # add it back and make sure everything is ok.
-        self.installed_db.add(concrete_spec, "")
+        self.installed_db.add(concrete_spec, spack.install_layout)
         installed = self.installed_db.query()
-        self.assertEqual(len(installed), len(original))
+        self.assertTrue(concrete_spec in installed)
+        self.assertEqual(installed, original)
 
         # sanity check against direcory layout and check ref counts.
         self._check_db_sanity()
@@ -233,7 +254,7 @@ def test_080_root_ref_counts(self):
         self.assertEqual(self.installed_db.get_record('mpich').ref_count, 1)
 
         # Put the spec back
-        self.installed_db.add(rec.spec, rec.path)
+        self.installed_db.add(rec.spec, spack.install_layout)
 
         # record is present again
         self.assertEqual(