Repository: spack/spack

Issue:
spack uninstall --dependents broken
This occurred for me on an older Spack clone, so I'm not sure whether or not it's been fixed already. But I noticed this odd behavior:
```
$ spack uninstall -y --dependents ncurses
==> Successfully uninstalled lua-luaposix@33.4.0%gcc@6.2.1 arch=linux-fedora25-x86_64 /hcrkzdn
==> Successfully uninstalled cmake@3.7.2%gcc@6.2.1~doc+ncurses+openssl+ownlibs~qt arch=linux-fedora25-x86_64 /kbq74gf
==> Successfully uninstalled gettext@0.19.8.1%gcc@6.2.1+bzip2+curses+git~libunistring+libxml2+tar+xz arch=linux-fedora25-x86_64 /yiqj5x3
==> Successfully uninstalled xcb-proto@1.12%gcc@6.2.1 arch=linux-fedora25-x86_64 /eqttirz
==> Successfully uninstalled boost@1.63.0%gcc@6.2.1+atomic+chrono+date_time~debug+filesystem~graph~icu+iostreams+locale+log+math~mpi+multithreaded+program_options+python+random+regex+serialization+shared+signals~singlethreaded+system~taggedlayout+test+thread+timer+wave arch=linux-fedora25-x86_64 /aywwnid
==> Successfully uninstalled py-setuptools@34.2.0%gcc@6.2.1 arch=linux-fedora25-x86_64 /lbwg7gi
==> Successfully uninstalled lua@5.3.2%gcc@6.2.1 arch=linux-fedora25-x86_64 /gf4j32n
==> Successfully uninstalled python@2.7.13%gcc@6.2.1~tk~ucs4 arch=linux-fedora25-x86_64 /r7mn5ia
==> Successfully uninstalled py-packaging@16.8%gcc@6.2.1 arch=linux-fedora25-x86_64 /jpjqk36
==> Successfully uninstalled py-appdirs@1.4.0%gcc@6.2.1 arch=linux-fedora25-x86_64 /t5eps6o
==> Successfully uninstalled py-six@1.10.0%gcc@6.2.1 arch=linux-fedora25-x86_64 /rxq43vd
==> Successfully uninstalled py-pyparsing@2.1.10%gcc@6.2.1 arch=linux-fedora25-x86_64 /2v3z55j
==> Successfully uninstalled python@2.7.13%gcc@6.2.1~tk~ucs4 arch=linux-fedora25-x86_64 /ldubumv
==> Error: Cannot uninstall readline@6.3%gcc@6.2.1 arch=linux-fedora25-x86_64 ^ncurses@6.0%gcc@6.2.1~symlinks arch=linux-fedora25-x86_64
$ spack uninstall -y --dependents ncurses
==> Successfully uninstalled lua@5.3.4%gcc@6.2.1 arch=linux-fedora25-x86_64 /t2szkj7
==> Successfully uninstalled readline@6.3%gcc@6.2.1 arch=linux-fedora25-x86_64 /vehx2ab
==> Error: Cannot uninstall ncurses@6.0%gcc@6.2.1~symlinks arch=linux-fedora25-x86_64
$ spack uninstall -y --dependents ncurses
==> Successfully uninstalled lmod@7.3%gcc@6.2.1 arch=linux-fedora25-x86_64 /tfc5zbg
==> Successfully uninstalled lua-luaposix@33.4.0%gcc@6.2.1 arch=linux-fedora25-x86_64 /ph5tf5y
==> Successfully uninstalled lua-luafilesystem@1_6_3%gcc@6.2.1 arch=linux-fedora25-x86_64 /t6vt2td
==> Successfully uninstalled lua@5.3.4%gcc@6.2.1 arch=linux-fedora25-x86_64 /jytyxod
==> Successfully uninstalled readline@7.0%gcc@6.2.1 arch=linux-fedora25-x86_64 /qgxtgq3
==> Successfully uninstalled ncurses@6.0%gcc@6.2.1~symlinks arch=linux-fedora25-x86_64 /ybsic3i
```
It seems like Spack isn't paying attention to the order of dependencies when it uninstalls something.


Discussion:
Ok, ran into this again on the latest develop:
```
$ spack uninstall --dependents ncurses%intel@17.0.2
==> The following packages will be uninstalled:

-- linux-centos6-x86_64 / intel@17.0.2 --------------------------
v4rd5em gettext@0.19.8.1%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" +bzip2+curses+git~libunistring+libxml2+tar+xz

frfeq3r glib@2.49.7%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

ehtay7y ncurses@6.0%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" ~symlinks

lwd2ety python@2.7.13%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" ~tk~ucs4

yxzkm4t readline@7.0%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

u4oo5ap xcb-proto@1.12%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

==> Do you want to proceed? [y/N] y
==> Successfully uninstalled glib@2.49.7%intel@17.0.2 arch=linux-centos6-x86_64 /frfeq3r
==> Successfully uninstalled xcb-proto@1.12%intel@17.0.2 arch=linux-centos6-x86_64 /u4oo5ap
==> Successfully uninstalled gettext@0.19.8.1%intel@17.0.2+bzip2+curses+git~libunistring+libxml2+tar+xz arch=linux-centos6-x86_64 /v4rd5em
==> Successfully uninstalled python@2.7.13%intel@17.0.2~tk~ucs4 arch=linux-centos6-x86_64 /lwd2ety
==> Successfully uninstalled readline@7.0%intel@17.0.2 arch=linux-centos6-x86_64 /yxzkm4t
==> Error: Cannot uninstall ncurses@6.0%intel@17.0.2 cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" ~symlinks arch=linux-centos6-x86_64
$ spack uninstall --dependents ncurses%intel@17.0.2
==> The following packages will be uninstalled:

-- linux-centos6-x86_64 / intel@17.0.2 --------------------------
ehtay7y ncurses@6.0%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" ~symlinks

mgpo2wx py-appdirs@1.4.0%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

lmiuqvw py-numpy@1.12.0%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" +blas+lapack

rubvtrx py-packaging@16.8%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

ugmabmy py-pyparsing@2.1.10%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

upqpkw5 py-setuptools@34.2.0%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

4f4gwyr py-six@1.10.0%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

skpx6c4 python@2.7.13%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" ~tk~ucs4

z3ogcil readline@6.3%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

==> Do you want to proceed? [y/N] y
==> Successfully uninstalled py-numpy@1.12.0%intel@17.0.2+blas+lapack arch=linux-centos6-x86_64 /lmiuqvw
==> Successfully uninstalled py-setuptools@34.2.0%intel@17.0.2 arch=linux-centos6-x86_64 /upqpkw5
==> Successfully uninstalled py-packaging@16.8%intel@17.0.2 arch=linux-centos6-x86_64 /rubvtrx
==> Successfully uninstalled py-appdirs@1.4.0%intel@17.0.2 arch=linux-centos6-x86_64 /mgpo2wx
==> Successfully uninstalled py-six@1.10.0%intel@17.0.2 arch=linux-centos6-x86_64 /4f4gwyr
==> Successfully uninstalled py-pyparsing@2.1.10%intel@17.0.2 arch=linux-centos6-x86_64 /ugmabmy
==> Successfully uninstalled python@2.7.13%intel@17.0.2~tk~ucs4 arch=linux-centos6-x86_64 /skpx6c4
==> Successfully uninstalled readline@6.3%intel@17.0.2 arch=linux-centos6-x86_64 /z3ogcil
==> Successfully uninstalled ncurses@6.0%intel@17.0.2~symlinks arch=linux-centos6-x86_64 /ehtay7y
```
Perhaps it has something to do with deptypes? @scheibelp 
Still seeing this. This time it took me **4** iterations to get Spack to uninstall everything:
```
$ spack uninstall sqlite
==> Error: sqlite matches multiple packages:

-- linux-centos6-x86_64 / gcc@4.4.7 -----------------------------
wew6vi2 sqlite@3.8.5%gcc


-- linux-centos6-x86_64 / gcc@5.3.0 -----------------------------
qtuvnlr sqlite@3.8.5%gcc


-- linux-centos6-x86_64 / gcc@6.1.0 -----------------------------
jiwxnpb sqlite@3.8.5%gcc


-- linux-centos6-x86_64 / intel@16.0.3 --------------------------
xlnqux3 sqlite@3.8.5%intel cflags="-axCORE-AVX2,AVX" cxxflags="-axCORE-AVX2,AVX" fflags="-axCORE-AVX2,AVX" 


-- linux-centos6-x86_64 / intel@17.0.1 --------------------------
pqundgy sqlite@3.8.5%intel cflags="-axCORE-AVX2,AVX" cxxflags="-axCORE-AVX2,AVX" fflags="-axCORE-AVX2,AVX" 


-- linux-centos6-x86_64 / intel@17.0.2 --------------------------
2udhmfm sqlite@3.8.5%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

cpd4kru sqlite@3.18.0%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 


==> Error: You can either:
    a) use a more specific spec, or
    b) use `spack uninstall --all` to uninstall ALL matching specs.

$ spack uninstall -a sqlite%intel@17.0.2
==> Error: Will not uninstall sqlite@3.18.0%intel@17.0.2/cpd4kru

The following packages depend on it:
-- linux-centos6-x86_64 / intel@17.0.2 --------------------------
t7zg6mu python@2.7.13%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" +shared~tk~ucs4


==> Error: Use `spack uninstall --dependents` to uninstall these dependencies as well.
$ spack uninstall -a --dependents sqlite%intel@17.0.2
==> The following packages will be uninstalled:

-- linux-centos6-x86_64 / intel@17.0.2 --------------------------
t7zg6mu python@2.7.13%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" +shared~tk~ucs4

2udhmfm sqlite@3.8.5%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

cpd4kru sqlite@3.18.0%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

==> Do you want to proceed? [y/N] y
==> Successfully uninstalled sqlite@3.8.5%intel@17.0.2 arch=linux-centos6-x86_64 /2udhmfm
==> Successfully uninstalled sqlite@3.8.5%intel@17.0.2 arch=linux-centos6-x86_64 /2udhmfm
==> Successfully uninstalled python@2.7.13%intel@17.0.2+shared~tk~ucs4 arch=linux-centos6-x86_64 /t7zg6mu
==> Successfully uninstalled python@2.7.13%intel@17.0.2+shared~tk~ucs4 arch=linux-centos6-x86_64 /t7zg6mu
==> Error: Cannot uninstall sqlite@3.18.0%intel@17.0.2 cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX"  arch=linux-centos6-x86_64
$ spack uninstall -a --dependents sqlite%intel@17.0.2
==> The following packages will be uninstalled:

-- linux-centos6-x86_64 / intel@17.0.2 --------------------------
hd3xdc2 py-appdirs@1.4.3%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

qw6dlqw py-lit@0.5.0%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

dl3seef py-packaging@16.8%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

tbag74l py-pyparsing@2.2.0%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

bwpszcq py-setuptools@34.4.1%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

i6xzqtt py-six@1.10.0%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

bed46oi python@2.7.13%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" ~tk~ucs4

cpd4kru sqlite@3.18.0%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

==> Do you want to proceed? [y/N] y
==> Successfully uninstalled py-setuptools@34.4.1%intel@17.0.2 arch=linux-centos6-x86_64 /bwpszcq
==> Successfully uninstalled py-setuptools@34.4.1%intel@17.0.2 arch=linux-centos6-x86_64 /bwpszcq
==> Successfully uninstalled py-lit@0.5.0%intel@17.0.2 arch=linux-centos6-x86_64 /qw6dlqw
==> Successfully uninstalled py-lit@0.5.0%intel@17.0.2 arch=linux-centos6-x86_64 /qw6dlqw
==> Successfully uninstalled py-appdirs@1.4.3%intel@17.0.2 arch=linux-centos6-x86_64 /hd3xdc2
==> Successfully uninstalled py-appdirs@1.4.3%intel@17.0.2 arch=linux-centos6-x86_64 /hd3xdc2
==> Successfully uninstalled py-packaging@16.8%intel@17.0.2 arch=linux-centos6-x86_64 /dl3seef
==> Successfully uninstalled py-packaging@16.8%intel@17.0.2 arch=linux-centos6-x86_64 /dl3seef
==> Successfully uninstalled py-pyparsing@2.2.0%intel@17.0.2 arch=linux-centos6-x86_64 /tbag74l
==> Successfully uninstalled py-pyparsing@2.2.0%intel@17.0.2 arch=linux-centos6-x86_64 /tbag74l
==> Successfully uninstalled py-six@1.10.0%intel@17.0.2 arch=linux-centos6-x86_64 /i6xzqtt
==> Successfully uninstalled py-six@1.10.0%intel@17.0.2 arch=linux-centos6-x86_64 /i6xzqtt
==> Successfully uninstalled python@2.7.13%intel@17.0.2~tk~ucs4 arch=linux-centos6-x86_64 /bed46oi
==> Successfully uninstalled python@2.7.13%intel@17.0.2~tk~ucs4 arch=linux-centos6-x86_64 /bed46oi
==> Error: Cannot uninstall sqlite@3.18.0%intel@17.0.2 cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX"  arch=linux-centos6-x86_64
$ spack uninstall -a --dependents sqlite%intel@17.0.2
==> The following packages will be uninstalled:

-- linux-centos6-x86_64 / intel@17.0.2 --------------------------
mph67ug libxml2@2.9.4%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" +python

uog3suv python@2.7.13%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" ~tk~ucs4

cpd4kru sqlite@3.18.0%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

==> Do you want to proceed? [y/N] y
==> Successfully uninstalled libxml2@2.9.4%intel@17.0.2+python arch=linux-centos6-x86_64 /mph67ug
==> Successfully uninstalled libxml2@2.9.4%intel@17.0.2+python arch=linux-centos6-x86_64 /mph67ug
==> Successfully uninstalled python@2.7.13%intel@17.0.2~tk~ucs4 arch=linux-centos6-x86_64 /uog3suv
==> Successfully uninstalled python@2.7.13%intel@17.0.2~tk~ucs4 arch=linux-centos6-x86_64 /uog3suv
==> Error: Cannot uninstall sqlite@3.18.0%intel@17.0.2 cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX"  arch=linux-centos6-x86_64
$ spack uninstall -a --dependents sqlite%intel@17.0.2
==> The following packages will be uninstalled:

-- linux-centos6-x86_64 / intel@17.0.2 --------------------------
3ggd67j python@3.6.0%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" +shared~tk~ucs4

cpd4kru sqlite@3.18.0%intel cflags="-axCORE-AVX512,CORE-AVX2,AVX" cxxflags="-axCORE-AVX512,CORE-AVX2,AVX" fflags="-axCORE-AVX512,CORE-AVX2,AVX" 

==> Do you want to proceed? [y/N] y
y
==> Successfully uninstalled python@3.6.0%intel@17.0.2+shared~tk~ucs4 arch=linux-centos6-x86_64 /3ggd67j
==> Successfully uninstalled python@3.6.0%intel@17.0.2+shared~tk~ucs4 arch=linux-centos6-x86_64 /3ggd67j
==> Successfully uninstalled sqlite@3.18.0%intel@17.0.2 arch=linux-centos6-x86_64 /cpd4kru
==> Successfully uninstalled sqlite@3.18.0%intel@17.0.2 arch=linux-centos6-x86_64 /cpd4kru
```
Also really weird message:
```
$ spack uninstall -y sqlite %intel@17.0.2
==> Successfully uninstalled sqlite@3.18.0%intel@17.0.2 arch=linux-centos6-x86_64 /cpd4kru
==> Successfully uninstalled sqlite@3.18.0%intel@17.0.2 arch=linux-centos6-x86_64 /cpd4kru
```
Why would it be uninstalling twice for the same hash?
Reproduced on Fedora 25:
```
$ spack uninstall automake
==> The following packages will be uninstalled:

-- linux-fedora25-x86_64 / gcc@6.2.1 ----------------------------
zwtptzu automake@1.15%gcc

==> Do you want to proceed? [y/N] y
==> Successfully uninstalled automake@1.15%gcc@6.2.1 arch=linux-fedora25-x86_64 /zwtptzu
==> Successfully uninstalled automake@1.15%gcc@6.2.1 arch=linux-fedora25-x86_64 /zwtptzu
$ spack uninstall autoconf
==> The following packages will be uninstalled:

-- linux-fedora25-x86_64 / gcc@6.2.1 ----------------------------
l7zxzsc autoconf@2.69%gcc

==> Do you want to proceed? [y/N] y
==> Successfully uninstalled autoconf@2.69%gcc@6.2.1 arch=linux-fedora25-x86_64 /l7zxzsc
==> Successfully uninstalled autoconf@2.69%gcc@6.2.1 arch=linux-fedora25-x86_64 /l7zxzsc
```
Are we maintaining 2 separate databases here?
@adamjstewart Do you have a reproducer? I tried to `fake` install a couple of complicated packages and couldn't reproduce the behavior above. For instance:
```console
$ spack install --fake trilinos
$ spack install --fake gcc
$ spack install --fake hdf5
...
$ spack uninstall -R libxml2
==> The following packages will be uninstalled:

-- linux-ubuntu14-x86_64 / gcc@4.8 ------------------------------
7axdpya binutils@2.28%gcc+gold~krellpatch~libiberty~plugins

g4ab7ky gcc@6.3.0%gcc+binutils~piclibs

yymghlh gettext@0.19.8.1%gcc+bzip2+curses+git~libunistring+libxml2+tar+xz

fbytriv hdf5@1.10.0-patch1%gcc+cxx~debug+fortran+mpi+pic+shared~szip~threadsafe

lkyfwot hwloc@1.11.6%gcc~cuda+libxml2+pci

4dbvktn hypre@2.11.2%gcc~int64~internal-superlu+shared

ryzxnsu libxml2@2.9.4%gcc~python

37aeczu matio@1.5.9%gcc+hdf5+shared+zlib

tkajlea mumps@5.1.1%gcc+complex+double+float~int64~metis+mpi~parmetis~ptscotch~scotch+shared

4iikmio netcdf@4.4.1.1%gcc~cdmremote~dap~hdf4 maxdims=1024  maxvars=8192 +mpi~parallel-netcdf+shared

lmesd22 netlib-scalapack@2.0.2%gcc~fpic+shared

55bmo53 openmpi@2.1.0%gcc~cuda~java~mxm~pmi~psm~psm2~slurm~sqlite3~thread_multiple~tm~verbs+vt

sewsq3x parmetis@4.0.3%gcc~debug~gdb+shared

bhyilnw superlu-dist@5.1.3%gcc~int64

yi7p3tr trilinos@12.10.1%gcc+boost~debug~exodus+hdf5+hypre+metis+mumps~python+shared+suite-sparse~superlu+superlu-dist+tpetra~xsdkflags

==> Do you want to proceed? [y/N] y
==> Successfully uninstalled trilinos@12.10.1%gcc@4.8+boost~debug~exodus+hdf5+hypre+metis+mumps~python+shared+suite-sparse~superlu+superlu-dist+tpetra~xsdkflags arch=linux-ubuntu14-x86_64 /yi7p3tr
==> Successfully uninstalled gcc@6.3.0%gcc@4.8+binutils~piclibs arch=linux-ubuntu14-x86_64 /g4ab7ky
==> Successfully uninstalled netcdf@4.4.1.1%gcc@4.8~cdmremote~dap~hdf4 maxdims=1024  maxvars=8192 +mpi~parallel-netcdf+shared arch=linux-ubuntu14-x86_64 /4iikmio
==> Successfully uninstalled matio@1.5.9%gcc@4.8+hdf5+shared+zlib arch=linux-ubuntu14-x86_64 /37aeczu
==> Successfully uninstalled hypre@2.11.2%gcc@4.8~int64~internal-superlu+shared arch=linux-ubuntu14-x86_64 /4dbvktn
==> Successfully uninstalled binutils@2.28%gcc@4.8+gold~krellpatch~libiberty~plugins arch=linux-ubuntu14-x86_64 /7axdpya
==> Successfully uninstalled superlu-dist@5.1.3%gcc@4.8~int64 arch=linux-ubuntu14-x86_64 /bhyilnw
==> Successfully uninstalled mumps@5.1.1%gcc@4.8+complex+double+float~int64~metis+mpi~parmetis~ptscotch~scotch+shared arch=linux-ubuntu14-x86_64 /tkajlea
==> Successfully uninstalled netlib-scalapack@2.0.2%gcc@4.8~fpic+shared arch=linux-ubuntu14-x86_64 /lmesd22
==> Successfully uninstalled gettext@0.19.8.1%gcc@4.8+bzip2+curses+git~libunistring+libxml2+tar+xz arch=linux-ubuntu14-x86_64 /yymghlh
==> Successfully uninstalled parmetis@4.0.3%gcc@4.8~debug~gdb+shared arch=linux-ubuntu14-x86_64 /sewsq3x
==> Successfully uninstalled hdf5@1.10.0-patch1%gcc@4.8+cxx~debug+fortran+mpi+pic+shared~szip~threadsafe arch=linux-ubuntu14-x86_64 /fbytriv
==> Successfully uninstalled openmpi@2.1.0%gcc@4.8~cuda~java~mxm~pmi~psm~psm2~slurm~sqlite3~thread_multiple~tm~verbs+vt arch=linux-ubuntu14-x86_64 /55bmo53
==> Successfully uninstalled hwloc@1.11.6%gcc@4.8~cuda+libxml2+pci arch=linux-ubuntu14-x86_64 /lkyfwot
==> Successfully uninstalled libxml2@2.9.4%gcc@4.8~python arch=linux-ubuntu14-x86_64 /ryzxnsu
```
Part of the problem is that some of these installations are old or were done on different branches and I've been continuously updating to develop. So reproducing this will be difficult. At the very least, you will need dozens of installations for the same version of a package with the same compiler. Aside from that, I'm not sure what the difference is.
Note sure how to provide reproducer but in my case this is externally installed Python package and I am not uninstalling any dependent packages. Below are three successive commands:

```
 → spack uninstall --all -y -f python
==> Successfully uninstalled python@2.6.6%gcc@4.4.7+shared~tk~ucs4 arch=bgq-rhel6-ppc64 /4tsndov
==> Successfully uninstalled python@2.6.7%xl@12.1+shared~tk~ucs4 arch=bgq-cnk-ppc64 /6n7wft6
==> Error: 'No such spec in database! python@2.6.7%xl@12.1+shared~tk~ucs4 arch=bgq-cnk-ppc64'

 → spack uninstall --all -y -f python
==> Successfully uninstalled python@2.6.7%xl@12.1+shared~tk~ucs4 arch=bgq-cnk-ppc64 /dwtx2ft
==> Successfully uninstalled python@2.7.5%xl@12.1+shared~tk~ucs4 arch=bgq-cnk-ppc64 /oxu757p
==> Error: 'No such spec in database! python@2.7.5%xl@12.1+shared~tk~ucs4 arch=bgq-cnk-ppc64'

→ spack uninstall --all -y -f python
==> Successfully uninstalled python@2.7.5%xl@12.1+shared~tk~ucs4 arch=bgq-cnk-ppc64 /32afiee
==> Successfully uninstalled python@2.7.5%gcc@4.4.7+shared~tk~ucs4 arch=bgq-rhel6-ppc64 /vyjai5m
==> Successfully uninstalled python@3.2%xl@12.1+shared~tk~ucs4 arch=bgq-cnk-ppc64 /si2e76z
```
For the record, I'm still seeing this. `spack uninstall --all` will take a few iterations to actually uninstall everything. Still have no idea how to reproduce this unfortunately.
Still seeing this:
```console
$ spack uninstall libsigsegv
==> Error: Will not uninstall libsigsegv@2.11%clang@9.0.0-apple/qlxyw6k

The following packages depend on it:
-- darwin-highsierra-x86_64 / clang@9.0.0-apple -----------------
hxmoo4q autoconf@2.69%clang

kww7vef bison@3.0.4%clang patches=57f972940a10d448efbd3d5ba46e65979ae4eea93681a85e1d998060b356e0d2,b72914fe38e54a6fc25f29019e0a0786705c4f61ce20d414cc2010c8d62448c7

gnvsi6r m4@1.4.18%clang patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00 +sigsegv


==> Error: Use `spack uninstall --dependents` to uninstall these dependencies as well.
$ spack uninstall --dependents libsigsegv
==> The following packages will be uninstalled:

-- darwin-highsierra-x86_64 / clang@9.0.0-apple -----------------
hxmoo4q autoconf@2.69%clang

kww7vef bison@3.0.4%clang patches=57f972940a10d448efbd3d5ba46e65979ae4eea93681a85e1d998060b356e0d2,b72914fe38e54a6fc25f29019e0a0786705c4f61ce20d414cc2010c8d62448c7

qlxyw6k libsigsegv@2.11%clang

gnvsi6r m4@1.4.18%clang patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00 +sigsegv

==> Do you want to proceed? [y/N] y
==> Successfully uninstalled bison@3.0.4%clang@9.0.0-apple patches=57f972940a10d448efbd3d5ba46e65979ae4eea93681a85e1d998060b356e0d2,b72914fe38e54a6fc25f29019e0a0786705c4f61ce20d414cc2010c8d62448c7 arch=darwin-highsierra-x86_64 /kww7vef
==> Successfully uninstalled autoconf@2.69%clang@9.0.0-apple arch=darwin-highsierra-x86_64 /hxmoo4q
==> Error: Cannot uninstall m4@1.4.18%clang@9.0.0-apple patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00 +sigsegv arch=darwin-highsierra-x86_64  ^libsigsegv@2.11%clang@9.0.0-apple arch=darwin-highsierra-x86_64
$ spack uninstall --dependents libsigsegv
==> The following packages will be uninstalled:

-- darwin-highsierra-x86_64 / clang@9.0.0-apple -----------------
2pkhuco autoconf@2.69%clang

qlxyw6k libsigsegv@2.11%clang

gnvsi6r m4@1.4.18%clang patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00 +sigsegv

==> Do you want to proceed? [y/N] y
==> Successfully uninstalled autoconf@2.69%clang@9.0.0-apple arch=darwin-highsierra-x86_64 /2pkhuco
==> Error: Cannot uninstall m4@1.4.18%clang@9.0.0-apple patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00 +sigsegv arch=darwin-highsierra-x86_64  ^libsigsegv@2.11%clang@9.0.0-apple arch=darwin-highsierra-x86_64
$ spack uninstall --dependents libsigsegv
==> The following packages will be uninstalled:

-- darwin-highsierra-x86_64 / clang@9.0.0-apple -----------------
47sqxor autoconf@2.69%clang

qlxyw6k libsigsegv@2.11%clang

gnvsi6r m4@1.4.18%clang patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00 +sigsegv

==> Do you want to proceed? [y/N] y
==> Successfully uninstalled autoconf@2.69%clang@9.0.0-apple arch=darwin-highsierra-x86_64 /47sqxor
==> Successfully uninstalled m4@1.4.18%clang@9.0.0-apple patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00 +sigsegv arch=darwin-highsierra-x86_64 /gnvsi6r
==> Successfully uninstalled libsigsegv@2.11%clang@9.0.0-apple arch=darwin-highsierra-x86_64 /qlxyw6k
```
It looks like this time, I had 3 versions of `autoconf` installed. No idea what the differences were. None of these were external packages.
@adamjstewart @tgamblin I got hit by this today in a very annoying way (had to give the same uninstall command 6 times). I got a moderately simple reproducer at c2de2558b6423711e3b2cdbaaca1c0b0dfb61757 :
```console
$ spack install --fake hdf5~mpi
...
$ spack install --fake quantum-espresso hdf5=serial
...
$ spack uninstall -ayR zlib
==> Successfully uninstalled quantum-espresso@6.4%gcc@9.0.1~elpa~epw hdf5=serial +mpi~openmp+scalapack arch=linux-ubuntu18.04-x86_64/ofunguj
==> Successfully uninstalled cmake@3.14.5%gcc@9.0.1~doc+ncurses+openssl+ownlibs~qt arch=linux-ubuntu18.04-x86_64/y7rjdgu
==> Successfully uninstalled netlib-scalapack@2.0.2%gcc@9.0.1 build_type=RelWithDebInfo patches=22ebf4e3d5a6356cd6086ea65bfdf30f9d0a2038136127590cd269d15bdb03af,e8f30dd1f26e523dfb552f8d7b8ad26ac88fc0c8d72e3d4f9a9717a3383e0b33 ~pic+shared arch=linux-ubuntu18.04-x86_64/ydmb5dx
==> Successfully uninstalled fftw@3.3.8%gcc@9.0.1+double+float~fma+long_double+mpi~openmp~pfft_patches~quad simd=avx,avx2,sse2 arch=linux-ubuntu18.04-x86_64/u264xpp
==> Successfully uninstalled openssl@1.1.1b%gcc@9.0.1+systemcerts arch=linux-ubuntu18.04-x86_64/j7pqzri
==> Successfully uninstalled hdf5@1.10.5%gcc@9.0.1~cxx~debug+fortran+hl~mpi+pic+shared~szip~threadsafe arch=linux-ubuntu18.04-x86_64/qkgrhmn
==> Successfully uninstalled openmpi@3.1.4%gcc@9.0.1~cuda+cxx_exceptions fabrics=none ~java~legacylaunchers~memchecker~pmi schedulers=none ~sqlite3~thread_multiple+vt arch=linux-ubuntu18.04-x86_64/rjm6svl
==> Successfully uninstalled hwloc@1.11.11%gcc@9.0.1~cairo~cuda~gl+libxml2~nvml+pci+shared arch=linux-ubuntu18.04-x86_64/4wetffy
==> Successfully uninstalled libxml2@2.9.9%gcc@9.0.1~python arch=linux-ubuntu18.04-x86_64/3gphlbz
==> Error: Cannot uninstall zlib@1.2.11%gcc@9.0.1+optimize+pic+shared arch=linux-ubuntu18.04-x86_64
$ spack uninstall -ayR zlib
==> Successfully uninstalled hdf5@1.10.5%gcc@9.0.1~cxx~debug~fortran~hl~mpi+pic+shared~szip~threadsafe arch=linux-ubuntu18.04-x86_64/xyhss36
==> Successfully uninstalled zlib@1.2.11%gcc@9.0.1+optimize+pic+shared arch=linux-ubuntu18.04-x86_64/tn4qvs7
```
Will look into this.

I think I got the issue. When we compute the list of dependents we end up building a `DependencyMap` object which maps a `str` (the name of the parent) to **a single** `DependencySpec` object:

https://github.com/spack/spack/blob/c2de2558b6423711e3b2cdbaaca1c0b0dfb61757/lib/spack/spack/spec.py#L1063-L1072

This is clearly insufficient for most cases. For instance in the reproducer above we had 2 different `hdf5` configurations pointing to the same `zlib` and one of them is overridden by the other when we set:
```python
# spec = 'zlib' , self = 'hdf5', dspec = 'edge hdf5 -> zlib'
spec._dependents[self.name] = dspec  
```
for the second time. When we uninstall we effectively lose information on some specs that have been overridden, and we stop when we try to remove stuff that still has a `ref_count > 0`.
@tgamblin @scheibelp I put this issue in high priority as I think currently we can't trust a traversal of dependents to be correct (some of them will be skipped).

Resolving patch:
diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py
--- a/lib/spack/spack/cmd/uninstall.py
+++ b/lib/spack/spack/cmd/uninstall.py
@@ -9,6 +9,7 @@
 
 import spack.cmd
 import spack.environment as ev
+import spack.error
 import spack.package
 import spack.cmd.common.arguments as arguments
 import spack.repo
@@ -126,9 +127,10 @@ def installed_dependents(specs, env):
 
     env_hashes = set(env.all_hashes()) if env else set()
 
+    all_specs_in_db = spack.store.db.query()
+
     for spec in specs:
-        installed = spack.store.db.installed_relatives(
-            spec, direction='parents', transitive=True)
+        installed = [x for x in all_specs_in_db if spec in x]
 
         # separate installed dependents into dpts in this environment and
         # dpts that are outside this environment
@@ -212,16 +214,23 @@ def do_uninstall(env, specs, force):
         if env:
             _remove_from_env(item, env)
 
-    # Sort packages to be uninstalled by the number of installed dependents
-    # This ensures we do things in the right order
-    def num_installed_deps(pkg):
-        dependents = spack.store.db.installed_relatives(
-            pkg.spec, 'parents', True)
-        return len(dependents)
+    # A package is ready to be uninstalled when nothing else references it,
+    # unless we are requested to force uninstall it.
+    is_ready = lambda x: not spack.store.db.query_by_spec_hash(x)[1].ref_count
+    if force:
+        is_ready = lambda x: True
+
+    while packages:
+        ready = [x for x in packages if is_ready(x.spec.dag_hash())]
+        if not ready:
+            msg = 'unexpected error [cannot proceed uninstalling specs with' \
+                  ' remaining dependents {0}]'
+            msg = msg.format(', '.join(x.name for x in packages))
+            raise spack.error.SpackError(msg)
 
-    packages.sort(key=num_installed_deps)
-    for item in packages:
-        item.do_uninstall(force=force)
+        packages = [x for x in packages if x not in ready]
+        for item in ready:
+            item.do_uninstall(force=force)
 
     # write any changes made to the active environment
     if env:
@@ -332,5 +341,5 @@ def uninstall(parser, args):
                 '  Use `spack uninstall --all` to uninstall ALL packages.')
 
     # [any] here handles the --all case by forcing all specs to be returned
-    uninstall_specs(
-        args, spack.cmd.parse_specs(args.packages) if args.packages else [any])
+    specs = spack.cmd.parse_specs(args.packages) if args.packages else [any]
+    uninstall_specs(args, specs)


Test patch:
diff --git a/lib/spack/spack/test/cmd/uninstall.py b/lib/spack/spack/test/cmd/uninstall.py
--- a/lib/spack/spack/test/cmd/uninstall.py
+++ b/lib/spack/spack/test/cmd/uninstall.py
@@ -37,7 +37,7 @@ def test_installed_dependents():
 
 
 @pytest.mark.db
-@pytest.mark.usefixtures('database')
+@pytest.mark.usefixtures('mutable_database')
 def test_recursive_uninstall():
     """Test recursive uninstall."""
     uninstall('-y', '-a', '--dependents', 'callpath')
@@ -52,3 +52,32 @@ def test_recursive_uninstall():
     assert len(mpileaks_specs) == 0
     assert len(callpath_specs) == 0
     assert len(mpi_specs) == 3
+
+
+@pytest.mark.db
+@pytest.mark.regression('3690')
+@pytest.mark.usefixtures('mutable_database')
+@pytest.mark.parametrize('constraint,expected_number_of_specs', [
+    ('dyninst', 7), ('libelf', 5)
+])
+def test_uninstall_spec_with_multiple_roots(
+        constraint, expected_number_of_specs
+):
+    uninstall('-y', '-a', '--dependents', constraint)
+
+    all_specs = spack.store.layout.all_specs()
+    assert len(all_specs) == expected_number_of_specs
+
+
+@pytest.mark.db
+@pytest.mark.usefixtures('mutable_database')
+@pytest.mark.parametrize('constraint,expected_number_of_specs', [
+    ('dyninst', 13), ('libelf', 13)
+])
+def test_force_uninstall_spec_with_ref_count_not_zero(
+        constraint, expected_number_of_specs
+):
+    uninstall('-f', '-y', constraint)
+
+    all_specs = spack.store.layout.all_specs()
+    assert len(all_specs) == expected_number_of_specs