Repository: spack/spack

Issue:
feature: Add option to terminate spack install after first failure
Several people have requested the ability to terminate "best effort" `spack install`'s (introduced in `v0.14.0`) after the first failure.  This feature would be useful when creating and debugging new packages.

Environment `spack install`'s continue to terminate when the installation of a package listed in the `spack.yaml` file fails.


Discussion:

Resolving patch:
diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py
--- a/lib/spack/spack/cmd/install.py
+++ b/lib/spack/spack/cmd/install.py
@@ -32,6 +32,7 @@ def update_kwargs_from_args(args, kwargs):
     that will be passed to Package.do_install API"""
 
     kwargs.update({
+        'fail_fast': args.fail_fast,
         'keep_prefix': args.keep_prefix,
         'keep_stage': args.keep_stage,
         'restage': not args.dont_restage,
@@ -78,6 +79,9 @@ def setup_parser(subparser):
     subparser.add_argument(
         '--overwrite', action='store_true',
         help="reinstall an existing spec, even if it has dependents")
+    subparser.add_argument(
+        '--fail-fast', action='store_true',
+        help="stop all builds if any build fails (default is best effort)")
     subparser.add_argument(
         '--keep-prefix', action='store_true',
         help="don't remove the install prefix if installation fails")
diff --git a/lib/spack/spack/installer.py b/lib/spack/spack/installer.py
--- a/lib/spack/spack/installer.py
+++ b/lib/spack/spack/installer.py
@@ -545,6 +545,9 @@ def package_id(pkg):
             dirty (bool): Don't clean the build environment before installing.
             explicit (bool): True if package was explicitly installed, False
                 if package was implicitly installed (as a dependency).
+            fail_fast (bool): Fail if any dependency fails to install;
+                otherwise, the default is to install as many dependencies as
+                possible (i.e., best effort installation).
             fake (bool): Don't really build; install fake stub files instead.
             force (bool): Install again, even if already installed.
             install_deps (bool): Install dependencies before installing this
@@ -1365,11 +1368,14 @@ def install(self, **kwargs):
 
         Args:"""
 
+        fail_fast = kwargs.get('fail_fast', False)
         install_deps = kwargs.get('install_deps', True)
         keep_prefix = kwargs.get('keep_prefix', False)
         keep_stage = kwargs.get('keep_stage', False)
         restage = kwargs.get('restage', False)
 
+        fail_fast_err = 'Terminating after first install failure'
+
         # install_package defaults True and is popped so that dependencies are
         # always installed regardless of whether the root was installed
         install_package = kwargs.pop('install_package', True)
@@ -1429,6 +1435,10 @@ def install(self, **kwargs):
             if pkg_id in self.failed or spack.store.db.prefix_failed(spec):
                 tty.warn('{0} failed to install'.format(pkg_id))
                 self._update_failed(task)
+
+                if fail_fast:
+                    raise InstallError(fail_fast_err)
+
                 continue
 
             # Attempt to get a write lock.  If we can't get the lock then
@@ -1502,14 +1512,28 @@ def install(self, **kwargs):
                 self._update_installed(task)
                 raise
 
-            except (Exception, KeyboardInterrupt, SystemExit) as exc:
-                # Assuming best effort installs so suppress the exception and
-                # mark as a failure UNLESS this is the explicit package.
+            except KeyboardInterrupt as exc:
+                # The build has been terminated with a Ctrl-C so terminate.
+                err = 'Failed to install {0} due to {1}: {2}'
+                tty.error(err.format(pkg.name, exc.__class__.__name__,
+                          str(exc)))
+                raise
+
+            except (Exception, SystemExit) as exc:
+                # Best effort installs suppress the exception and mark the
+                # package as a failure UNLESS this is the explicit package.
                 err = 'Failed to install {0} due to {1}: {2}'
                 tty.error(err.format(pkg.name, exc.__class__.__name__,
                           str(exc)))
+
                 self._update_failed(task, True, exc)
 
+                if fail_fast:
+                    # The user requested the installation to terminate on
+                    # failure.
+                    raise InstallError('{0}: {1}'
+                                       .format(fail_fast_err, str(exc)))
+
                 if pkg_id == self.pkg_id:
                     raise
 


Test patch:
diff --git a/lib/spack/spack/test/installer.py b/lib/spack/spack/test/installer.py
--- a/lib/spack/spack/test/installer.py
+++ b/lib/spack/spack/test/installer.py
@@ -531,6 +531,61 @@ def test_install_failed(install_mockery, monkeypatch, capsys):
     assert 'Warning: b failed to install' in out
 
 
+def test_install_fail_on_interrupt(install_mockery, monkeypatch):
+    """Test ctrl-c interrupted install."""
+    err_msg = 'mock keyboard interrupt'
+
+    def _interrupt(installer, task, **kwargs):
+        raise KeyboardInterrupt(err_msg)
+
+    spec, installer = create_installer('a')
+
+    # Raise a KeyboardInterrupt error to trigger early termination
+    monkeypatch.setattr(inst.PackageInstaller, '_install_task', _interrupt)
+
+    with pytest.raises(KeyboardInterrupt, match=err_msg):
+        installer.install()
+
+
+def test_install_fail_fast_on_detect(install_mockery, monkeypatch, capsys):
+    """Test fail_fast install when an install failure is detected."""
+    spec, installer = create_installer('a')
+
+    # Make sure the package is identified as failed
+    #
+    # This will prevent b from installing, which will cause the build of a
+    # to be skipped.
+    monkeypatch.setattr(spack.database.Database, 'prefix_failed', _true)
+
+    with pytest.raises(spack.installer.InstallError):
+        installer.install(fail_fast=True)
+
+    out = str(capsys.readouterr())
+    assert 'Skipping build of a' in out
+
+
+def test_install_fail_fast_on_except(install_mockery, monkeypatch, capsys):
+    """Test fail_fast install when an install failure results from an error."""
+    err_msg = 'mock patch failure'
+
+    def _patch(installer, task, **kwargs):
+        raise RuntimeError(err_msg)
+
+    spec, installer = create_installer('a')
+
+    # Raise a non-KeyboardInterrupt exception to trigger fast failure.
+    #
+    # This will prevent b from installing, which will cause the build of a
+    # to be skipped.
+    monkeypatch.setattr(spack.package.PackageBase, 'do_patch', _patch)
+
+    with pytest.raises(spack.installer.InstallError, matches=err_msg):
+        installer.install(fail_fast=True)
+
+    out = str(capsys.readouterr())
+    assert 'Skipping build of a' in out
+
+
 def test_install_lock_failures(install_mockery, monkeypatch, capfd):
     """Cover basic install lock failure handling in a single pass."""
     def _requeued(installer, task):