Repository: spack/spack

Issue:
`spack test` tries to run tests for external packages that require additional files
### Steps to reproduce the issue

1. Add `openmpi` as an external package
2. Run `spack test openmpi`
3. Test fails because `examples` directory isn't found in installation prefix

### Error Message

```
Command exited with status 2:
    '/usr/bin/make' 'all'
make: *** No rule to make target `all'.  Stop.
```

### Recommended Solution

We should skip tests for external packages that require additional files.

### Information on your system

* **Spack:** 0.14.1-6842-6f534ac
* **Python:** 3.6.8
* **Platform:** linux-centos7-haswell
* **Concretizer:** original

```yaml
packages:
  openmpi:
    buildable: false
    externals:
    - spec: openmpi@3.1.6%gcc@9.3.0
      modules:
      - gnu/openmpi-3.1.6-gnu-9.3.0
```

### Additional information

- [x] I have run `spack debug report` and reported the version of Spack/Python/Platform
- [x] I have searched the issues of this repo and believe this is not a duplicate
- [x] I have run the failing commands in debug mode and reported the output


Discussion:
Should we even be testing external packages?

Resolving patch:
diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py
--- a/lib/spack/spack/package.py
+++ b/lib/spack/spack/package.py
@@ -1791,7 +1791,7 @@ def cache_extra_test_sources(self, srcs):
                 fsys.mkdirp(os.path.dirname(dest_path))
                 fsys.copy(src_path, dest_path)
 
-    def do_test(self, dirty=False):
+    def do_test(self, dirty=False, externals=False):
         if self.test_requires_compiler:
             compilers = spack.compilers.compilers_for_spec(
                 self.spec.compiler, arch_spec=self.spec.architecture)
@@ -1808,6 +1808,12 @@ def do_test(self, dirty=False):
         self.tested_file = self.test_suite.tested_file_for_spec(self.spec)
         fsys.touch(self.test_log_file)  # Otherwise log_parse complains
 
+        if self.spec.external and not externals:
+            with open(self.test_log_file, 'w') as ofd:
+                ofd.write('Testing package {0}\n'
+                          .format(self.test_suite.test_pkg_id(self.spec)))
+            return
+
         kwargs = {'dirty': dirty, 'fake': False, 'context': 'test'}
         spack.build_environment.start_build_process(self, test_process, kwargs)
 
diff --git a/lib/spack/spack/report.py b/lib/spack/spack/report.py
--- a/lib/spack/spack/report.py
+++ b/lib/spack/spack/report.py
@@ -171,8 +171,15 @@ def wrapper(instance, *args, **kwargs):
                 value = None
                 try:
                     value = do_fn(instance, *args, **kwargs)
-                    package['result'] = 'success'
-                    package['stdout'] = fetch_log(pkg, do_fn, self.dir)
+
+                    externals = kwargs.get('externals', False)
+                    skip_externals = pkg.spec.external and not externals
+                    if do_fn.__name__ == 'do_test' and skip_externals:
+                        package['result'] = 'skipped'
+                        package['stdout'] = 'Skipped external package'
+                    else:
+                        package['result'] = 'success'
+                        package['stdout'] = fetch_log(pkg, do_fn, self.dir)
                     package['installed_from_binary_cache'] = \
                         pkg.installed_from_binary_cache
                     if do_fn.__name__ == '_install_task' and installed_already:


Test patch:
diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py
--- a/lib/spack/spack/cmd/test.py
+++ b/lib/spack/spack/cmd/test.py
@@ -54,6 +54,10 @@ def setup_parser(subparser):
         '--fail-first', action='store_true',
         help="Stop after the first failed package."
     )
+    run_parser.add_argument(
+        '--externals', action='store_true',
+        help="Test packages that are externally installed."
+    )
     run_parser.add_argument(
         '--keep-stage',
         action='store_true',
@@ -204,7 +208,8 @@ def test_run(args):
     with reporter('test', test_suite.stage):
         test_suite(remove_directory=not args.keep_stage,
                    dirty=args.dirty,
-                   fail_first=args.fail_first)
+                   fail_first=args.fail_first,
+                   externals=args.externals)
 
 
 def has_test_method(pkg):
diff --git a/lib/spack/spack/install_test.py b/lib/spack/spack/install_test.py
--- a/lib/spack/spack/install_test.py
+++ b/lib/spack/spack/install_test.py
@@ -128,6 +128,7 @@ def __call__(self, *args, **kwargs):
         remove_directory = kwargs.get('remove_directory', True)
         dirty = kwargs.get('dirty', False)
         fail_first = kwargs.get('fail_first', False)
+        externals = kwargs.get('externals', False)
 
         for spec in self.specs:
             try:
@@ -149,9 +150,7 @@ def __call__(self, *args, **kwargs):
                 fs.mkdirp(test_dir)
 
                 # run the package tests
-                spec.package.do_test(
-                    dirty=dirty
-                )
+                spec.package.do_test(dirty=dirty, externals=externals)
 
                 # Clean up on success
                 if remove_directory:
@@ -160,7 +159,18 @@ def __call__(self, *args, **kwargs):
                 # Log test status based on whether any non-pass-only test
                 # functions were called
                 tested = os.path.exists(self.tested_file_for_spec(spec))
-                status = 'PASSED' if tested else 'NO-TESTS'
+                if tested:
+                    status = 'PASSED'
+                else:
+                    self.ensure_stage()
+                    if spec.external and not externals:
+                        status = 'SKIPPED'
+                        msg = 'Skipped external package'
+                    else:
+                        status = 'NO-TESTS'
+                        msg = 'No tests to run'
+                    _add_msg_to_file(self.log_file_for_spec(spec), msg)
+
                 self.write_test_result(spec, status)
             except BaseException as exc:
                 self.fails += 1
diff --git a/lib/spack/spack/test/test_suite.py b/lib/spack/spack/test/test_suite.py
--- a/lib/spack/spack/test/test_suite.py
+++ b/lib/spack/spack/test/test_suite.py
@@ -87,6 +87,34 @@ def test_do_test(mock_packages, install_mockery, mock_test_stage):
     assert os.path.exists(data_filename)
 
 
+@pytest.mark.parametrize('arguments,status,msg', [
+    ({}, 'SKIPPED', 'Skipped'),
+    ({'externals': True}, 'NO-TESTS', 'No tests'),
+])
+def test_test_external(mock_packages, install_mockery, mock_test_stage,
+                       arguments, status, msg):
+    def ensure_results(filename, expected):
+        assert os.path.exists(filename)
+        with open(filename, 'r') as fd:
+            lines = fd.readlines()
+            have = False
+            for line in lines:
+                if expected in line:
+                    have = True
+                    break
+            assert have
+
+    name = 'trivial-smoke-test'
+    spec = spack.spec.Spec(name).concretized()
+    spec.external_path = '/path/to/external/{0}'.format(name)
+
+    test_suite = spack.install_test.TestSuite([spec])
+    test_suite(**arguments)
+
+    ensure_results(test_suite.results_file, status)
+    ensure_results(test_suite.log_file_for_spec(spec), msg)
+
+
 def test_test_stage_caches(mock_packages, install_mockery, mock_test_stage):
     def ensure_current_cache_fail(test_suite):
         with pytest.raises(spack.install_test.TestSuiteSpecError):