Repository: innolitics/hdat

Issue:
Hook up hdat_main_script in setup.py
After running `pip install hdat` we should have the `hdat_main_script` available as `hdat` on the shell PATH.  I am not sure how to do this, but we will probably need to update our `setup.py` and hook up either a "script" or an "entrypoint".

Discussion:

Patch:
diff --git a/README.md b/README.md
index 5714ec4..b296333 100644
--- a/README.md
+++ b/README.md
@@ -103,9 +103,9 @@ A casespec is a string that selects one or more test cases.  A casespec may spec
 
 Here are several casespecs along with the test cases they would select:
 
-- `` Selects all test cases in all suites.
-- `a` Selects all cases in the test suite with id "a".
-- `a/b` Selects test case with id "b" in the suite with id "b".
+"" - Selects all test cases in all suites.
+"a" - Selects all cases in the test suite with id "a".
+"a/b" - Selects test case with id "b" in the suite with id "b".
 
 # Resultspecs
 
@@ -113,10 +113,10 @@ A resultspec is a string that selects one or more test results.  A result spec m
 
 Here are several resultspec along with the test cases they would select:
 
-- `` Selects the most recent result for every test case in every test suite.
-- `a` Selects the most recent results for every test case in the test suite with id "a".
-- `a/b` Selects the most recent result for the test case with id "b" in the test suite with id "a".
-- `a/b/c` Selects the test result with id "c" for the test case with id "b" in the test suite with id "a".
-- `a/b/~0` Selects the most recent result for the test case with id "b" in the test suite with id "a".
-- `a/b/~1` Selects the previous result for the test case with id "b" in the test suite with id "a".
-- `a/b/~4` Selects the 4 test older than the previous result for the test case with id "b" in the test suite with id "a".
+"" - Selects the most recent result for every test case in every test suite.
+"a" - Selects the most recent results for every test case in the test suite with id "a".
+"a/b" - Selects the most recent result for the test case with id "b" in the test suite with id "a".
+"a/b/c" - Selects the test result with id "c" for the test case with id "b" in the test suite with id "a".
+"a/b/~0" - Selects the most recent result for the test case with id "b" in the test suite with id "a".
+"a/b/~1" - Selects the previous result for the test case with id "b" in the test suite with id "a".
+"a/b/~4" - Selects the 4 test older than the previous result for the test case with id "b" in the test suite with id "a".
diff --git a/hdat/hdat_cli.py b/hdat/hdat_cli.py
new file mode 100644
index 0000000..a995af4
--- /dev/null
+++ b/hdat/hdat_cli.py
@@ -0,0 +1,111 @@
+import argparse
+import traceback
+
+from .resultspec import resolve_resultspecs, print_resultspec
+from .casespec import resolve_casespecs, select_suite
+from .runner import run_cases
+from .util import AbortError
+
+
+def parse_arguments(arguments):
+    parser = argparse.ArgumentParser(prog='hdat')
+    subparsers = parser.add_subparsers(dest='command', metavar='<command>')
+
+    run_help = 'run cases, store results in archive, compare against goldens'
+    run_parser = subparsers.add_parser('run', help=run_help)
+    run_parser.add_argument('casespecs', nargs='*', default=[''], metavar='<case>')
+    run_parser.add_argument('--collect-only', default=False, action='store_true')
+
+    show_help = 'visualize a single result'
+    show_parser = subparsers.add_parser('show', help=show_help)
+    show_result_help = 'result specifier to show'
+    show_parser.add_argument('resultspec', nargs="?", default='', metavar='<result>', help=show_result_help)
+
+    runshow_help = 'run then visualize a single result'
+    runshow_parser = subparsers.add_parser('runshow', help=runshow_help)
+    runshow_parser.add_argument('casespecs', nargs=1, default='', metavar='<result>')
+
+    diff_help = 'compare two results'
+    diff_parser = subparsers.add_parser('diff', help=diff_help)
+    diff_result_help = 'results being compared'
+    diff_parser.add_argument('resultspec', nargs=2, metavar='<result>', help=diff_result_help)
+
+    verify_help = 'move result metrics from archive to the golden store'
+    verify_parser = subparsers.add_parser('verify', help=verify_help)
+    verify_result_help = 'results to be stripped and moved into the golden store'
+    verify_parser.add_argument('resultspec', nargs='?', default='', metavar='<result>', help=verify_result_help)
+
+    return parser.parse_args(arguments)
+
+
+def _format_cases_status(cases_status):
+    return 'PASS: {}, FAIL: {}, UNKNOWN: {}, ERROR: {}'.format(
+        cases_status['pass'],
+        cases_status['fail'],
+        cases_status['unknown'],
+        cases_status['error'],
+    )
+
+
+def hdat_cli(arguments, suites, golden_store, archive, git_info):
+    args = parse_arguments(arguments)
+
+    if args.command is None:
+        parse_arguments(['-h'])
+
+    if args.command == 'run' and args.collect_only:
+        cases = resolve_casespecs(suites, args.casespecs)
+        print("\n".join(['{}/{}'.format(suite_id, case_id) for suite_id, case_id in cases]))
+    elif args.command == 'run':
+        cases = resolve_casespecs(suites, args.casespecs)
+        cases_status = run_cases(suites, golden_store, archive, git_info, cases)
+        if cases_status['pass'] < len(cases):
+            raise AbortError(_format_cases_status(cases_status))
+    elif args.command == 'show':
+        results = resolve_resultspecs(archive, args.resultspec)
+        for result in results:
+            show_result(suites, result)
+    elif args.command == 'runshow':
+        cases = resolve_casespecs(suites, args.casespecs)
+        cases_status = run_cases(suites, golden_store, archive, git_info, cases)
+        if cases_status['error'] > 0:
+            raise AbortError(_format_cases_status(cases_status))
+        results = resolve_resultspecs(archive, args.casespecs)
+        for result in results:
+            show_result(suites, result)
+    elif args.command == 'diff':
+        golden_results = resolve_resultspecs(archive, args.resultspec[0])
+        results = resolve_resultspecs(archive, args.resultspec[1])
+        for golden_result, result in zip(golden_results, results):
+            diff_results(suites, golden_result, result)
+    elif args.command == 'verify':
+        results = resolve_resultspecs(archive, args.resultspec)
+        for result in results:
+            golden_store.insert(result)
+
+
+def show_result(suites, result):
+    suite = select_suite(suites, result['suite_id'])
+    try:
+        suite.show(result)
+    except Exception as e:
+        traceback.print_exc()
+        msg = 'Error when attempting to show "{}": {}'
+        raise AbortError(msg.format(print_resultspec(result), e))
+
+
+def diff_results(suites, golden_result, result):
+    suite_id = result['suite_id']
+    golden_suite_id = golden_result['suite_id']
+
+    if golden_suite_id != suite_id:
+        msg = 'Can not diff results from different suites "{}" and "{}"'
+        raise AbortError(msg.format(golden_suite_id, suite_id))
+
+    suite = select_suite(suites, suite_id)
+    try:
+        suite.diff(golden_result, result)
+    except Exception as e:
+        traceback.print_exc()
+        msg = 'Error when attempting to show "{}": {}'
+        raise AbortError(msg.format(print_resultspec(result), e))
diff --git a/hdat/main.py b/hdat/main.py
old mode 100644
new mode 100755
index 2e763c6..b35ba0b
--- a/hdat/main.py
+++ b/hdat/main.py
@@ -1,132 +1,41 @@
-import argparse
-import traceback
-from collections import OrderedDict
+#!/usr/bin/env python3
+import sys
+import os
 
-import tabulate
+from hdat.hdat_cli import hdat_cli
+from hdat.suite import collect_suites
+from hdat.source_control import git_info_from_directory
+from hdat.util import repository_root, print_error, AbortError
+from hdat.store import Archive, GoldenStore
 
-from .resultspec import resolve_resultspec, print_resultspec
-from .casespec import resolve_casespecs, select_suite
-from .runner import run_cases
-from .util import AbortError
 
+def main():
+    cwd = os.getcwd()
 
-def parse_arguments(arguments):
-    parser = argparse.ArgumentParser(prog='hdatt')
-    subparsers = parser.add_subparsers(dest='command', metavar='<command>')
-
-    run_help = 'run cases, store results in archive, compare against goldens'
-    run_parser = subparsers.add_parser('run', help=run_help)
-    run_parser.add_argument('casespecs', nargs='*', default=[''], metavar='<case>')
-    run_parser.add_argument('--collect-only', default=False, action='store_true')
-
-    show_help = 'visualize a single result'
-    show_parser = subparsers.add_parser('show', help=show_help)
-    show_result_help = 'result specifier to show'
-    show_parser.add_argument('resultspec', nargs="?", default='', metavar='<result>', help=show_result_help)
-
-    runshow_help = 'run then visualize a single result'
-    runshow_parser = subparsers.add_parser('runshow', help=runshow_help)
-    runshow_parser.add_argument('casespecs', nargs=1, default='', metavar='<result>')
-
-    diff_help = 'compare two results'
-    diff_parser = subparsers.add_parser('diff', help=diff_help)
-    diff_golden_help = 'result to compare to (defaults to current golden for the case)'
-    diff_parser.add_argument('goldenspec', nargs='?', default=None, metavar='<golden>', help=diff_golden_help)
-    diff_result_help = 'result being compared'
-    diff_parser.add_argument('resultspec', nargs=1, metavar='<result>', help=diff_result_help)
-
-    verify_help = 'move result metrics from archive to the golden store'
-    verify_parser = subparsers.add_parser('verify', help=verify_help)
-    verify_result_help = 'results to be stripped and moved into the golden store'
-    verify_parser.add_argument('resultspec', nargs='?', default='', metavar='<result>', help=verify_result_help)
-
-    report_help = 'Print a report from the most recent results for a given suite.'
-    report_parser = subparsers.add_parser('report', help=report_help)
-    report_help = 'suite to generate and print out a report for.'
-    report_parser.add_argument('suitespec', default='', metavar='<suite>', help=report_help)
-
-    return parser.parse_args(arguments)
-
-
-def _format_cases_status(cases_status):
-    return 'PASS: {}, FAIL: {}, UNKNOWN: {}, ERROR: {}'.format(
-        cases_status['pass'],
-        cases_status['fail'],
-        cases_status['unknown'],
-        cases_status['error'],
-    )
-
-
-def main(arguments, suites, golden_store, archive, git_info):
-    args = parse_arguments(arguments)
+    try:
+        git_info = git_info_from_directory(cwd)
+        repo_directory = repository_root(cwd)
 
-    if args.command is None:
-        parse_arguments(['-h'])
+        if 'HDATT_ARCHIVE' in os.environ:
+            archive_location = os.environ['HDATT_ARCHIVE']
+        else:
+            archive_location = os.path.join(repo_directory, '.hdattarchive')
+        archive = Archive(archive_location)
 
-    if args.command == 'run' and args.collect_only:
-        cases = resolve_casespecs(suites, args.casespecs)
-        print("\n".join(['{}/{}'.format(s, c) for s, c in cases]))
-    elif args.command == 'run':
-        cases = resolve_casespecs(suites, args.casespecs)
-        cases_status = run_cases(suites, golden_store, archive, git_info, cases)
-        if cases_status['pass'] < len(cases):
-            raise AbortError(_format_cases_status(cases_status))
-    elif args.command == 'show':
-        result = resolve_resultspec(archive, args.resultspec)
-        show_result(suites, result)
-    elif args.command == 'runshow':
-        cases = resolve_casespecs(suites, args.casespecs)
-        cases_status = run_cases(suites, golden_store, archive, git_info, cases)
-        if cases_status['error'] > 0:
-            raise AbortError(_format_cases_status(cases_status))
-        result = resolve_resultspec(archive, args.casespecs[0])
-        show_result(suites, result)
-    elif args.command == 'diff':
-        golden_result = resolve_resultspec(archive, args.goldenspec)
-        result = resolve_resultspec(archive, args.resultspec)
-        diff_results(suites, golden_result, result)
-    elif args.command == 'verify':
-        result = resolve_resultspec(archive, args.resultspec)
-        golden_store.insert(result)
-    elif args.command == 'report':
-        suite = select_suite(suites, args.suitespec)
-        result_specs = ['{}/{}'.format(args.suitespec, case_id) for case_id in suite.collect()]
-        results = [resolve_resultspec(archive, result_spec) for result_spec in result_specs]
-        table = []
-        for result in results:
-            row = OrderedDict({'case id': result['case_id']})
-            row['TPF'] = result['metrics']['TPF']
-            row['FPF'] = result['metrics']['FPF']
-            table.append(row)
+        golden_store_location = os.path.join(repo_directory, 'golden_results')
+        golden_store = GoldenStore(golden_store_location)
 
-        print(tabulate.tabulate(table, headers="keys", tablefmt="psql", floatfmt=".5f"))
-        # TODO: Find better way to print results table. Currently it only works
-        # with feature detection hdat.  e.g. allow the user to specify two
-        # metrics (e.g. using jsonrefs) for the x and y axis of the table
+        suites = collect_suites(cwd)
 
+        hdat_cli(sys.argv[1:], suites, golden_store, archive, git_info)
 
-def show_result(suites, result):
-    suite = select_suite(suites, result['suite_id'])
-    try:
-        suite.show(result)
-    except Exception as e:
-        traceback.print_exc()
-        msg = 'Error when attempting to show "{}": {}'
-        raise AbortError(msg.format(print_resultspec(result), e))
+        sys.exit(0)
 
+    except AbortError as e:
+        print_error(e)
 
-def diff_results(suites, golden_result, result):
-    suite_id = result['suite_id']
-    golden_suite_id = golden_result['result_id']
+        sys.exit(1)
 
-    if golden_suite_id != suite_id:
-        msg = 'Can not diff results from different suites "{}" and "{}"'
-        raise AbortError(msg.format(golden_suite_id, suite_id))
 
-    suite = select_suite(suites, suite_id)
-    try:
-        suite.diff(golden_result, result)
-    except Exception as e:
-        traceback.print_exc()
-        msg = 'Error when attempting to show "{}": {}'
-        raise AbortError(msg.format(print_resultspec(result), e))
+if __name__ == '__main__':
+    main()
diff --git a/hdat/resultspec.py b/hdat/resultspec.py
index afcf739..9851c8a 100644
--- a/hdat/resultspec.py
+++ b/hdat/resultspec.py
@@ -4,10 +4,10 @@ import traceback
 from .util import AbortError
 
 
-def resolve_resultspec(archive, resultspec):
+def resolve_resultspecs(archive, resultspec):
     if os.path.isfile(resultspec):
         try:
-            return archive.read_result(resultspec)
+            return [archive.read_result(resultspec)]
         except Exception:
             traceback.print_exc()
             msg = 'Unable to read resultspec "{}"'
@@ -18,12 +18,14 @@ def resolve_resultspec(archive, resultspec):
     else:
         resultspec_parts = resultspec.split('/')
 
-    pick_latest = len(resultspec_parts) == 2
+    pick_recents_all_suites = len(resultspec_parts) == 0
+    pick_recents_one_suite = len(resultspec_parts) == 1
+    pick_recent = len(resultspec_parts) == 2
     pick_by_index = len(resultspec_parts) == 3 and resultspec_parts[2].startswith('~')
     pick_by_result_id = len(resultspec_parts) == 3 and not pick_by_index
 
-    if pick_latest or pick_by_index:
-        if pick_latest:
+    if pick_recent or pick_by_index:
+        if pick_recent:
             i = -1
         else:
             try:
@@ -32,27 +34,23 @@ def resolve_resultspec(archive, resultspec):
                 msg = 'Invalid resultspec "{}"; the third part must be a valid result_id' + \
                     'or a tilde followed by an integer, but not "{}"'
                 raise AbortError(msg.format(resultspec, resultspec_parts[2][1]))
-
-        # TODO: make this faster in the event there are many results
-        results = archive.select_all(*resultspec_parts[:2])
-        if len(results) == 0:
-            msg = 'Unable to locate any results matching "{}" in the archive at "{}"'
-            raise AbortError(msg.format(resultspec, archive.root))
-        else:
-            try:
-                return results[i]
-            except IndexError:
-                msg = 'Unable to locate any results matching "{}", there are only {} results present.'
-                raise AbortError(msg.format(resultspec, i))
-
+        try:
+            result = archive.select_recent(i, *resultspec_parts[:2])
+            return [result]
+        except IndexError:
+            msg = 'Unable to locate any results matching "{}", there are more than {} results present.'
+            raise AbortError(msg.format(resultspec, i))
     elif pick_by_result_id:
         result = archive.select(*resultspec_parts)
         if result is None:
             msg = 'Unable to locate result "{}" in the archive at "{}"'
             raise AbortError(msg.format(resultspec, archive.root))
         else:
-            return result
-
+            return [result]
+    elif pick_recents_all_suites:
+        return archive.select_recents_all()
+    elif pick_recents_one_suite:
+        return archive.select_recents_suite(resultspec_parts[0])
     else:
         msg = 'Invalid result spec "{}". ' + \
               'Resultspecs must point to a result file, ' + \
diff --git a/hdat/runner.py b/hdat/runner.py
index e57ba50..db8e82b 100644
--- a/hdat/runner.py
+++ b/hdat/runner.py
@@ -6,7 +6,7 @@ from .casespec import print_casespec
 
 def run_cases(suites, golden_store, archive, git_info, cases):
     '''
-    Run a list of cases, store results in the archive, and verify against
+    Run a list of cases, store results in the archive, and check against
     results in the golden_store.
 
     Cases are specified by a list of tuples of the form `(suite_id, case_id)`.
@@ -47,7 +47,7 @@ def run_case(suite, golden_store, archive, git_info, case_id):
     if golden_result is None:
         status, comments = 'unknown', 'No golden result present'
     else:
-        passed, comments = suite.verify(golden_result['metrics'], metrics)
+        passed, comments = suite.check(golden_result['metrics'], metrics)
         status = 'pass' if passed else 'fail'
 
     result = build_result(suite, git_info, case_id, case_input, metrics, context, status)
@@ -61,6 +61,10 @@ def validate_result(run_result):
         raise ValueError(msg.format(repr(run_result)))
 
 
+def build_result_id(result):
+        return '{}_{}'.format(result['ran_on'], result['commit'])
+
+
 def build_result(suite, git_info, case_id, case_input, metrics, context, status):
     run_datetime = datetime.datetime.utcnow()
     result = {
@@ -75,5 +79,5 @@ def build_result(suite, git_info, case_id, case_input, metrics, context, status)
         'status': status,
     }
 
-    result['result_id'] = suite.build_result_id(result)
+    result['result_id'] = build_result_id(result)
     return result
diff --git a/hdat/store.py b/hdat/store.py
index 5f29d80..e1b890c 100644
--- a/hdat/store.py
+++ b/hdat/store.py
@@ -2,6 +2,9 @@ import os
 import json
 import copy
 import pickle
+from collections import namedtuple
+
+from .util import AbortError
 
 
 class Archive:
@@ -17,21 +20,50 @@ class Archive:
         else:
             return self.read_result(result_filename)
 
-    def select_all(self, *args):
+    def select_recent(self, i, *args):
         top_directory = os.path.join(self.root, *args)
-
-        result_filenames = []
-        for dirpath, _, filenames in os.walk(top_directory):
-            result_filenames.extend([os.path.join(dirpath, p) for p in filenames])
+        if not os.path.isdir(top_directory):
+            msg = "Selected case directory {} does not exist or is not a directory"
+            raise AbortError(msg.format(top_directory))
 
         results = []
-        for filename in result_filenames:
-            result = self.read_result(filename)
-            results.append(result)
-
-        results_sorted = sorted(results, key=lambda r: r['ran_on'])
+        ResultDesc = namedtuple('ResultDesc', ('id', 'ran_on'))
+        id_to_full_result = dict()
+
+        for entry in os.listdir(top_directory):
+            if not entry.startswith('.') and os.path.isfile(os.path.join(top_directory, entry)):
+                # check for ran_on timestamp as part of <timestamp>_<commit_id> result ID format
+                try:
+                    ran_on = float(entry.split('_')[0])
+                    results.append(ResultDesc(entry, ran_on))
+                except ValueError:
+                    result = self.read_result(os.path.join(top_directory, entry))
+                    ran_on = result['ran_on']
+                    id_to_full_result[entry] = result
+                    results.append(ResultDesc(entry, ran_on))
+
+        results_sorted = sorted(results, key=lambda r: r.ran_on)
+        recent_id = results_sorted[i].id
+
+        if recent_id in id_to_full_result:
+            return id_to_full_result[recent_id]
+        else:
+            return self.read_result(os.path.join(top_directory, recent_id))
 
-        return results_sorted
+    def select_recents_suite(self, *args):
+        top_directory = os.path.join(self.root, *args)
+        if not os.path.isdir(top_directory):
+            msg = "Selected suite directory {} does not exist or is not a directory"
+            raise AbortError(msg.format(top_directory))
+
+        for entry in os.listdir(top_directory):
+            if not entry.startswith('.') and os.path.isdir(os.path.join(top_directory, entry)):
+                yield self.select_recent(-1, *(args+(entry,)))
+
+    def select_recents_all(self):
+        for entry in os.listdir(self.root):
+            if not entry.startswith('.') and os.path.isdir(os.path.join(self.root, entry)):
+                yield from self.select_recents_suite(entry)
 
     def insert(self, result):
         suite_id = result['suite_id']
diff --git a/hdat/suite.py b/hdat/suite.py
index b730350..a45c80c 100644
--- a/hdat/suite.py
+++ b/hdat/suite.py
@@ -7,7 +7,7 @@ class Suite:
     '''
     Base class for a suite of algorithm test cases.
 
-    Is responsible for collecting, running, verifying, and visualizing the
+    Is responsible for collecting, running, checking, and visualizing the
     results of running the algorithm against its test cases.
     '''
     def cases(self):
@@ -18,9 +18,9 @@ class Suite:
         '''
         raise NotImplementedError()
 
-    def verify(self, golden_metrics, metrics):
+    def check(self, golden_metrics, metrics):
         '''
-        Given two result comparable outputs, verify if the second result passes based on
+        Given two result comparable outputs, check if the second result passes based on
         the first result.  Should return a tuple with a boolean and a string
         with any comments.
         '''
@@ -47,9 +47,6 @@ class Suite:
     def id(self):
         return type(self).__name__
 
-    def build_result_id(self, result):
-        return '{}_{}'.format(result['ran_on'], result['commit'])
-
 
 def collect_suites(directory):
     suite_classes = _collect_suite_classes(directory)
diff --git a/hdat_main_script b/hdat_main_script
deleted file mode 100755
index 9e40c89..0000000
--- a/hdat_main_script
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/env python
-import sys
-import os
-
-from hdatt.main import main
-from hdatt.suite import collect_suites
-from hdatt.source_control import git_info_from_directory
-from hdatt.util import repository_root, print_error, AbortError
-from hdatt.store import Archive, GoldenStore
-
-
-if __name__ == '__main__':
-    cwd = os.getcwd()
-
-    try:
-        git_info = git_info_from_directory(cwd)
-        repo_directory = repository_root(cwd)
-
-        if 'HDATT_ARCHIVE' in os.environ:
-            archive_location = os.environ['HDATT_ARCHIVE']
-        else:
-            archive_location = os.path.join(repo_directory, '.hdattarchive')
-        archive = Archive(archive_location)
-
-        golden_store_location = os.path.join(repo_directory, 'golden_results')
-        golden_store = GoldenStore(golden_store_location)
-
-        suites = collect_suites(cwd)
-
-        main(sys.argv[1:], suites, golden_store, archive, git_info)
-
-        sys.exit(0)
-
-    except AbortError as e:
-        print_error(e)
-
-        sys.exit(1)
diff --git a/setup.py b/setup.py
index acce29c..2621d2f 100644
--- a/setup.py
+++ b/setup.py
@@ -38,7 +38,7 @@ setup(
 
     packages=find_packages(exclude=['contrib', 'docs', 'tests']),
 
-    install_requires=['gitpython', 'tabulate'],
+    install_requires=['gitpython'],
 
     extras_require={
         'dev': ['check-manifest', 'sphinx', 'sphinx-autobuild', 'mock'],
@@ -47,5 +47,9 @@ setup(
 
     package_data={},
     data_files=[],
-    entry_points={},
+    entry_points={
+        'console_scripts': [
+            'hdat = hdat.main:main'
+        ]
+    },
 )


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
index 7de6259..8374aa9 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -23,7 +23,7 @@ def tmp_archive():
 
 
 class BaseSuite(Suite):
-    def verify(self, old, new):
+    def check(self, old, new):
         return old == new, 'Looks good!'
 
     def run(self, case_input):
@@ -81,3 +81,40 @@ def mock_suites(basic_suite_a, basic_suite_b):
         'a': basic_suite_a(),
         'b': basic_suite_b(),
     }
+
+
+@pytest.fixture
+def mock_results():
+    return [
+        {
+            'suite_id': 'a',
+            'case_id': '1',
+            'result_id': 'r1',
+            'ran_on': 100,
+        },
+        {
+            'suite_id': 'a',
+            'case_id': '1',
+            'result_id': '101_r2',
+            'ran_on': 101,
+        },
+        {
+            'suite_id': 'a',
+            'case_id': '2',
+            'result_id': '103_r3',
+            'ran_on': 103,
+        },
+        {
+            'suite_id': 'b',
+            'case_id': '1',
+            'result_id': '103_r4',
+            'ran_on': 104,
+        },
+    ]
+
+
+@pytest.fixture
+def archive(tmp_archive, mock_results):
+    for result in mock_results:
+        tmp_archive.insert(result)
+    return tmp_archive
diff --git a/tests/main_test.py b/tests/main_test.py
index 52d4e81..399dd97 100644
--- a/tests/main_test.py
+++ b/tests/main_test.py
@@ -1,37 +1,43 @@
 import pytest
 
-from hdat.main import main
+from hdat.hdat_cli import hdat_cli
 from hdat.util import AbortError
 
 
 @pytest.fixture
-def main_with_mocks(mock_suites, tmp_golden_store, tmp_archive, mock_git_info):
-    return lambda args: main(args, mock_suites, tmp_golden_store, tmp_archive, mock_git_info)
+def hdat_cli_with_mocks(mock_suites, tmp_golden_store, tmp_archive, mock_git_info):
+    return lambda args: hdat_cli(args, mock_suites, tmp_golden_store, tmp_archive, mock_git_info)
 
 
 class TestMainRun:
-    def test_run_all_verify_all_rerun(self, main_with_mocks):
+    def test_run_all_verify_all_rerun(self, hdat_cli_with_mocks):
         with pytest.raises(AbortError) as e:
-            main_with_mocks(['run'])
+            hdat_cli_with_mocks(['run'])
         assert 'UNKNOWN: 3' in str(e)
 
-        main_with_mocks(['verify', 'a/1'])
+        hdat_cli_with_mocks(['verify', 'a/1'])
 
         with pytest.raises(AbortError) as e:
-            main_with_mocks(['run'])
+            hdat_cli_with_mocks(['run'])
         assert 'UNKNOWN: 2' in str(e)
         assert 'PASS: 1' in str(e)
 
-        main_with_mocks(['verify', 'a/2'])
-        main_with_mocks(['verify', 'b/3'])
+        hdat_cli_with_mocks(['verify', 'a/2'])
+        hdat_cli_with_mocks(['verify', 'b/3'])
 
-        main_with_mocks(['run'])
+        hdat_cli_with_mocks(['run'])
 
-    def test_show_most_recent(self, main_with_mocks):
+    def test_show_most_recent(self, hdat_cli_with_mocks):
         with pytest.raises(AbortError) as e:
-            main_with_mocks(['run', 'a/1'])
+            hdat_cli_with_mocks(['run', 'a/1'])
 
         with pytest.raises(AbortError) as e:
-            main_with_mocks(['show', 'a/1'])
+            hdat_cli_with_mocks(['show', 'a/1'])
 
         assert 'showing "a/1' in str(e)
+
+    def test_diff(self, hdat_cli_with_mocks, archive, mock_results):
+        with pytest.raises(AbortError) as e:
+            hdat_cli_with_mocks(['diff', 'a/1/r1', 'a/1/101_r2'])
+
+        assert 'diffing "a/1/r1" and "a/1/101_r2"' in str(e)
diff --git a/tests/resultspec_test.py b/tests/resultspec_test.py
index 27de696..1505f1b 100644
--- a/tests/resultspec_test.py
+++ b/tests/resultspec_test.py
@@ -1,50 +1,13 @@
 import pytest
 
-from hdat.resultspec import resolve_resultspec
+from hdat.resultspec import resolve_resultspecs
 from hdat.util import AbortError
 
 
-@pytest.fixture
-def mock_results():
-    return [
-        {
-            'suite_id': 'a',
-            'case_id': '1',
-            'result_id': 'r1',
-            'ran_on': 100,
-        },
-        {
-            'suite_id': 'a',
-            'case_id': '1',
-            'result_id': 'r2',
-            'ran_on': 101,
-        },
-        {
-            'suite_id': 'a',
-            'case_id': '2',
-            'result_id': 'r3',
-            'ran_on': 103,
-        },
-        {
-            'suite_id': 'b',
-            'case_id': '1',
-            'result_id': 'r4',
-            'ran_on': 104,
-        },
-    ]
-
-
-@pytest.fixture
-def archive(tmp_archive, mock_results):
-    for result in mock_results:
-        tmp_archive.insert(result)
-    return tmp_archive
-
-
 class TestResolveResultSpec:
     def test_existing_file(self, archive, mock_results):
         resultspec = archive._result_filename('a', '1', 'r1')
-        assert resolve_resultspec(archive, resultspec) == mock_results[0]
+        assert list(resolve_resultspecs(archive, resultspec)) == [mock_results[0]]
 
     @pytest.mark.skip
     def test_existing_file_invalid(self, archive):
@@ -54,32 +17,43 @@ class TestResolveResultSpec:
     def test_nonexistant_file(self, archive):
         resultspec = archive._result_filename('a', '1', 'huh')
         with pytest.raises(AbortError):
-            resolve_resultspec(archive, resultspec)
+            resolve_resultspecs(archive, resultspec)
 
     def test_fully_qualified(self, archive, mock_results):
-        resultspec = 'a/1/r2'
-        assert resolve_resultspec(archive, resultspec) == mock_results[1]
+        resultspec = 'a/1/101_r2'
+        assert list(resolve_resultspecs(archive, resultspec)) == [mock_results[1]]
+
+    def test_relative_index_1(self, archive, mock_results):
+        resultspec = 'a/1/~0'
+        assert list(resolve_resultspecs(archive, resultspec)) == [mock_results[1]]
+
+    def test_relative_index_2(self, archive, mock_results):
+        resultspec = 'a/1/~1'
+        assert list(resolve_resultspecs(archive, resultspec)) == [mock_results[0]]
+
+    def test_relative_index_3(self, archive, mock_results):
+        resultspec = 'a/1/~2'
+        with pytest.raises(AbortError):
+            assert resolve_resultspecs(archive, resultspec)
 
     def test_fully_qualified_missing(self, archive):
         resultspec = 'a/1/huh'
         with pytest.raises(AbortError):
-            resolve_resultspec(archive, resultspec)
+            resolve_resultspecs(archive, resultspec)
 
     def test_most_recent_in_case(self, archive, mock_results):
         resultspec = 'a/1'
-        assert resolve_resultspec(archive, resultspec) == mock_results[1]
+        assert list(resolve_resultspecs(archive, resultspec)) == [mock_results[1]]
 
-    @pytest.mark.xfail  # TODO not implemented
     def test_most_recent_in_suite(self, archive, mock_results):
         resultspec = 'a'
-        assert resolve_resultspec(archive, resultspec) == mock_results[2]
+        assert list(resolve_resultspecs(archive, resultspec)) == [mock_results[1], mock_results[2]]
 
-    @pytest.mark.xfail  # TODO not implemented
-    def test_most_recent_overall(self, archive, mock_results):
+    def test_most_recent_all(self, archive, mock_results):
         resultspec = ''
-        assert resolve_resultspec(archive, resultspec) == mock_results[3]
+        assert list(resolve_resultspecs(archive, resultspec)) == [mock_results[1], mock_results[2], mock_results[3]]
 
     def test_most_recent_missing(self, archive):
         resultspec = 'a/huh'
         with pytest.raises(AbortError):
-            resolve_resultspec(archive, resultspec)
+            resolve_resultspecs(archive, resultspec)