Repository: innolitics/hdat

Issue:
Create a small assertion library for metrics
See

https://github.com/softcirs/cirs/blob/master/process/full_algorithm_hdats.py#L198
https://github.com/softcirs/cirs/blob/master/process/feature_detection_hdats.py#L111
https://github.com/softcirs/cirs/blob/master/process/registration_hdats.py#L239

for inspiration.  We would want to keep the number of supported assertions quite small.

Comments:

Patch:
diff --git a/hdat/suite.py b/hdat/suite.py
index f67d04d..de6dbd2 100644
--- a/hdat/suite.py
+++ b/hdat/suite.py
@@ -97,3 +97,82 @@ def _collect_suite_classes(directory):
                     if hdat_suite_class in inspect.getmro(value) and hdat_suite_class != value:
                         suite_classes.append(value)
     return suite_classes
+
+
+def ignore_key_errors(decoratee):
+    def decorated(*args, **kwargs):
+        try:
+            return decoratee(*args, **kwargs)
+        except KeyError:
+            pass
+    return decorated
+
+
+class MetricsChecker:
+    '''
+    Helper class for comparing metrics of new results vs golden results in suite.check()
+    '''
+    def __init__(self, old, new):
+        self._old = old
+        self._new = new
+        self._match = True
+        self._msgs = []
+        missing_metrics = set(old.keys()).difference(set(new.keys()))
+        new_metrics = set(new.keys()).difference(set(old.keys()))
+        if missing_metrics:
+            msg = 'Metric(s) in golden result {} were missing from the new run'
+            self._msgs.append(msg.format(missing_metrics))
+            self._match = False
+        if new_metrics:
+            msg = 'New metric(s) {} added to hdat test suite'
+            self._msgs.append(msg.format(new_metrics))
+
+    def _isclose(self, a, b, rel_tol=1e-09, abs_tol=0.0):
+        '''Fills in for math.isclose in 3.4'''
+        return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)
+
+    @ignore_key_errors
+    def close(self, metric, **kwargs):
+        if not self._isclose(self._old[metric], self._new[metric], **kwargs):
+            self._match = False
+            msg = 'Metric {} value {} was not close to golden value {} within {}'
+            self._msgs.append(msg.format(metric, self._new[metric], self._old[metric], kwargs))
+
+    @ignore_key_errors
+    def exact(self, metric):
+        if self._old[metric] != self._new[metric]:
+            self._match = False
+            msg = 'Metric {} value {} did not match golden value {}'
+            self._msgs.append(msg.format(metric, self._new[metric], self._old[metric]))
+
+    @ignore_key_errors
+    def custom(self, metric, func=None):
+        result, msg = func(self._old[metric], self._new[metric])
+        if not result:
+            self._match = False
+            self._msgs.append(msg)
+
+    @ignore_key_errors
+    def can_increase(self, metric, abs_tol=0.0):
+        if (self._new[metric] + abs_tol < self._old[metric] or
+                self._isclose(self._new[metric], self._old[metric])):
+            self._match = False
+            msg = 'Metric {} value {} decreased over golden value {} more than {}'
+            self._msgs.append(msg.format(metric, self._new[metric], self._old[metric], abs_tol))
+
+    @ignore_key_errors
+    def can_decrease(self, metric, abs_tol=0.0):
+        if (self._new[metric] - abs_tol > self._old[metric] or
+                self._isclose(self._new[metric], self._old[metric])):
+            self._match = False
+            msg = 'Metric {} value {} increased over golden value {} more than {}'
+            self._msgs.append(msg.format(metric, self._new[metric], self._old[metric], abs_tol))
+
+    def msgs(self):
+        return self._msgs
+
+    def match(self):
+        return self._match
+
+    def result(self):
+        return self._match, self._msgs


Test patch:
diff --git a/tests/main_test.py b/tests/main_test.py
index 16e9363..399dd97 100644
--- a/tests/main_test.py
+++ b/tests/main_test.py
@@ -1,9 +1,7 @@
 import pytest
-import os
 
 from hdat.hdat_cli import hdat_cli
 from hdat.util import AbortError
-from hdat.suite import collect_suites
 
 
 @pytest.fixture
@@ -43,8 +41,3 @@ class TestMainRun:
             hdat_cli_with_mocks(['diff', 'a/1/r1', 'a/1/101_r2'])
 
         assert 'diffing "a/1/r1" and "a/1/101_r2"' in str(e)
-
-    def test_collect_suites(self):
-        test_path = os.path.dirname(__file__)
-        suites = collect_suites(test_path)
-        assert suites.keys() == set(['BaseSuite', 'a', 'b'])
diff --git a/tests/suite_test.py b/tests/suite_test.py
new file mode 100644
index 0000000..cffacb8
--- /dev/null
+++ b/tests/suite_test.py
@@ -0,0 +1,103 @@
+import pytest
+
+import os
+
+from hdat.suite import collect_suites, MetricsChecker
+
+
+class TestSuite:
+
+    def test_collect_suites(self):
+        test_path = os.path.dirname(__file__)
+        suites = collect_suites(test_path)
+        assert suites.keys() == set(['BaseSuite', 'a', 'b'])
+
+    def test_metrics_checker_bad_keys(self):
+        old = {'key1': 1, 'key3': 'value1'}
+        new = {'key2': 1, 'key3': 'value2'}
+        checker = MetricsChecker(old, new)
+        checker.exact('key3')
+        assert checker.match() is False
+        assert len(checker.msgs()) == 3  # key3 is also checked + new key2
+
+    def test_metrics_checker_new_key(self):
+        old = {'oldkey': 1}
+        new = {'oldkey': 1, 'newkey': 'value'}
+        checker = MetricsChecker(old, new)
+        checker.exact('oldkey')
+        assert checker.match() is True
+        assert len(checker.msgs()) == 1 and 'newkey' in checker.msgs()[0]
+
+    @pytest.mark.parametrize("old, new, method, expected", [
+        ({'item': 1.00}, {'item': 1.000000001}, 'exact', False),
+        ({'item': 1}, {'item': 1}, 'exact', True),
+        ({'item': 1}, {'item': 2}, 'close', False),
+        ({'item': 1.00}, {'item': 1.0000000001}, 'close', True),
+        ({'item': 4}, {'item': 3}, 'can_increase', False),
+        ({'item': 4}, {'item': 5}, 'can_increase', True),
+        ({'item': 4}, {'item': 5}, 'can_decrease', False),
+        ({'item': 4}, {'item': 3}, 'can_decrease', True)
+    ])
+    def test_checker_single(self, old, new, method, expected):
+        checker = MetricsChecker(old, new)
+        check_func = getattr(checker, method)
+        check_func(list(old.keys())[0])
+        assert checker.match() is expected and checker.result()[0] is expected
+
+    def test_custom_check(self):
+        old = {'item': 10}
+        new_good = {'item': 5}
+        new_bad = {'item': 4}
+        checker_good = MetricsChecker(old, new_good)
+        checker_bad = MetricsChecker(old, new_bad)
+        checker_good.custom('item', func=lambda x, y: (x % y == 0 or y % x == 0, "no factor"))
+        checker_bad.custom('item', func=lambda x, y: (x % y == 0 or y % x == 0, "no factor"))
+        assert checker_good.match() is True
+        assert checker_bad.match() is False
+
+    def test_can_increase_with_tol(self):
+        old = {'item': 10}
+        new_good = {'item': 9}
+        new_bad = {'item': 8}
+        checker_good = MetricsChecker(old, new_good)
+        checker_bad = MetricsChecker(old, new_bad)
+        checker_good.can_increase('item', abs_tol=1)
+        checker_bad.can_increase('item', abs_tol=1)
+        assert checker_good.match() is True
+        assert checker_bad.match() is False
+        assert True
+
+    def test_can_decrease_with_tol(self):
+        old = {'item': 10}
+        new_good = {'item': 11}
+        new_bad = {'item': 12}
+        checker_good = MetricsChecker(old, new_good)
+        checker_bad = MetricsChecker(old, new_bad)
+        checker_good.can_decrease('item', abs_tol=1)
+        checker_bad.can_decrease('item', abs_tol=1)
+        assert checker_good.match() is True
+        assert checker_bad.match() is False
+        assert True
+
+    def test_metrics_checker_multi_success(self):
+        old = {'float': 1.00, 'float_increase': 1.00, 'float_decrease': 1.00, 'string': 'one', 'custom': 2}
+        new = {'float': 1.0000000001, 'float_increase': 1.1, 'float_decrease': 0.9, 'string': 'one', 'custom': 4}
+        checker = MetricsChecker(old, new)
+        checker.close('float', abs_tol=0.01)
+        checker.can_increase('float_increase')
+        checker.can_decrease('float_decrease')
+        checker.exact('string')
+        checker.custom('custom', lambda x, y: (x % 2 == 0 and y % 2 == 0, 'not even'))
+        assert checker.match() is True and checker.result()[0] is True
+
+    def test_metrics_checker_multi_fail(self):
+        old = {'float': 1.00, 'float_increase': 1.00, 'float_decrease': 1.00, 'string': 'one', 'custom': 2}
+        new = {'float': 1.01, 'float_increase': 0.89, 'float_decrease': 1.11, 'string': 'two', 'custom': 5}
+        checker = MetricsChecker(old, new)
+        checker.close('float')
+        checker.can_increase('float_increase', abs_tol=0.1)
+        checker.can_decrease('float_decrease', abs_tol=0.1)
+        checker.exact('string')
+        checker.custom('custom', lambda x, y: (x % 2 == 0 and y % 2 == 0, 'not even'))
+        assert checker.match() is False and checker.result()[0] is False
+        assert len(checker.msgs()) == 5