Repository: CORE-GATECH-GROUP/serpent-tools
Pull request #228 (2018-08-10 17:48:57)

[API] Implement comparison methods for detectors
Addresses some aspects of #150 by allowing Detector objects and DetectorReaders to compare against each other.

Grids are checked before checking the tallies to raise any differences, e.g. slightly different energy grids or totally different bin structures. Tallies are compared only if their shapes agree. Scores are compared only if both have a score attribute.

Added testing for modified and unmodified grid and tally data for both a detector object and a reader.

- [x] PR fits the [project scope](http://serpent-tools.readthedocs.io/en/latest/develop/contributing.html#project-scope)
- [x] Code to be committed is [DRY](https://en.wikipedia.org/wiki/Don%27t_repeat_yourself) and matches the [code style](http://serpent-tools.readthedocs.io/en/latest/develop/codestyle.html#code-style) of the project
- [x] New and/or changed features are [well documented](http://serpent-tools.readthedocs.io/en/latest/develop/documentation.html#documentation) and have adequate testing


Patch:
diff --git a/serpentTools/objects/base.py b/serpentTools/objects/base.py
index 7244751..eadb0fe 100644
--- a/serpentTools/objects/base.py
+++ b/serpentTools/objects/base.py
@@ -8,11 +8,20 @@ from six import add_metaclass
 from numpy import arange, hstack, log, divide
 from matplotlib.pyplot import axes
 
-from serpentTools.messages import debug, warning, SerpentToolsException, info
+from serpentTools.messages import (
+    debug, warning, SerpentToolsException, info,
+    error,
+    BAD_OBJ_SUBJ,
+
+)
 from serpentTools.plot import plot, cartMeshPlot
 from serpentTools.utils import (
     magicPlotDocDecorator, formatPlot, DETECTOR_PLOT_LABELS,
-    compareDocDecorator, DEF_COMP_LOWER, DEF_COMP_SIGMA, DEF_COMP_UPPER,
+    compareDocDecorator, DEF_COMP_LOWER, DEF_COMP_SIGMA,
+    DEF_COMP_UPPER, compareDictOfArrays,
+)
+from serpentTools.utils.compare import (
+    getLogOverlaps, finalCompareMsg,
 )
 from serpentTools.settings import rc
 
@@ -85,6 +94,11 @@ class BaseObject(object):
 
         areSimilar = self._compare(other, lower, upper, sigma)
 
+        if areSimilar:
+            herald = info
+        else:
+            herald = warning
+        herald(finalCompareMsg(self, other, areSimilar))
         if previousVerb is not None:
             rc['verbosity'] = previousVerb
 
@@ -523,3 +537,34 @@ class DetectorBase(NamedObject):
         if qty in self.indexes:
             return self.indexes[qty], xlabel
         return fallbackX, xlabel
+
+    def _compare(self, other, lower, upper, sigma):
+        myShape = self.tallies.shape
+        otherShape = other.tallies.shape
+        if myShape != otherShape:
+            error("Detector tallies do not have identical shapes"
+                  + BAD_OBJ_SUBJ.format('tallies', myShape, otherShape))
+            return False
+        similar = compareDictOfArrays(self.grids, other.grids, 'grids',
+                                      lower=lower, upper=upper)
+
+        similar &= getLogOverlaps('tallies', self.tallies, other.tallies,
+                                  self.errors, other.errors, sigma,
+                                  relative=True)
+        hasScores = [obj.scores is not None for obj in (self, other)]
+
+        similar &= hasScores[0] == hasScores[1]
+
+        if not any(hasScores):
+            return similar
+        if all(hasScores):
+            similar &= getLogOverlaps('scores', self.scores, other.scores,
+                                      self.errors, other.errors, sigma,
+                                      relative=True)
+            return similar
+        firstK, secondK = "first", "second"
+        if hasScores[1]:
+            firstK, secondK = secondK, firstK
+        error("{} detector has scores while {} does not"
+              .format(firstK.capitalize(), secondK))
+        return similar
diff --git a/serpentTools/parsers/detector.py b/serpentTools/parsers/detector.py
index 9053962..e63bfaa 100644
--- a/serpentTools/parsers/detector.py
+++ b/serpentTools/parsers/detector.py
@@ -4,6 +4,7 @@ from six import iteritems
 from numpy import empty
 
 from serpentTools.utils import str2vec
+from serpentTools.utils.compare import getKeyMatchingShapes
 from serpentTools.engines import KeywordParser
 from serpentTools.objects.detectors import detectorFactory
 from serpentTools.parsers.base import BaseReader
@@ -95,6 +96,20 @@ class DetectorReader(BaseReader):
         if not self.detectors:
             warning("No detectors stored from file {}".format(self.filePath))
 
+    def _compare(self, other, lower, upper, sigma):
+        """Compare two detector readers."""
+        similar = len(self.detectors) == len(other.detectors)
+
+        commonKeys = getKeyMatchingShapes(self.detectors, other.detectors,
+                                          'detectors')
+        similar &= len(commonKeys) == len(self.detectors)
+
+        for detName in sorted(commonKeys):
+            myDetector = self[detName]
+            otherDetector = other[detName]
+            similar &= myDetector.compare(otherDetector, lower, upper, sigma)
+        return similar
+
 
 def cleanDetChunk(chunk):
     """
diff --git a/serpentTools/tests/test_compare.py b/serpentTools/tests/compare/test_results.py
similarity index 100%
rename from serpentTools/tests/test_compare.py
rename to serpentTools/tests/compare/test_results.py
diff --git a/serpentTools/utils/compare.py b/serpentTools/utils/compare.py
index 9b958b2..4c69b08 100644
--- a/serpentTools/utils/compare.py
+++ b/serpentTools/utils/compare.py
@@ -535,3 +535,28 @@ def compareDictOfArrays(d0, d1, desc, lower=DEF_COMP_LOWER,
             continue
         similar &= logDirectCompare(val0, val1, lower, upper, key)
     return similar
+
+
+FINAL_COMPARE_MSG = "Objects {} and {}{} agree within given tolerances"
+
+
+def finalCompareMsg(obj0, obj1, similar):
+    """
+    Return the string used to signify the conclusion of a comparison
+
+    Mainly exposed to developers for testing purposes.
+
+    Parameters
+    ----------
+    obj0:
+    obj1: Subclass of :class:`~serpentTools.objects.base.BaseObject`
+        Objects that have been compared
+    similar: bool
+        Result of comparison
+
+    Returns
+    -------
+    str:
+        Concluding remark about the comparison.
+    """
+    return FINAL_COMPARE_MSG.format(obj0, obj1, "" if similar else "do not")


Test patch:
diff --git a/serpentTools/tests/compare/__init__.py b/serpentTools/tests/compare/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/serpentTools/tests/compare/test_detector.py b/serpentTools/tests/compare/test_detector.py
new file mode 100644
index 0000000..61dd824
--- /dev/null
+++ b/serpentTools/tests/compare/test_detector.py
@@ -0,0 +1,202 @@
+"""
+Test the comparison utilities for detectors
+"""
+
+from serpentTools.tests.utils import TestCaseWithLogCapture
+from serpentTools.utils.compare import finalCompareMsg
+from serpentTools import readDataFile
+
+TEST_FILE = 'bwr_det0.m'
+DET_TO_MODIFY = 'xymesh'
+UNMODIFIED_DETS = [
+    'spectrum',
+]
+# All other detectors that shall be unmodified
+
+
+def compareObjs(ref, other):
+    """Fixture to call identical comparisons across tests."""
+    return ref.compare(other, verbosity='debug')
+
+
+class DetCompareHelper(TestCaseWithLogCapture):
+    """
+    Helper case for testing the detector compare methods
+
+    setUpClass:
+        *. Read the ``bwr_det0.m`` file
+        *. Store this reader on the class
+
+    setUp:
+        *. Read the same file, but store this reader as the
+           one to mess up.
+        *. Call setUp from TestCaseWithLogCapture to capture
+           log messages
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        cls.refReader = readDataFile(TEST_FILE)
+        cls.refDetector = cls.refReader.detectors[DET_TO_MODIFY]
+
+    def setUp(self):
+        TestCaseWithLogCapture.setUp(self)
+        self.otherReader = readDataFile(TEST_FILE)
+        self.otherDetector = self.otherReader.detectors[DET_TO_MODIFY]
+
+
+class DetGridsCompareTester(DetCompareHelper):
+    """
+    Tweak the grid structure on a detector and ensure that logs
+    capture this.
+    """
+
+    GRID_FMT = "Values for {} are {}"
+
+    def setUp(self):
+        DetCompareHelper.setUp(self)
+        self.gridKeys = sorted(list(self.refDetector.grids.keys()))
+
+    def test_identicalGrids(self):
+        """Verify that the inner grid compare works."""
+        similar = compareObjs(self.refDetector, self.otherDetector)
+        self.assertTrue(similar, msg='output from comparison')
+        for gridKey in self.gridKeys:
+            self.assertMsgInLogs(
+                "DEBUG", self.GRID_FMT.format(gridKey, 'identical'),
+                partial=True)
+
+    def test_modifiedGrids(self):
+        """Pick up on some errors due to modified grids."""
+        # Changes
+        # X grid unchanged
+        # E grid slightly modified, but inside tolerances
+        # Y grid modified to be outside tolerances
+        # Z grid set to be of different shape
+        # Added a bad grid that should not exist in the reference
+        grids = self.otherDetector.grids
+        missingKey = 'NOT PRESENT'
+        grids['E'] *= 1.05
+        grids['Z'] = grids['X']
+        grids['Y'] *= 2
+        grids[missingKey] = list(range(5))
+        similar = compareObjs(self.refDetector, self.otherDetector)
+        self.assertFalse(similar,
+                         msg="Mismatched grids did not induce failure.")
+        self.assertMsgInLogs(
+            "WARNING",
+            self.GRID_FMT.format('E', 'different, but within tolerances'),
+            partial=True)
+        self.assertMsgInLogs(
+            "ERROR",
+            self.GRID_FMT.format('Y', 'outside acceptable tolerances'),
+            partial=True)
+        self.assertMsgInLogs(
+            "ERROR",
+            "Z: {} - {}".format(
+                self.refDetector.grids['Z'].shape,
+                grids['Z'].shape),
+            partial=True)
+        self.assertMsgInLogs(
+            'DEBUG',
+            self.GRID_FMT.format('X', 'identical'),
+            partial=True)
+
+
+class TallyModifier(DetCompareHelper):
+    """Base class that modifies detectors and checks the comparisons."""
+
+    IDENTICAL_TALLY_MSG = "Expected values for tallies are identical"
+    INSIDE_CONF_MSG = "Confidence intervals for tallies overlap"
+    OUTISDE_CONF_MSG = ("Values for tallies are outside acceptable "
+                        "statistical limits")
+
+    def compare(self):
+        """Compare the two test objects and return the result."""
+        return compareObjs(self.refObj, self.otherObj)
+
+    @property
+    def refObj(self):
+        raise AttributeError
+
+    @property
+    def otherObj(self):
+        raise AttributeError
+
+    def checkUnmodifiedDetectors(self):
+        """Check all other detectors in the comparison."""
+        pass
+
+    def checkFinalStatus(self, obj0, obj1, status):
+        """Assert that the correct final status is logged."""
+        expected = finalCompareMsg(obj0, obj1, status)
+        level = "INFO" if status else "WARNING"
+        self.assertMsgInLogs(level, expected)
+
+    def test_unmodifiedCompare(self):
+        """Verify that w/o modifications the test passes"""
+        similar = self.compare()
+        self.assertTrue(similar)
+        self.assertMsgInLogs(
+            "DEBUG", self.IDENTICAL_TALLY_MSG,
+            partial=True)
+        self.checkFinalStatus(self.refObj, self.otherObj, True)
+        self.checkUnmodifiedDetectors()
+
+    def test_withinConfIntervals(self):
+        """Verify that slight differences in tallies are logged."""
+        self.otherDetector.tallies *= 1.01
+        similar = self.compare()
+        self.assertTrue(similar)
+        self.assertMsgInLogs(
+            "DEBUG", self.INSIDE_CONF_MSG, partial=True)
+        self.checkFinalStatus(self.refObj, self.otherObj, True)
+        self.checkUnmodifiedDetectors()
+
+    def test_outsideConfIntervals(self):
+        """Verify that large differences in tallies are logged."""
+        self.otherDetector.tallies *= 2.0
+        similar = self.compare()
+        self.assertFalse(similar)
+        self.assertMsgInLogs(
+            "ERROR", self.OUTISDE_CONF_MSG, partial=True)
+        self.checkFinalStatus(self.refObj, self.otherObj, False)
+        self.checkUnmodifiedDetectors()
+
+
+class DetectorCompareTester(TallyModifier):
+    """Class that tests a compare across detectors"""
+
+    @property
+    def refObj(self):
+        return self.refDetector
+
+    @property
+    def otherObj(self):
+        return self.otherDetector
+
+
+class DetectorReaderCompareTester(TallyModifier):
+    """Class that also tests the reader-level compare for completeness."""
+
+    @property
+    def refObj(self):
+        return self.refReader
+
+    @property
+    def otherObj(self):
+        return self.otherReader
+
+    def checkUnmodifiedDetectors(self):
+        for detName in UNMODIFIED_DETS:
+            myDet = self.refReader.detectors[detName]
+            otherDet = self.otherReader.detectors[detName]
+            finalMsg = finalCompareMsg(myDet, otherDet, True)
+            self.assertMsgInLogs("INFO", finalMsg)
+
+
+del TallyModifier
+
+if __name__ == '__main__':
+    from unittest import main
+    main()