Repository: ESMCI/cime

Issue:
Rework output of compare/bless_test_results to look more like TestStatus output
Moving a comment from PR #413 because it isn't critical to address this for that PR:

It would help if the output of compare_test_results (as well as bless_test_results, though that's less important) looked basically the same as the output put in the TestStatus file - although possibly just for the COMPARE_baseline phase. That is, I'd want to see something like:

PASS   SOMETEST  BASELINE_COMPARE

or

FAIL   SOMETEST  BASELINE_COMPARE

That's approximately what happened with the earlier version of these tools.

This comment applies both to the main output from the comparison, and also to various other extraneous information that is printed (or not). So, for example, I'd prefer not to have this output for every test:

``` python
                logging.info("Comparing results for test: %s, most recent result: %s" % (test_name, overall_result))
```

-- because I'd like to be able to parse this output in the same way that I parse the normal TestStatus or cs.status output. 

However, I agree that it does make sense to report on failed tests here. One possibility would be to print out the full testStatus output for each test that failed.



Discussion:
It's possible that we could accomplish all of this cleanly by making more use of the TestStatus class here: Rather than having compare_test_results be responsible for output, it could instead just update the ts object with the results of the BASELINE_COMPARE (or "COMPARE_baseline") phase. Then you can output the updated TestStatus results.

One complication with this is that I don't think you want to change the TestStatus file in the test itself (it seems like a Bad Idea to have some external script change the TestStatus file). In this usage, we instead would want the TestStatus to be written for each test via the logger. 

I think this could be accomplished by adding a __str__ method to TestStatus. This would contain the bulk of what is currently TestStatus.flush, and then TestStatus.flush could simply write out the results of str(self). That refactoring would have other benefits as well: for example, cs.status could just output the string representation of each TestStatus object, rather than needing to do a loop using the __iter__ protocol (which leads to a different formatting of the output for cs.status vs. TestStatus).


Resolving patch:
diff --git a/utils/python/CIME/hist_utils.py b/utils/python/CIME/hist_utils.py
--- a/utils/python/CIME/hist_utils.py
+++ b/utils/python/CIME/hist_utils.py
@@ -173,7 +173,7 @@ def _hists_match(model, hists1, hists2, suffix1="", suffix2=""):
 
     return one_not_two, two_not_one, match_ups
 
-def _compare_hists(case, from_dir1, from_dir2, suffix1="", suffix2=""):
+def _compare_hists(case, from_dir1, from_dir2, suffix1="", suffix2="", outfile_suffix=""):
     if from_dir1 == from_dir2:
         expect(suffix1 != suffix2, "Comparing files to themselves?")
 
@@ -204,7 +204,9 @@ def _compare_hists(case, from_dir1, from_dir2, suffix1="", suffix2=""):
         num_compared += len(match_ups)
 
         for hist1, hist2 in match_ups:
-            success, cprnc_comments = cprnc(model, hist1, hist2, case, from_dir1, multiinst_cpl_compare)
+            success, cprnc_comments = cprnc(model, hist1, hist2, case, from_dir1,
+                                            multiinst_cpl_compare = multiinst_cpl_compare,
+                                            outfile_suffix = outfile_suffix)
             if success:
                 comments += "    %s matched %s\n" % (hist1, hist2)
             else:
@@ -232,7 +234,7 @@ def compare_test(case, suffix1, suffix2):
 
     return _compare_hists(case, rundir, rundir, suffix1, suffix2)
 
-def cprnc(model, file1, file2, case, rundir, multiinst_cpl_compare=False):
+def cprnc(model, file1, file2, case, rundir, multiinst_cpl_compare=False, outfile_suffix=""):
     """
     Run cprnc to compare two individual nc files
 
@@ -240,6 +242,8 @@ def cprnc(model, file1, file2, case, rundir, multiinst_cpl_compare=False):
     file2 - the full or relative path of the second file
     case - the case containing the files
     rundir - the rundir for the case
+    outfile_suffix - if non-blank, then the output file name ends with this
+        suffix (with a '.' added before the given suffix)
 
     returns True if the files matched
     """
@@ -259,7 +263,10 @@ def cprnc(model, file1, file2, case, rundir, multiinst_cpl_compare=False):
     if mstr1 != mstr2:
         mstr = mstr1+mstr2
 
-    cpr_stat, out, _ = run_cmd("%s -m %s %s 2>&1 | tee %s/%s%s.cprnc.out" % (cprnc_exe, file1, file2, rundir, basename, mstr))
+    output_filename = "%s%s.cprnc.out"%(basename, mstr)
+    if outfile_suffix:
+        output_filename += ".%s"%(outfile_suffix)
+    cpr_stat, out, _ = run_cmd("%s -m %s %s 2>&1 | tee %s/%s" % (cprnc_exe, file1, file2, rundir, output_filename))
     if multiinst_cpl_compare:
         #  In a multiinstance test the cpl hist file will have a different number of
         # dimensions and so cprnc will indicate that the files seem to be DIFFERENT
@@ -269,12 +276,14 @@ def cprnc(model, file1, file2, case, rundir, multiinst_cpl_compare=False):
     else:
         return (cpr_stat == 0 and "files seem to be IDENTICAL" in out, out)
 
-def compare_baseline(case, baseline_dir=None):
+def compare_baseline(case, baseline_dir=None, outfile_suffix=""):
     """
     compare the current test output to a baseline result
 
     case - The case containing the hist files to be compared against baselines
     baseline_dir - Optionally, specify a specific baseline dir, otherwise it will be computed from case config
+    outfile_suffix - if non-blank, then the cprnc output file name ends with
+        this suffix (with a '.' added before the given suffix)
 
     returns (SUCCESS, comments)
     SUCCESS means all hist files matched their corresponding baseline
@@ -292,7 +301,7 @@ def compare_baseline(case, baseline_dir=None):
         if not os.path.isdir(bdir):
             return False, "ERROR %s baseline directory '%s' does not exist" % (TEST_NO_BASELINES_COMMENT,bdir)
 
-    return _compare_hists(case, rundir, basecmp_dir)
+    return _compare_hists(case, rundir, basecmp_dir, outfile_suffix = outfile_suffix)
 
 def get_extension(model, filepath):
     """


Test patch:
diff --git a/scripts/Tools/compare_test_results b/scripts/Tools/compare_test_results
--- a/scripts/Tools/compare_test_results
+++ b/scripts/Tools/compare_test_results
@@ -1,11 +1,24 @@
 #!/usr/bin/env python
 
-"""
-Analyze results from a test root area, comparing non-BFB
-changes. Purpose is, instead of re-running tests
-in compare mode, which is very slow, allow for very fast analsis of diffs.
+"""Analyze results from a test root area, comparing non-BFB changes.
+
+Purpose is, instead of re-running tests in compare mode, which is very slow,
+allow for very fast analysis of diffs.
+
+Outputs results for each test to stdout (one line per test); possible status
+codes are: PASS, FAIL, SKIP. (A SKIP denotes a test that did not make it to the
+run phase or a test for which the run phase did not pass: we skip baseline
+comparisons in this case.)
+
+In addition, creates files named compare.log.BASELINE_NAME.TIMESTAMP in each
+test directory, which contain more detailed output. Also creates
+*.cprnc.out.BASELINE_NAME.TIMESTAMP files in each run directory.
+
+Returns a 0 exit status if all tests are bit-for-bit, and a non-zero exit status
+(TESTS_FAILED_ERR_CODE) if any tests differed from the baseline.
 
 You may need to load modules for cprnc to work.
+
 """
 
 from standard_script_setup import *
@@ -88,7 +101,7 @@ def _main_func(description):
         parse_command_line(sys.argv, description)
 
     success = compare_test_results(baseline_name, baseline_root, test_root, compiler, test_id, compare_tests)
-    sys.exit(0 if success else 1)
+    sys.exit(0 if success else CIME.utils.TESTS_FAILED_ERR_CODE)
 
 ###############################################################################
 
diff --git a/utils/python/CIME/compare_test_results.py b/utils/python/CIME/compare_test_results.py
--- a/utils/python/CIME/compare_test_results.py
+++ b/utils/python/CIME/compare_test_results.py
@@ -1,3 +1,5 @@
+from __future__ import print_function
+
 import CIME.compare_namelists, CIME.simple_compare
 from CIME.utils import expect
 from CIME.test_status import *
@@ -7,64 +9,95 @@
 import os, glob
 
 ###############################################################################
-def compare_history(testcase_dir_for_test, baseline_name, baseline_root):
+def compare_history(testcase_dir_for_test, baseline_name, baseline_root, log_id):
 ###############################################################################
     with Case(testcase_dir_for_test) as case:
         baseline_full_dir = os.path.join(baseline_root, baseline_name, case.get_value("CASEBASEID"))
-        result, comments = compare_baseline(case, baseline_dir=baseline_full_dir)
-        if result:
-            return True, None
-        else:
-            logging.info(comments)
-            return False, "Diff'd"
+        outfile_suffix = "%s.%s"%(baseline_name, log_id)
+        result, comments = compare_baseline(case, baseline_dir=baseline_full_dir,
+                                            outfile_suffix=outfile_suffix)
+
+        return result, comments
 
 ###############################################################################
 def compare_test_results(baseline_name, baseline_root, test_root, compiler, test_id=None, compare_tests=None):
+    """Compares with baselines for all matching tests
+
+    Outputs results for each test to stdout (one line per test); possible status
+    codes are: PASS, FAIL, SKIP. (A SKIP denotes a test that did not make it to
+    the run phase or a test for which the run phase did not pass: we skip
+    baseline comparisons in this case.)
+
+    In addition, creates files named compare.log.BASELINE_NAME.TIMESTAMP in each
+    test directory, which contain more detailed output. Also creates
+    *.cprnc.out.BASELINE_NAME.TIMESTAMP files in each run directory.
+
+    Returns True if all tests generated either PASS or SKIP results, False if
+    there was at least one FAIL result.
+
+    """
 ###############################################################################
     test_id_glob = "*%s*%s*" % (compiler, baseline_name) if test_id is None else "*%s" % test_id
     test_status_files = glob.glob("%s/%s/%s" % (test_root, test_id_glob, TEST_STATUS_FILENAME))
     expect(test_status_files, "No matching test cases found in for %s/%s/%s" % (test_root, test_id_glob, TEST_STATUS_FILENAME))
 
-    broken_compares = []
+    # ID to use in the log file names, to avoid file name collisions with
+    # earlier files that may exist.
+    log_id = CIME.utils.get_timestamp()
+    logfile_name = "compare.log.%s.%s"%(baseline_name, log_id)
+
+    all_pass_or_skip = True
+
     for test_status_file in test_status_files:
         test_dir = os.path.dirname(test_status_file)
         ts = TestStatus(test_dir=test_dir)
         test_name = ts.get_name()
         if (compare_tests in [[], None] or CIME.utils.match_any(test_name, compare_tests)):
-            overall_result = ts.get_overall_test_status()
+            CIME.utils.append_status(
+                msg = ("Comparing against baseline with compare_test_results:\n" +
+                       "Baseline: %s\n"%(baseline_name) +
+                       "In baseline_root: %s"%(baseline_root)),
+                caseroot = test_dir,
+                sfile = logfile_name)
 
-            # Compute hist status, False implies it diffed
             run_result = ts.get_status(RUN_PHASE)
+            compare_comment = ""
+            detailed_comments = ""
             if (run_result is None):
-                broken_compares.append((test_name, "no run phase"))
-                logging.warning("Test '%s' did not make it to run phase" % test_name)
-                hist_no_compare = True
+                compare_result = "SKIP"
+                compare_comment = "Test did not make it to run phase"
+                do_compare = False
             elif (run_result != TEST_PASS_STATUS):
-                broken_compares.append((test_name, "test did not pass"))
-                logging.warning("Test '%s' did not pass, not safe to compare" % test_name)
-                hist_no_compare = True
+                compare_result = "SKIP"
+                compare_comment = "Run phase did not pass"
+                do_compare = False
             else:
-                hist_no_compare = False
+                do_compare = True
 
-            # Now, do the compare
-            if hist_no_compare:
-                logging.info("Cannot compare test: %s, overall status: %s" % (test_name, overall_result))
-            else:
+            if do_compare:
+                # Compare hist files
+                success, detailed_comments = compare_history(test_dir, baseline_name, baseline_root, log_id)
+                if success:
+                    compare_result = TEST_PASS_STATUS
+                else:
+                    compare_result = TEST_FAIL_STATUS
+                    all_pass_or_skip = False
 
-                logging.info("###############################################################################")
-                logging.info("Comparing results for test: %s, most recent result: %s" % (test_name, overall_result))
-                logging.info("###############################################################################")
+            brief_result = "%s %s %s"%(compare_result, test_name, BASELINE_PHASE)
+            if compare_comment:
+                brief_result += " %s"%(compare_comment)
+            print(brief_result)
+
+            CIME.utils.append_status(
+                msg = brief_result,
+                caseroot = test_dir,
+                sfile = logfile_name)
+
+            if detailed_comments:
+                CIME.utils.append_status(
+                    msg = "Detailed comments:\n" + detailed_comments,
+                    caseroot = test_dir,
+                    sfile = logfile_name)
+
+    return all_pass_or_skip
 
-                # Compare hist files
-                if (not hist_no_compare):
-                    success, reason = compare_history(test_dir, baseline_name, baseline_root)
-                    if (not success):
-                        broken_compares.append((test_name, reason))
-
-    # Make sure user knows that some tests were not compareed
-    success = True
-    for broken_compare, reason in broken_compares:
-        logging.warning("COMPARE FAILED FOR TEST: %s, reason %s" % (broken_compare, reason))
-        success = False
-
-    return success
diff --git a/utils/python/tests/scripts_regression_tests.py b/utils/python/tests/scripts_regression_tests.py
--- a/utils/python/tests/scripts_regression_tests.py
+++ b/utils/python/tests/scripts_regression_tests.py
@@ -886,10 +886,10 @@ def test_bless_test_results(self):
 
         # compare_test_results should detect the fail
         cpr_cmd = "%s/compare_test_results -b %s -t %s 2>&1" % (TOOLS_DIR, self._baseline_name, test_id)
-        output = run_cmd_assert_result(self, cpr_cmd, expected_stat=1)
+        output = run_cmd_assert_result(self, cpr_cmd, expected_stat=CIME.utils.TESTS_FAILED_ERR_CODE)
 
         # use regex
-        expected_pattern = re.compile(r'COMPARE FAILED FOR TEST: %s[^\s]* reason Diff' % self._test_name)
+        expected_pattern = re.compile(r'FAIL %s[^\s]* BASELINE' % self._test_name)
         the_match = expected_pattern.search(output)
         self.assertNotEqual(the_match, None,
                             msg="Cmd '%s' failed to display failed test in output:\n%s" % (cpr_cmd, output))