Repository: ESMCI/cime

Issue:
bless_test_results - can not find create_test command
I am using tag:
 https://github.com/CESM-Development/cime/tags/cime5.2.0-alpha.10

When I issue the command:
 /glade/u/home/cacraig/cam5_4_94/cime/scripts/Tools/bless_test_results  -t '' -c '' -r /glade/scratch/cacraig/aux_cam_20161123172411 -b cam5_4_94-copy6 -f

I get the following error:
FAILED TO BLESS TEST: SMS_Lm13.f09_f09.F2000_DEV.yellowstone_intel.cam-outfrq1m_clm5, reason Namelist regen failed: '/bin/sh: create_test: command not found'

The baseline directory created by this script has the netCDF files in it, but no namelists are copied.  I've tried running this command from various locations including the cime/scripts directory where create_test resides, and can not eliminate this error.




Discussion:
It seems like the script should be run from the cime/scripts directory and you need to add a ./ in front of the create_test call at line 21 of bless_test_results.py
@cacraigucar thanks, the current implementation is very fragile. I will fix.
CAM is planning on making this utility an integral part of our testing framework.  Should we reconsider our plans on using this tool (I'm asking because of the comment that it is very fragile).
I think that that tool just hasn't had a lot of users yet, now that you are integrating it with your testing we will assure that it works robustly.   

Resolving patch:
diff --git a/utils/python/CIME/case_cmpgen_namelists.py b/utils/python/CIME/case_cmpgen_namelists.py
--- a/utils/python/CIME/case_cmpgen_namelists.py
+++ b/utils/python/CIME/case_cmpgen_namelists.py
@@ -14,10 +14,10 @@
 
 logger = logging.getLogger(__name__)
 
-def _do_full_nl_comp(case, test, compare_name):
+def _do_full_nl_comp(case, test, compare_name, baseline_root=None):
     test_dir       = case.get_value("CASEROOT")
     casedoc_dir    = os.path.join(test_dir, "CaseDocs")
-    baseline_root  = case.get_value("BASELINE_ROOT")
+    baseline_root  = case.get_value("BASELINE_ROOT") if baseline_root is None else baseline_root
 
     all_match         = True
     baseline_dir      = os.path.join(baseline_root, compare_name, test)
@@ -52,10 +52,10 @@ def _do_full_nl_comp(case, test, compare_name):
     logging.info(comments)
     return all_match, comments
 
-def _do_full_nl_gen(case, test, generate_name):
+def _do_full_nl_gen(case, test, generate_name, baseline_root=None):
     test_dir       = case.get_value("CASEROOT")
     casedoc_dir    = os.path.join(test_dir, "CaseDocs")
-    baseline_root  = case.get_value("BASELINE_ROOT")
+    baseline_root  = case.get_value("BASELINE_ROOT") if baseline_root is None else baseline_root
 
     baseline_dir      = os.path.join(baseline_root, generate_name, test)
     baseline_casedocs = os.path.join(baseline_dir, "CaseDocs")
@@ -79,7 +79,7 @@ def _do_full_nl_gen(case, test, generate_name):
         shutil.copy2(item, baseline_dir)
         os.chmod(preexisting_baseline, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP)
 
-def case_cmpgen_namelists(case, compare=False, generate=False, compare_name=None, generate_name=None):
+def case_cmpgen_namelists(case, compare=False, generate=False, compare_name=None, generate_name=None, baseline_root=None, logfile_name="TestStatus.log"):
     expect(case.get_value("TEST"), "Only makes sense to run this for a test case")
 
     caseroot, casebaseid = case.get_value("CASEROOT"), case.get_value("CASEBASEID")
@@ -117,9 +117,9 @@ def case_cmpgen_namelists(case, compare=False, generate=False, compare_name=None
             success = True
             output = ""
             if compare:
-                success, output = _do_full_nl_comp(case, test_name, compare_name)
+                success, output = _do_full_nl_comp(case, test_name, compare_name, baseline_root)
             if generate:
-                _do_full_nl_gen(case, test_name, generate_name)
+                _do_full_nl_gen(case, test_name, generate_name, baseline_root)
         except:
             ts.set_status(NAMELIST_PHASE, TEST_FAIL_STATUS)
             success = False
@@ -128,7 +128,7 @@ def case_cmpgen_namelists(case, compare=False, generate=False, compare_name=None
             logging.warning(warn)
         finally:
             ts.set_status(NAMELIST_PHASE, TEST_PASS_STATUS if success else TEST_FAIL_STATUS)
-            append_status(output, caseroot=caseroot, sfile="TestStatus.log")
+            append_status(output, caseroot=caseroot, sfile=logfile_name)
 
         return success
 


Test patch:
diff --git a/scripts/Tools/compare_test_results b/scripts/Tools/compare_test_results
--- a/scripts/Tools/compare_test_results
+++ b/scripts/Tools/compare_test_results
@@ -66,6 +66,12 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter
 
     CIME.utils.setup_standard_logging_options(parser)
 
+    parser.add_argument("-n", "--namelists-only", action="store_true",
+                        help="Only analyze namelists.")
+
+    parser.add_argument("--hist-only", action="store_true",
+                        help="Only analyze history files.")
+
     parser.add_argument("-b", "--baseline-name", default=default_baseline_name,
                         help="Name of baselines to use, corresponds to branch used.")
 
@@ -88,7 +94,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter
 
     CIME.utils.handle_standard_logging_options(args)
 
-    return args.baseline_name, args.baseline_root, args.test_root, args.compiler, args.test_id, args.compare_tests
+    return args.baseline_name, args.baseline_root, args.test_root, args.compiler, args.test_id, args.compare_tests, args.namelists_only, args.hist_only
 
 ###############################################################################
 def _main_func(description):
@@ -97,10 +103,10 @@ def _main_func(description):
         test_results = doctest.testmod(verbose=True)
         sys.exit(1 if test_results.failed > 0 else 0)
 
-    baseline_name, baseline_root, test_root, compiler, test_id, compare_tests = \
+    baseline_name, baseline_root, test_root, compiler, test_id, compare_tests, namelists_only, hist_only = \
         parse_command_line(sys.argv, description)
 
-    success = compare_test_results(baseline_name, baseline_root, test_root, compiler, test_id, compare_tests)
+    success = compare_test_results(baseline_name, baseline_root, test_root, compiler, test_id, compare_tests, namelists_only, hist_only)
     sys.exit(0 if success else CIME.utils.TESTS_FAILED_ERR_CODE)
 
 ###############################################################################
diff --git a/utils/python/CIME/bless_test_results.py b/utils/python/CIME/bless_test_results.py
--- a/utils/python/CIME/bless_test_results.py
+++ b/utils/python/CIME/bless_test_results.py
@@ -1,6 +1,6 @@
 import CIME.compare_namelists, CIME.simple_compare
 from CIME.test_scheduler import NAMELIST_PHASE
-from CIME.utils import run_cmd, expect
+from CIME.utils import run_cmd, expect, get_scripts_root
 from CIME.test_status import *
 from CIME.hist_utils import generate_baseline, compare_baseline
 from CIME.case import Case
@@ -18,7 +18,7 @@ def bless_namelists(test_name, report_only, force, baseline_name, baseline_root)
     print "Test '%s' had a namelist diff" % test_name
     if (not report_only and
         (force or raw_input("Update namelists (y/n)? ").upper() in ["Y", "YES"])):
-        stat, _, err = run_cmd("create_test -n -g %s -b %s --baseline-root %s" % (test_name, baseline_name, baseline_root))
+        stat, _, err = run_cmd("%s/create_test -n -g %s -b %s --baseline-root %s -o" % (get_scripts_root(), test_name, baseline_name, baseline_root))
         if stat != 0:
             return False, "Namelist regen failed: '%s'" % err
         else:
diff --git a/utils/python/CIME/compare_test_results.py b/utils/python/CIME/compare_test_results.py
--- a/utils/python/CIME/compare_test_results.py
+++ b/utils/python/CIME/compare_test_results.py
@@ -1,26 +1,33 @@
-from __future__ import print_function
-
 import CIME.compare_namelists, CIME.simple_compare
 from CIME.utils import expect
 from CIME.test_status import *
 from CIME.hist_utils import compare_baseline
+from CIME.case_cmpgen_namelists import case_cmpgen_namelists
 from CIME.case import Case
 
-import os, glob
+import os, glob, logging
+
+###############################################################################
+def compare_namelists(case, baseline_name, baseline_root, logfile_name):
+###############################################################################
+    log_lvl = logging.getLogger().getEffectiveLevel()
+    logging.disable(logging.CRITICAL)
+    success = case_cmpgen_namelists(case, compare=True, compare_name=baseline_name, baseline_root=baseline_root, logfile_name=logfile_name)
+    logging.getLogger().setLevel(log_lvl)
+    return success
 
 ###############################################################################
-def compare_history(testcase_dir_for_test, baseline_name, baseline_root, log_id):
+def compare_history(case, baseline_name, baseline_root, log_id):
 ###############################################################################
-    with Case(testcase_dir_for_test) as case:
-        baseline_full_dir = os.path.join(baseline_root, baseline_name, case.get_value("CASEBASEID"))
-        outfile_suffix = "%s.%s"%(baseline_name, log_id)
-        result, comments = compare_baseline(case, baseline_dir=baseline_full_dir,
-                                            outfile_suffix=outfile_suffix)
+    baseline_full_dir = os.path.join(baseline_root, baseline_name, case.get_value("CASEBASEID"))
+    outfile_suffix = "%s.%s" % (baseline_name, log_id)
+    result, comments = compare_baseline(case, baseline_dir=baseline_full_dir,
+                                        outfile_suffix=outfile_suffix)
 
-        return result, comments
+    return result, comments
 
 ###############################################################################
-def compare_test_results(baseline_name, baseline_root, test_root, compiler, test_id=None, compare_tests=None):
+def compare_test_results(baseline_name, baseline_root, test_root, compiler, test_id=None, compare_tests=None, namelists_only=False, hist_only=False):
     """Compares with baselines for all matching tests
 
     Outputs results for each test to stdout (one line per test); possible status
@@ -44,7 +51,7 @@ def compare_test_results(baseline_name, baseline_root, test_root, compiler, test
     # ID to use in the log file names, to avoid file name collisions with
     # earlier files that may exist.
     log_id = CIME.utils.get_timestamp()
-    logfile_name = "compare.log.%s.%s"%(baseline_name, log_id)
+    logfile_name = "compare.log.%s.%s" % (baseline_name.replace("/", "_"), log_id)
 
     all_pass_or_skip = True
 
@@ -60,39 +67,74 @@ def compare_test_results(baseline_name, baseline_root, test_root, compiler, test
                 caseroot = test_dir,
                 sfile = logfile_name)
 
-            run_result = ts.get_status(RUN_PHASE)
-            compare_comment = ""
-            detailed_comments = ""
-            if (run_result is None):
-                compare_result = "SKIP"
-                compare_comment = "Test did not make it to run phase"
-                do_compare = False
-            elif (run_result != TEST_PASS_STATUS):
-                compare_result = "SKIP"
-                compare_comment = "Run phase did not pass"
-                do_compare = False
+            if (not hist_only):
+                nl_compare_result = None
+                nl_compare_comment = ""
+                nl_result = ts.get_status(SETUP_PHASE)
+                if (nl_result is None):
+                    nl_compare_result = "SKIP"
+                    nl_compare_comment = "Test did not make it to setup phase"
+                    nl_do_compare = False
+                else:
+                    nl_do_compare = True
             else:
-                do_compare = True
+                nl_do_compare = False
 
-            if do_compare:
-                # Compare hist files
-                success, detailed_comments = compare_history(test_dir, baseline_name, baseline_root, log_id)
-                if success:
-                    compare_result = TEST_PASS_STATUS
+            detailed_comments = ""
+            if (not namelists_only):
+                compare_result = None
+                compare_comment = ""
+                run_result = ts.get_status(RUN_PHASE)
+                if (run_result is None):
+                    compare_result = "SKIP"
+                    compare_comment = "Test did not make it to run phase"
+                    do_compare = False
+                elif (run_result != TEST_PASS_STATUS):
+                    compare_result = "SKIP"
+                    compare_comment = "Run phase did not pass"
+                    do_compare = False
                 else:
-                    compare_result = TEST_FAIL_STATUS
-                    all_pass_or_skip = False
-
-                # Following the logic in SystemTestsCommon._compare_baseline:
-                # We'll print the comment if it's a brief one-liner; otherwise
-                # the comment will only appear in the log file
-                if "\n" not in detailed_comments:
-                    compare_comment = detailed_comments
+                    do_compare = True
+            else:
+                do_compare = False
 
-            brief_result = "%s %s %s"%(compare_result, test_name, BASELINE_PHASE)
-            if compare_comment:
-                brief_result += " %s"%(compare_comment)
-            print(brief_result)
+            if nl_do_compare or do_compare:
+                with Case(test_dir) as case:
+                    if nl_do_compare:
+                        nl_success = compare_namelists(case, baseline_name, baseline_root, logfile_name)
+                        if nl_success:
+                            nl_compare_result = TEST_PASS_STATUS
+                            nl_compare_comment = ""
+                        else:
+                            nl_compare_result = TEST_FAIL_STATUS
+                            nl_compare_comment = "See %s/%s" % (test_dir, logfile_name)
+                            all_pass_or_skip = False
+
+                    if do_compare:
+                        success, detailed_comments = compare_history(case, baseline_name, baseline_root, log_id)
+                        if success:
+                            compare_result = TEST_PASS_STATUS
+                        else:
+                            compare_result = TEST_FAIL_STATUS
+                            all_pass_or_skip = False
+
+                        # Following the logic in SystemTestsCommon._compare_baseline:
+                        # We'll print the comment if it's a brief one-liner; otherwise
+                        # the comment will only appear in the log file
+                        if "\n" not in detailed_comments:
+                            compare_comment = detailed_comments
+
+            brief_result = ""
+            if not hist_only:
+                brief_result += "%s %s %s %s\n" % (nl_compare_result, test_name, NAMELIST_PHASE, nl_compare_comment)
+
+            if not namelists_only:
+                brief_result += "%s %s %s" % (compare_result, test_name, BASELINE_PHASE)
+                if compare_comment:
+                    brief_result += " %s" % compare_comment
+                brief_result += "\n"
+
+            print brief_result,
 
             CIME.utils.append_status(
                 msg = brief_result,
@@ -106,4 +148,3 @@ def compare_test_results(baseline_name, baseline_root, test_root, compiler, test
                     sfile = logfile_name)
 
     return all_pass_or_skip
-
diff --git a/utils/python/tests/scripts_regression_tests.py b/utils/python/tests/scripts_regression_tests.py
--- a/utils/python/tests/scripts_regression_tests.py
+++ b/utils/python/tests/scripts_regression_tests.py
@@ -473,78 +473,6 @@ def tearDown(self):
                 else:
                     os.remove(file_to_clean)
 
-###############################################################################
-class N_TestCreateTest(TestCreateTestCommon):
-###############################################################################
-
-    ###########################################################################
-    def simple_test(self, expect_works, extra_args):
-    ###########################################################################
-        if NO_BATCH:
-            extra_args += " --no-batch"
-        run_cmd_assert_result(self, "%s/create_test cime_test_only_pass %s" % (SCRIPT_DIR, extra_args),
-                              expected_stat=(0 if expect_works else CIME.utils.TESTS_FAILED_ERR_CODE))
-
-    ###############################################################################
-    def test_create_test_rebless_namelist(self):
-    ###############################################################################
-        # Generate some namelist baselines
-        if CIME.utils.get_model() == "acme":
-            genarg = "-g -o -b %s" % self._baseline_name
-            comparg = "-c -b %s" % self._baseline_name
-        else:
-            genarg = "-g %s -o" % self._baseline_name
-            comparg = "-c %s" % self._baseline_name
-
-        self.simple_test(True, "%s -n -t %s-%s" % (genarg, self._baseline_name, CIME.utils.get_timestamp()))
-
-        # Basic namelist compare
-        test_id = "%s-%s" % (self._baseline_name, CIME.utils.get_timestamp())
-        self.simple_test(True, "%s -n -t %s" % (comparg, test_id))
-
-        # Check standalone case.cmpgen_namelists
-        casedir = os.path.join(self._testroot,
-                               "%s.C.%s" % (CIME.utils.get_full_test_name("TESTRUNPASS_P1.f19_g16_rx1.A", machine=self._machine, compiler=self._compiler), test_id))
-        run_cmd_assert_result(self, "./case.cmpgen_namelists", from_dir=casedir)
-
-        # Modify namelist
-        fake_nl = """
- &fake_nml
-   fake_item = 'fake'
-   fake = .true.
-/"""
-        baseline_area = self._baseline_area
-        baseline_glob = glob.glob(os.path.join(baseline_area, self._baseline_name, "TEST*"))
-        self.assertEqual(len(baseline_glob), 3, msg="Expected three matches, got:\n%s" % "\n".join(baseline_glob))
-
-        import stat
-        for baseline_dir in baseline_glob:
-            nl_path = os.path.join(baseline_dir, "CaseDocs", "datm_in")
-            self.assertTrue(os.path.isfile(nl_path), msg="Missing file %s" % nl_path)
-
-            os.chmod(nl_path, stat.S_IRUSR | stat.S_IWUSR)
-            with open(nl_path, "a") as nl_file:
-                nl_file.write(fake_nl)
-
-        # Basic namelist compare should now fail
-        test_id = "%s-%s" % (self._baseline_name, CIME.utils.get_timestamp())
-        self.simple_test(False, "%s -n -t %s" % (comparg, test_id))
-        casedir = os.path.join(self._testroot,
-                               "%s.C.%s" % (CIME.utils.get_full_test_name("TESTRUNPASS_P1.f19_g16_rx1.A", machine=self._machine, compiler=self._compiler), test_id))
-        run_cmd_assert_result(self, "./case.cmpgen_namelists", from_dir=casedir, expected_stat=100)
-
-        # preview namelists should work
-        run_cmd_assert_result(self, "./preview_namelists", from_dir=casedir)
-
-        # This should still fail
-        run_cmd_assert_result(self, "./case.cmpgen_namelists", from_dir=casedir, expected_stat=100)
-
-        # Regen
-        self.simple_test(True, "%s -n -t %s-%s" % (genarg, self._baseline_name, CIME.utils.get_timestamp()))
-
-        # Basic namelist compare should now pass again
-        self.simple_test(True, "%s -n -t %s-%s" % (comparg, self._baseline_name, CIME.utils.get_timestamp()))
-
 ###############################################################################
 class O_TestTestScheduler(TestCreateTestCommon):
 ###############################################################################
@@ -838,8 +766,6 @@ def test_jenkins_generic_job_kill(self):
 class Q_TestBlessTestResults(TestCreateTestCommon):
 ###############################################################################
 
-    _test_name = "TESTRUNDIFF_P1.f19_g16_rx1.A"
-
     ###########################################################################
     def tearDown(self):
     ###########################################################################
@@ -854,10 +780,10 @@ def simple_test(self, expect_works, extra_args):
         if NO_BATCH:
             extra_args += " --no-batch"
 
-        run_cmd_assert_result(self, "%s/create_test %s %s" % (SCRIPT_DIR, self._test_name, extra_args),
+        run_cmd_assert_result(self, "%s/create_test %s" % (SCRIPT_DIR, extra_args),
                               expected_stat=(0 if expect_works or self._hasbatch else CIME.utils.TESTS_FAILED_ERR_CODE))
 
-        if self._hasbatch:
+        if self._hasbatch and " -n " not in extra_args:
             test_id = extra_args.split()[extra_args.split().index("-t") + 1]
             run_cmd_assert_result(self, "%s/wait_for_tests *%s/TestStatus" % (TOOLS_DIR, test_id),
                                   from_dir=self._testroot, expected_stat=(0 if expect_works else CIME.utils.TESTS_FAILED_ERR_CODE))
@@ -865,13 +791,15 @@ def simple_test(self, expect_works, extra_args):
     ###############################################################################
     def test_bless_test_results(self):
     ###############################################################################
-        # Generate some namelist baselines
+        # Generate some baselines
+        test_name = "TESTRUNDIFF_P1.f19_g16_rx1.A"
+
         if CIME.utils.get_model() == "acme":
-            genarg = "-g -o -b %s"%self._baseline_name
-            comparg = "-c -b %s"%self._baseline_name
+            genarg = "-g -o -b %s %s" % (self._baseline_name, test_name)
+            comparg = "-c -b %s %s" % (self._baseline_name, test_name)
         else:
-            genarg = "-g %s -o"%self._baseline_name
-            comparg = "-c %s"%self._baseline_name
+            genarg = "-g %s -o %s" % (self._baseline_name, test_name)
+            comparg = "-c %s %s" % (self._baseline_name, test_name)
 
         self.simple_test(True, "%s -t %s-%s" % (genarg, self._baseline_name, CIME.utils.get_timestamp()))
 
@@ -890,7 +818,7 @@ def test_bless_test_results(self):
         output = run_cmd_assert_result(self, cpr_cmd, expected_stat=CIME.utils.TESTS_FAILED_ERR_CODE)
 
         # use regex
-        expected_pattern = re.compile(r'FAIL %s[^\s]* BASELINE' % self._test_name)
+        expected_pattern = re.compile(r'FAIL %s[^\s]* BASELINE' % test_name)
         the_match = expected_pattern.search(output)
         self.assertNotEqual(the_match, None,
                             msg="Cmd '%s' failed to display failed test in output:\n%s" % (cpr_cmd, output))
@@ -901,6 +829,88 @@ def test_bless_test_results(self):
         # Hist compare should now pass again
         self.simple_test(True, "%s -t %s-%s" % (comparg, self._baseline_name, CIME.utils.get_timestamp()))
 
+    ###############################################################################
+    def test_rebless_namelist(self):
+    ###############################################################################
+        # Generate some namelist baselines
+        test_to_change = "TESTRUNPASS_P1.f19_g16_rx1.A"
+        if CIME.utils.get_model() == "acme":
+            genarg = "-g -o -b %s cime_test_only_pass" % self._baseline_name
+            comparg = "-c -b %s cime_test_only_pass" % self._baseline_name
+        else:
+            genarg = "-g %s -o cime_test_only_pass" % self._baseline_name
+            comparg = "-c %s cime_test_only_pass" % self._baseline_name
+
+        self.simple_test(True, "%s -n -t %s-%s" % (genarg, self._baseline_name, CIME.utils.get_timestamp()))
+
+        # Basic namelist compare
+        test_id = "%s-%s" % (self._baseline_name, CIME.utils.get_timestamp())
+        self.simple_test(True, "%s -n -t %s" % (comparg, test_id))
+
+        # Check standalone case.cmpgen_namelists
+        casedir = os.path.join(self._testroot,
+                               "%s.C.%s" % (CIME.utils.get_full_test_name(test_to_change, machine=self._machine, compiler=self._compiler), test_id))
+        run_cmd_assert_result(self, "./case.cmpgen_namelists", from_dir=casedir)
+
+        # compare_test_results should pass
+        cpr_cmd = "%s/compare_test_results -n -b %s -t %s 2>&1" % (TOOLS_DIR, self._baseline_name, test_id)
+        output = run_cmd_assert_result(self, cpr_cmd)
+
+        # use regex
+        expected_pattern = re.compile(r'PASS %s[^\s]* NLCOMP' % test_to_change)
+        the_match = expected_pattern.search(output)
+        self.assertNotEqual(the_match, None,
+                            msg="Cmd '%s' failed to display passed test in output:\n%s" % (cpr_cmd, output))
+
+
+        # Modify namelist
+        fake_nl = """
+ &fake_nml
+   fake_item = 'fake'
+   fake = .true.
+/"""
+        baseline_area = self._baseline_area
+        baseline_glob = glob.glob(os.path.join(baseline_area, self._baseline_name, "TEST*"))
+        self.assertEqual(len(baseline_glob), 3, msg="Expected three matches, got:\n%s" % "\n".join(baseline_glob))
+
+        import stat
+        for baseline_dir in baseline_glob:
+            nl_path = os.path.join(baseline_dir, "CaseDocs", "datm_in")
+            self.assertTrue(os.path.isfile(nl_path), msg="Missing file %s" % nl_path)
+
+            os.chmod(nl_path, stat.S_IRUSR | stat.S_IWUSR)
+            with open(nl_path, "a") as nl_file:
+                nl_file.write(fake_nl)
+
+        # Basic namelist compare should now fail
+        test_id = "%s-%s" % (self._baseline_name, CIME.utils.get_timestamp())
+        self.simple_test(False, "%s -n -t %s" % (comparg, test_id))
+        casedir = os.path.join(self._testroot,
+                               "%s.C.%s" % (CIME.utils.get_full_test_name(test_to_change, machine=self._machine, compiler=self._compiler), test_id))
+        run_cmd_assert_result(self, "./case.cmpgen_namelists", from_dir=casedir, expected_stat=100)
+
+        # preview namelists should work
+        run_cmd_assert_result(self, "./preview_namelists", from_dir=casedir)
+
+        # This should still fail
+        run_cmd_assert_result(self, "./case.cmpgen_namelists", from_dir=casedir, expected_stat=100)
+
+        # compare_test_results should fail
+        cpr_cmd = "%s/compare_test_results -n -b %s -t %s 2>&1" % (TOOLS_DIR, self._baseline_name, test_id)
+        output = run_cmd_assert_result(self, cpr_cmd, expected_stat=CIME.utils.TESTS_FAILED_ERR_CODE)
+
+        # use regex
+        expected_pattern = re.compile(r'FAIL %s[^\s]* NLCOMP' % test_to_change)
+        the_match = expected_pattern.search(output)
+        self.assertNotEqual(the_match, None,
+                            msg="Cmd '%s' failed to display passed test in output:\n%s" % (cpr_cmd, output))
+
+        # Bless
+        run_cmd_no_fail("%s/bless_test_results -n --force -b %s -t %s" % (TOOLS_DIR, self._baseline_name, test_id))
+
+        # Basic namelist compare should now pass again
+        self.simple_test(True, "%s -n -t %s-%s" % (comparg, self._baseline_name, CIME.utils.get_timestamp()))
+
 ###############################################################################
 @unittest.skip("Disabling this test until we figure out how to integrate ACME tests and CIME xml files.")
 class R_TestUpdateACMETests(unittest.TestCase):