Repository: ESMCI/cime

Issue:
IRT test in cime_developer test suite fails in generate phase
I just ran the cime_developer test suite with the --generate flag. This produced:

```
    FAIL IRT_N2_Ln9.f19_g16_rx1.A.cheyenne_intel GENERATE exception
```

The problem seems to be that there are no history files in the run directory that can be used for generation. It looks like the test *does* produce a history file (IRT_N2_Ln9.f19_g16_rx1.A.cheyenne_intel.G.20210521_174039_x66o19.cpl.hi.0001-01-01-16200.nc in my case) but it is sent to the archive directory. I thought there is general logic to copy the last set of history files back into the run directory, but maybe it doesn't apply to coupler history files?


Discussion:
I ran into this same problem when trying to run an IRT test in the aux_glc test suite in CESM.

Resolving patch:
diff --git a/scripts/lib/CIME/SystemTests/irt.py b/scripts/lib/CIME/SystemTests/irt.py
--- a/scripts/lib/CIME/SystemTests/irt.py
+++ b/scripts/lib/CIME/SystemTests/irt.py
@@ -7,6 +7,7 @@
 (3) Recover first interim restart to the case2 run directory
 (4) Start case2 from restart and run to the end of case1
 (5) compare results.
+(6) this test does not save or compare history files in baselines.
 
 """
 
diff --git a/scripts/lib/CIME/hist_utils.py b/scripts/lib/CIME/hist_utils.py
--- a/scripts/lib/CIME/hist_utils.py
+++ b/scripts/lib/CIME/hist_utils.py
@@ -34,6 +34,8 @@
 COMPARISON_FAILURE_COMMENT_OPTIONS = (COMPARISON_COMMENT_OPTIONS
                                       - set([NO_COMPARE, FIELDLISTS_DIFFER]))
 
+NO_HIST_TESTS = ["IRT", "PFS", "TSC"]
+
 def _iter_model_file_substrs(case):
     models = case.get_compset_components()
     models.append('cpl')
@@ -273,8 +275,8 @@ def _compare_hists(case, from_dir1, from_dir2, suffix1="", suffix2="", outfile_s
                         logger.warning("Could not copy {} to {}".format(cprnc_log_file, casedir))
 
                 all_success = False
-    # PFS test may not have any history files to compare.
-    if num_compared == 0 and testcase != "PFS":
+    # Some tests don't save history files. 
+    if num_compared == 0 and testcase not in NO_HIST_TESTS:
         all_success = False
         comments += "Did not compare any hist files! Missing baselines?\n"
 
@@ -494,7 +496,7 @@ def _generate_baseline_impl(case, baseline_dir=None, allow_baseline_overwrite=Fa
     testname = case.get_value("TESTCASE")
     testopts = parse_test_name(case.get_value("CASEBASEID"))[1]
     testopts = [] if testopts is None else testopts
-    expect(num_gen > 0 or (testname in ["PFS", "TSC"] or "B" in testopts),
+    expect(num_gen > 0 or (testname in NO_HIST_TESTS or "B" in testopts),
            "Could not generate any hist files for case '{}', something is seriously wrong".format(os.path.join(rundir, testcase)))
 
     if get_model() == "e3sm":


Test patch:
diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py
--- a/scripts/tests/scripts_regression_tests.py
+++ b/scripts/tests/scripts_regression_tests.py
@@ -382,7 +382,7 @@ def test_a_createnewcase(self):
         self.assertTrue(os.path.exists(os.path.join(testdir, "case.setup")))
 
         run_cmd_assert_result(self, "./case.setup", from_dir=testdir)
-        run_cmd_assert_result(self, "./case.build", from_dir=testdir)
+        run_cmd_assert_result(self, "./case.build --skip-provenance-check", from_dir=testdir)
 
         with Case(testdir, read_only=False) as case:
             ntasks = case.get_value("NTASKS_ATM")
@@ -393,12 +393,12 @@ def test_a_createnewcase(self):
                               from_dir=testdir, expected_stat=1)
 
         run_cmd_assert_result(self, "./case.setup --reset", from_dir=testdir)
-        run_cmd_assert_result(self, "./case.build", from_dir=testdir)
+        run_cmd_assert_result(self, "./case.build --skip-provenance-check", from_dir=testdir)
         with Case(testdir, read_only=False) as case:
             case.set_value("CHARGE_ACCOUNT", "fred")
 
         # this should not fail with a locked file issue
-        run_cmd_assert_result(self, "./case.build", from_dir=testdir)
+        run_cmd_assert_result(self, "./case.build --skip-provenance-check", from_dir=testdir)
 
         run_cmd_assert_result(self, "./case.st_archive --test-all", from_dir=testdir)
 
@@ -611,7 +611,7 @@ def test_f_createnewcase_with_user_compset(self):
 
         run_cmd_assert_result(self, "%s/create_newcase %s"%(SCRIPT_DIR, args), from_dir=SCRIPT_DIR)
         run_cmd_assert_result(self, "./case.setup", from_dir=testdir)
-        run_cmd_assert_result(self, "./case.build", from_dir=testdir)
+        run_cmd_assert_result(self, "./case.build --skip-provenance-check", from_dir=testdir)
 
         cls._do_teardown.append(testdir)
 
@@ -2553,7 +2553,7 @@ def test_self_build_cprnc(self):
 
         run_cmd_assert_result(self, "./xmlchange CCSM_CPRNC=this_is_a_broken_cprnc",
                               from_dir=casedir)
-        run_cmd_assert_result(self, "./case.build", from_dir=casedir)
+        run_cmd_assert_result(self, "./case.build --skip-provenance-check", from_dir=casedir)
         run_cmd_assert_result(self, "./case.submit", from_dir=casedir)
 
         self._wait_for_tests(self._baseline_name, always_wait=True)
@@ -2604,7 +2604,7 @@ def test_clean_rebuild(self):
         run_cmd_assert_result(self, "./case.build --clean atm", from_dir=casedir)
         run_cmd_assert_result(self, "./case.build --clean gptl", from_dir=casedir)
 
-        run_cmd_assert_result(self, "./case.build", from_dir=casedir)
+        run_cmd_assert_result(self, "./case.build --skip-provenance-check", from_dir=casedir)
 
 ###############################################################################
 class L_TestSaveTimings(TestCreateTestCommon):