Repository: ESMCI/cime

Issue:
PET test (and maybe others) are acting like smoke tests
From cime5.6.6, a PET test (e.g., PET_P36x2_D.f10_f10_musgs.I1850Clm50BgcCrop.cheyenne_gnu.clm-default but I expect you'd see this with any PET test) is acting like a smoke test: It only runs the first case, doesn't run the second, and doesn't do any comparison between the cases. You can see this by seeing the empty rundir from case2, the lack of a run in case2's CaseStatus, and the lack of a compare phase in TestStatus.

My suspicion is that something is broken with the multisubmit mode of system_tests_compare_two.


Discussion:

Resolving patch:
diff --git a/scripts/lib/CIME/SystemTests/pet.py b/scripts/lib/CIME/SystemTests/pet.py
--- a/scripts/lib/CIME/SystemTests/pet.py
+++ b/scripts/lib/CIME/SystemTests/pet.py
@@ -39,4 +39,4 @@ def _case_two_setup(self):
             self._case.set_value("NTHRDS_{}".format(comp), 1)
 
         # Need to redo case_setup because we may have changed the number of threads
-
+        self._case.case_setup(reset=True, test_mode=True)
diff --git a/scripts/lib/CIME/case/case_st_archive.py b/scripts/lib/CIME/case/case_st_archive.py
--- a/scripts/lib/CIME/case/case_st_archive.py
+++ b/scripts/lib/CIME/case/case_st_archive.py
@@ -630,6 +630,7 @@ def case_st_archive(self, last_date_str=None, archive_incomplete_logs=True, copy
 
     # resubmit case if appropriate
     resubmit_cnt = self.get_value("RESUBMIT")
+    logger.debug("resubmit_cnt {} resubmit {}".format(resubmit_cnt, resubmit))
     if resubmit_cnt > 0 and resubmit:
         logger.info("resubmitting from st_archive, resubmit={:d}".format(resubmit_cnt))
         if self.get_value("MACH") == "mira":


Test patch:
diff --git a/config/cesm/machines/template.case.test b/config/cesm/machines/template.case.test
--- a/config/cesm/machines/template.case.test
+++ b/config/cesm/machines/template.case.test
@@ -48,6 +48,9 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter
     parser.add_argument("--reset", action="store_true",
                         help="Reset the case to its original state as defined by config_tests.xml")
 
+    parser.add_argument("--resubmit", action="store_true",
+                        help="Ignored in tests, but needed for all templates")
+
     parser.add_argument("--skip-preview-namelist", action="store_true",
                         help="Skip calling preview-namelist during case.run")
 
diff --git a/scripts/lib/CIME/SystemTests/system_tests_common.py b/scripts/lib/CIME/SystemTests/system_tests_common.py
--- a/scripts/lib/CIME/SystemTests/system_tests_common.py
+++ b/scripts/lib/CIME/SystemTests/system_tests_common.py
@@ -223,7 +223,7 @@ def run_indv(self, suffix="base", st_archive=False):
 
         logger.info(infostr)
 
-        self._case.case_run(skip_pnl=self._skip_pnl)
+        self._case.case_run(skip_pnl=self._skip_pnl, submit_resubmits=True)
 
         if not self._coupler_log_indicates_run_complete():
             expect(False, "Coupler did not indicate run passed")
@@ -232,7 +232,7 @@ def run_indv(self, suffix="base", st_archive=False):
             self._component_compare_copy(suffix)
 
         if st_archive:
-            self._case.case_st_archive()
+            self._case.case_st_archive(resubmit=True)
 
     def _coupler_log_indicates_run_complete(self):
         newestcpllogfiles = self._get_latest_cpl_logs()
diff --git a/scripts/lib/CIME/SystemTests/system_tests_compare_two.py b/scripts/lib/CIME/SystemTests/system_tests_compare_two.py
--- a/scripts/lib/CIME/SystemTests/system_tests_compare_two.py
+++ b/scripts/lib/CIME/SystemTests/system_tests_compare_two.py
@@ -221,6 +221,7 @@ def run_phase(self, success_change=False):  # pylint: disable=arguments-differ
             self._case_one_custom_postrun_action()
 
         # Second run
+        logger.info("_multisubmit {} first phase {}".format(self._multisubmit, first_phase))
         if not self._multisubmit or not first_phase:
             # Subtle issue: case1 is already in a writeable state since it tends to be opened
             # with a with statement in all the API entrances in CIME. case2 was created via clone,