Repository: ESMCI/cime

Issue:
proposed test of short term archiving 
A test should be created that does an initial run with the save interum restart option on and writes restart at some time which is not an even divisor of the total.   It would then clean the run directory and replace it with the files in the archive directory and restart from those files and run to the original runtime comparing history files from the initial run to the restart run.    Perhaps the ERS test could be refactored to do this instead of introducing an entirely new test?


Discussion:

Resolving patch:
diff --git a/scripts/lib/CIME/SystemTests/irt.py b/scripts/lib/CIME/SystemTests/irt.py
new file mode 100644
--- /dev/null
+++ b/scripts/lib/CIME/SystemTests/irt.py
@@ -0,0 +1,55 @@
+"""
+Implementation of the CIME IRT. (Interim Restart Test)
+This test the model's restart capability as well as the short term archiver's interim restart capability
+
+(1) Do a Run of length N with restart at N/2 and DOUT_S_SAVE_INTERIM_RESTART set to TRUE
+(2) Archive Run using ST archive tools
+(3) Recover first interim restart to the case2 run directory
+(4) Start case2 from restart and run to the end of case1
+(5) compare results.
+
+"""
+
+from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo
+from CIME.XML.standard_module_setup import *
+from CIME.case_st_archive import case_st_archive, restore_from_archive
+from CIME.utils import ls_sorted_by_mtime
+
+logger = logging.getLogger(__name__)
+
+class IRT(SystemTestsCompareTwo):
+
+    def __init__(self, case):
+        SystemTestsCompareTwo.__init__(self, case,
+                                       separate_builds=False,
+                                       run_two_suffix = 'restart',
+                                       run_one_description = 'initial',
+                                       run_two_description = 'restart',
+                                       multisubmit = False)
+
+    def _case_one_setup(self):
+        stop_n = self._case1.get_value("STOP_N")
+        expect(stop_n >= 3,"STOP_N must be at least 3, STOP_N = {}".format(stop_n))
+
+    def _case_two_setup(self):
+        rest_n = self._case1.get_value("REST_N")
+        stop_n = self._case1.get_value("STOP_N")
+        stop_new = stop_n - rest_n
+        expect(stop_new > 0, "ERROR: stop_n value {:d} too short {:d} {:d}".format(stop_new,stop_n,rest_n))
+        # hist_n is set to the stop_n value of case1
+        self._case.set_value("HIST_N", stop_n)
+        self._case.set_value("STOP_N", stop_new)
+        self._case.set_value("CONTINUE_RUN",True)
+        self._case.set_value("REST_OPTION", "never")
+
+    def _case_one_custom_postrun_action(self):
+        case_st_archive(self._case)
+        # Since preview namelist is run before _case_two_prerun_action, we need to do this here.
+        dout_s_root = self._case1.get_value("DOUT_S_ROOT")
+        restart_list = ls_sorted_by_mtime(os.path.join(dout_s_root,"rest"))
+        logger.info("Restart directory list is {}".format(restart_list))
+        expect(len(restart_list) >=2,"Expected at least two restart directories")
+        # Get the older of the two restart directories
+        restore_from_archive(self._case2,
+                             rest_dir=os.path.abspath(
+                                 os.path.join(dout_s_root, "rest", restart_list[0])))
diff --git a/scripts/lib/CIME/case_st_archive.py b/scripts/lib/CIME/case_st_archive.py
--- a/scripts/lib/CIME/case_st_archive.py
+++ b/scripts/lib/CIME/case_st_archive.py
@@ -7,7 +7,7 @@
 from CIME.XML.standard_module_setup import *
 from CIME.case_submit               import submit
 from CIME.XML.env_archive           import EnvArchive
-from CIME.utils                     import run_and_log_case_status
+from CIME.utils                     import run_and_log_case_status, ls_sorted_by_mtime
 from os.path                        import isdir, join
 import datetime
 
@@ -90,13 +90,16 @@ def _archive_rpointer_files(case, archive, archive_entry, archive_restdir,
 
             # loop through the possible rpointer files and contents
             for rpointer_file, rpointer_content in rpointer_items:
-
+                temp_rpointer_file = rpointer_file
+                temp_rpointer_content = rpointer_content
                 # put in a temporary setting for ninst_strings if they are empty
                 # in order to have just one loop over ninst_strings below
                 if rpointer_content is not 'unset':
                     if not ninst_strings:
                         ninst_strings = ["empty"]
                 for ninst_string in ninst_strings:
+                    rpointer_file = temp_rpointer_file
+                    rpointer_content = temp_rpointer_content
                     if ninst_string == 'empty':
                         ninst_string = ""
                     for key, value in [('$CASE', casename),
@@ -384,16 +387,21 @@ def _archive_process(case, archive, last_date, archive_incomplete_logs, copy_onl
                                        archive_file_fn)
 
 ###############################################################################
-def restore_from_archive(case):
+def restore_from_archive(case, rest_dir=None):
 ###############################################################################
     """
-    Take most recent archived restart files and load them into current case.
+    Take archived restart files and load them into current case.  Use rest_dir if provided otherwise use most recent
     """
     dout_sr = case.get_value("DOUT_S_ROOT")
     rundir = case.get_value("RUNDIR")
-    most_recent_rest = run_cmd_no_fail("ls -1dt {}/rest/* | head -1".format(dout_sr))
+    if rest_dir is not None:
+        if not os.path.isabs(rest_dir):
+            rest_dir = os.path.join(dout_sr, "rest", rest_dir)
+    else:
+        rest_dir = ls_sorted_by_mtime(os.path.join(dout_sr, "rest"))[-1]
 
-    for item in glob.glob("{}/*".format(most_recent_rest)):
+    logger.info("Restoring from {} to {}".format(rest_dir, rundir))
+    for item in glob.glob("{}/*".format(rest_dir)):
         base = os.path.basename(item)
         dst = os.path.join(rundir, base)
         if os.path.exists(dst):
diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py
--- a/scripts/lib/CIME/utils.py
+++ b/scripts/lib/CIME/utils.py
@@ -1224,6 +1224,11 @@ def _get_most_recent_lid_impl(files):
 
     return sorted(results)
 
+def ls_sorted_by_mtime(path):
+    ''' return list of path sorted by timestamp oldest first'''
+    mtime = lambda f: os.stat(os.path.join(path, f)).st_mtime
+    return list(sorted(os.listdir(path), key=mtime))
+
 def get_lids(case):
     model = case.get_value("MODEL")
     logdir = case.get_value("LOGDIR")


Test patch:
diff --git a/config/config_tests.xml b/config/config_tests.xml
--- a/config/config_tests.xml
+++ b/config/config_tests.xml
@@ -273,6 +273,19 @@ NODEFAIL          Tests restart upon detected node failure. Generates fake failu
     <CONTINUE_RUN>FALSE</CONTINUE_RUN>
   </test>
 
+  <test NAME="IRT">
+    <DESC>exact restart from startup, default 4 days + 7 days with restart from interim file</DESC>
+    <INFO_DBUG>1</INFO_DBUG>
+    <STOP_OPTION>ndays</STOP_OPTION>
+    <STOP_N>11</STOP_N>
+    <REST_N>$STOP_N / 2 - 1</REST_N>
+    <REST_OPTION>$STOP_OPTION</REST_OPTION>
+    <HIST_N>$STOP_N</HIST_N>
+    <HIST_OPTION>$STOP_OPTION</HIST_OPTION>
+    <DOUT_S_SAVE_INTERIM_RESTART_FILES>TRUE</DOUT_S_SAVE_INTERIM_RESTART_FILES>
+    <CONTINUE_RUN>FALSE</CONTINUE_RUN>
+  </test>
+
   <test NAME="ERIO">
     <DESC>exact restart from startup with different PIO methods, default 6 days + 5 days</DESC>
     <INFO_DBUG>1</INFO_DBUG>
diff --git a/scripts/lib/CIME/SystemTests/system_tests_compare_two.py b/scripts/lib/CIME/SystemTests/system_tests_compare_two.py
--- a/scripts/lib/CIME/SystemTests/system_tests_compare_two.py
+++ b/scripts/lib/CIME/SystemTests/system_tests_compare_two.py
@@ -14,13 +14,24 @@
 (2) _case_two_setup
     This method will be called to set up case 2, the "test" case
 
-In addition, they MAY require the following method:
+In addition, they MAY require the following methods:
 
 (1) _common_setup
     This method will be called to set up both cases. It should contain any setup
     that's needed in both cases. This is called before _case_one_setup or
     _case_two_setup.
 
+(2) _case_one_custom_prerun_action(self):
+    Use this to do arbitrary actions immediately before running case one
+
+(3) _case_two_custom_prerun_action(self):
+    Use this to do arbitrary actions immediately before running case two
+
+(4) _case_one_custom_postrun_action(self):
+    Use this to do arbitrary actions immediately after running case one
+
+(5) _case_two_custom_postrun_action(self):
+    Use this to do arbitrary actions immediately after running case two
 """
 
 from CIME.XML.standard_module_setup import *
@@ -39,7 +50,8 @@ def __init__(self,
                  separate_builds,
                  run_two_suffix = 'test',
                  run_one_description = '',
-                 run_two_description = ''):
+                 run_two_description = '',
+                 multisubmit = False):
         """
         Initialize a SystemTestsCompareTwo object. Individual test cases that
         inherit from SystemTestsCompareTwo MUST call this __init__ method.
@@ -56,6 +68,8 @@ def __init__(self,
                 when starting the first run. Defaults to ''.
             run_two_description (str, optional): Description printed to log file
                 when starting the second run. Defaults to ''.
+            multisubmit (bool): Do first and second runs as different submissions.
+                Designed for tests with RESUBMIT=1
         """
         SystemTestsCommon.__init__(self, case)
 
@@ -92,6 +106,8 @@ def __init__(self,
         self._case2 = None
 
         self._setup_cases_if_not_yet_done()
+
+        self._multisubmit = multisubmit
     # ========================================================================
     # Methods that MUST be implemented by specific tests that inherit from this
     # base class
@@ -131,6 +147,30 @@ def _common_setup(self):
         """
         pass
 
+    def _case_one_custom_prerun_action(self):
+        """
+        Use to do arbitrary actions immediately before running case one
+        """
+        pass
+
+    def _case_two_custom_prerun_action(self):
+        """
+        Use to do arbitrary actions immediately before running case two
+        """
+        pass
+
+    def _case_one_custom_postrun_action(self):
+        """
+        Use to do arbitrary actions immediately after running case one
+        """
+        pass
+
+    def _case_two_custom_postrun_action(self):
+        """
+        Use to do arbitrary actions immediately after running case two
+        """
+        pass
+
     # ========================================================================
     # Main public methods
     # ========================================================================
@@ -160,21 +200,33 @@ def run_phase(self, success_change=False):  # pylint: disable=arguments-differ
         Runs both phases of the two-phase test and compares their results
         If success_change is True, success requires some files to be different
         """
-
-        # First run
-        logger.info('Doing first run: ' + self._run_one_description)
-        self._activate_case1()
+        first_phase = self._case1.get_value("RESUBMIT") == 1 # Only relevant for multi-submit tests
         run_type = self._case1.get_value("RUN_TYPE")
-        self.run_indv(suffix = self._run_one_suffix)
+        # First run
+        if not self._multisubmit or first_phase:
+            logger.info('Doing first run: ' + self._run_one_description)
+            self._activate_case1()
+            self._case_one_custom_prerun_action()
+            self.run_indv(suffix = self._run_one_suffix)
+            self._case_one_custom_postrun_action()
 
         # Second run
-        logger.info('Doing second run: ' + self._run_two_description)
-        self._activate_case2()
-        # we need to make sure run2 is properly staged.
-        if run_type != "startup":
-            check_case(self._case2, self._caseroot2)
-        self._force_case2_settings()
-        self.run_indv(suffix = self._run_two_suffix)
+        if not self._multisubmit or not first_phase:
+            # Subtle issue: case1 is already in a writeable state since it tends to be opened
+            # with a with statement in all the API entrances in CIME. case2 was created via clone,
+            # not a with statement, so it's not in a writeable state, so we need to use a with
+            # statement here to put it in a writeable state.
+            with self._case2:
+                logger.info('Doing second run: ' + self._run_two_description)
+                self._activate_case2()
+                # we need to make sure run2 is properly staged.
+                if run_type != "startup":
+                    check_case(self._case2, self._caseroot2)
+                self._force_case2_settings()
+
+                self._case_two_custom_prerun_action()
+                self.run_indv(suffix = self._run_two_suffix)
+                self._case_two_custom_postrun_action()
 
         # Compare results
         # Case1 is the "main" case, and we need to do the comparisons from there
@@ -236,7 +288,7 @@ def _setup_cases_if_not_yet_done(self):
                                                   self._case1.get_value("CASE"), "case2")
                 self._case2 = self._case1.create_clone(
                     newcase = self._caseroot2,
-                    keepexe = self._separate_builds==False,
+                    keepexe = not self._separate_builds,
                     cime_output_root = case2_output_root)
                 self._setup_cases()
             except:
diff --git a/scripts/lib/CIME/tests/SystemTests/test_system_tests_compare_two.py b/scripts/lib/CIME/tests/SystemTests/test_system_tests_compare_two.py
--- a/scripts/lib/CIME/tests/SystemTests/test_system_tests_compare_two.py
+++ b/scripts/lib/CIME/tests/SystemTests/test_system_tests_compare_two.py
@@ -396,3 +396,6 @@ def test_run2_fails(self):
         # Verify
         self.assertEqual(test_status.TEST_FAIL_STATUS,
                          mytest._test_status.get_status(test_status.RUN_PHASE))
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2, catchbreak=True)
diff --git a/scripts/lib/CIME/tests/case_fake.py b/scripts/lib/CIME/tests/case_fake.py
--- a/scripts/lib/CIME/tests/case_fake.py
+++ b/scripts/lib/CIME/tests/case_fake.py
@@ -102,3 +102,9 @@ def set_rundir(self):
         inside CASEROOT)
         """
         self.set_value('RUNDIR', os.path.join(self.get_value('CASEROOT'), 'run'))
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, *_):
+        pass
diff --git a/scripts/lib/update_acme_tests.py b/scripts/lib/update_acme_tests.py
--- a/scripts/lib/update_acme_tests.py
+++ b/scripts/lib/update_acme_tests.py
@@ -43,7 +43,7 @@
                              "ERIO.f09_g16.X",
                              "SEQ_Ln9.f19_g16_rx1.A",
                              "ERS.ne30_g16_rx1.A",
-                             "ERS_N2.f19_g16_rx1.A",
+                             "IRT_N2.f19_g16_rx1.A",
                              "ERR.f45_g37_rx1.A",
                              "ERP.f45_g37_rx1.A",
                              "SMS_D_Ln9.f19_g16_rx1.A",