Repository: ESMCI/cime

Issue:
scripts_regression_tests.py Y_TestUserConcurrentMods is failing for cesm 
It seems to archive the cesm.log before the file is grepped to look for the run successful message. 


Discussion:
The test is also writing to the default archive directory instead of creating one in the TEST_ROOT and is not cleaning up after itself. 
It seems like a race condition, sometimes the test will pass and other times I get:
ERROR: Model did not complete, no /glade/scratch/jedwards/scripts_regression_test.20201005_115759/TESTRUNUSERXMLCHANGE.f19_g16.X.cheyenne_intel.fake_testing_only_20201005_115804/run/cesm.log.4498193.chadmin1.ib0.cheyenne.ucar.edu.201005-120142 log file 
However this file is in the archive/TESTRUNUSERXMLCHANGE.f19_g16.X.cheyenne_intel.fake_testing_only_20201005_115804/logs/cesm.log.4498193.chadmin1.ib0.cheyenne.ucar.edu.201005-120142
The test cannot be rerun from the test directory because the number of occurrences of string
"model execution success" will increase with each run.  
@jedwards4b , thx, I will take a look.

Resolving patch:
diff --git a/scripts/lib/CIME/case/case_run.py b/scripts/lib/CIME/case/case_run.py
--- a/scripts/lib/CIME/case/case_run.py
+++ b/scripts/lib/CIME/case/case_run.py
@@ -87,7 +87,6 @@ def _run_model_impl(case, lid, skip_pnl=False, da_cycle=0):
     logger.info("run command is {} ".format(cmd))
 
     rundir = case.get_value("RUNDIR")
-    loop = True
 
     # MPIRUN_RETRY_REGEX allows the mpi command to be reattempted if the
     # failure described by that regular expression is matched in the model log
@@ -102,6 +101,7 @@ def _run_model_impl(case, lid, skip_pnl=False, da_cycle=0):
     if node_fail_re:
         node_fail_regex = re.compile(re.escape(node_fail_re))
 
+    loop = True
     while loop:
         loop = False
 
@@ -155,6 +155,7 @@ def _run_model_impl(case, lid, skip_pnl=False, da_cycle=0):
                     logger.warning("Detected model run failed, restarting")
                     retry_count -= 1
                     loop = True
+
                 if loop:
                     # Archive the last consistent set of restart files and restore them
                     if case.get_value("DOUT_S"):
diff --git a/scripts/lib/CIME/case/case_st_archive.py b/scripts/lib/CIME/case/case_st_archive.py
--- a/scripts/lib/CIME/case/case_st_archive.py
+++ b/scripts/lib/CIME/case/case_st_archive.py
@@ -15,6 +15,11 @@
 
 logger = logging.getLogger(__name__)
 
+###############################################################################
+def _get_archive_fn_desc(archive_fn):
+###############################################################################
+    return "moving" if archive_fn is shutil.move else "copying"
+
 ###############################################################################
 def _get_archive_file_fn(copy_only):
 ###############################################################################
@@ -196,8 +201,8 @@ def _archive_log_files(dout_s_root, rundir, archive_incomplete, archive_file_fn)
     for logfile in logfiles:
         srcfile = join(rundir, os.path.basename(logfile))
         destfile = join(archive_logdir, os.path.basename(logfile))
+        logger.info("{} {} to {}".format(_get_archive_fn_desc(archive_file_fn), srcfile, destfile))
         archive_file_fn(srcfile, destfile)
-        logger.info("moving {} to {}".format(srcfile, destfile))
 
 ###############################################################################
 def _archive_history_files(archive, compclass, compname, histfiles_savein_rundir,
@@ -234,7 +239,7 @@ def _archive_history_files(archive, compclass, compname, histfiles_savein_rundir
             for rbldfile in rbldfiles:
                 srcfile = join(rundir, rbldfile)
                 destfile = join(archive_rblddir, rbldfile)
-                logger.info("moving {} to {} ".format(srcfile, destfile))
+                logger.info("{} {} to {} ".format(_get_archive_fn_desc(archive_file_fn), srcfile, destfile))
                 archive_file_fn(srcfile, destfile)
 
         sfxhst = casename + r'_[0-9][mdy]_' + r'[0-9]*'
@@ -246,7 +251,7 @@ def _archive_history_files(archive, compclass, compname, histfiles_savein_rundir
             for hstfile in hstfiles:
                 srcfile = join(rundir, hstfile)
                 destfile = join(archive_histdir, hstfile)
-                logger.info("moving {} to {} ".format(srcfile, destfile))
+                logger.info("{} {} to {} ".format(_get_archive_fn_desc(archive_file_fn), srcfile, destfile))
                 archive_file_fn(srcfile, destfile)
 
     # determine ninst and ninst_string
@@ -267,7 +272,7 @@ def _archive_history_files(archive, compclass, compname, histfiles_savein_rundir
                     logger.info("copying {} to {} ".format(srcfile, destfile))
                     safe_copy(srcfile, destfile)
                 else:
-                    logger.info("moving {} to {} ".format(srcfile, destfile))
+                    logger.info("{} {} to {} ".format(_get_archive_fn_desc(archive_file_fn), srcfile, destfile))
                     archive_file_fn(srcfile, destfile)
 
 ###############################################################################
@@ -462,8 +467,8 @@ def _archive_restarts_date_comp(case, casename, rundir, archive, archive_entry,
                     destfile = os.path.join(archive_restdir, rfile)
                     expect(os.path.isfile(srcfile),
                            "restart file {} does not exist ".format(srcfile))
+                    logger.info("{} file {} to {}".format(_get_archive_fn_desc(archive_file_fn), srcfile, destfile))
                     archive_file_fn(srcfile, destfile)
-                    logger.info("moving file {} to {}".format(srcfile, destfile))
 
                     # need to copy the history files needed for interim restarts - since
                     # have not archived all of the history files yet


Test patch:
diff --git a/config/config_tests.xml b/config/config_tests.xml
--- a/config/config_tests.xml
+++ b/config/config_tests.xml
@@ -517,10 +517,10 @@ NODEFAIL          Tests restart upon detected node failure. Generates fake failu
     <STOP_N>3</STOP_N>
     <NTASKS>1</NTASKS>
     <CHECK_TIMING>FALSE</CHECK_TIMING>
-    <DOUT_S>TRUE</DOUT_S>
     <REST_OPTION>none</REST_OPTION>
     <HIST_OPTION>$STOP_OPTION</HIST_OPTION>
     <HIST_N>$STOP_N</HIST_N>
+    <DOUT_S>FALSE</DOUT_S>
   </test>
 
   <test NAME="NODEFAIL" INFRASTRUCTURE_TEST="TRUE">
diff --git a/scripts/lib/CIME/SystemTests/system_tests_common.py b/scripts/lib/CIME/SystemTests/system_tests_common.py
--- a/scripts/lib/CIME/SystemTests/system_tests_common.py
+++ b/scripts/lib/CIME/SystemTests/system_tests_common.py
@@ -260,7 +260,7 @@ def _set_active_case(self, case):
         self._case.load_env(reset=True)
         self._caseroot = case.get_value("CASEROOT")
 
-    def run_indv(self, suffix="base", st_archive=False):
+    def run_indv(self, suffix="base", st_archive=False, submit_resubmits=None):
         """
         Perform an individual run. Raises an EXCEPTION on fail.
         """
@@ -268,7 +268,10 @@ def run_indv(self, suffix="base", st_archive=False):
         stop_option = self._case.get_value("STOP_OPTION")
         run_type    = self._case.get_value("RUN_TYPE")
         rundir      = self._case.get_value("RUNDIR")
-        is_batch    = self._case.get_value("BATCH_SYSTEM") != "none"
+        if submit_resubmits is None:
+            do_resub = self._case.get_value("BATCH_SYSTEM") != "none"
+        else:
+            do_resub = submit_resubmits
 
         # remove any cprnc output leftover from previous runs
         for compout in glob.iglob(os.path.join(rundir,"*.cprnc.out")):
@@ -285,7 +288,7 @@ def run_indv(self, suffix="base", st_archive=False):
 
         logger.info(infostr)
 
-        self._case.case_run(skip_pnl=self._skip_pnl, submit_resubmits=is_batch)
+        self._case.case_run(skip_pnl=self._skip_pnl, submit_resubmits=do_resub)
 
         if not self._coupler_log_indicates_run_complete():
             expect(False, "Coupler did not indicate run passed")
@@ -610,12 +613,12 @@ def build_phase(self, sharedlib_only=False, model_only=False):
                 expect(os.path.exists(modelexe),"Could not find expected file {}".format(modelexe))
                 logger.info("FakeTest build_phase complete {} {}".format(modelexe, self._requires_exe))
 
-    def run_indv(self, suffix="base", st_archive=False):
+    def run_indv(self, suffix="base", st_archive=False, submit_resubmits=False):
         mpilib = self._case.get_value("MPILIB")
         # This flag is needed by mpt to run a script under mpiexec
         if mpilib == "mpt":
             os.environ["MPI_SHEPHERD"] = "true"
-        super(FakeTest, self).run_indv(suffix, st_archive)
+        super(FakeTest, self).run_indv(suffix, st_archive=st_archive, submit_resubmits=submit_resubmits)
 
 class TESTRUNPASS(FakeTest):
 
@@ -747,22 +750,13 @@ def build_phase(self, sharedlib_only=False, model_only=False):
         script = \
 """
 cd {caseroot}
-./xmlchange DOUT_S=TRUE
-./xmlchange DOUT_S=TRUE --file env_test.xml
-./xmlchange RESUBMIT=1
-./xmlchange STOP_N={stopn}
-./xmlchange CONTINUE_RUN=FALSE
-./xmlchange RESUBMIT_SETS_CONTINUE_RUN=FALSE
+./xmlchange RESUBMIT=1,STOP_N={stopn},CONTINUE_RUN=FALSE,RESUBMIT_SETS_CONTINUE_RUN=FALSE
 cd -
-sleep 5
 {modelexe}.real "$@"
-sleep 5
 
 # Need to remove self in order to avoid infinite loop
-ls $(dirname {modelexe})
 mv {modelexe} {modelexe}.old
 mv {modelexe}.real {modelexe}
-ls $(dirname {modelexe})
 sleep 5
 """.format(caseroot=caseroot, modelexe=modelexe, stopn=str(new_stop_n))
         self._set_script(script, requires_exe=True)
@@ -770,7 +764,7 @@ def build_phase(self, sharedlib_only=False, model_only=False):
                              sharedlib_only=sharedlib_only, model_only=model_only)
 
     def run_phase(self):
-        self.run_indv(st_archive=True)
+        self.run_indv(submit_resubmits=True)
 
 class TESTRUNSLOWPASS(FakeTest):
 
diff --git a/scripts/lib/CIME/tests/SystemTests/test_system_tests_compare_two.py b/scripts/lib/CIME/tests/SystemTests/test_system_tests_compare_two.py
--- a/scripts/lib/CIME/tests/SystemTests/test_system_tests_compare_two.py
+++ b/scripts/lib/CIME/tests/SystemTests/test_system_tests_compare_two.py
@@ -156,7 +156,7 @@ def _init_case_setup(self):
     # SystemTestsCommon
     # ------------------------------------------------------------------------
 
-    def run_indv(self, suffix="base", st_archive=False):
+    def run_indv(self, suffix="base", st_archive=False, submit_resubmits=None):
         """
         This fake implementation appends to the log and raises an exception if
         it's supposed to
diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py
--- a/scripts/tests/scripts_regression_tests.py
+++ b/scripts/tests/scripts_regression_tests.py
@@ -22,7 +22,7 @@
 
 import collections
 
-from CIME.utils import run_cmd, run_cmd_no_fail, get_lids, get_current_commit, safe_copy, CIMEError, get_cime_root
+from CIME.utils import run_cmd, run_cmd_no_fail, get_lids, get_current_commit, safe_copy, CIMEError, get_cime_root, Timeout
 import get_tests
 import CIME.test_scheduler, CIME.wait_for_tests
 from  CIME.test_scheduler import TestScheduler
@@ -1906,17 +1906,17 @@ def test_user_concurrent_mods(self):
         if (FAST_ONLY):
             self.skipTest("Skipping slow test")
 
-        casedir = self._create_test(["--walltime=0:30:00", "TESTRUNUSERXMLCHANGE.f19_g16.X"], test_id=self._baseline_name)
+        casedir = self._create_test(["--walltime=0:30:00", "TESTRUNUSERXMLCHANGE_Mmpi-serial.f19_g16.X"], test_id=self._baseline_name)
 
-        # We need to be careful since we are running a resubmit. The first run should
-        # report a RUN PASS, which will cause our waiting code to stop waiting. But we
-        # want to wait for the second run too.
-        time.sleep(10) # give second run a chance to begin
-        self._wait_for_tests(self._baseline_name) # wait for second run
+        with Timeout(3000):
+            while True:
+                with open(os.path.join(casedir, "CaseStatus"), "r") as fd:
+                    self._wait_for_tests(self._baseline_name)
+                    contents = fd.read()
+                    if contents.count("model execution success") == 2:
+                        break
 
-        with open(os.path.join(casedir, "CaseStatus"), "r") as fd:
-            contents = fd.read()
-            self.assertEqual(contents.count("model execution success"), 2)
+                time.sleep(5)
 
         rundir = run_cmd_no_fail("./xmlquery RUNDIR --value", from_dir=casedir)
         with open(os.path.join(rundir, "drv_in"), "r") as fd:
@@ -3484,6 +3484,10 @@ def write_provenance_info():
 
 def cleanup():
     # if the TEST_ROOT directory exists and is empty, remove it
+    testreporter = os.path.join(TEST_ROOT,"testreporter")
+    files = os.listdir(TEST_ROOT)
+    if len(files)==1 and os.path.isfile(testreporter):
+        os.unlink(testreporter)
     if os.path.exists(TEST_ROOT) and not os.listdir(TEST_ROOT):
         print("All pass, removing directory:", TEST_ROOT)
         os.rmdir(TEST_ROOT)