Repository: ESMCI/cime

Issue:
scripts_regression_tests.py test TESTRUNUSERXMLCHANGE problematic 
The test writes a script which overwrites itself 
[here](https://github.com/ESMCI/cime/blob/master/scripts/lib/CIME/SystemTests/system_tests_common.py#L757)
On cheyenne the line mv {modelexe} {modelexe}.old overwrites the script currently being executed so that the next line
mv {modelexe}.real {modelexe} 
is never executed and the test fails. 


Discussion:

Resolving patch:
diff --git a/scripts/lib/CIME/case/case_submit.py b/scripts/lib/CIME/case/case_submit.py
--- a/scripts/lib/CIME/case/case_submit.py
+++ b/scripts/lib/CIME/case/case_submit.py
@@ -130,7 +130,7 @@ def _submit(case, job=None, no_batch=False, prereq=None, allow_fail=False, resub
         unlock_file(os.path.basename(env_batch.filename))
         lock_file(os.path.basename(env_batch.filename))
 
-        case.check_case()
+        case.check_case(skip_pnl=skip_pnl)
         if job == case.get_primary_job():
             case.check_DA_settings()
             if case.get_value("MACH") == "mira":
@@ -212,9 +212,10 @@ def submit(self, job=None, no_batch=False, prereq=None, allow_fail=False, resubm
 
         raise
 
-def check_case(self):
+def check_case(self, skip_pnl=False):
     self.check_lockedfiles()
-    self.create_namelists() # Must be called before check_all_input_data
+    if not skip_pnl:
+        self.create_namelists() # Must be called before check_all_input_data
     logger.info("Checking that inputdata is available as part of case submission")
     self.check_all_input_data()
 
diff --git a/scripts/lib/CIME/case/preview_namelists.py b/scripts/lib/CIME/case/preview_namelists.py
--- a/scripts/lib/CIME/case/preview_namelists.py
+++ b/scripts/lib/CIME/case/preview_namelists.py
@@ -62,8 +62,6 @@ def create_namelists(self, component=None):
 
     self.stage_refcase()
 
-    logger.info("Creating component namelists")
-
     # Create namelists - must have cpl last in the list below
     # Note - cpl must be last in the loop below so that in generating its namelist,
     # it can use xml vars potentially set by other component's buildnml scripts
@@ -87,6 +85,7 @@ def create_namelists(self, component=None):
                 # otherwise look in the component config_dir
                 cmd = os.path.join(config_dir, "buildnml")
             expect(os.path.isfile(cmd), "Could not find buildnml file for component {}".format(compname))
+            logger.info("Create namelist for component {}".format(compname))
             run_sub_or_cmd(cmd, (caseroot), "buildnml",
                            (self, caseroot, compname), case=self)
 


Test patch:
diff --git a/config/config_tests.xml b/config/config_tests.xml
--- a/config/config_tests.xml
+++ b/config/config_tests.xml
@@ -513,10 +513,11 @@ NODEFAIL          Tests restart upon detected node failure. Generates fake failu
   <test NAME="TESTRUNUSERXMLCHANGE" INFRASTRUCTURE_TEST="TRUE">
     <DESC>For testing infra only. Test simultaneous xmlchanges while first run is running.</DESC>
     <INFO_DBUG>1</INFO_DBUG>
-    <STOP_OPTION>ndays</STOP_OPTION>
+    <STOP_OPTION>nsteps</STOP_OPTION>
     <STOP_N>3</STOP_N>
+    <NTASKS>1</NTASKS>
     <CHECK_TIMING>FALSE</CHECK_TIMING>
-    <DOUT_S>FALSE</DOUT_S>
+    <DOUT_S>TRUE</DOUT_S>
     <REST_OPTION>none</REST_OPTION>
     <HIST_OPTION>$STOP_OPTION</HIST_OPTION>
     <HIST_N>$STOP_N</HIST_N>
diff --git a/scripts/lib/CIME/SystemTests/system_tests_common.py b/scripts/lib/CIME/SystemTests/system_tests_common.py
--- a/scripts/lib/CIME/SystemTests/system_tests_common.py
+++ b/scripts/lib/CIME/SystemTests/system_tests_common.py
@@ -3,7 +3,7 @@
 """
 from CIME.XML.standard_module_setup import *
 from CIME.XML.env_run import EnvRun
-from CIME.utils import append_testlog, get_model, safe_copy, get_timestamp, CIMEError
+from CIME.utils import append_testlog, get_model, safe_copy, get_timestamp, CIMEError, expect
 from CIME.test_status import *
 from CIME.hist_utils import copy_histfiles, compare_test, generate_teststatus, \
     compare_baseline, get_ts_synopsis, generate_baseline
@@ -11,7 +11,7 @@
 from CIME.locked_files import LOCKED_DIR, lock_file, is_locked
 import CIME.build as build
 
-import glob, gzip, time, traceback, six, shutil
+import glob, gzip, time, traceback, six, os
 
 logger = logging.getLogger(__name__)
 
@@ -129,6 +129,7 @@ def build_indv(self, sharedlib_only=False, model_only=False):
                          sharedlib_only=sharedlib_only, model_only=model_only,
                          save_build_provenance=not model=='cesm',
                          use_old=self._old_build, ninja=self._ninja, dry_run=self._dry_run)
+        logger.info("build_indv complete")
 
     def clean_build(self, comps=None):
         if comps is None:
@@ -591,8 +592,10 @@ def build_phase(self, sharedlib_only=False, model_only=False):
             modelexe = os.path.join(exeroot, "{}.exe".format(cime_model))
             real_modelexe = modelexe + ".real"
 
-            if self._requires_exe:
-                shutil.move(modelexe, real_modelexe)
+            if self._requires_exe and os.path.exists(modelexe):
+                logger.info("moving {} to {}".format(modelexe, real_modelexe))
+                os.rename(modelexe, real_modelexe)
+                expect(os.path.exists(real_modelexe),"Could not find expected file {}".format(real_modelexe))
 
             with open(modelexe, 'w') as f:
                 f.write("#!/bin/bash\n")
@@ -602,6 +605,9 @@ def build_phase(self, sharedlib_only=False, model_only=False):
 
             if not self._requires_exe:
                 build.post_build(self._case, [], build_complete=True)
+            else:
+                expect(os.path.exists(modelexe),"Could not find expected file {}".format(modelexe))
+                logger.info("FakeTest build_phase complete {} {}".format(modelexe, self._requires_exe))
 
     def run_indv(self, suffix="base", st_archive=False):
         mpilib = self._case.get_value("MPILIB")
diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py
--- a/scripts/tests/scripts_regression_tests.py
+++ b/scripts/tests/scripts_regression_tests.py
@@ -1906,12 +1906,12 @@ def test_user_concurrent_mods(self):
         if (FAST_ONLY):
             self.skipTest("Skipping slow test")
 
-        casedir = self._create_test(["--walltime=0:30:00", "TESTRUNUSERXMLCHANGE_P1.f09_g16.X"], test_id=self._baseline_name)
+        casedir = self._create_test(["--walltime=0:30:00", "TESTRUNUSERXMLCHANGE.f19_g16.X"], test_id=self._baseline_name)
 
         # We need to be careful since we are running a resubmit. The first run should
         # report a RUN PASS, which will cause our waiting code to stop waiting. But we
         # want to wait for the second run too.
-        time.sleep(60) # give second run a chance to begin
+        time.sleep(10) # give second run a chance to begin
         self._wait_for_tests(self._baseline_name) # wait for second run
 
         with open(os.path.join(casedir, "CaseStatus"), "r") as fd:
@@ -2334,7 +2334,12 @@ def test_cime_case_test_walltime_mgmt_8(self):
     def test_cime_case_test_custom_project(self):
     ###########################################################################
         test_name = "ERS_P1.f19_g16_rx1.A"
-        machine, compiler = "mappy", "gnu" # have to use a machine both models know and one that doesn't put PROJECT in any key paths
+        # have to use a machine both models know and one that doesn't put PROJECT in any key paths
+        if CIME.utils.get_model() == "e3sm":
+            machine = "mappy"
+        else:
+            machine = "melvin"
+        compiler = "gnu"
         casedir = self._create_test(["--no-setup", "--machine={}".format(machine), "--compiler={}".format(compiler), "--project=testproj", test_name, "--mpilib=mpi-serial"],
                                     test_id=self._baseline_name,
                                     env_changes="unset CIME_GLOBAL_WALLTIME &&")