Repository: ESMCI/cime

Issue:
Running case.st_archive with RESUBMIT > 0 submits case.run jobs
One of our users complained about this, and it seems very unintuitive from a UI perspective for people who run case.st_archive from the login node.
My thought on a solution to this is to add an argument to case.st_archive which enables resubmitting, and have case.submit set that option for the archiving jobs.


Discussion:
Hmmm - if case.st_archive is run from the command line instead of from case.submit then it should not invoke case.run - I don't think that this requires a new argument, just a minor change in logic.
I don't see an obvious mechanism for determining if it's being run from the command line as opposed to case.submit? If there is one, I'd expect the mechanism for determining this to be in env_run.py or env_batch.py?
I could make that method if you'd prefer that

Resolving patch:
diff --git a/scripts/lib/CIME/case/case_run.py b/scripts/lib/CIME/case/case_run.py
--- a/scripts/lib/CIME/case/case_run.py
+++ b/scripts/lib/CIME/case/case_run.py
@@ -3,7 +3,7 @@
 '"""
 from CIME.XML.standard_module_setup import *
 from CIME.utils                     import gzip_existing_file, new_lid, run_and_log_case_status
-from CIME.utils                     import run_sub_or_cmd, append_status, safe_copy
+from CIME.utils                     import run_sub_or_cmd, append_status, safe_copy, model_log
 from CIME.get_timing                import get_timing
 from CIME.provenance                import save_prerun_provenance, save_postrun_provenance
 
@@ -50,14 +50,14 @@ def _pre_run_check(case, lid, skip_pnl=False, da_cycle=0):
     # This needs to be done everytime the LID changes in order for log files to be set up correctly
     # The following also needs to be called in case a user changes a user_nl_xxx file OR an env_run.xml
     # variable while the job is in the queue
-    logger.debug("{} NAMELIST CREATION BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+    model_log("e3sm", logger, "{} NAMELIST CREATION BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
     if skip_pnl:
         case.create_namelists(component='cpl')
     else:
         logger.info("Generating namelists for {}".format(caseroot))
         case.create_namelists()
 
-    logger.debug("{} NAMELIST CREATION HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+    model_log("e3sm", logger, "{} NAMELIST CREATION HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
 
     logger.info("-------------------------------------------------------------------------")
     logger.info(" - Prestage required restarts into {}".format(rundir))
@@ -69,9 +69,9 @@ def _pre_run_check(case, lid, skip_pnl=False, da_cycle=0):
 def _run_model_impl(case, lid, skip_pnl=False, da_cycle=0):
 ###############################################################################
 
-    logger.debug("{} PRE_RUN_CHECK BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+    model_log("e3sm", logger, "{} PRE_RUN_CHECK BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
     _pre_run_check(case, lid, skip_pnl=skip_pnl, da_cycle=da_cycle)
-    logger.debug("{} PRE_RUN_CHECK HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+    model_log("e3sm", logger, "{} PRE_RUN_CHECK HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
 
     model = case.get_value("MODEL")
 
@@ -101,14 +101,14 @@ def _run_model_impl(case, lid, skip_pnl=False, da_cycle=0):
     while loop:
         loop = False
 
-        logger.debug("{} SAVE_PRERUN_PROVENANCE BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+        model_log("e3sm", logger, "{} SAVE_PRERUN_PROVENANCE BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
         save_prerun_provenance(case)
-        logger.debug("{} SAVE_PRERUN_PROVENANCE HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+        model_log("e3sm", logger, "{} SAVE_PRERUN_PROVENANCE HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
 
-        logger.debug("{} MODEL EXECUTION BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+        model_log("e3sm", logger, "{} MODEL EXECUTION BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
         run_func = lambda: run_cmd(cmd, from_dir=rundir)[0]
         stat = run_and_log_case_status(run_func, "model execution", caseroot=case.get_value("CASEROOT"))
-        logger.debug("{} MODEL EXECUTION HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+        model_log("e3sm", logger, "{} MODEL EXECUTION HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
 
         model_logfile = os.path.join(rundir, model + ".log." + lid)
         # Determine if failure was due to a failed node, if so, try to restart
@@ -147,9 +147,9 @@ def _run_model_impl(case, lid, skip_pnl=False, da_cycle=0):
             # We failed and we're not restarting
             expect(False, "RUN FAIL: Command '{}' failed\nSee log file for details: {}".format(cmd, model_logfile))
 
-    logger.debug("{} POST_RUN_CHECK BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+    model_log("e3sm", logger, "{} POST_RUN_CHECK BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
     _post_run_check(case, lid)
-    logger.debug("{} POST_RUN_CHECK HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+    model_log("e3sm", logger, "{} POST_RUN_CHECK HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
 
     return lid
 
@@ -262,7 +262,7 @@ def _do_data_assimilation(da_script, caseroot, cycle, lid, rundir):
 ###############################################################################
 def case_run(self, skip_pnl=False, set_continue_run=False, submit_resubmits=False):
 ###############################################################################
-    logger.debug("{} CASE.RUN BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+    model_log("e3sm", logger, "{} CASE.RUN BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
     # Set up the run, run the model, do the postrun steps
     prerun_script = self.get_value("PRERUN_SCRIPT")
     postrun_script = self.get_value("POSTRUN_SCRIPT")
@@ -280,12 +280,12 @@ def case_run(self, skip_pnl=False, set_continue_run=False, submit_resubmits=Fals
     lid = new_lid()
 
     if prerun_script:
-        logger.debug("{} PRERUN_SCRIPT BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+        model_log("e3sm", logger, "{} PRERUN_SCRIPT BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
         self.flush()
         _do_external(prerun_script, self.get_value("CASEROOT"), self.get_value("RUNDIR"),
                     lid, prefix="prerun")
         self.read_xml()
-        logger.debug("{} PRERUN_SCRIPT HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+        model_log("e3sm", logger, "{} PRERUN_SCRIPT HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
 
     for cycle in range(data_assimilation_cycles):
         # After the first DA cycle, runs are restart runs
@@ -294,41 +294,41 @@ def case_run(self, skip_pnl=False, set_continue_run=False, submit_resubmits=Fals
             self.set_value("CONTINUE_RUN",
                            self.get_value("RESUBMIT_SETS_CONTINUE_RUN"))
 
-        logger.debug("{} RUN_MODEL BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+        model_log("e3sm", logger, "{} RUN_MODEL BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
         lid = _run_model(self, lid, skip_pnl, da_cycle=cycle)
-        logger.debug("{} RUN_MODEL HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+        model_log("e3sm", logger, "{} RUN_MODEL HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
 
         # TODO mvertens: remove the hard-wiring for nuopc below
         if driver != 'nuopc':
             if self.get_value("CHECK_TIMING") or self.get_value("SAVE_TIMING"):
-                logger.debug("{} GET_TIMING BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+                model_log("e3sm", logger, "{} GET_TIMING BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
                 get_timing(self, lid)     # Run the getTiming script
-                logger.debug("{} GET_TIMING HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+                model_log("e3sm", logger, "{} GET_TIMING HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
         else:
             self.set_value("CHECK_TIMING",False)
 
         if data_assimilation:
-            logger.debug("{} DO_DATA_ASSIMILATION BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+            model_log("e3sm", logger, "{} DO_DATA_ASSIMILATION BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
             self.flush()
             _do_data_assimilation(data_assimilation_script, self.get_value("CASEROOT"), cycle, lid,
                                  self.get_value("RUNDIR"))
             self.read_xml()
-            logger.debug("{} DO_DATA_ASSIMILATION HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+            model_log("e3sm", logger, "{} DO_DATA_ASSIMILATION HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
 
         _save_logs(self, lid)       # Copy log files back to caseroot
 
-        logger.debug("{} SAVE_POSTRUN_PROVENANCE BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+        model_log("e3sm", logger, "{} SAVE_POSTRUN_PROVENANCE BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
         save_postrun_provenance(self)
-        logger.debug("{} SAVE_POSTRUN_PROVENANCE HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+        model_log("e3sm", logger, "{} SAVE_POSTRUN_PROVENANCE HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
 
     if postrun_script:
-        logger.debug("{} POSTRUN_SCRIPT BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+        model_log("e3sm", logger, "{} POSTRUN_SCRIPT BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
         self.flush()
         _do_external(postrun_script, self.get_value("CASEROOT"), self.get_value("RUNDIR"),
                     lid, prefix="postrun")
         self.read_xml()
         _save_logs(self, lid)
-        logger.debug("{} POSTRUN_SCRIPT HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+        model_log("e3sm", logger, "{} POSTRUN_SCRIPT HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
 
     if set_continue_run:
         self.set_value("CONTINUE_RUN",
@@ -338,6 +338,6 @@ def case_run(self, skip_pnl=False, set_continue_run=False, submit_resubmits=Fals
     if submit_resubmits:
         _resubmit_check(self)
 
-    logger.debug("{} CASE.RUN HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
+    model_log("e3sm", logger, "{} CASE.RUN HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
 
     return True
diff --git a/scripts/lib/CIME/get_timing.py b/scripts/lib/CIME/get_timing.py
--- a/scripts/lib/CIME/get_timing.py
+++ b/scripts/lib/CIME/get_timing.py
@@ -65,13 +65,13 @@ def gettime2(self, heading_padded):
 
         heading = '"' + heading_padded.strip() + '"'
         for line in self.finlines:
-            m = re.match(r'\s*{}\s*(\d+)\s*\d+\s*(\S+)'.format(heading), line)
+            m = re.match(r'\s*{}\s+\S\s+(\d+)\s*\d+\s*(\S+)'.format(heading), line)
             if m:
                 nprocs = int(float(m.groups()[0]))
                 ncount = int(float(m.groups()[1]))
                 return (nprocs, ncount)
             else:
-                m = re.match(r'\s*{}\s+(\d+)\s'.format(heading), line)
+                m = re.match(r'\s*{}\s+\S\s+(\d+)\s'.format(heading), line)
                 if m:
                     nprocs = 1
                     ncount = int(float(m.groups()[0]))
@@ -85,7 +85,7 @@ def gettime(self, heading_padded):
         maxval = 0
 
         for line in self.finlines:
-            m = re.match(r'\s*{}\s*\d+\s*\d+\s*\S+\s*\S+\s*(\d*\.\d+)\s*\(.*\)\s*(\d*\.\d+)\s*\(.*\)'.format(heading), line)
+            m = re.match(r'\s*{}\s+\S\s+\d+\s*\d+\s*\S+\s*\S+\s*(\d*\.\d+)\s*\(.*\)\s*(\d*\.\d+)\s*\(.*\)'.format(heading), line)
             if m:
                 maxval = float(m.groups()[0])
                 minval = float(m.groups()[1])
diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py
--- a/scripts/lib/CIME/utils.py
+++ b/scripts/lib/CIME/utils.py
@@ -1787,3 +1787,9 @@ def string_in_list(_string, _list):
         if _string.lower() == x.lower():
             return x
     return None
+
+def model_log(model, arg_logger, msg, debug_others=True):
+    if get_model() == model:
+        arg_logger.info(msg)
+    elif debug_others:
+        arg_logger.debug(msg)


Test patch:
diff --git a/config/e3sm/tests.py b/config/e3sm/tests.py
--- a/config/e3sm/tests.py
+++ b/config/e3sm/tests.py
@@ -65,6 +65,12 @@
                         )
                        ),
 
+    #atmopheric nbfb tests
+    "e3sm_atm_nbfb" : (None, None,
+                                 ("PGN_P1x1.ne4_ne4.FC5AV1C-L",
+                                  "TSC.ne4_ne4.FC5AV1C-L")
+                                 ),
+
     "e3sm_developer" : (("e3sm_land_developer","e3sm_atm_developer"), "0:45:00",
                         ("ERS.f19_g16_rx1.A",
                          "ERS.ne30_g16_rx1.A",