Repository: ESMCI/cime

Issue:
Misleading output in CaseStatus
This is a two-fer, because there are a couple of confusing entries in `CaseStatus` (at least with 5.8.0). Aside from cutting the `xmlchange` entries, here are the complete contents of a test that encounters a run-time error:

```
2019-04-08 16:05:00: case.setup starting
 ---------------------------------------------------
2019-04-08 16:05:04: case.setup success
 ---------------------------------------------------
2019-04-08 16:05:05: case.build starting
 ---------------------------------------------------
2019-04-08 16:06:21: case.build success
 ---------------------------------------------------
2019-04-08 16:06:22: case.build starting
 ---------------------------------------------------
2019-04-08 16:07:08: case.build success
 ---------------------------------------------------
2019-04-08 16:07:08: case.submit starting
 ---------------------------------------------------
2019-04-08 16:07:10: case.submit success case.test:4946980.chadmin1
 ---------------------------------------------------
2019-04-08 16:15:50: case.run starting
 ---------------------------------------------------
2019-04-08 16:15:52: model execution starting
 ---------------------------------------------------
2019-04-08 16:16:02: model execution success
 ---------------------------------------------------
2019-04-08 16:16:02: case.run error
ERROR: RUN FAIL: Command 'mpiexec_mpt -p "%g:"  -np 144  omplace -tm open64  /glade/scratch/mlevy/SMS_D_P144x1_Ld1.T62_g17.C1850ECO.cheyenne_intel.pop-ecosys_144blocks_320x384_spacecurve.POP_trunk/bld/cesm.exe  >> cesm.log.$LID 2>&1 ' failed
See log file for details: /glade/scratch/mlevy/SMS_D_P144x1_Ld1.T62_g17.C1850ECO.cheyenne_intel.pop-ecosys_144blocks_320x384_spacecurve.POP_trunk/run/cesm.log.4946980.chadmin1.190408-161550
 ---------------------------------------------------
```

The first issue is cosmetic: I suspect the first `case.build` is building the shared libraries and then the second `case.build` is building the model itself... but changing the log to something more like

```
2019-04-08 16:05:05: case.build (SHAREDLIB_BUILD) starting
 ---------------------------------------------------
2019-04-08 16:06:21: case.build (SHAREDLIB_BUILD) success
 ---------------------------------------------------
2019-04-08 16:06:22: case.build (MODEL_BUILD) starting
 ---------------------------------------------------
2019-04-08 16:07:08: case.build (MODEL_BUILD) success
```

Would be a little clearer and align with what we see in stdout:

```
Starting SHAREDLIB_BUILD for test ... with 1 procs
Finished SHAREDLIB_BUILD for test ... in 77.041030 seconds (PASS)
Starting MODEL_BUILD for test ... with 4 procs
Finished MODEL_BUILD for test ... in 46.444216 seconds (PASS)
```

The bigger issue is reporting

```
2019-04-08 16:16:02: model execution success
```

when the model did not successfully execute. Note that `CaseStatus` looks right if I kill a job while running (this is from a different test):

```
2019-04-08 15:34:27: model execution starting
 ---------------------------------------------------
2019-04-08 15:40:46: model execution error
ERROR: Job killed due to receiving signal 15 (SIGTERM)
 ---------------------------------------------------
2019-04-08 15:40:46: case.run error
ERROR: Job killed due to receiving signal 15 (SIGTERM)
```


Discussion:
I just noticed the "model execution success" statement followed immediately by a "case.run error" in a run that failed.  That is annoying and could lead to bigger problems.  Any user's control script parsing the CaseStatus file to figure out what is happening will get confused.
This was in a regular create_newcase case.

Resolving patch:
diff --git a/scripts/lib/CIME/build.py b/scripts/lib/CIME/build.py
--- a/scripts/lib/CIME/build.py
+++ b/scripts/lib/CIME/build.py
@@ -558,7 +558,12 @@ def case_build(caseroot, case, sharedlib_only=False, model_only=False, buildlist
 ###############################################################################
     functor = lambda: _case_build_impl(caseroot, case, sharedlib_only, model_only, buildlist,
                                        save_build_provenance)
-    return run_and_log_case_status(functor, "case.build", caseroot=caseroot)
+    cb = "case.build"
+    if (sharedlib_only == True):
+        cb = cb + " (SHAREDLIB_BUILD)"
+    if (model_only == True):
+        cb = cb + " (MODEL_BUILD)"
+    return run_and_log_case_status(functor, cb, caseroot=caseroot)
 
 ###############################################################################
 def clean(case, cleanlist=None, clean_all=False, clean_depends=None):
diff --git a/scripts/lib/CIME/case/case_run.py b/scripts/lib/CIME/case/case_run.py
--- a/scripts/lib/CIME/case/case_run.py
+++ b/scripts/lib/CIME/case/case_run.py
@@ -3,7 +3,7 @@
 '"""
 from CIME.XML.standard_module_setup import *
 from CIME.utils                     import gzip_existing_file, new_lid, run_and_log_case_status
-from CIME.utils                     import run_sub_or_cmd, append_status, safe_copy, model_log
+from CIME.utils                     import run_sub_or_cmd, append_status, safe_copy, model_log, CIMEError
 from CIME.utils                     import get_model
 from CIME.get_timing                import get_timing
 from CIME.provenance                import save_prerun_provenance, save_postrun_provenance
@@ -110,8 +110,13 @@ def _run_model_impl(case, lid, skip_pnl=False, da_cycle=0):
         model_log("e3sm", logger, "{} SAVE_PRERUN_PROVENANCE HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
 
         model_log("e3sm", logger, "{} MODEL EXECUTION BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
-        run_func = lambda: run_cmd(cmd, from_dir=rundir)[0]
-        stat = run_and_log_case_status(run_func, "model execution", caseroot=case.get_value("CASEROOT"))
+        run_func = lambda: run_cmd_no_fail(cmd, from_dir=rundir)
+        try:
+            run_and_log_case_status(run_func, "model execution", caseroot=case.get_value("CASEROOT"))
+            cmd_success = True
+        except CIMEError:
+            cmd_success = False
+
         model_log("e3sm", logger, "{} MODEL EXECUTION HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S")))
 
         model_logfile = os.path.join(rundir, model + ".log." + lid)
@@ -147,7 +152,7 @@ def _run_model_impl(case, lid, skip_pnl=False, da_cycle=0):
                     lid = new_lid()
                     case.create_namelists()
 
-        if stat != 0 and not loop:
+        if not cmd_success and not loop:
             # We failed and we're not restarting
             expect(False, "RUN FAIL: Command '{}' failed\nSee log file for details: {}".format(cmd, model_logfile))
 


Test patch: