Repository: ESMCI/cime

Issue:
TestStatus.log confusing with runtime error 
When a cime based test run fails at runtime the TestStatus.log gets something like

```
2016-09-14 13:23:39: Exception during run:
ERROR: Command: 'mpiexec --machinefile /var/spool/torque/aux//3236.hobart.cgd.ucar.edu  -n 24  /scratch/cluster/mvertens/SMS_D_Ld3.f45_g37_rx1.A.hobart_nag.C.20160914_130855/bld/cesm.exe  >> cesm.log.$LID 2>&1 ' failed with error '' from dir '/scratch/cluster/mvertens/SMS_D_Ld3.f45_g37_rx1.A.hobart_nag.C.20160914_130855/run'
Traceback (most recent call last):
  File "/fs/cgd/data0/mvertens/src/cesm2_0_beta01/cime/utils/python/CIME/SystemTests/system_tests_common.py", line 139, in run
    self.run_phase()
  File "/fs/cgd/data0/mvertens/src/cesm2_0_beta01/cime/utils/python/CIME/SystemTests/system_tests_common.py", line 171, in run_phase
    self.run_indv()
  File "/fs/cgd/data0/mvertens/src/cesm2_0_beta01/cime/utils/python/CIME/SystemTests/system_tests_common.py", line 203, in run_indv
    case_run(self._case)
  File "/fs/cgd/data0/mvertens/src/cesm2_0_beta01/cime/utils/python/CIME/case_run.py", line 215, in case_run
    run_model(case)
  File "/fs/cgd/data0/mvertens/src/cesm2_0_beta01/cime/utils/python/CIME/case_run.py", line 98, in run_model
    run_cmd_no_fail(cmd, from_dir=rundir)
  File "/fs/cgd/data0/mvertens/src/cesm2_0_beta01/cime/utils/python/CIME/utils.py", line 210, in run_cmd_no_fail
    (cmd, errput, "" if from_dir is None else " from dir '%s'" % from_dir))
  File "/fs/cgd/data0/mvertens/src/cesm2_0_beta01/cime/utils/python/CIME/utils.py", line 26, in expect
    raise exc_type("ERROR: %s" % error_msg)
SystemExit: ERROR: Command: 'mpiexec --machinefile /var/spool/torque/aux//3236.hobart.cgd.ucar.edu  -n 24  /scratch/cluster/mvertens/SMS_D_Ld3.f45_g37_rx1.A.hobart_nag.C.20160914_130855/bld/cesm.exe  >> cesm.log.$LID 2>&1 ' failed with error '' from dir '/scratch/cluster/mvertens/SMS_D_Ld3.f45_g37_rx1.A.hobart_nag.C.20160914_130855/run'
```

this is a really confusing message and many users don't know that they should look in the cesm.log.$LID file for the real error message - can this be improved?



Discussion:

Resolving patch:
diff --git a/scripts/lib/CIME/aprun.py b/scripts/lib/CIME/aprun.py
old mode 100755
new mode 100644
diff --git a/scripts/lib/CIME/case_run.py b/scripts/lib/CIME/case_run.py
--- a/scripts/lib/CIME/case_run.py
+++ b/scripts/lib/CIME/case_run.py
@@ -102,6 +102,7 @@ def _run_model_impl(case, lid):
     while loop:
         loop = False
         stat = run_cmd(cmd, from_dir=rundir)[0]
+        model_logfile = os.path.join(rundir, model + ".log." + lid)
         # Determine if failure was due to a failed node, if so, try to restart
         if stat != 0:
             node_fail_re = case.get_value("NODE_FAIL_REGEX")
@@ -130,7 +131,7 @@ def _run_model_impl(case, lid):
 
             if not loop:
                 # We failed and we're not restarting
-                expect(False, "Command '%s' failed" % cmd)
+                expect(False, "RUN FAIL: Command '%s' failed\nSee log file for details: %s" % (cmd, model_logfile))
 
     logger.info("%s MODEL EXECUTION HAS FINISHED" %(time.strftime("%Y-%m-%d %H:%M:%S")))
 
diff --git a/scripts/lib/CIME/case_st_archive.py b/scripts/lib/CIME/case_st_archive.py
old mode 100755
new mode 100644


Test patch:
diff --git a/scripts/lib/CIME/SystemTests/system_tests_common.py b/scripts/lib/CIME/SystemTests/system_tests_common.py
--- a/scripts/lib/CIME/SystemTests/system_tests_common.py
+++ b/scripts/lib/CIME/SystemTests/system_tests_common.py
@@ -146,7 +146,13 @@ def run(self):
 
         except:
             success = False
-            excmsg = "Exception during run:\n%s\n%s" % (sys.exc_info()[1], traceback.format_exc())
+            msg = sys.exc_info()[1].message
+            if "RUN FAIL" in msg:
+                # Don't want to print stacktrace for a model failure since that
+                # is not a CIME/infrastructure problem.
+                excmsg = msg
+            else:
+                excmsg = "Exception during run:\n%s\n%s" % (sys.exc_info()[1], traceback.format_exc())
             logger.warning(excmsg)
             append_testlog(excmsg)
 
diff --git a/scripts/lib/CIME/test_scheduler.py b/scripts/lib/CIME/test_scheduler.py
--- a/scripts/lib/CIME/test_scheduler.py
+++ b/scripts/lib/CIME/test_scheduler.py
@@ -325,14 +325,14 @@ def _update_test_status(self, test, phase, status):
     def _shell_cmd_for_phase(self, test, cmd, phase, from_dir=None):
     ###########################################################################
         while True:
-            rc, output, errput = run_cmd(cmd, from_dir=from_dir)
+            rc, output, _ = run_cmd(cmd + " 2>&1", from_dir=from_dir)
             if rc != 0:
                 self._log_output(test,
-                                 "%s FAILED for test '%s'.\nCommand: %s\nOutput: %s\n\nErrput: %s" %
-                                 (phase, test, cmd, output, errput))
+                                 "%s FAILED for test '%s'.\nCommand: %s\nOutput: %s\n" %
+                                 (phase, test, cmd, output))
                 # Temporary hack to get around odd file descriptor use by
                 # buildnml scripts.
-                if "bad interpreter" in errput:
+                if "bad interpreter" in output:
                     time.sleep(1)
                     continue
                 else:
@@ -340,10 +340,10 @@ def _shell_cmd_for_phase(self, test, cmd, phase, from_dir=None):
             else:
                 # We don't want "RUN PASSED" in the TestStatus.log if the only thing that
                 # succeeded was the submission.
-                if phase != RUN_PHASE or self._no_batch:
-                    self._log_output(test,
-                                     "%s PASSED for test '%s'.\nCommand: %s\nOutput: %s" %
-                                     (phase, test, cmd, output))
+                phase = "SUBMIT" if phase == RUN_PHASE else phase
+                self._log_output(test,
+                                 "%s PASSED for test '%s'.\nCommand: %s\nOutput: %s\n" %
+                                 (phase, test, cmd, output))
                 break
 
         return rc == 0
@@ -545,8 +545,8 @@ def _setup_phase(self, test):
         rv = self._shell_cmd_for_phase(test, "./case.setup", SETUP_PHASE, from_dir=test_dir)
 
         # It's OK for this command to fail with baseline diffs but not catastrophically
-        cmdstat, output, errput = run_cmd("./case.cmpgen_namelists", from_dir=test_dir)
-        expect(cmdstat in [0, TESTS_FAILED_ERR_CODE], "Fatal error in case.cmpgen_namelists: %s" % (output + "\n" + errput))
+        cmdstat, output, _ = run_cmd("./case.cmpgen_namelists 2>&1", from_dir=test_dir)
+        expect(cmdstat in [0, TESTS_FAILED_ERR_CODE], "Fatal error in case.cmpgen_namelists: %s" % output)
 
         return rv