Repository: ESMCI/cime

Issue:
ERR test does not always report failures correctly
The ERR test runs four separate jobs, if one of these jobs completes but the next fails to launch, the test reports PASS.   To reproduce this problem its enough to edit the jobid_pattern field in config batch so that the dependency is incorrect - this causes the first job to exit and the second to fail to launch.   But the TestStatus file indicates all PASS. 


Discussion:

Resolving patch:
diff --git a/utils/python/CIME/case_submit.py b/utils/python/CIME/case_submit.py
--- a/utils/python/CIME/case_submit.py
+++ b/utils/python/CIME/case_submit.py
@@ -12,10 +12,11 @@
 from CIME.check_lockedfiles         import check_lockedfiles
 from CIME.check_input_data          import check_all_input_data
 from CIME.case_cmpgen_namelists     import case_cmpgen_namelists
+from CIME.test_status               import *
 
 logger = logging.getLogger(__name__)
 
-def submit(case, job=None, resubmit=False, no_batch=False):
+def _submit(case, job=None, resubmit=False, no_batch=False):
     caseroot = case.get_value("CASEROOT")
 
     if job is None:
@@ -61,11 +62,27 @@ def submit(case, job=None, resubmit=False, no_batch=False):
     case.set_value("RUN_WITH_SUBMIT",True)
     case.flush()
 
-    logger.warn("submit_jobs %s"%job)
+    logger.warn("submit_jobs %s" % job)
     job_ids = case.submit_jobs(no_batch=no_batch, job=job)
-    msg = "Submitted jobs %s"%job_ids
+    msg = "Submitted jobs %s" % job_ids
     append_status(msg, caseroot=caseroot, sfile="CaseStatus")
 
+def submit(case, job=None, resubmit=False, no_batch=False):
+    try:
+        _submit(case, job=job, resubmit=resubmit, no_batch=no_batch)
+    except:
+        # If something failed in the batch system, make sure to mark
+        # the test as failed if we are running a test.
+        if case.get_value("TEST"):
+            caseroot = case.get_value("CASEROOT")
+            casebaseid = case.get_value("CASEBASEID")
+            with TestStatus(test_dir=caseroot, test_name=casebaseid, lock=True) as ts:
+                ts.set_status(RUN_PHASE, TEST_FAIL_STATUS, comments="batch system failure")
+
+            append_status("Batch submission failed, TestStatus file changed to read-only", caseroot=caseroot, sfile="TestStatus.log")
+
+        raise
+
 def check_case(case, caseroot):
     check_lockedfiles(caseroot)
     create_namelists(case) # Must be called before check_all_input_data


Test patch:
diff --git a/utils/python/CIME/test_status.py b/utils/python/CIME/test_status.py
--- a/utils/python/CIME/test_status.py
+++ b/utils/python/CIME/test_status.py
@@ -27,6 +27,9 @@
 
 from collections import OrderedDict
 
+import os
+from stat import S_IREAD, S_IRGRP, S_IROTH
+
 TEST_STATUS_FILENAME = "TestStatus"
 
 # The statuses that a phase can be in
@@ -95,7 +98,7 @@ def _test_helper2(file_contents, wait_for_run=False, check_throughput=False, che
 
 class TestStatus(object):
 
-    def __init__(self, test_dir=None, test_name=None, no_io=False):
+    def __init__(self, test_dir=None, test_name=None, no_io=False, lock=False):
         """
         Create a TestStatus object
 
@@ -110,6 +113,7 @@ def __init__(self, test_dir=None, test_name=None, no_io=False):
         self._test_name = test_name
         self._ok_to_modify = False
         self._no_io = no_io
+        self._lock = lock
 
         if os.path.exists(self._filename):
             self._parse_test_status_file()
@@ -221,6 +225,9 @@ def flush(self):
             with open(self._filename, "w") as fd:
                 self.phase_statuses_dump(fd)
 
+            if self._lock:
+                os.chmod(self._filename, S_IREAD | S_IRGRP | S_IROTH)
+
     def _parse_test_status(self, file_contents):
         """
         >>> contents = '''