Repository: ESMCI/cime

Issue:
Support --prereq argument to case.submit

The --prereq argument is still giving an error.
```
blogin1[247]: ./case.submit --prereq 1662097
ERROR: --prereq not currently supported
```

We should support or temporarily remove it from --help string.


Discussion:
A few questions:
1. ACME scientists would like to be able to chain jobs to run regardless of failure/crashing; which the depends_string (config_batch.xml) explicitly does not do on most batch systems. Would it be reasonable to change this to chain regardless of failure on batch systems that support this? This variable is currently not used anywhere else.
An alternative would be to rename it to depends_success_string, and add a new variable depends_run_string which would be used for dependencies running regardless of failure.

2. It looks like the existing depends_string for cobalt is not set correctly (it doesn't contain "jobid"). Is cobalt still being supported by ACME and/or CESM? Should I fix this?

Resolving patch:
diff --git a/scripts/lib/CIME/XML/env_batch.py b/scripts/lib/CIME/XML/env_batch.py
--- a/scripts/lib/CIME/XML/env_batch.py
+++ b/scripts/lib/CIME/XML/env_batch.py
@@ -21,7 +21,6 @@ def __init__(self, case_root=None, infile="env_batch.xml"):
         """
         initialize an object interface to file env_batch.xml in the case directory
         """
-        self._prereq_jobid = None
         self._batchtype = None
         # This arbitrary setting should always be overwritten
         self._default_walltime = "00:20:00"
@@ -316,9 +315,9 @@ def get_submit_args(self, case, job):
 
         return submitargs
 
-    def submit_jobs(self, case, no_batch=False, job=None, skip_pnl=False,
-                    mail_user=None, mail_type='never', batch_args=None,
-                    dry_run=False):
+    def submit_jobs(self, case, no_batch=False, job=None, user_prereq=None,
+                    skip_pnl=False, mail_user=None, mail_type='never',
+                    batch_args=None, dry_run=False):
         alljobs = self.get_jobs()
         startindex = 0
         jobs = []
@@ -355,25 +354,16 @@ def submit_jobs(self, case, no_batch=False, job=None, skip_pnl=False,
                 deps = dependency.split()
             else:
                 deps = []
-            jobid = ""
-            if self._prereq_jobid is not None:
-                jobid = self._prereq_jobid
+            dep_jobs = []
+            if user_prereq is not None:
+                dep_jobs.append(user_prereq)
             for dep in deps:
-                if dep in depid and depid[dep] is not None:
-                    jobid += " " + str(depid[dep])
-#TODO: doubt these will be used
-#               elif dep == "and":
-#                   jobid += " && "
-#               elif dep == "or":
-#                   jobid += " || "
+                if dep in depid.keys() and depid[dep] is not None:
+                    dep_jobs.append(str(depid[dep]))
 
-
-            slen = len(jobid)
-            if slen == 0:
-                jobid = None
-
-            logger.warning("job is {}".format(job))
-            result = self._submit_single_job(case, job, jobid,
+            logger.warning("job {} depends on {}".format(job, dep_jobs))
+            result = self._submit_single_job(case, job,
+                                             dep_jobs=dep_jobs,
                                              no_batch=no_batch,
                                              skip_pnl=skip_pnl,
                                              mail_user=mail_user,
@@ -391,7 +381,7 @@ def submit_jobs(self, case, no_batch=False, job=None, skip_pnl=False,
         else:
             return depid
 
-    def _submit_single_job(self, case, job, depid=None, no_batch=False,
+    def _submit_single_job(self, case, job, dep_jobs=None, no_batch=False,
                            skip_pnl=False, mail_user=None, mail_type='never',
                            batch_args=None, dry_run=False):
         logger.warning("Submit job {}".format(job))
@@ -415,9 +405,15 @@ def _submit_single_job(self, case, job, depid=None, no_batch=False,
         if args_override:
             submitargs = args_override
 
-        if depid is not None:
+        if dep_jobs is not None and len(dep_jobs) > 0:
+            logger.info("dependencies: {}".format(dep_jobs))
             dep_string = self.get_value("depend_string", subgroup=None)
-            dep_string = dep_string.replace("jobid",depid.strip()) # pylint: disable=maybe-no-member
+            separator_string = self.get_value("depend_separator", subgroup=None)
+            expect("jobid" in dep_string, "depend_string is missing jobid for prerequisite jobs")
+            dep_ids_str = str(dep_jobs[0])
+            for dep_id in dep_jobs[1:]:
+                dep_ids_str += separator_string + str(dep_id)
+            dep_string = dep_string.replace("jobid",dep_ids_str.strip()) # pylint: disable=maybe-no-member
             submitargs += " " + dep_string
 
         if batch_args is not None:
diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py
--- a/scripts/lib/CIME/case.py
+++ b/scripts/lib/CIME/case.py
@@ -1165,11 +1165,11 @@ def _get_comp_user_mods(self, component):
         else:
             return comp_user_mods
 
-    def submit_jobs(self, no_batch=False, job=None, skip_pnl=False,
+    def submit_jobs(self, no_batch=False, job=None, prereq=None, skip_pnl=False,
                     mail_user=None, mail_type='never', batch_args=None,
                     dry_run=False):
         env_batch = self.get_env('batch')
-        return env_batch.submit_jobs(self, no_batch=no_batch, job=job,
+        return env_batch.submit_jobs(self, no_batch=no_batch, job=job, user_prereq=prereq,
                                      skip_pnl=skip_pnl, mail_user=mail_user,
                                      mail_type=mail_type, batch_args=batch_args,
                                      dry_run=dry_run)
diff --git a/scripts/lib/CIME/case_submit.py b/scripts/lib/CIME/case_submit.py
--- a/scripts/lib/CIME/case_submit.py
+++ b/scripts/lib/CIME/case_submit.py
@@ -15,8 +15,8 @@
 
 logger = logging.getLogger(__name__)
 
-def _submit(case, job=None, resubmit=False, no_batch=False, skip_pnl=False,
-            mail_user=None, mail_type='never', batch_args=None):
+def _submit(case, job=None, no_batch=False, prereq=None, resubmit=False,
+            skip_pnl=False, mail_user=None, mail_type='never', batch_args=None):
     if job is None:
         if case.get_value("TEST"):
             job = "case.test"
@@ -64,8 +64,8 @@ def _submit(case, job=None, resubmit=False, no_batch=False, skip_pnl=False,
 
     logger.warning("submit_jobs {}".format(job))
     job_ids = case.submit_jobs(no_batch=no_batch, job=job, skip_pnl=skip_pnl,
-                               mail_user=mail_user, mail_type=mail_type,
-                               batch_args=batch_args)
+                               prereq=prereq, mail_user=mail_user,
+                               mail_type=mail_type, batch_args=batch_args)
 
     xml_jobids = []
     for jobname, jobid in job_ids.items():
@@ -77,8 +77,8 @@ def _submit(case, job=None, resubmit=False, no_batch=False, skip_pnl=False,
     if xml_jobid_text:
         case.set_value("JOB_IDS", xml_jobid_text)
 
-def submit(case, job=None, resubmit=False, no_batch=False, skip_pnl=False,
-           mail_user=None, mail_type='never', batch_args=None):
+def submit(case, job=None, no_batch=False, prereq=None, resubmit=False,
+           skip_pnl=False, mail_user=None, mail_type='never', batch_args=None):
     if case.get_value("TEST"):
         caseroot = case.get_value("CASEROOT")
         casebaseid = case.get_value("CASEBASEID")
@@ -93,8 +93,10 @@ def submit(case, job=None, resubmit=False, no_batch=False, skip_pnl=False,
                 ts.set_status(SUBMIT_PHASE, TEST_PASS_STATUS)
 
     try:
-        functor = lambda: _submit(case, job, resubmit, no_batch, skip_pnl,
-                                  mail_user, mail_type, batch_args)
+        functor = lambda: _submit(case, job=job, no_batch=no_batch, prereq=prereq,
+                                  resubmit=resubmit, skip_pnl=skip_pnl,
+                                  mail_user=mail_user, mail_type=mail_type,
+                                  batch_args=batch_args)
         run_and_log_case_status(functor, "case.submit", caseroot=case.get_value("CASEROOT"))
     except:
         # If something failed in the batch system, make sure to mark


Test patch:
diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py
--- a/scripts/tests/scripts_regression_tests.py
+++ b/scripts/tests/scripts_regression_tests.py
@@ -13,6 +13,8 @@
 from six import assertRaisesRegex
 import six
 
+import collections
+
 from CIME.utils import run_cmd, run_cmd_no_fail, get_lids, get_current_commit
 import update_acme_tests
 import CIME.test_scheduler, CIME.wait_for_tests
@@ -1394,6 +1396,47 @@ def test_cime_case(self):
             # Test some test properties
             self.assertEqual(case.get_value("TESTCASE"), "TESTRUNPASS")
 
+    ###########################################################################
+    def test_cime_case_prereq(self):
+    ###########################################################################
+        testcase_name = 'prereq_test'
+        testdir = os.path.join(TEST_ROOT, testcase_name)
+        if os.path.exists(testdir):
+            shutil.rmtree(testdir)
+        run_cmd_assert_result(self, ("%s/create_newcase --case %s --script-root %s --compset X --res f19_g16 --output-root %s"
+                                     % (SCRIPT_DIR, testcase_name, testdir, testdir)),
+                              from_dir=SCRIPT_DIR)
+
+        with Case(testdir, read_only=False) as case:
+            job_name = "case.run"
+            prereq_name = 'prereq_test'
+            batch_commands = case.submit_jobs(prereq=prereq_name, job=job_name, skip_pnl=True, dry_run=True)
+            self.assertTrue(isinstance(batch_commands, collections.Sequence), "case.submit_jobs did not return a sequence for a dry run")
+            self.assertTrue(len(batch_commands) > 0, "case.submit_jobs did not return any job submission string")
+            # The first element in the internal sequence should just be the job name
+            # The second one (batch_cmd_index) should be the actual batch submission command
+            batch_cmd_index = 1
+            # The prerequisite should be applied to all jobs, though we're only expecting one
+            for batch_cmd in batch_commands:
+                self.assertTrue(isinstance(batch_cmd, collections.Sequence), "case.submit_jobs did not return a sequence of sequences")
+                self.assertTrue(len(batch_cmd) > batch_cmd_index, "case.submit_jobs returned internal sequences with length <= {}".format(batch_cmd_index))
+                self.assertTrue(isinstance(batch_cmd[1], str), "case.submit_jobs returned internal sequences without the batch command string as the second parameter: {}".format(batch_cmd[1]))
+                batch_cmd_args = batch_cmd[1]
+
+                jobid_ident = 'jobid'
+                dep_str_fmt = case.get_env('batch').get_value('depend_string', subgroup=None)
+                self.assertTrue(jobid_ident in dep_str_fmt, "dependency string doesn't include the jobid identifier {}".format(jobid_ident))
+                dep_str = dep_str_fmt[:-len(jobid_ident)]
+
+                while dep_str in batch_cmd_args:
+                    dep_id_pos = batch_cmd_args.find(dep_str) + len(dep_str)
+                    batch_cmd_args = batch_cmd_args[dep_id_pos:]
+                    prereq_substr = batch_cmd_args[:len(prereq_name)]
+                    if prereq_substr == prereq_name:
+                        break
+
+                self.assertTrue(prereq_name in prereq_substr, "Dependencies added, but not the user specified one")
+
     ###########################################################################
     def test_cime_case_build_threaded_1(self):
     ###########################################################################