Repository: ESMCI/cime

Issue:
Add a "case.qstatus" command

"case.submit" provides a flexible wrapper around the various qsub, brun, llsubmit, etc. commands that various schedulers use.

To unify the CIME user experience, we should add a "case.qstatus" command that will execute the appropriate "qstat, llq, whatever-slurm-uses" command to show the status of the submitted jobs in the queue.




Discussion:
"case.qstatus" instead of "case.qstat" so that is doesn't look PBS-specific.

"case.qstatus" instead of "case.status" to avoid confusion with CaseStatus. 
Requesting this because I can never remember the slurm one.

Resolving patch:
diff --git a/scripts/lib/CIME/XML/env_batch.py b/scripts/lib/CIME/XML/env_batch.py
--- a/scripts/lib/CIME/XML/env_batch.py
+++ b/scripts/lib/CIME/XML/env_batch.py
@@ -1,14 +1,15 @@
 """
 Interface to the env_batch.xml file.  This class inherits from EnvBase
 """
-import stat
-import re
-import math
+
 from CIME.XML.standard_module_setup import *
 from CIME.utils import format_time
 from CIME.XML.env_base import EnvBase
 from CIME.utils import transform_vars, get_cime_root, convert_to_seconds
+
 from copy import deepcopy
+from collections import OrderedDict
+import stat, re, math
 
 logger = logging.getLogger(__name__)
 
@@ -350,7 +351,7 @@ def submit_jobs(self, case, no_batch=False, job=None, skip_pnl=False,
             if self.batchtype == "cobalt":
                 break
 
-        depid = {}
+        depid = OrderedDict()
         jobcmds = []
 
         for job, dependency in jobs:
@@ -392,7 +393,7 @@ def submit_jobs(self, case, no_batch=False, job=None, skip_pnl=False,
         if dry_run:
             return jobcmds
         else:
-            return sorted(list(depid.values()))
+            return depid
 
     def _submit_single_job(self, case, job, depid=None, no_batch=False,
                            skip_pnl=False, mail_user=None, mail_type='never',
@@ -559,3 +560,20 @@ def get_nodes(self, nodename, attributes=None, root=None, xpath=None):
         else:
             nodes =  EnvBase.get_nodes(self, nodename, attributes, root, xpath)
         return nodes
+
+    def get_status(self, jobid):
+        batch_query = self.get_optional_node("batch_query")
+        if batch_query is None:
+            logger.warning("Batch queries not supported on this platform")
+        else:
+            cmd = batch_query.text + " "
+            if "per_job_arg" in batch_query.attrib:
+                cmd += batch_query.get("per_job_arg") + " "
+
+            cmd += jobid
+
+            stat, out, err = run_cmd(cmd)
+            if stat != 0:
+                logger.warning("Batch query command '{}' failed with error '{}'".format(cmd, err))
+            else:
+                return out.strip()
diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py
--- a/scripts/lib/CIME/case.py
+++ b/scripts/lib/CIME/case.py
@@ -970,6 +970,7 @@ def _create_caseroot_tools(self):
         exefiles = (os.path.join(toolsdir, "case.setup"),
                     os.path.join(toolsdir, "case.build"),
                     os.path.join(toolsdir, "case.submit"),
+                    os.path.join(toolsdir, "case.qstatus"),
                     os.path.join(toolsdir, "case.cmpgen_namelists"),
                     os.path.join(toolsdir, "preview_namelists"),
                     os.path.join(toolsdir, "preview_run"),
@@ -1245,6 +1246,20 @@ def submit_jobs(self, no_batch=False, job=None, skip_pnl=False,
                                      mail_type=mail_type, batch_args=batch_args,
                                      dry_run=dry_run)
 
+    def report_job_status(self):
+        xml_job_ids = self.get_value("JOB_IDS")
+        if not xml_job_ids:
+            logger.info("No job ids associated with this case. Either case.submit was not run or was run with no-batch")
+        else:
+            job_infos = xml_job_ids.split(", ") # pylint: disable=no-member
+            for job_info in job_infos:
+                jobname, jobid = job_info.split(":")
+                status = self.get_env("batch").get_status(jobid)
+                if status:
+                    logger.info("{}: {}".format(jobname, status))
+                else:
+                    logger.info("{}: Unable to get status. Job may be complete already.".format(jobname))
+
     def get_mpirun_cmd(self, job="case.run"):
         env_mach_specific = self.get_env('mach_specific')
         run_exe = env_mach_specific.get_value("run_exe")
diff --git a/scripts/lib/CIME/case_submit.py b/scripts/lib/CIME/case_submit.py
--- a/scripts/lib/CIME/case_submit.py
+++ b/scripts/lib/CIME/case_submit.py
@@ -68,7 +68,16 @@ def _submit(case, job=None, resubmit=False, no_batch=False, skip_pnl=False,
     job_ids = case.submit_jobs(no_batch=no_batch, job=job, skip_pnl=skip_pnl,
                                mail_user=mail_user, mail_type=mail_type,
                                batch_args=batch_args)
-    logger.info("Submitted job ids {}".format(job_ids))
+
+    xml_jobids = []
+    for jobname, jobid in job_ids.iteritems():
+        logger.info("Submitted job {} with id {}".format(jobname, jobid))
+        if jobid:
+            xml_jobids.append("{}:{}".format(jobname, jobid))
+
+    xml_jobid_text = ", ".join(xml_jobids)
+    if xml_jobid_text:
+        case.set_value("JOB_IDS", xml_jobid_text)
 
 def submit(case, job=None, resubmit=False, no_batch=False, skip_pnl=False,
            mail_user=None, mail_type='never', batch_args=None):


Test patch: