Repository: ESMCI/cime

Issue:
Want ability to select default queue based on number of tasks, not just number of nodes
We used to have the ability for cime to select a default queue based on the number of tasks. With @jgfouca 's changes in #1918 (to fix #1913), this is no longer possible.

This is important for jobs that use less than a full node - such as single-point or other jobs that use a single task. CLM, CISM and possibly other components have used this capability in some configurations. Now these single-processor jobs are using a full node - so, on cheyenne, they are getting charged 36x what they should be.

Can we get back the capability to select queues based on the number of tasks?

@jedwards4b @ekluzek 


Discussion:
Would a special case for single task jobs be an adequate solution? 
> Would a special case for single task jobs be an adequate solution?

That would be better than nothing, but in the past we settled on something like half a node as a good compromise: there are some jobs that use more than one task but still significantly less than a full node - for example, a few ensemble members of a single-point case (we have some things like that in the test suite).

If we could have just one special case, I think the best special case would be for jobs using less than half of a node.
Need to decide on how to handle this usecase.
I don't think #2326 actually addressed this, since my request related to the core job, and #2326 only addressed non-core jobs.

Resolving patch:
diff --git a/scripts/lib/CIME/XML/env_batch.py b/scripts/lib/CIME/XML/env_batch.py
--- a/scripts/lib/CIME/XML/env_batch.py
+++ b/scripts/lib/CIME/XML/env_batch.py
@@ -191,22 +191,23 @@ def set_job_defaults(self, batch_jobs, case):
             walltime    = case.get_value("USER_REQUESTED_WALLTIME", subgroup=job) if case.get_value("USER_REQUESTED_WALLTIME", subgroup=job) else None
             force_queue = case.get_value("USER_REQUESTED_QUEUE", subgroup=job) if case.get_value("USER_REQUESTED_QUEUE", subgroup=job) else None
             logger.info("job is {} USER_REQUESTED_WALLTIME {} USER_REQUESTED_QUEUE {}".format(job, walltime, force_queue))
-            task_count = jsect["task_count"] if "task_count" in jsect else None
+            task_count = int(jsect["task_count"]) if "task_count" in jsect else case.total_tasks
             walltime = jsect["walltime"] if ("walltime" in jsect and walltime is None) else walltime
-            if task_count is None:
-                node_count = case.num_nodes
-            else:
+            if "task_count" in jsect:
+                # job is using custom task_count, need to compute a node_count based on this
                 node_count = int(math.ceil(float(task_count)/float(case.tasks_per_node)))
+            else:
+                node_count = case.num_nodes
 
             if force_queue:
-                if not self.queue_meets_spec(force_queue, node_count, walltime=walltime, job=job):
+                if not self.queue_meets_spec(force_queue, node_count, task_count, walltime=walltime, job=job):
                     logger.warning("WARNING: User-requested queue '{}' does not meet requirements for job '{}'".format(force_queue, job))
                 queue = force_queue
             else:
-                queue = self.select_best_queue(node_count, walltime=walltime, job=job)
+                queue = self.select_best_queue(node_count, task_count, walltime=walltime, job=job)
                 if queue is None and walltime is not None:
                     # Try to see if walltime was the holdup
-                    queue = self.select_best_queue(node_count, walltime=None, job=job)
+                    queue = self.select_best_queue(node_count, task_count, walltime=None, job=job)
                     if queue is not None:
                         # It was, override the walltime if a test, otherwise just warn the user
                         new_walltime = self.get_queue_specs(queue)[3]
@@ -514,20 +515,22 @@ def get_job_id(self, output):
         jobid = search_match.group(1)
         return jobid
 
-    def queue_meets_spec(self, queue, num_nodes, walltime=None, job=None):
+    def queue_meets_spec(self, queue, num_nodes, num_tasks, walltime=None, job=None):
         specs = self.get_queue_specs(queue)
         if specs is None:
             logger.warning("WARNING: queue '{}' is unknown to this system".format(queue))
             return True
 
-        nodemin, nodemax, jobname, walltimemax, strict = specs
+        nodemin, nodemax, jobname, walltimemax, jobmin, jobmax, strict = specs
 
         # A job name match automatically meets spec
         if job is not None and jobname is not None:
             return jobname == job
 
-        if nodemin is not None and num_nodes < int(nodemin) or \
-           nodemax is not None and num_nodes > int(nodemax):
+        if nodemin is not None and num_nodes < nodemin or \
+           nodemax is not None and num_nodes > nodemax or \
+           jobmin is not None  and num_tasks < jobmin or \
+           jobmax is not None  and num_tasks > jobmax:
             return False
 
         if walltime is not None and walltimemax is not None and strict:
@@ -538,7 +541,7 @@ def queue_meets_spec(self, queue, num_nodes, walltime=None, job=None):
 
         return True
 
-    def select_best_queue(self, num_nodes, walltime=None, job=None):
+    def select_best_queue(self, num_nodes, num_tasks, walltime=None, job=None):
         # Make sure to check default queue first.
         all_queues = []
         all_queues.append( self.get_default_queue())
@@ -546,7 +549,7 @@ def select_best_queue(self, num_nodes, walltime=None, job=None):
         for queue in all_queues:
             if queue is not None:
                 qname = self.text(queue)
-                if self.queue_meets_spec(qname, num_nodes, walltime=walltime, job=job):
+                if self.queue_meets_spec(qname, num_nodes, num_tasks, walltime=walltime, job=job):
                     return qname
 
         return None
@@ -555,17 +558,28 @@ def get_queue_specs(self, queue):
         """
         Get queue specifications by name.
 
-        Returns (nodemin, nodemax, jobname, walltimemax, is_strict)
+        Returns (nodemin, nodemax, jobname, walltimemax, jobmin, jobmax, is_strict)
         """
         for queue_node in self.get_all_queues():
             if self.text(queue_node) == queue:
                 nodemin = self.get(queue_node, "nodemin")
+                nodemin = None if nodemin is None else int(nodemin)
                 nodemax = self.get(queue_node, "nodemax")
+                nodemax = None if nodemax is None else int(nodemax)
+
+                jobmin = self.get(queue_node, "jobmin")
+                jobmin = None if jobmin is None else int(jobmin)
+                jobmax = self.get(queue_node, "jobmax")
+                jobmax = None if jobmax is None else int(jobmax)
+
+                expect( nodemin is None or jobmin is None, "Cannot specify both nodemin and jobmin for a queue")
+                expect( nodemax is None or jobmax is None, "Cannot specify both nodemax and jobmax for a queue")
+
                 jobname = self.get(queue_node, "jobname")
                 walltimemax = self.get(queue_node, "walltimemax")
                 strict = self.get(queue_node, "strict") == "true"
 
-                return nodemin, nodemax, jobname, walltimemax, strict
+                return nodemin, nodemax, jobname, walltimemax, jobmin, jobmax, strict
 
         return None
 
diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py
--- a/scripts/lib/CIME/case.py
+++ b/scripts/lib/CIME/case.py
@@ -730,15 +730,7 @@ def _setup_mach_pes(self, pecount, multi_driver, ninst, machine_name, mpilib):
             mach_pes_obj.set_value(rootpe_str, rootpe)
             mach_pes_obj.set_value(pstrid_str, pstrid)
 
-        pesize = 1
-        max_mpitasks_per_node = self.get_value("MAX_MPITASKS_PER_NODE")
-        for val in totaltasks:
-            if val < 0:
-                val = -1*val*max_mpitasks_per_node
-            if val > pesize:
-                pesize = val
         if multi_driver:
-            pesize *= int(ninst)
             mach_pes_obj.set_value("MULTI_DRIVER", True)
 
         # Make sure that every component has been accounted for
@@ -756,8 +748,6 @@ def _setup_mach_pes(self, pecount, multi_driver, ninst, machine_name, mpilib):
             if compclass not in pes_nthrds:
                 mach_pes_obj.set_value(compclass,1)
 
-        return pesize
-
     def configure(self, compset_name, grid_name, machine_name=None,
                   project=None, pecount=None, compiler=None, mpilib=None,
                   pesfile=None,user_grid=False, gridfile=None,


Test patch:
diff --git a/scripts/create_test b/scripts/create_test
--- a/scripts/create_test
+++ b/scripts/create_test
@@ -526,7 +526,7 @@ def single_submit_impl(machine_name, test_id, proc_pool, _, args, job_cost_map,
     else:
         wall_time_bab = wall_time
 
-    queue = env_batch.select_best_queue(proc_pool, num_nodes, wall_time_bab)
+    queue = env_batch.select_best_queue(num_nodes, proc_pool, walltime=wall_time_bab)
     wall_time_max_bab = env_batch.get_queue_specs(queue)[3]
     if wall_time_max_bab is not None:
         wall_time_max = convert_to_seconds(wall_time_max_bab)