Repository: ESMCI/cime

Issue:
Want ability to select default queue based on number of tasks, not just number of nodes
We used to have the ability for cime to select a default queue based on the number of tasks. With @jgfouca 's changes in #1918 (to fix #1913), this is no longer possible.

This is important for jobs that use less than a full node - such as single-point or other jobs that use a single task. CLM, CISM and possibly other components have used this capability in some configurations. Now these single-processor jobs are using a full node - so, on cheyenne, they are getting charged 36x what they should be.

Can we get back the capability to select queues based on the number of tasks?

@jedwards4b @ekluzek 


Discussion:
Would a special case for single task jobs be an adequate solution? 
> Would a special case for single task jobs be an adequate solution?

That would be better than nothing, but in the past we settled on something like half a node as a good compromise: there are some jobs that use more than one task but still significantly less than a full node - for example, a few ensemble members of a single-point case (we have some things like that in the test suite).

If we could have just one special case, I think the best special case would be for jobs using less than half of a node.
Need to decide on how to handle this usecase.

Resolving patch:
diff --git a/scripts/lib/CIME/XML/env_batch.py b/scripts/lib/CIME/XML/env_batch.py
--- a/scripts/lib/CIME/XML/env_batch.py
+++ b/scripts/lib/CIME/XML/env_batch.py
@@ -112,7 +112,7 @@ def create_job_groups(self, batch_jobs):
 
         childnodes = []
         for child in reversed(orig_group_children):
-            childnodes.append(self.copy(child))
+            childnodes.append(child)
 
         self.remove_child(orig_group)
 
@@ -124,7 +124,7 @@ def create_job_groups(self, batch_jobs):
                 self.make_child("type", root=node, text="char")
 
             for child in childnodes:
-                self.add_child(child, root=new_job_group)
+                self.add_child(self.copy(child), root=new_job_group)
 
     def cleanupnode(self, node):
         if self.get(node, "id") == "batch_system":
@@ -164,7 +164,6 @@ def make_batch_script(self, input_template, job, case):
             overrides["total_tasks"] = int(task_count)
             overrides["num_nodes"]   = int(math.ceil(float(task_count)/float(case.tasks_per_node)))
 
-        overrides["pedocumentation"] = "" # TODO?
         overrides["job_id"] = case.get_value("CASE") + os.path.splitext(job)[1]
         if "pleiades" in case.get_value("MACH"):
             # pleiades jobname needs to be limited to 15 chars
@@ -193,6 +192,7 @@ def set_job_defaults(self, batch_jobs, case):
             force_queue = case.get_value("USER_REQUESTED_QUEUE", subgroup=job) if case.get_value("USER_REQUESTED_QUEUE", subgroup=job) else None
             logger.info("job is {} USER_REQUESTED_WALLTIME {} USER_REQUESTED_QUEUE {}".format(job, walltime, force_queue))
             task_count = jsect["task_count"] if "task_count" in jsect else None
+            walltime = jsect["walltime"] if ("walltime" in jsect and walltime is None) else walltime
             if task_count is None:
                 node_count = case.num_nodes
             else:
diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py
--- a/scripts/lib/CIME/case.py
+++ b/scripts/lib/CIME/case.py
@@ -944,11 +944,6 @@ def configure(self, compset_name, grid_name, machine_name=None,
         #--------------------------------------------
         # batch system (must come after initialize_derived_attributes)
         #--------------------------------------------
-        if walltime:
-            self.set_value("USER_REQUESTED_WALLTIME", walltime)
-        if queue:
-            self.set_value("USER_REQUESTED_QUEUE", queue)
-
         env_batch = self.get_env("batch")
 
         batch_system_type = machobj.get_value("BATCH_SYSTEM")
@@ -957,6 +952,12 @@ def configure(self, compset_name, grid_name, machine_name=None,
 
         env_batch.set_batch_system(batch, batch_system_type=batch_system_type)
         env_batch.create_job_groups(bjobs)
+
+        if walltime:
+            self.set_value("USER_REQUESTED_WALLTIME", walltime, subgroup=("case.test" if test else "case.run"))
+        if queue:
+            self.set_value("USER_REQUESTED_QUEUE", queue, subgroup=("case.test" if test else "case.run"))
+
         env_batch.set_job_defaults(bjobs, self)
         self.schedule_rewrite(env_batch)
 


Test patch: