Repository: ESMCI/cime

Issue:
Add a true walltimedef option to config_batch.xml
Sometimes we need walltimemax to actually be the maximum to help set the rules for picking the right queue (using the "strict=true" setting).   But then the max becomes the default time when someone doesn't specify a time in their create_newcase and the max time can usually result in long wait times.  So add a walltimedef option to specify a more reasonable default time for a queue.


Discussion:
The user already has an option to modify wall time on the command line of create_newcase as well as with xmlchange in the case.   For production experiments we most often want the max possible walltime.   I don't think that this change is warranted.   
In production experiments, you should specify the walltime and not rely on the default.

Resolving patch:
diff --git a/scripts/lib/CIME/XML/env_batch.py b/scripts/lib/CIME/XML/env_batch.py
--- a/scripts/lib/CIME/XML/env_batch.py
+++ b/scripts/lib/CIME/XML/env_batch.py
@@ -245,7 +245,7 @@ def set_job_defaults(self, batch_jobs, case):
                 queue = self.select_best_queue(node_count, task_count, name=force_queue, walltime=None, job=job)
                 if queue is not None:
                     # It was, override the walltime if a test, otherwise just warn the user
-                    new_walltime = self.get_queue_specs(queue)[4]
+                    new_walltime = self.get_queue_specs(queue)[5]
                     expect(new_walltime is not None, "Should never make it here")
                     logger.warning("WARNING: Requested walltime '{}' could not be matched by any {} queue".format(walltime, force_queue))
                     if case.get_value("TEST"):
@@ -257,36 +257,39 @@ def set_job_defaults(self, batch_jobs, case):
             if queue is None:
                 logger.warning("WARNING: No queue on this system met the requirements for this job. Falling back to defaults")
                 queue = self.get_default_queue()
-                walltime = self.get_queue_specs(queue)[4]
+                walltime = self.get_queue_specs(queue)[5]
 
-            specs = self.get_queue_specs(queue)
+            _, _, _, walltimedef, walltimemin, walltimemax, _, _, _ = \
+                self.get_queue_specs(queue)
 
             if walltime is None:
-                # Figure out walltime
-                if specs is None:
-                    # Queue is unknown, use specs from default queue
-                    walltime = self.get(self.get_default_queue(), "walltimemax")
-
-                    logger.debug("Using walltimemax {!r} from default "
-                                 "queue {!r}".format(
-                                     walltime, self.text(queue)))
+                # Use default walltime if available for queue
+                if walltimedef is not None:
+                    walltime = walltimedef
                 else:
-                    walltime = specs[4]
+                    # Last chance to figure out a walltime
+                    # No default for queue, take max if available
+                    if walltime is None and walltimemax is not None:
+                        walltime = walltimemax
 
-                    logger.debug("Using walltimemax {!r} from queue "
-                                 "{!r}".format(walltime, self.text(queue)))
+                    # Still no walltime, try max from the default queue
+                    if walltime is None:
+                        # Queue is unknown, use specs from default queue
+                        walltime = self.get(self.get_default_queue(), "walltimemax")
 
-                if walltime is None:
-                    walltime = self._default_walltime
+                        logger.debug("Using walltimemax {!r} from default "
+                                    "queue {!r}".format(
+                                        walltime, self.text(queue)))
 
-                    logger.debug("Last resort using default walltime "
-                                 "{!r}".format(walltime))
+                    # Still no walltime, use the hardcoded default
+                    if walltime is None:
+                        walltime = self._default_walltime
+
+                        logger.debug("Last resort using default walltime "
+                                    "{!r}".format(walltime))
 
             # only enforce when not running a test
             if not case.get_value("TEST"):
-                walltimemin = specs[3]
-                walltimemax = specs[4]
-
                 walltime_seconds = convert_to_seconds(walltime)
 
                 # walltime must not be less than walltimemin
@@ -319,7 +322,8 @@ def set_job_defaults(self, batch_jobs, case):
                 full_bab_time = convert_to_babylonian_time(seconds)
                 walltime = format_time(walltime_format, "%H:%M:%S", full_bab_time)
 
-            env_workflow.set_value("JOB_QUEUE", self.text(queue), subgroup=job, ignore_type=specs is None)
+            env_workflow.set_value("JOB_QUEUE", self.text(queue), subgroup=job,
+                                   ignore_type=False)
             env_workflow.set_value("JOB_WALLCLOCK_TIME", walltime, subgroup=job)
             logger.debug("Job {} queue {} walltime {}".format(job, self.text(queue), walltime))
 
@@ -796,7 +800,7 @@ def get_job_id(self, output):
     def queue_meets_spec(self, queue, num_nodes, num_tasks, walltime=None, job=None):
         specs = self.get_queue_specs(queue)
 
-        nodemin, nodemax, jobname, _, walltimemax, jobmin, \
+        nodemin, nodemax, jobname, _, _, walltimemax, jobmin, \
             jobmax, strict = specs
 
         # A job name match automatically meets spec
@@ -863,12 +867,13 @@ def get_queue_specs(self, qnode):
         expect( nodemax is None or jobmax is None, "Cannot specify both nodemax and jobmax for a queue")
 
         jobname = self.get(qnode, "jobname")
+        walltimedef = self.get(qnode, "walltimedef")
         walltimemin = self.get(qnode, "walltimemin")
         walltimemax = self.get(qnode, "walltimemax")
         strict = self.get(qnode, "strict") == "true"
 
-        return nodemin, nodemax, jobname, walltimemin, walltimemax, jobmin, \
-            jobmax, strict
+        return nodemin, nodemax, jobname, walltimedef, walltimemin, \
+            walltimemax, jobmin, jobmax, strict
 
     def get_default_queue(self):
         bs_nodes = self.get_children("batch_system")


Test patch:
diff --git a/scripts/lib/CIME/tests/test_xml_env_batch.py b/scripts/lib/CIME/tests/test_xml_env_batch.py
--- a/scripts/lib/CIME/tests/test_xml_env_batch.py
+++ b/scripts/lib/CIME/tests/test_xml_env_batch.py
@@ -14,15 +14,17 @@ def test_get_queue_specs(self, get):
         batch = EnvBatch()
 
         get.side_effect = [
-            "1", "1", None, None, "case.run", "05:00:00", "12:00:00", "false",
+            "1", "1", None, None, "case.run", "08:00:00", "05:00:00", \
+            "12:00:00", "false",
         ]
 
-        nodemin, nodemax, jobname, walltimemin, walltimemax, jobmin, jobmax, \
-            strict = batch.get_queue_specs(node)
+        nodemin, nodemax, jobname, walltimedef, walltimemin, walltimemax, \
+            jobmin, jobmax, strict = batch.get_queue_specs(node)
 
         self.assertTrue(nodemin == 1)
         self.assertTrue(nodemax == 1)
         self.assertTrue(jobname == "case.run")
+        self.assertTrue(walltimedef == "08:00:00")
         self.assertTrue(walltimemin == "05:00:00")
         self.assertTrue(walltimemax == "12:00:00")
         self.assertTrue(jobmin == None)
@@ -32,7 +34,7 @@ def test_get_queue_specs(self, get):
     @mock.patch("CIME.XML.env_batch.EnvBatch.text", return_value="default")
     # nodemin, nodemax, jobname, walltimemin, walltimemax, jobmin, jobmax, strict
     @mock.patch("CIME.XML.env_batch.EnvBatch.get_queue_specs", return_value=[
-        1, 1, "case.run", "08:00:00", "12:00:00", 1, 1, False,
+        1, 1, "case.run", "10:00:00", "08:00:00", "12:00:00", 1, 1, False,
     ])
     @mock.patch("CIME.XML.env_batch.EnvBatch.select_best_queue")
     @mock.patch("CIME.XML.env_batch.EnvBatch.get_default_queue")
@@ -72,7 +74,7 @@ def get_value(*args, **kwargs):
     @mock.patch("CIME.XML.env_batch.EnvBatch.text", return_value="default")
     # nodemin, nodemax, jobname, walltimemin, walltimemax, jobmin, jobmax, strict
     @mock.patch("CIME.XML.env_batch.EnvBatch.get_queue_specs", return_value=[
-        1, 1, "case.run", "08:00:00", "12:00:00", 1, 1, False,
+        1, 1, "case.run", "10:00:00", "08:00:00", "12:00:00", 1, 1, False,
     ])
     @mock.patch("CIME.XML.env_batch.EnvBatch.select_best_queue")
     @mock.patch("CIME.XML.env_batch.EnvBatch.get_default_queue")
@@ -112,7 +114,7 @@ def get_value(*args, **kwargs):
     @mock.patch("CIME.XML.env_batch.EnvBatch.text", return_value="default")
     # nodemin, nodemax, jobname, walltimemax, jobmin, jobmax, strict
     @mock.patch("CIME.XML.env_batch.EnvBatch.get_queue_specs", return_value=[
-        1, 1, "case.run", "08:00:00", "12:00:00", 1, 1, False,
+        1, 1, "case.run", "10:00:00", "08:00:00", "12:00:00", 1, 1, False,
     ])
     @mock.patch("CIME.XML.env_batch.EnvBatch.select_best_queue")
     @mock.patch("CIME.XML.env_batch.EnvBatch.get_default_queue")
@@ -153,7 +155,7 @@ def get_value(*args, **kwargs):
     @mock.patch("CIME.XML.env_batch.EnvBatch.text", return_value="default")
     # nodemin, nodemax, jobname, walltimemax, jobmin, jobmax, strict
     @mock.patch("CIME.XML.env_batch.EnvBatch.get_queue_specs", return_value=[
-        1, 1, "case.run", "05:00:00", None, 1, 1, False,
+        1, 1, "case.run", "10:00:00", "05:00:00", None, 1, 1, False,
     ])
     @mock.patch("CIME.XML.env_batch.EnvBatch.select_best_queue")
     @mock.patch("CIME.XML.env_batch.EnvBatch.get_default_queue")
@@ -193,7 +195,7 @@ def get_value(*args, **kwargs):
     @mock.patch("CIME.XML.env_batch.EnvBatch.text", return_value="default")
     # nodemin, nodemax, jobname, walltimemax, jobmin, jobmax, strict
     @mock.patch("CIME.XML.env_batch.EnvBatch.get_queue_specs", return_value=[
-        1, 1, "case.run", None, "12:00:00", 1, 1, False,
+        1, 1, "case.run", "10:00:00", None, "12:00:00", 1, 1, False,
     ])
     @mock.patch("CIME.XML.env_batch.EnvBatch.select_best_queue")
     @mock.patch("CIME.XML.env_batch.EnvBatch.get_default_queue")
@@ -233,7 +235,47 @@ def get_value(*args, **kwargs):
     @mock.patch("CIME.XML.env_batch.EnvBatch.text", return_value="default")
     # nodemin, nodemax, jobname, walltimemax, jobmin, jobmax, strict
     @mock.patch("CIME.XML.env_batch.EnvBatch.get_queue_specs", return_value=[
-        1, 1, "case.run", "08:00:00", "12:00:00", 1, 1, False,
+        1, 1, "case.run", "10:00:00", "08:00:00", "12:00:00", 1, 1, False,
+    ])
+    @mock.patch("CIME.XML.env_batch.EnvBatch.select_best_queue")
+    @mock.patch("CIME.XML.env_batch.EnvBatch.get_default_queue")
+    def test_set_job_defaults_walltimedef(self, get_default_queue, select_best_queue,
+                              get_queue_specs, text):
+        case = mock.MagicMock()
+
+        batch_jobs = [
+            ("case.run", {
+                "template": "template.case.run",
+                "prereq": "$BUILD_COMPLETE and not $TEST"
+            })
+        ]
+
+        def get_value(*args, **kwargs):
+            if args[0] == "USER_REQUESTED_WALLTIME":
+                return None
+
+            return mock.MagicMock()
+
+        case.get_value = get_value
+
+        case.get_env.return_value.get_jobs.return_value = ["case.run"]
+
+        batch = EnvBatch()
+
+        batch.set_job_defaults(batch_jobs, case)
+
+        env_workflow = case.get_env.return_value
+
+        env_workflow.set_value.assert_any_call("JOB_QUEUE", "default",
+                                               subgroup="case.run",
+                                               ignore_type=False)
+        env_workflow.set_value.assert_any_call("JOB_WALLCLOCK_TIME", "10:00:00",
+                                               subgroup="case.run")
+
+    @mock.patch("CIME.XML.env_batch.EnvBatch.text", return_value="default")
+    # nodemin, nodemax, jobname, walltimemax, jobmin, jobmax, strict
+    @mock.patch("CIME.XML.env_batch.EnvBatch.get_queue_specs", return_value=[
+        1, 1, "case.run", None, "08:00:00", "12:00:00", 1, 1, False,
     ])
     @mock.patch("CIME.XML.env_batch.EnvBatch.select_best_queue")
     @mock.patch("CIME.XML.env_batch.EnvBatch.get_default_queue")