Repository: ESMCI/cime

Issue:
Set order that test suite builds / submits?
This is a very low priority issue that I'm bring over from the CESM-Development/cime issue page because it would be nice if this got addressed at some point.

> The aux_pop_MARBL test suite takes 40 minutes to build / submit 8 tests from a geyser node. Most of these are short, but we do have an ERI test that takes significantly longer than any other test to run (50 minutes instead of 5 or less)... so it would be nice to have that be the first test built and submitted. Currently it's the 5th test built, 20 minutes after the first job is submitted, and I don't see an easy way to change that. Perhaps a "priority" field in the xml file?

I'm not familiar with the parallelism in `create_test`; my impression is that the individual tests are built in parallel but processed serially (i.e. Test A is built in parallel and submitted, then Test B is built in parallel and submitted, ...). If the goal is to have all the tests processed in parallel, then this issue is not as important... though I'd argue being able to say "build / submit tests 1-4 first and 5-8 next" would still be useful.


Discussion:
@mnlevy1981 create_test has parallelism both at the test level and for the individual builds.
@mnlevy1981 this is a good point, the most expensive tests should be started first.
@jgfouca I was about to say that this is a testlist issue and not really acme, but I just checked the list returned from test_utils and it is ordered correctly:
```
Testnames: ['ERI.T62_g16.CECO.yellowstone_intel.pop-ciso', 'ERS_Ld5_D.T62_g16.CECO.yellowstone_intel.pop-ciso', 'SMS_Ld20.T62_g16.CECO.yellowstone_intel.pop-ecosys_shr_stream', 'ERS.T62_g37.CECO.yellowstone_intel.pop-ecosys_restore_gx3v7', 'ERS.T62_g37.CECO.yellowstone_intel.pop-ciso_transient', 'SMS_D.T62_g37.CECO.yellowstone_intel.pop-ecosys', 'SMS_Ld2_D.T62_g37.CECO.yellowstone_intel.pop-ciso_daily_r4_tavg', 'SMS.T62_g16.GECO.yellowstone_intel.pop-ecosys_daily_r4_tavg', 'ERS.T62_g37.GECO.yellowstone_intel.pop-cice_ciso', 'ERS.T62_g37.GECO.yellowstone_intel.pop-cice_ecosys_dust_flux_driver', 'ERS.T62_g37.GECO.yellowstone_intel.pop-cice_ecosys_ladjust_bury_coeff', 'ERS.T62_g37.GECO.yellowstone_intel.pop-cice_ecosys_apply_nhx_surface_emis', 'ERS_Ld5_D.T62_g37.GECO.yellowstone_intel.pop-cice_ciso', 'SMS_D.T62_g37.GECO.yellowstone_intel.pop-cice_ecosys']
@jedwards4b so testlists already returns test in order of test time (longest to shortest)? Does that mean test_scheduler isn't honoring this order?
Yes - I think that's what is happening.

Resolving patch:
diff --git a/scripts/lib/CIME/XML/grids.py b/scripts/lib/CIME/XML/grids.py
--- a/scripts/lib/CIME/XML/grids.py
+++ b/scripts/lib/CIME/XML/grids.py
@@ -642,8 +642,6 @@ def _get_all_values_v2(self):
             name = domain_node.get("name")
             if name == 'null':
                 continue
-            nx = self.get_node("nx", root=domain_node).text
-            ny = self.get_node("ny", root=domain_node).text
             desc = self.get_node("desc", root=domain_node).text
             ##support = self.get_optional_node("support", root=domain_node).text
             files = ""


Test patch:
diff --git a/scripts/create_test b/scripts/create_test
--- a/scripts/create_test
+++ b/scripts/create_test
@@ -363,6 +363,13 @@ OR
         dot_count = name.count('.')
         expect(dot_count > 1 and dot_count <= 4, "Invalid test Name, '{}'".format(name))
 
+    # for acme, sort by walltime
+    if model == "acme":
+        if args.walltime is None:
+            test_names.sort(cmp=update_acme_tests.sort_by_time)
+        else:
+            test_names.sort()
+
     return test_names, test_extra_data, args.compiler, mach_obj.get_machine_name(), args.no_run, args.no_build, args.no_setup, args.no_batch,\
         args.test_root, args.baseline_root, args.clean, baseline_cmp_name, baseline_gen_name, \
         args.namelists_only, args.project, args.test_id, args.parallel_jobs, args.walltime, \
diff --git a/scripts/lib/CIME/test_scheduler.py b/scripts/lib/CIME/test_scheduler.py
--- a/scripts/lib/CIME/test_scheduler.py
+++ b/scripts/lib/CIME/test_scheduler.py
@@ -9,6 +9,7 @@
 """
 
 import traceback, stat, threading, time, glob
+from collections import OrderedDict
 from CIME.XML.standard_module_setup import *
 import CIME.compare_namelists
 import CIME.utils
@@ -192,7 +193,7 @@ def __init__(self, test_names, test_data=None,
         # Each test has it's own value and setting/retrieving items from a dict
         # is atomic, so this should be fine to use without mutex.
         # name -> (phase, status)
-        self._tests = {}
+        self._tests = OrderedDict()
         for test_name in test_names:
             self._tests[test_name] = (TEST_START, TEST_PASS_STATUS)
 
@@ -708,12 +709,14 @@ def _producer(self):
             num_threads_launched_this_iteration = 0
             for test in self._tests:
                 logger.debug("test_name: " + test)
-                # If we have no workers available, immediately wait
-                if len(threads_in_flight) == self._parallel_jobs:
-                    self._wait_for_something_to_finish(threads_in_flight)
 
                 if self._work_remains(test):
                     work_to_do = True
+
+                    # If we have no workers available, immediately break out of loop so we can wait
+                    if len(threads_in_flight) == self._parallel_jobs:
+                        break
+
                     if test not in threads_in_flight:
                         test_phase, test_status = self._get_test_data(test)
                         expect(test_status != TEST_PEND_STATUS, test)
diff --git a/scripts/lib/update_acme_tests.py b/scripts/lib/update_acme_tests.py
--- a/scripts/lib/update_acme_tests.py
+++ b/scripts/lib/update_acme_tests.py
@@ -340,3 +340,25 @@ def get_recommended_test_time(test_full_name):
                     best_time = rec_time
 
     return best_time
+
+###############################################################################
+def sort_by_time(test_one, test_two):
+###############################################################################
+    """
+    >>> tests = get_full_test_names(["cime_tiny"], "melvin", "gnu")
+    >>> tests.extend(get_full_test_names(["cime_developer"], "melvin", "gnu"))
+    >>> tests.append("A.f19_f19.A.melvin_gnu")
+    >>> tests.sort(cmp=sort_by_time)
+    >>> tests
+    ['DAE.f19_f19.A.melvin_gnu', 'ERI.f09_g16.X.melvin_gnu', 'ERIO.f09_g16.X.melvin_gnu', 'ERP.f45_g37_rx1.A.melvin_gnu', 'ERR.f45_g37_rx1.A.melvin_gnu', 'ERS.ne30_g16_rx1.A.melvin_gnu', 'IRT_N2.f19_g16_rx1.A.melvin_gnu', 'NCK_Ld3.f45_g37_rx1.A.melvin_gnu', 'PET_P32.f19_f19.A.melvin_gnu', 'PRE.f19_f19.ADESP.melvin_gnu', 'PRE.f19_f19.ADESP_TEST.melvin_gnu', 'SEQ_Ln9.f19_g16_rx1.A.melvin_gnu', 'SMS.T42_T42.S.melvin_gnu', 'SMS_D_Ln9.f19_g16_rx1.A.melvin_gnu', 'ERS.f19_g16_rx1.A.melvin_gnu', 'NCK.f19_g16_rx1.A.melvin_gnu', 'A.f19_f19.A.melvin_gnu']
+    """
+    rec1, rec2 = get_recommended_test_time(test_one), get_recommended_test_time(test_two)
+    if rec1 == rec2:
+        return cmp(test_one, test_two)
+    else:
+        if rec2 is None:
+            return -1
+        elif rec1 is None:
+            return 1
+        else:
+            return cmp(convert_to_seconds(rec2), convert_to_seconds(rec1))