Repository: ESMCI/cime

Issue:
wait_for_tests needs a timeout option
We run scripts_regression_tests.py from cron on some batch systems.  Jobs submitted to the batch system from the test may end up in the queue and not run if resources are not available.  I think that we need to add a timeout to wait_for_tests so that the nightly cdash testing can report the partial results even if batch jobs fail to run within a defined time window.  This option needs to be available at the scripts_regression_tests.py command line. 


Discussion:
@jedwards4b we have some experience with this problem on the ACME side. We currently rely on Jenkins' timeout system, which sends a SIGTERM to the process when it's out of time. Wait_for_tests is designed to handle this signal gracefully (stop waiting on stuff and report to dashboard). Jenkins will wait a couple minutes for your processes to stop and, if not, will then finish them off with a SIGKILL.

This is a long way of saying most of the infrastructure you're asking for already exists and you might be able to implement what you need in bash.
Your solution implies access to tools that we don't have, I am suggesting an alternative that can be completely contained within cime.
@jedwards4b sorry, I didn't read your comment closely enough. I gather that you want timeouts in the individual wait_for_tests calls in scripts_regression_tests.
@jgfouca right

Resolving patch:
diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py
--- a/scripts/lib/CIME/utils.py
+++ b/scripts/lib/CIME/utils.py
@@ -2,7 +2,7 @@
 Common functions used by cime python scripts
 Warning: you cannot use CIME Classes in this module as it causes circular dependencies
 """
-import logging, gzip, sys, os, time, re, shutil, glob, string, random, imp, errno
+import logging, gzip, sys, os, time, re, shutil, glob, string, random, imp, errno, signal
 import stat as statlib
 import warnings
 from contextlib import contextmanager
@@ -1346,3 +1346,25 @@ def __enter__(self):
 
     def __exit__(self, *_):
         os.umask(self._orig_umask)
+
+class Timeout(object):
+    """
+    A context manager that implements a timeout. By default, it
+    will raise exception, but a custon function call can be provided.
+    Provided None as seconds makes this class a no-op
+    """
+    def __init__(self, seconds, action=None):
+        self._seconds = seconds
+        self._action  = action if action is not None else self._handle_timeout
+
+    def _handle_timeout(self, *_):
+        raise RuntimeError("Timeout expired")
+
+    def __enter__(self):
+        if self._seconds is not None:
+            signal.signal(signal.SIGALRM, self._action)
+            signal.alarm(self._seconds)
+
+    def __exit__(self, *_):
+        if self._seconds is not None:
+            signal.alarm(0)


Test patch:
diff --git a/scripts/Tools/wait_for_tests b/scripts/Tools/wait_for_tests
--- a/scripts/Tools/wait_for_tests
+++ b/scripts/Tools/wait_for_tests
@@ -66,9 +66,12 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter
     parser.add_argument("-g", "--cdash-build-group", default=CIME.wait_for_tests.CDASH_DEFAULT_BUILD_GROUP,
                         help="The build group to be used to display results on the CDash dashboard.")
 
+    parser.add_argument("--timeout", type=int,
+                        help="Timeout wait in seconds.")
+
     args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)
 
-    return args.paths, args.no_wait, args.check_throughput, args.check_memory, args.ignore_namelist_diffs, args.ignore_memleak, args.cdash_build_name, args.cdash_project, args.cdash_build_group
+    return args.paths, args.no_wait, args.check_throughput, args.check_memory, args.ignore_namelist_diffs, args.ignore_memleak, args.cdash_build_name, args.cdash_project, args.cdash_build_group, args.timeout
 
 ###############################################################################
 def _main_func(description):
@@ -77,18 +80,19 @@ def _main_func(description):
         CIME.utils.run_cmd_no_fail("python -m doctest %s/wait_for_tests.py -v" % CIME.utils.get_python_libs_root(), arg_stdout=None, arg_stderr=None)
         return
 
-    test_paths, no_wait, check_throughput, check_memory, ignore_namelist_diffs, ignore_memleak, cdash_build_name, cdash_project, cdash_build_group = \
+    test_paths, no_wait, check_throughput, check_memory, ignore_namelist_diffs, ignore_memleak, cdash_build_name, cdash_project, cdash_build_group, timeout = \
         parse_command_line(sys.argv, description)
 
     sys.exit(0 if CIME.wait_for_tests.wait_for_tests(test_paths,
-                                                no_wait=no_wait,
-                                                check_throughput=check_throughput,
-                                                check_memory=check_memory,
-                                                ignore_namelists=ignore_namelist_diffs,
-                                                ignore_memleak=ignore_memleak,
-                                                cdash_build_name=cdash_build_name,
-                                                cdash_project=cdash_project,
-                                                cdash_build_group=cdash_build_group)
+                                                     no_wait=no_wait,
+                                                     check_throughput=check_throughput,
+                                                     check_memory=check_memory,
+                                                     ignore_namelists=ignore_namelist_diffs,
+                                                     ignore_memleak=ignore_memleak,
+                                                     cdash_build_name=cdash_build_name,
+                                                     cdash_project=cdash_project,
+                                                     cdash_build_group=cdash_build_group,
+                                                     timeout=timeout)
              else CIME.utils.TESTS_FAILED_ERR_CODE)
 
 ###############################################################################
diff --git a/scripts/lib/CIME/wait_for_tests.py b/scripts/lib/CIME/wait_for_tests.py
--- a/scripts/lib/CIME/wait_for_tests.py
+++ b/scripts/lib/CIME/wait_for_tests.py
@@ -3,7 +3,7 @@
 import xml.etree.ElementTree as xmlet
 
 import CIME.utils
-from CIME.utils import expect
+from CIME.utils import expect, Timeout
 from CIME.XML.machines import Machines
 from CIME.test_status import *
 
@@ -337,13 +337,15 @@ def wait_for_tests(test_paths,
                    ignore_memleak=False,
                    cdash_build_name=None,
                    cdash_project=ACME_MAIN_CDASH,
-                   cdash_build_group=CDASH_DEFAULT_BUILD_GROUP):
+                   cdash_build_group=CDASH_DEFAULT_BUILD_GROUP,
+                   timeout=None):
 ###############################################################################
     # Set up signal handling, we want to print results before the program
     # is terminated
     set_up_signal_handlers()
 
-    test_results = wait_for_tests_impl(test_paths, no_wait, check_throughput, check_memory, ignore_namelists, ignore_memleak)
+    with Timeout(timeout, action=signal_handler):
+        test_results = wait_for_tests_impl(test_paths, no_wait, check_throughput, check_memory, ignore_namelists, ignore_memleak)
 
     all_pass = True
     for test_name, test_data in sorted(test_results.iteritems()):
diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py
--- a/scripts/tests/scripts_regression_tests.py
+++ b/scripts/tests/scripts_regression_tests.py
@@ -25,6 +25,7 @@
 SCRIPT_DIR  = CIME.utils.get_scripts_root()
 TOOLS_DIR   = os.path.join(SCRIPT_DIR,"Tools")
 TEST_COMPILER = None
+GLOBAL_TIMEOUT = None
 TEST_MPILIB = None
 MACHINE     = Machines()
 FAST_ONLY   = False
@@ -609,6 +610,12 @@ def test_wait_for_test_no_wait(self):
         expected_results = ["PEND" if item == 5 else "PASS" for item in range(10)]
         self.simple_test(self._testdir_unfinished, expected_results, "-n")
 
+    ###########################################################################
+    def test_wait_for_test_timeout(self):
+    ###########################################################################
+        expected_results = ["PEND" if item == 5 else "PASS" for item in range(10)]
+        self.simple_test(self._testdir_unfinished, expected_results, "--timeout=3")
+
     ###########################################################################
     def test_wait_for_test_wait_for_pend(self):
     ###########################################################################
@@ -735,15 +742,7 @@ def setUp(self):
         self._thread_error      = None
         self._unset_proxy       = setup_proxy()
         self._machine           = MACHINE.get_machine_name()
-        if TEST_COMPILER is None:
-            self._compiler          = MACHINE.get_default_compiler()
-        else:
-            self._compiler = TEST_COMPILER
-        if TEST_MPILIB is None:
-            self._mpilib = MACHINE.get_default_MPIlib()
-        else:
-            self._mpilib = TEST_MPILIB
-
+        self._compiler          = MACHINE.get_default_compiler() if TEST_COMPILER is None else TEST_COMPILER
         self._baseline_name     = "fake_testing_only_%s" % CIME.utils.get_timestamp()
         self._baseline_area     = MACHINE.get_value("BASELINE_ROOT")
         self._testroot          = TEST_ROOT
@@ -786,6 +785,41 @@ def tearDown(self):
                 else:
                     os.remove(file_to_clean)
 
+    ###########################################################################
+    def _create_test(self, extra_args, test_id=None, pre_run_errors=False, run_errors=False, env_changes=""):
+    ###########################################################################
+        test_id = CIME.utils.get_timestamp() if test_id is None else test_id
+        extra_args.append("-t {}".format(test_id))
+        if NO_BATCH:
+            extra_args.append("--no-batch")
+        if TEST_COMPILER:
+            extra_args.append("--compiler={}".format(TEST_COMPILER))
+        if TEST_MPILIB:
+            extra_args.append("--mpilib={}".format(TEST_MPILIB))
+        extra_args.append("--test-root={0} --output-root={0}".format(TEST_ROOT))
+
+        full_run = (set(extra_args) & set(["-n", "--namelist-only", "--no-setup", "--no-build"])) == set()
+
+        if self._hasbatch:
+            expected_stat = 0 if not pre_run_errors else CIME.utils.TESTS_FAILED_ERR_CODE
+        else:
+            expected_stat = 0 if not pre_run_errors and not run_errors else CIME.utils.TESTS_FAILED_ERR_CODE
+
+        run_cmd_assert_result(self, "{} {}/create_test {}".format(env_changes, SCRIPT_DIR, " ".join(extra_args)),
+                              expected_stat=expected_stat)
+
+        if full_run:
+            self._wait_for_tests(test_id, expect_works=(not pre_run_errors and not run_errors))
+
+    ###########################################################################
+    def _wait_for_tests(self, test_id, expect_works=True):
+    ###########################################################################
+        if self._hasbatch:
+            timeout_arg = "--timeout={}".format(GLOBAL_TIMEOUT) if GLOBAL_TIMEOUT is not None else ""
+            expected_stat = 0 if expect_works else CIME.utils.TESTS_FAILED_ERR_CODE
+            run_cmd_assert_result(self, "{}/wait_for_tests {} *{}/TestStatus".format(TOOLS_DIR, timeout_arg, test_id),
+                                  from_dir=self._testroot, expected_stat=expected_stat)
+
 ###############################################################################
 class O_TestTestScheduler(TestCreateTestCommon):
 ###############################################################################
@@ -803,7 +837,7 @@ def test_a_phases(self):
                                                       self._machine, self._compiler)
         self.assertEqual(len(tests), 3)
         ct = TestScheduler(tests, test_root=TEST_ROOT, output_root=TEST_ROOT,
-                           compiler=self._compiler, mpilib=self._mpilib)
+                           compiler=self._compiler, mpilib=TEST_MPILIB)
 
         build_fail_test = [item for item in tests if "TESTBUILDFAIL" in item][0]
         run_fail_test   = [item for item in tests if "TESTRUNFAIL" in item][0]
@@ -874,7 +908,7 @@ def test_b_full(self):
         tests = update_acme_tests.get_full_test_names(["cime_test_only"], self._machine, self._compiler)
         test_id="%s-%s" % (self._baseline_name, CIME.utils.get_timestamp())
         ct = TestScheduler(tests, test_id=test_id, no_batch=NO_BATCH, test_root=TEST_ROOT,
-                           output_root=TEST_ROOT,compiler=self._compiler, mpilib=self._mpilib)
+                           output_root=TEST_ROOT,compiler=self._compiler, mpilib=TEST_MPILIB)
 
         build_fail_test     = [item for item in tests if "TESTBUILDFAIL_" in item][0]
         build_fail_exc_test = [item for item in tests if "TESTBUILDFAILEXC" in item][0]
@@ -892,9 +926,7 @@ def test_b_full(self):
         finally:
             logging.getLogger().setLevel(log_lvl)
 
-        if (self._hasbatch):
-            run_cmd_assert_result(self, "%s/wait_for_tests *%s/TestStatus" % (TOOLS_DIR, test_id), from_dir=self._testroot,
-                                  expected_stat=CIME.utils.TESTS_FAILED_ERR_CODE)
+        self._wait_for_tests(test_id, expect_works=False)
 
         test_statuses = glob.glob("%s/*%s/TestStatus" % (self._testroot, test_id))
         self.assertEqual(len(tests), len(test_statuses))
@@ -943,7 +975,7 @@ def test_c_use_existing(self):
                                                       self._machine, self._compiler)
         test_id="%s-%s" % (self._baseline_name, CIME.utils.get_timestamp())
         ct = TestScheduler(tests, test_id=test_id, no_batch=NO_BATCH, test_root=TEST_ROOT,
-                           output_root=TEST_ROOT,compiler=self._compiler, mpilib=self._mpilib)
+                           output_root=TEST_ROOT,compiler=self._compiler, mpilib=TEST_MPILIB)
 
         build_fail_test     = [item for item in tests if "TESTBUILDFAIL" in item][0]
         run_fail_test       = [item for item in tests if "TESTRUNFAIL" in item][0]
@@ -959,9 +991,7 @@ def test_c_use_existing(self):
         test_statuses = glob.glob("%s/*%s/TestStatus" % (self._testroot, test_id))
         self.assertEqual(len(tests), len(test_statuses))
 
-        if (self._hasbatch):
-            run_cmd_assert_result(self, "%s/wait_for_tests *%s/TestStatus" % (TOOLS_DIR, test_id), from_dir=self._testroot,
-                                  expected_stat=CIME.utils.TESTS_FAILED_ERR_CODE)
+        self._wait_for_tests(test_id, expect_works=False)
 
         for test_status in test_statuses:
             casedir = os.path.dirname(test_status)
@@ -985,7 +1015,7 @@ def test_c_use_existing(self):
         os.environ["TESTRUNFAIL_PASS"] = "True"
         ct2 = TestScheduler(tests, test_id=test_id, no_batch=NO_BATCH, use_existing=True,
                             test_root=TEST_ROOT,output_root=TEST_ROOT,compiler=self._compiler,
-                            mpilib=self._mpilib)
+                            mpilib=TEST_MPILIB)
 
         log_lvl = logging.getLogger().getEffectiveLevel()
         logging.disable(logging.CRITICAL)
@@ -994,8 +1024,7 @@ def test_c_use_existing(self):
         finally:
             logging.getLogger().setLevel(log_lvl)
 
-        if (self._hasbatch):
-            run_cmd_assert_result(self, "%s/wait_for_tests *%s/TestStatus" % (TOOLS_DIR, test_id), from_dir=self._testroot)
+        self._wait_for_tests(test_id)
 
         for test_status in test_statuses:
             ts = TestStatus(test_dir=os.path.dirname(test_status))
@@ -1105,12 +1134,7 @@ class T_TestRunRestart(TestCreateTestCommon):
     ###########################################################################
     def test_run_restart(self):
     ###########################################################################
-        run_cmd_assert_result(self, "{}/create_test --test-root {} --output-root {} -t {} --compiler {} --mpilib {} NODEFAIL_P1.f09_g16.X".
-                              format(SCRIPT_DIR, TEST_ROOT, TEST_ROOT, self._baseline_name,
-                                     self._compiler, self._mpilib))
-        if self._hasbatch:
-            run_cmd_assert_result(self, "%s/wait_for_tests *%s/TestStatus" % (TOOLS_DIR, self._baseline_name),
-                                  from_dir=self._testroot)
+        self._create_test(["NODEFAIL_P1.f09_g16.X"], test_id=self._baseline_name)
 
         casedir = os.path.join(self._testroot,
                                "{}.{}".format(CIME.utils.get_full_test_name("NODEFAIL_P1.f09_g16.X", machine=self._machine, compiler=self._compiler), self._baseline_name))
@@ -1123,14 +1147,7 @@ def test_run_restart(self):
     ###########################################################################
     def test_run_restart_too_many_fails(self):
     ###########################################################################
-        os.environ["NODEFAIL_NUM_FAILS"] = "5"
-        run_cmd_assert_result(self, "{}/create_test --test-root {} --output-root {} -t {} --compiler {} --mpilib {} NODEFAIL_P1.f09_g16.X".
-                              format(SCRIPT_DIR, TEST_ROOT, TEST_ROOT, self._baseline_name,
-                                     self._compiler, self._mpilib),
-                              expected_stat=(0 if self._hasbatch else CIME.utils.TESTS_FAILED_ERR_CODE))
-        if self._hasbatch:
-            run_cmd_assert_result(self, "%s/wait_for_tests *%s/TestStatus" % (TOOLS_DIR, self._baseline_name),
-                                  from_dir=self._testroot, expected_stat=CIME.utils.TESTS_FAILED_ERR_CODE)
+        self._create_test(["NODEFAIL_P1.f09_g16.X"], test_id=self._baseline_name, env_changes="NODEFAIL_NUM_FAILS=5", run_errors=True)
 
         casedir = os.path.join(self._testroot,
                                "{}.{}".format(CIME.utils.get_full_test_name("NODEFAIL_P1.f09_g16.X", machine=self._machine, compiler=self._compiler), self._baseline_name))
@@ -1151,26 +1168,6 @@ def tearDown(self):
         if "TESTRUNDIFF_ALTERNATE" in os.environ:
             del os.environ["TESTRUNDIFF_ALTERNATE"]
 
-    ###########################################################################
-    def simple_test(self, expect_works, extra_args):
-    ###########################################################################
-        if NO_BATCH:
-            extra_args += " --no-batch"
-
-        if " -n " in extra_args:
-            run_cmd_assert_result(self, "%s/create_test --test-root %s --output-root %s %s --compiler %s --mpilib %s"
-                                  % (SCRIPT_DIR, TEST_ROOT, TEST_ROOT, extra_args, self._compiler, self._mpilib),
-                                  expected_stat=(0 if expect_works else CIME.utils.TESTS_FAILED_ERR_CODE))
-        else:
-            run_cmd_assert_result(self, "%s/create_test --test-root %s --output-root %s %s --compiler %s --mpilib %s"
-                                  % (SCRIPT_DIR, TEST_ROOT, TEST_ROOT, extra_args, self._compiler, self._mpilib),
-                                  expected_stat=(0 if expect_works or self._hasbatch else CIME.utils.TESTS_FAILED_ERR_CODE))
-
-            if self._hasbatch:
-                test_id = extra_args.split()[extra_args.split().index("-t") + 1]
-                run_cmd_assert_result(self, "%s/wait_for_tests *%s/TestStatus" % (TOOLS_DIR, test_id),
-                                      from_dir=self._testroot, expected_stat=(0 if expect_works else CIME.utils.TESTS_FAILED_ERR_CODE))
-
     ###############################################################################
     def test_bless_test_results(self):
     ###############################################################################
@@ -1178,23 +1175,23 @@ def test_bless_test_results(self):
         test_name = "TESTRUNDIFF_P1.f19_g16_rx1.A"
 
         if CIME.utils.get_model() == "acme":
-            genarg = "-g -o -b %s %s" % (self._baseline_name, test_name)
-            comparg = "-c -b %s %s" % (self._baseline_name, test_name)
+            genargs = ["-g", "-o", "-b", self._baseline_name, test_name]
+            compargs = ["-c", "-b", self._baseline_name, test_name]
         else:
-            genarg = "-g %s -o %s" % (self._baseline_name, test_name)
-            comparg = "-c %s %s" % (self._baseline_name, test_name)
+            genargs = ["-g", self._baseline_name, "-o", test_name]
+            compargs = ["-c", self._baseline_name, test_name]
 
-        self.simple_test(True, "%s -t %s-%s" % (genarg, self._baseline_name, CIME.utils.get_timestamp()))
+        self._create_test(genargs)
 
         # Hist compare should pass
-        self.simple_test(True, "%s -t %s-%s" % (comparg, self._baseline_name, CIME.utils.get_timestamp()))
+        self._create_test(compargs)
 
         # Change behavior
         os.environ["TESTRUNDIFF_ALTERNATE"] = "True"
 
         # Hist compare should now fail
         test_id = "%s-%s" % (self._baseline_name, CIME.utils.get_timestamp())
-        self.simple_test(False, "%s -t %s" % (comparg, test_id))
+        self._create_test(compargs, test_id=test_id, run_errors=True)
 
         # compare_test_results should detect the fail
         cpr_cmd = "%s/compare_test_results --test-root %s -b %s -t %s 2>&1" \
@@ -1212,7 +1209,7 @@ def test_bless_test_results(self):
                             % (TOOLS_DIR, TEST_ROOT, self._baseline_name, test_id))
 
         # Hist compare should now pass again
-        self.simple_test(True, "%s -t %s-%s" % (comparg, self._baseline_name, CIME.utils.get_timestamp()))
+        self._create_test(compargs)
 
     ###############################################################################
     def test_rebless_namelist(self):
@@ -1220,17 +1217,17 @@ def test_rebless_namelist(self):
         # Generate some namelist baselines
         test_to_change = "TESTRUNPASS_P1.f19_g16_rx1.A"
         if CIME.utils.get_model() == "acme":
-            genarg = "-g -o -b %s cime_test_only_pass" % self._baseline_name
-            comparg = "-c -b %s cime_test_only_pass" % self._baseline_name
+            genargs = ["-n", "-g", "-o", "-b", self._baseline_name, "cime_test_only_pass"]
+            compargs = ["-n", "-c", "-b", self._baseline_name, "cime_test_only_pass"]
         else:
-            genarg = "-g %s -o cime_test_only_pass" % self._baseline_name
-            comparg = "-c %s cime_test_only_pass" % self._baseline_name
+            genargs = ["-n", "-g", self._baseline_name, "-o",  "cime_test_only_pass"]
+            compargs = ["-n", "-c", self._baseline_name, "cime_test_only_pass"]
 
-        self.simple_test(True, "%s -n -t %s-%s" % (genarg, self._baseline_name, CIME.utils.get_timestamp()))
+        self._create_test(genargs)
 
         # Basic namelist compare
         test_id = "%s-%s" % (self._baseline_name, CIME.utils.get_timestamp())
-        self.simple_test(True, "%s -n -t %s" % (comparg, test_id))
+        self._create_test(compargs, test_id=test_id)
 
         # Check standalone case.cmpgen_namelists
         casedir = os.path.join(self._testroot,
@@ -1270,7 +1267,7 @@ def test_rebless_namelist(self):
 
         # Basic namelist compare should now fail
         test_id = "%s-%s" % (self._baseline_name, CIME.utils.get_timestamp())
-        self.simple_test(False, "%s -n -t %s" % (comparg, test_id))
+        self._create_test(compargs, test_id=test_id, pre_run_errors=True)
         casedir = os.path.join(self._testroot,
                                "%s.C.%s" % (CIME.utils.get_full_test_name(test_to_change, machine=self._machine, compiler=self._compiler), test_id))
         run_cmd_assert_result(self, "./case.cmpgen_namelists", from_dir=casedir, expected_stat=100)
@@ -1297,80 +1294,9 @@ def test_rebless_namelist(self):
                             % (TOOLS_DIR, TEST_ROOT, self._baseline_name, test_id))
 
         # Basic namelist compare should now pass again
-        self.simple_test(True, "%s -n -t %s-%s" % (comparg, self._baseline_name, CIME.utils.get_timestamp()))
+        self._create_test(compargs)
 
 ###############################################################################
-@unittest.skip("Disabling this test until we figure out how to integrate ACME tests and CIME xml files.")
-class R_TestUpdateACMETests(unittest.TestCase):
-###############################################################################
-
-    ###########################################################################
-    def setUp(self):
-    ###########################################################################
-        # Grab all active tests
-        files = Files()
-        self._testlist_drv = files.get_value("TESTS_SPEC_FILE", {"component":"drv"})
-        shutil.copy2(self._testlist_drv, ".")
-
-    ###########################################################################
-    def tearDown(self):
-    ###########################################################################
-        shutil.copy2("testlist_drv.xml", self._testlist_drv)
-
-    ###########################################################################
-    def test_update_acme_tests(self):
-    ###########################################################################
-        # Add some testable stuff to acme tests
-        pass
-        # update_acme_tests._TEST_SUITES["cime_tiny"] = \
-        #     (None, (("ERS.f19_g16_rx1.A", "jgftestmodtest/test_mod"),
-        #             ("NCK.f19_g16_rx1.A", "jgftestmodtest/test_mod"))
-        #      )
-
-        # try:
-        #     update_acme_tests.update_acme_tests(os.path.basename(self._testlist_drv), update_acme_tests.get_test_suites())
-        # except:
-        #     traceback.print_tb(sys.exc_info()[2])
-        #     self.assertTrue(False, str(sys.exc_info()[1]))
-
-        # stat = run_cmd("grep 'jgftestmodtest/test_mod' %s" % os.path.basename(self._testlist_drv))[0]
-        # self.assertEqual(stat, 0, msg="update_acme_tests did not update XML")
-
-    ###########################################################################
-    def test_update_acme_tests_test_mods(self):
-    ###########################################################################
-        pass
-        # machine = "melvin"
-        # not_my_machine = "%s_jgftest" % machine
-
-        # # Add some testable stuff to acme tests
-        # update_acme_tests._TEST_SUITES["cime_tiny"] = \
-        #     (None, (("ERS.f19_g16_rx1.A", "test_mod"),
-        #             ("ERS.f19_g16_rx1.B", "test_mod", machine),
-        #             ("ERS.f19_g16_rx1.C", "test_mod", (machine, not_my_machine)),
-        #             ("ERS.f19_g16_rx1.D", "test_mod", not_my_machine),
-        #             "ERS.f19_g16_rx1.E")
-        #      )
-
-        # tests = update_acme_tests.get_test_suite("cime_tiny", compiler="gnu")
-
-        # self.assertEqual(5, len(tests))
-        # self.assertTrue("ERS.f19_g16_rx1.A.melvin_gnu.test_mod" in tests)
-        # self.assertTrue("ERS.f19_g16_rx1.B.melvin_gnu.test_mod" in tests)
-        # self.assertTrue("ERS.f19_g16_rx1.C.melvin_gnu.test_mod" in tests)
-        # self.assertTrue("ERS.f19_g16_rx1.D.melvin_gnu" in tests)
-        # self.assertTrue("ERS.f19_g16_rx1.E.melvin_gnu" in tests)
-
-        # if (CIME.utils.does_machine_have_batch()):
-        #     stat, output, errput = run_cmd("%s/wait_for_tests *%s*/TestStatus" % (TOOLS_DIR, self._baseline_name), from_dir=self._testroot)
-        #     self.assertEqual(stat, 0,
-        #                      msg="COMMAND SHOULD HAVE WORKED\nwait_for_tests output:\n%s\n\nerrput:\n%s\n\ncode: %d" % (output, errput, stat))
-
-
-        # stat, output, errput = run_cmd("%s/cs.status.%s" % (self._testroot, self._baseline_name))
-        # self.assertEqual(stat, 0,
-        #                  msg="COMMAND SHOULD HAVE WORKED\ncs.status output:\n%s\n\nerrput:\n%s\n\ncode: %d" % (output, errput, stat))
-
 class Z_FullSystemTest(TestCreateTestCommon):
 ###############################################################################
 
@@ -1381,16 +1307,7 @@ def test_full_system(self):
         if (FAST_ONLY):
             self.skipTest("Skipping slow test")
 
-        create_test_cmd =  "%s/create_test cime_developer --test-root %s --output-root %s --walltime 0:15:00 -t %s --compiler %s --mpilib %s" \
-            % (SCRIPT_DIR, TEST_ROOT, TEST_ROOT, self._baseline_name, self._compiler, self._mpilib)
-        if NO_BATCH:
-            create_test_cmd += " --no-batch"
-
-        run_cmd_assert_result(self, create_test_cmd)
-
-        if (self._hasbatch):
-            run_cmd_assert_result(self, "%s/wait_for_tests *%s/TestStatus" % (TOOLS_DIR, self._baseline_name),
-                                  from_dir=self._testroot)
+        self._create_test(["--walltime=0:15:00", "cime_developer"], test_id=self._baseline_name)
 
         run_cmd_assert_result(self, "%s/cs.status.%s" % (self._testroot, self._baseline_name),
                               from_dir=self._testroot)
@@ -1421,9 +1338,7 @@ def test_full_system(self):
 
             run_cmd_assert_result(self, "./case.submit", from_dir=casedir)
 
-        if (self._hasbatch):
-            run_cmd_assert_result(self, "%s/wait_for_tests *%s/TestStatus" % (TOOLS_DIR, self._baseline_name),
-                                  from_dir=self._testroot)
+        self._wait_for_tests(self._baseline_name)
 
 ###############################################################################
 class K_TestCimeCase(TestCreateTestCommon):
@@ -1432,8 +1347,7 @@ class K_TestCimeCase(TestCreateTestCommon):
     ###########################################################################
     def test_cime_case(self):
     ###########################################################################
-        run_cmd_assert_result(self, "%s/create_test TESTRUNPASS_P1.f19_g16_rx1.A -t %s --no-build --test-root %s --output-root %s --compiler %s --mpilib %s"
-                              % (SCRIPT_DIR, self._baseline_name, TEST_ROOT, TEST_ROOT, self._compiler, self._mpilib))
+        self._create_test(["--no-build", "TESTRUNPASS_P1.f19_g16_rx1.A"], test_id=self._baseline_name)
 
         self.assertEqual(type(MACHINE.get_value("MAX_TASKS_PER_NODE")), int)
         self.assertTrue(type(MACHINE.get_value("PROJECT_REQUIRED")) in [type(None) , bool])
@@ -1474,8 +1388,7 @@ def test_cime_case(self):
     ###########################################################################
     def test_cime_case_build_threaded_1(self):
     ###########################################################################
-        run_cmd_assert_result(self, "%s/create_test TESTRUNPASS_P1x1.f19_g16_rx1.A -t %s --no-build --test-root %s --output-root %s --compiler %s --mpilib %s"
-                              % (SCRIPT_DIR, self._baseline_name, TEST_ROOT, TEST_ROOT, self._compiler, self._mpilib))
+        self._create_test(["--no-build", "TESTRUNPASS_P1x1.f19_g16_rx1.A"], test_id=self._baseline_name)
 
         casedir = os.path.join(self._testroot,
                                "%s.%s" % (CIME.utils.get_full_test_name("TESTRUNPASS_P1x1.f19_g16_rx1.A", machine=self._machine, compiler=self._compiler), self._baseline_name))
@@ -1496,8 +1409,7 @@ def test_cime_case_build_threaded_1(self):
     ###########################################################################
     def test_cime_case_build_threaded_2(self):
     ###########################################################################
-        run_cmd_assert_result(self, "%s/create_test TESTRUNPASS_P1x2.f19_g16_rx1.A -t %s --no-build --test-root %s --output-root %s --compiler %s --mpilib %s"
-                              % (SCRIPT_DIR, self._baseline_name, TEST_ROOT, TEST_ROOT, self._compiler, self._mpilib))
+        self._create_test(["--no-build", "TESTRUNPASS_P1x2.f19_g16_rx1.A"], test_id=self._baseline_name)
 
         casedir = os.path.join(self._testroot,
                                "%s.%s" % (CIME.utils.get_full_test_name("TESTRUNPASS_P1x2.f19_g16_rx1.A", machine=self._machine, compiler=self._compiler), self._baseline_name))
@@ -1513,8 +1425,7 @@ def test_cime_case_build_threaded_2(self):
     ###########################################################################
     def test_cime_case_mpi_serial(self):
     ###########################################################################
-        run_cmd_assert_result(self, "%s/create_test TESTRUNPASS_Mmpi-serial.f19_g16_rx1.A -t %s --no-build --test-root %s --output-root %s --compiler %s"
-                              % (SCRIPT_DIR, self._baseline_name, self._testroot, self._testroot, self._compiler))
+        self._create_test(["--no-build", "TESTRUNPASS_Mmpi-serial.f19_g16_rx1.A"], test_id=self._baseline_name)
 
         casedir = os.path.join(self._testroot,
                                "%s.%s" % (CIME.utils.get_full_test_name("TESTRUNPASS_Mmpi-serial.f19_g16_rx1.A", machine=self._machine, compiler=self._compiler), self._baseline_name))
@@ -1531,8 +1442,7 @@ def test_cime_case_mpi_serial(self):
     ###########################################################################
     def test_cime_case_force_pecount(self):
     ###########################################################################
-        run_cmd_assert_result(self, "%s/create_test TESTRUNPASS_Mmpi-serial.f19_g16_rx1.A -t %s --no-build --test-root %s --output-root %s --force-procs 16 --force-threads 8 --compiler %s"
-                              % (SCRIPT_DIR, self._baseline_name, self._testroot, self._testroot, self._compiler))
+        self._create_test(["--no-build", "--force-procs=16", "--force-threads=8", "TESTRUNPASS_Mmpi-serial.f19_g16_rx1.A"], test_id=self._baseline_name)
 
         casedir = os.path.join(self._testroot,
                                "%s.%s" % (CIME.utils.get_full_test_name("TESTRUNPASS_Mmpi-serial_P16x8.f19_g16_rx1.A", machine=self._machine, compiler=self._compiler), self._baseline_name))
@@ -1546,11 +1456,10 @@ def test_cime_case_force_pecount(self):
     ###########################################################################
     def test_cime_case_xmlchange_append(self):
     ###########################################################################
-        run_cmd_assert_result(self, "%s/create_test TESTRUNPASS_Mmpi-serial.f19_g16_rx1.A -t %s --no-build --test-root %s --output-root %s --force-procs 16 --force-threads 8 --compiler %s"
-                              % (SCRIPT_DIR, self._baseline_name, self._testroot, self._testroot, self._compiler))
+        self._create_test(["--no-build", "TESTRUNPASS_P1x1.f19_g16_rx1.A"], test_id=self._baseline_name)
 
         casedir = os.path.join(self._testroot,
-                               "%s.%s" % (CIME.utils.get_full_test_name("TESTRUNPASS_Mmpi-serial_P16x8.f19_g16_rx1.A", machine=self._machine, compiler=self._compiler), self._baseline_name))
+                               "%s.%s" % (CIME.utils.get_full_test_name("TESTRUNPASS_P1x1.f19_g16_rx1.A", machine=self._machine, compiler=self._compiler), self._baseline_name))
         self.assertTrue(os.path.isdir(casedir), msg="Missing casedir '%s'" % casedir)
 
         run_cmd_assert_result(self, "./xmlchange --id PIO_CONFIG_OPTS --val='-opt1'", from_dir=casedir)
@@ -1569,8 +1478,8 @@ def test_cime_case_test_walltime_mgmt_1(self):
 
         test_name = "ERS.f19_g16_rx1.A"
         machine, compiler = "blues", "gnu"
-        run_cmd_assert_result(self, "unset CIME_GLOBAL_WALLTIME && %s/create_test --no-setup --machine %s %s -t %s --test-root %s --output-root %s " %
-                              (SCRIPT_DIR, machine, test_name, self._baseline_name, self._testroot, self._testroot))
+        self._create_test(["--no-setup", "--machine={}".format(machine), test_name], test_id=self._baseline_name,
+                          env_changes="unset CIME_GLOBAL_WALLTIME &&")
 
         casedir = os.path.join(self._testroot,
                                "%s.%s" % (CIME.utils.get_full_test_name(test_name, machine=machine, compiler=compiler), self._baseline_name))
@@ -1590,8 +1499,8 @@ def test_cime_case_test_walltime_mgmt_2(self):
 
         test_name = "ERS_P64.f19_g16_rx1.A"
         machine, compiler = "blues", "gnu"
-        run_cmd_assert_result(self, "unset CIME_GLOBAL_WALLTIME && %s/create_test --no-setup --machine %s %s -t %s --test-root %s --output-root %s " %
-                              (SCRIPT_DIR, machine, test_name, self._baseline_name, self._testroot, self._testroot))
+        self._create_test(["--no-setup", "--machine={}".format(machine), test_name], test_id=self._baseline_name,
+                          env_changes="unset CIME_GLOBAL_WALLTIME &&")
 
         casedir = os.path.join(self._testroot,
                                "%s.%s" % (CIME.utils.get_full_test_name(test_name, machine=machine, compiler=compiler), self._baseline_name))
@@ -1611,8 +1520,8 @@ def test_cime_case_test_walltime_mgmt_3(self):
 
         test_name = "ERS_P64.f19_g16_rx1.A"
         machine, compiler = "blues", "gnu"
-        run_cmd_assert_result(self, "unset CIME_GLOBAL_WALLTIME && %s/create_test --no-setup --machine %s %s -t %s --test-root %s --output-root %s --walltime='0:10:00' " %
-                              (SCRIPT_DIR, machine, test_name, self._baseline_name, self._testroot, self._testroot))
+        self._create_test(["--no-setup", "--machine={}".format(machine), "--walltime=0:10:00", test_name], test_id=self._baseline_name,
+                          env_changes="unset CIME_GLOBAL_WALLTIME &&")
 
         casedir = os.path.join(self._testroot,
                                "%s.%s" % (CIME.utils.get_full_test_name(test_name, machine=machine, compiler=compiler), self._baseline_name))
@@ -1632,8 +1541,8 @@ def test_cime_case_test_walltime_mgmt_4(self):
 
         test_name = "ERS_P1.f19_g16_rx1.A"
         machine, compiler = "blues", "gnu"
-        run_cmd_assert_result(self, "unset CIME_GLOBAL_WALLTIME && %s/create_test --no-setup --machine %s %s -t %s --test-root %s --output-root %s --walltime='2:00:00' " %
-                              (SCRIPT_DIR, machine, test_name, self._baseline_name, self._testroot, self._testroot))
+        self._create_test(["--no-setup", "--machine={}".format(machine), "--walltime=2:00:00", test_name], test_id=self._baseline_name,
+                          env_changes="unset CIME_GLOBAL_WALLTIME &&")
 
         casedir = os.path.join(self._testroot,
                                "%s.%s" % (CIME.utils.get_full_test_name(test_name, machine=machine, compiler=compiler), self._baseline_name))
@@ -1653,8 +1562,8 @@ def test_cime_case_test_walltime_mgmt_5(self):
 
         test_name = "ERS_P1.f19_g16_rx1.A"
         machine, compiler = "blues", "gnu"
-        run_cmd_assert_result(self, "unset CIME_GLOBAL_WALLTIME && %s/create_test --no-setup --machine %s %s -t %s --test-root %s --output-root %s --queue slartibartfast" %
-                              (SCRIPT_DIR, machine, test_name, self._baseline_name, self._testroot, self._testroot))
+        self._create_test(["--no-setup", "--machine={}".format(machine), "--queue=slartibartfast", test_name], test_id=self._baseline_name,
+                          env_changes="unset CIME_GLOBAL_WALLTIME &&")
 
         casedir = os.path.join(self._testroot,
                                "%s.%s" % (CIME.utils.get_full_test_name(test_name, machine=machine, compiler=compiler), self._baseline_name))
@@ -1666,9 +1575,10 @@ def test_cime_case_test_walltime_mgmt_5(self):
         result = run_cmd_assert_result(self, "./xmlquery JOB_QUEUE --subgroup=case.test --value", from_dir=casedir)
         self.assertEqual(result, "slartibartfast")
 
+    ###########################################################################
     def test_create_test_longname(self):
-        create_test_cmd =  "%s/create_test SMS.f19_g16.2000_SATM_XLND_SICE_SOCN_XROF_XGLC_SWAV --no-build --output-root %s --compiler %s --mpilib %s" % (SCRIPT_DIR, TEST_ROOT, self._compiler, self._mpilib)
-        run_cmd_assert_result(self, create_test_cmd)
+    ###########################################################################
+        self._create_test(["SMS.f19_g16.2000_SATM_XLND_SICE_SOCN_XROF_XGLC_SWAV", "--no-build"])
 
 ###############################################################################
 class X_TestSingleSubmit(TestCreateTestCommon):
@@ -1686,11 +1596,8 @@ def test_single_submit(self):
             self.skipTest("Skipping single submit. Only works on skybridge")
 
         # Keep small enough for now that we don't have to worry about load balancing
-        run_cmd_assert_result(self,
-                              "unset CIME_GLOBAL_WALLTIME && %s/create_test SMS_Ln9_P8.f45_g37_rx1.A SMS_Ln9_P8.f19_g16_rx1.A  -t %s --single-submit --test-root %s --output-root %s --compiler %s --mpilib %s"
-                              % (SCRIPT_DIR, self._baseline_name, TEST_ROOT, TEST_ROOT, self._compiler, self._mpilib))
-        run_cmd_assert_result(self, "%s/wait_for_tests *%s/TestStatus" % (TOOLS_DIR, self._baseline_name),
-                              from_dir=self._testroot)
+        self._create_test(["--single-submit", "SMS_Ln9_P8.f45_g37_rx1.A", "SMS_Ln9_P8.f19_g16_rx1.A"],
+                          env_changes="unset CIME_GLOBAL_WALLTIME &&")
 
 ###############################################################################
 class L_TestSaveTimings(TestCreateTestCommon):
@@ -1700,16 +1607,7 @@ class L_TestSaveTimings(TestCreateTestCommon):
     def simple_test(self, manual_timing=False):
     ###########################################################################
         timing_flag = "" if manual_timing else "--save-timing"
-        create_test_cmd =  "%s/create_test SMS_Ln9_P1.f19_g16_rx1.A %s --walltime 0:15:00 -t %s --test-root %s --output-root %s --compiler %s --mpilib %s " \
-            % (SCRIPT_DIR, timing_flag, self._baseline_name, TEST_ROOT, TEST_ROOT,
-               self._compiler, self._mpilib)
-        if NO_BATCH:
-            create_test_cmd += " --no-batch"
-
-        run_cmd_assert_result(self, create_test_cmd)
-        if (self._hasbatch):
-            run_cmd_assert_result(self, "%s/wait_for_tests *%s/TestStatus" % (TOOLS_DIR, self._baseline_name),
-                                  from_dir=self._testroot)
+        self._create_test(["SMS_Ln9_P1.f19_g16_rx1.A", timing_flag, "--walltime=0:15:00"], test_id=self._baseline_name)
 
         statuses = glob.glob("%s/*%s/TestStatus" % (self._testroot, self._baseline_name))
         self.assertEqual(len(statuses), 1, msg="Should have had exactly one match, found %s" % statuses)
@@ -2494,6 +2392,10 @@ def _main_func():
         TEST_ROOT = os.path.join(MACHINE.get_value("CIME_OUTPUT_ROOT"),
                                  "scripts_regression_test.%s"% CIME.utils.get_timestamp())
 
+    if "--timeout" in sys.argv:
+        global GLOBAL_TIMEOUT
+        tidx = sys.argv.index("--machine")
+        GLOBAL_TIMEOUT = int(sys.argv[tidx + 1])
 
     args = lambda: None # just something to set attrs on
     for log_param in ["debug", "silent", "verbose"]: