Repository: ESMCI/cime

Issue:
cs.status reset to force rebuild
I would like an additional option to cs.status or perhaps create_test that
would reset all cases in a test suite to the PEND SHAREDLIB_BUILD state so that 
all tests are rebuilt before being restarted. 


Discussion:
This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.
@jasonb5 is this something you might take on?
This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.

Resolving patch:
diff --git a/CIME/cs_status.py b/CIME/cs_status.py
--- a/CIME/cs_status.py
+++ b/CIME/cs_status.py
@@ -6,7 +6,7 @@
 from __future__ import print_function
 from CIME.XML.standard_module_setup import *
 from CIME.XML.expected_fails_file import ExpectedFailsFile
-from CIME.test_status import TestStatus
+from CIME.test_status import TestStatus, SHAREDLIB_BUILD_PHASE, TEST_PEND_STATUS
 import os
 import sys
 from collections import defaultdict
@@ -20,6 +20,7 @@ def cs_status(
     check_throughput=False,
     check_memory=False,
     expected_fails_filepath=None,
+    force_rebuild=False,
     out=sys.stdout,
 ):
     """Print the test statuses of all tests in test_paths. The default
@@ -56,6 +57,11 @@ def cs_status(
     for test_path in test_paths:
         test_dir = os.path.dirname(test_path)
         ts = TestStatus(test_dir=test_dir)
+
+        if force_rebuild:
+            with ts:
+                ts.set_status(SHAREDLIB_BUILD_PHASE, TEST_PEND_STATUS)
+
         test_id = os.path.basename(test_dir).split(".")[-1]
         if summary:
             output = _overall_output(


Test patch:
diff --git a/CIME/scripts/create_test.py b/CIME/scripts/create_test.py
--- a/CIME/scripts/create_test.py
+++ b/CIME/scripts/create_test.py
@@ -473,12 +473,25 @@ def parse_command_line(args, description):
         f"The default is {srcroot_default}",
     )
 
+    parser.add_argument(
+        "--force-rebuild",
+        action="store_true",
+        help="When used with 'use-existing' and 'test-id', the"
+        "tests will have their 'BUILD_SHAREDLIB' phase reset to 'PEND'.",
+    )
+
     CIME.utils.add_mail_type_args(parser)
 
     args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)
 
     CIME.utils.resolve_mail_type_args(args)
 
+    if args.force_rebuild:
+        expect(
+            args.use_existing and args.test_id,
+            "Cannot force a rebuild without 'use-existing' and 'test-id'",
+        )
+
     # generate and compare flags may not point to the same directory
     if model_config.create_test_flag_mode == "cesm":
         if args.generate is not None:
@@ -754,6 +767,7 @@ def parse_command_line(args, description):
         args.single_exe,
         args.workflow,
         args.chksum,
+        args.force_rebuild,
     )
 
 
@@ -913,6 +927,7 @@ def create_test(
     single_exe,
     workflow,
     chksum,
+    force_rebuild,
 ):
     ###############################################################################
     impl = TestScheduler(
@@ -953,6 +968,7 @@ def create_test(
         single_exe=single_exe,
         workflow=workflow,
         chksum=chksum,
+        force_rebuild=force_rebuild,
     )
 
     success = impl.run_tests(
@@ -1054,6 +1070,7 @@ def _main_func(description=None):
         single_exe,
         workflow,
         chksum,
+        force_rebuild,
     ) = parse_command_line(sys.argv, description)
 
     success = False
@@ -1105,6 +1122,7 @@ def _main_func(description=None):
             single_exe,
             workflow,
             chksum,
+            force_rebuild,
         )
         run_count += 1
 
diff --git a/CIME/test_scheduler.py b/CIME/test_scheduler.py
--- a/CIME/test_scheduler.py
+++ b/CIME/test_scheduler.py
@@ -209,6 +209,7 @@ def __init__(
         single_exe=False,
         workflow=None,
         chksum=False,
+        force_rebuild=False,
     ):
         ###########################################################################
         self._cime_root = get_cime_root()
@@ -396,6 +397,9 @@ def __init__(
         if use_existing:
             for test in self._tests:
                 with TestStatus(self._get_test_dir(test)) as ts:
+                    if force_rebuild:
+                        ts.set_status(SHAREDLIB_BUILD_PHASE, TEST_PEND_STATUS)
+
                     for phase, status in ts:
                         if phase in CORE_PHASES:
                             if status in [TEST_PEND_STATUS, TEST_FAIL_STATUS]:
diff --git a/CIME/tests/test_sys_test_scheduler.py b/CIME/tests/test_sys_test_scheduler.py
--- a/CIME/tests/test_sys_test_scheduler.py
+++ b/CIME/tests/test_sys_test_scheduler.py
@@ -279,6 +279,60 @@ def test_b_full(self):
                         test_status.TEST_PASS_STATUS,
                     )
 
+    def test_force_rebuild(self):
+        tests = get_tests.get_full_test_names(
+            [
+                "TESTBUILDFAIL_P1.f19_g16_rx1.A",
+                "TESTRUNFAIL_P1.f19_g16_rx1.A",
+                "TESTRUNPASS_P1.f19_g16_rx1.A",
+            ],
+            self._machine,
+            self._compiler,
+        )
+        test_id = "%s-%s" % (self._baseline_name, utils.get_timestamp())
+        ct = test_scheduler.TestScheduler(
+            tests,
+            test_id=test_id,
+            no_batch=self.NO_BATCH,
+            test_root=self._testroot,
+            output_root=self._testroot,
+            compiler=self._compiler,
+            mpilib=self.TEST_MPILIB,
+            machine_name=self.MACHINE.get_machine_name(),
+        )
+
+        log_lvl = logging.getLogger().getEffectiveLevel()
+        logging.disable(logging.CRITICAL)
+        try:
+            ct.run_tests()
+        finally:
+            logging.getLogger().setLevel(log_lvl)
+
+        ct = test_scheduler.TestScheduler(
+            tests,
+            test_id=test_id,
+            no_batch=self.NO_BATCH,
+            test_root=self._testroot,
+            output_root=self._testroot,
+            compiler=self._compiler,
+            mpilib=self.TEST_MPILIB,
+            machine_name=self.MACHINE.get_machine_name(),
+            force_rebuild=True,
+            use_existing=True,
+        )
+
+        test_statuses = glob.glob("%s/*%s/TestStatus" % (self._testroot, test_id))
+
+        for x in test_statuses:
+            casedir = os.path.dirname(x)
+
+            ts = test_status.TestStatus(test_dir=casedir)
+
+            self.assertTrue(
+                ts.get_status(test_status.SHAREDLIB_BUILD_PHASE)
+                == test_status.TEST_PEND_STATUS
+            )
+
     def test_c_use_existing(self):
         tests = get_tests.get_full_test_names(
             [
diff --git a/CIME/tests/test_unit_cs_status.py b/CIME/tests/test_unit_cs_status.py
--- a/CIME/tests/test_unit_cs_status.py
+++ b/CIME/tests/test_unit_cs_status.py
@@ -71,6 +71,23 @@ def set_phase_to_status(test_dir_path, test_name, phase, status):
     # Begin actual tests
     # ------------------------------------------------------------------------
 
+    def test_force_rebuild(self):
+        test_name = "my.test.name"
+        test_dir = "my.test.name.testid"
+        test_dir_path = self.create_test_dir(test_dir)
+        self.create_test_status_core_passes(test_dir_path, test_name)
+        cs_status(
+            [os.path.join(test_dir_path, "TestStatus")],
+            force_rebuild=True,
+            out=self._output,
+        )
+        self.assert_status_of_phase(
+            self._output.getvalue(),
+            test_status.TEST_PEND_STATUS,
+            test_status.SHAREDLIB_BUILD_PHASE,
+            test_name,
+        )
+
     def test_single_test(self):
         """cs_status for a single test should include some minimal expected output"""
         test_name = "my.test.name"