Repository: ESMCI/cime

Issue:
Unsupported option in testlist schema
We have cases in CAM where we create tests to protect new functionality which is not ready for prime time. We would like these tests run regularly which is easiest if they are in aux_cam. However, currently, that makes that compset / resolution combination 'supported' for `create_newcase`.

I propose a new `attribute` in the `test` element in testlist.xsd:
```
<xs:attribute name="supported" use="optional" type="xs:boolean" default="true"/>
```
I am fine with leaving off the `use="optional"` property if that is not standard usage in CIME.

Adding a new, optional argument to Testlist.get_tests (`supported_only=False`) which would trigger culling of those tests which have an `supported=false` attribute.
@jedwards4b, @jgfouca: thoughts?


Discussion:
Should functionality which is not ready for primetime be included in the cesm2 release?   
If not then I would suggest waiting until after the release and since there has been clambering to 
reconsider the testlist format this should be included in that task.
Some might argue that CAM is not ready for primetime :)
However, another use case is that we have a refined mesh test with a test grid which **should not be used** (it was created just to be able to run fast tests). We really do _not_ want to support that resolution but we need to test the refined mesh functionality as a CAM regression test.
I can't give guidance since E3SM doesn't use testlists.
I don't have any objection to adding this and the optional argument is fine.

Resolving patch:
diff --git a/scripts/lib/CIME/XML/testlist.py b/scripts/lib/CIME/XML/testlist.py
--- a/scripts/lib/CIME/XML/testlist.py
+++ b/scripts/lib/CIME/XML/testlist.py
@@ -1,6 +1,6 @@
 """
 Interface to the config_files.xml file.  This class inherits from generic_xml.py
-It supports versions 1.0 and 2.0 of the testlist.xml file
+It supports version 2.0 of the testlist.xml file
 
 In version 2 of the file options can be specified to further refine a test or
 set of tests. They can be specified either at the top level, in which case they
@@ -53,47 +53,7 @@ def __init__(self,infile, files=None):
         schema = files.get_schema("TESTS_SPEC_FILE")
         GenericXML.__init__(self, infile, schema=schema)
 
-    def _get_testsv1(self, machine=None, category=None, compiler=None, compset=None, grid=None):
-        tests = []
-
-        machatts = {}
-        if category is not None:
-            machatts["testtype"] = category
-        if compiler is not None:
-            machatts["compiler"] = compiler
-
-        compsetnodes = self.get_children("compset")
-        logger.debug("compsetnodes {:d}".format(len(compsetnodes)))
-        for cnode in compsetnodes:
-            gridnodes = self.get_children("grid", root=cnode)
-            logger.debug("  gridnodes {:d}".format(len(gridnodes)))
-            for gnode in gridnodes:
-                testnamenodes = self.get_children("test",root=gnode)
-                logger.debug("    testnamenodes {:d}".format(len(testnamenodes)))
-                for tnode in testnamenodes:
-                    machnodes = self.get_children("machine",machatts,root=tnode)
-                    logger.debug("      machnodes {:d}".format(len(machnodes)))
-                    for mach in machnodes:
-                        thistest = {}
-                        save_this = True
-                        if machine is None or (machine is not None and self.text(mach) == machine):
-                            thistest["compiler"] = self.get(mach, "compiler")
-                            thistest["category"] = self.get(mach, "testtype")
-                            thistest["machine"] = self.text(mach)
-                            thistest["testname"] = self.get(tnode, "name")
-                            thistest["grid"] = self.get(gnode, "name")
-                            thistest["compset"] = self.get(cnode, "name")
-                            if self.has(mach, "testmods"):
-                                thistest["testmods"] = self.get(mach, "testmods")
-                            if compset is not None and compset != thistest["compset"]:
-                                save_this = False
-                            if grid is not None and grid != thistest["grid"]:
-                                save_this = False
-                            if save_this:
-                                tests.append(thistest)
-        return tests
-
-    def _get_testsv2(self, machine=None, category=None, compiler=None, compset=None, grid=None):
+    def get_tests(self, machine=None, category=None, compiler=None, compset=None, grid=None, supported_only=False):
         tests = []
         attributes = {}
         if compset is not None:
@@ -111,7 +71,11 @@ def _get_testsv2(self, machine=None, category=None, compiler=None, compset=None,
         if compiler is not None:
             machatts["compiler"]  = compiler
 
+
         for tnode in testnodes:
+            if supported_only and self.has(tnode, "supported") and not self.get(tnode, "support"):
+                continue
+
             machnode = self.get_optional_child("machines", root=tnode)
             machnodes = None if machnode is None else self.get_children("machine",machatts,root=machnode)
             if machnodes:
@@ -122,6 +86,8 @@ def _get_testsv2(self, machine=None, category=None, compiler=None, compset=None,
                     else:
                         this_test_node[key] = value
 
+
+
                 # Get options that apply to all machines/compilers for this test
                 options = self.get_children("options", root=tnode)
                 if len(options) > 0:
@@ -150,12 +116,3 @@ def _get_testsv2(self, machine=None, category=None, compiler=None, compset=None,
                     tests.append(this_test)
 
         return tests
-
-    def get_tests(self, machine=None, category=None, compiler=None, compset=None, grid=None):
-        if self.get_version() == 1.0:
-            return self._get_testsv1(machine, category, compiler, compset, grid)
-        elif self.get_version() >= 2.0:
-            return self._get_testsv2(machine, category, compiler, compset, grid)
-        else:
-            logger.critical("Did not recognize testlist file version {} for file {}"
-                             .format(self.get_version(), self.filename))
diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py
--- a/scripts/lib/CIME/case.py
+++ b/scripts/lib/CIME/case.py
@@ -1297,8 +1297,10 @@ def _check_testlists(self, compset_alias, grid_name, files):
             # It's important that we not try to find matching tests if
             # compset_alias is None, since compset=None tells get_tests to find
             # tests of all compsets!
+            # Only collect supported tests as this _check_testlists is only
+            #   called if run_unsupported is False.
             tests = Testlist(tests_spec_file, files)
-            testlist = tests.get_tests(compset=compset_alias, grid=grid_name)
+            testlist = tests.get_tests(compset=compset_alias, grid=grid_name, supported_only=True)
             for test in testlist:
                 if test["category"] == "prealpha" or test["category"] == "prebeta" or "aux_" in test["category"]:
                     testcnt += 1


Test patch: