Repository: ESMCI/cime

Issue:
Want better documentation for new testlist xml format
Moving some comments from #620 

Can someone document how to do the following with the testlist xml files:
1. Add a comment that appears when using the tool that queries testlists (i.e., not just a comment in the xml, but something with a special comment tag). I don't see any examples of this in the new xml formatted files. Incidentally, this would be another tag that is good to have nested under a specific machine / compiler / category, because my comments are often specific to that level.
2. Have the exact same test appear in multiple categories: Is the correct way to do this to have two lines that are identical except for their category, or to have something like ?
3. Similar question to (2), but how do you have the same test run with multiple compilers? Again, is the expectation that there should be multiple lines or one machine line with a '|' in the match?
4. Document the different ways you can specify options - either nested under a machine tag or not.

I'm not sure what the best place would be for this documentation, but testlist.py seems like a reasonable option.

cc @jedwards4b 



Discussion:

Resolving patch:
diff --git a/utils/python/CIME/XML/testlist.py b/utils/python/CIME/XML/testlist.py
--- a/utils/python/CIME/XML/testlist.py
+++ b/utils/python/CIME/XML/testlist.py
@@ -2,10 +2,36 @@
 Interface to the config_files.xml file.  This class inherits from generic_xml.py
 It supports versions 1.0 and 2.0 of the testlist.xml file
 
-In version 2 of the file options can be specified to further refine a test or set of tests
-they can be specified at the <machines> level or at the level of a particular machine.
-currently supported options are walltime which sets the wallclock time for a given test
-in the queing system, and memleak_tolerance which specifies the relative memory growth expected for a given test.
+In version 2 of the file options can be specified to further refine a test or
+set of tests. They can be specified either at the top level, in which case they
+apply to all machines/compilers for this test:
+
+<test ...>
+  <options>
+    <option name="wallclock">00:20</option>
+  </options>
+  ...
+</test>
+
+or at the level of a particular machine/compiler:
+
+<test ...>
+  <machines>
+    <machine ...>
+      <options>
+        <option name="wallclock">00:20</option>
+      </options>
+    </machine>
+  </machines>
+</test>
+
+Currently supported options are:
+
+- walltime: sets the wallclock limit in the queuing system
+
+- memleak_tolerance: specifies the relative memory growth expected for this test
+
+- comment: has no effect, but is written out when printing the test list
 
 """
 from CIME.XML.standard_module_setup import *
@@ -86,11 +112,16 @@ def _get_testsv2(self, machine=None, category=None, compiler=None):
                         else:
                             this_test[key] = value
                     this_test["options"] = {}
-                    # option xpath here gets only the children of tnode and ignores grandchildren
-                    optionnodes = self.get_nodes("option", root=tnode, xpath="options/option")
 
+                    # Get options that apply to all machines/compilers for this test
+                    #
+                    # option xpath here prevents us from looking within the
+                    # machines block
+                    optionnodes = self.get_nodes("option", root=tnode, xpath="options/option")
                     for onode in optionnodes:
                         this_test['options'][onode.get('name')] = onode.text
+
+                    # Now get options specific to this machine/compiler
                     optionnodes = self.get_nodes("option", root=mach)
                     for onode in optionnodes:
                         this_test['options'][onode.get('name')] = onode.text


Test patch:
diff --git a/utils/python/CIME/test_utils.py b/utils/python/CIME/test_utils.py
--- a/utils/python/CIME/test_utils.py
+++ b/utils/python/CIME/test_utils.py
@@ -1,5 +1,6 @@
 """
-Utility functions used in test_scheduler.py
+Utility functions used in test_scheduler.py, and by other utilities that need to
+get test lists.
 """
 
 from CIME.XML.standard_module_setup import *
@@ -50,3 +51,42 @@ def get_tests_from_xml(xml_machine=None,xml_category=None,xml_compiler=None, xml
         logger.debug("Found %d tests"% len(listoftests))
 
     return listoftests
+
+def test_to_string(test, category_field_width=0, test_field_width=0, show_options=False):
+    """Given a test dictionary, return a string representation suitable for printing
+
+    Args:
+        test (dict): dictionary for a single test - e.g., one element from the
+                     list returned by get_tests_from_xml
+        category_field_width (int): minimum amount of space to use for printing the test category
+        test_field_width (int): minimum amount of space to use for printing the test category
+        show_options (bool): if True, print test options, too (note that the 'comment'
+                             option is always printed, if present)
+
+    Basic functionality:
+    >>> mytest = {'name': 'SMS.f19_g16.A.yellowstone_intel', 'category': 'prealpha', 'options': {}}
+    >>> test_to_string(mytest, 10)
+    'prealpha  : SMS.f19_g16.A.yellowstone_intel'
+
+    Printing comments:
+    >>> mytest = {'name': 'SMS.f19_g16.A.yellowstone_intel', 'category': 'prealpha', 'options': {'comment': 'my remarks'}}
+    >>> test_to_string(mytest, 10)
+    'prealpha  : SMS.f19_g16.A.yellowstone_intel # my remarks'
+
+    Printing other options, too:
+    >>> mytest = {'name': 'SMS.f19_g16.A.yellowstone_intel', 'category': 'prealpha', 'options': {'comment': 'my remarks', 'wallclock': '0:20', 'memleak_tolerance': 0.2}}
+    >>> test_to_string(mytest, 10, show_options=True)
+    'prealpha  : SMS.f19_g16.A.yellowstone_intel # my remarks  # memleak_tolerance: 0.2  # wallclock: 0:20'
+    """
+
+    mystr = "%-*s: %-*s"%(category_field_width, test['category'], test_field_width, test['name'])
+    if 'options' in test:
+        myopts = test['options'].copy()
+        comment = myopts.pop('comment', None)
+        if comment:
+            mystr += " # %s"%(comment)
+        if show_options:
+            for one_opt in sorted(myopts):
+                mystr += "  # %s: %s"%(one_opt, myopts[one_opt])
+
+    return mystr
diff --git a/utils/python/tests/scripts_regression_tests.py b/utils/python/tests/scripts_regression_tests.py
--- a/utils/python/tests/scripts_regression_tests.py
+++ b/utils/python/tests/scripts_regression_tests.py
@@ -1724,7 +1724,7 @@ def test_query_testlists_runs(self):
         when it runs. This helps ensure that changes in other utilities don't
         break query_testlists.
         """
-        self._run_and_assert_query_testlist()
+        self._run_and_assert_query_testlist(extra_args="--show-options")
 
     def test_query_testlists_count_runs(self):
         """Make sure that query_testlists runs successfully with the --count argument"""