Repository: ESMCI/cime

Issue:
Duplicate call to set_nodes in nmlgen:set_defaults looks suspicious
I'm not 100% sure this is a bug, but it looks like one to me... if so, my guess is that it's just slowing things down a bit, probably not causing incorrect results, but it could still be worth fixing:

It looks to me like set_nodes is unnecessarily called twice here, and that only the second call (which saves the return results) should be done:

https://github.com/ESMCI/cime/blob/c258cdf083e55c950b47562408e96a27a8411b74/scripts/lib/CIME/nmlgen.py#L106-L109

But again, I'm not 100% sure that this is a bug: it may be that it needs to be called twice for some reason.

It looks like this came in with @mvertens 's overhaul of this code in #965. @mvertens can you comment on this?


Discussion:
Here's another suspicious duplication: It looks like `self._entry_nodes.append(node)` will be called twice for each node:

https://github.com/ESMCI/cime/blob/c258cdf083e55c950b47562408e96a27a8411b74/scripts/lib/CIME/XML/namelist_definition.py#L103-L116
@billsacks
It looks like both cases cause duplications. 
https://github.com/ESMCI/cime/blob/83fbedf011b5e53a6cbc73449355ef79fd183c52/scripts/lib/CIME/XML/namelist_definition.py#L85-L86
The **first** case causes duplications in `_entry_nodes` and `_entry_ids`.
The **second** case causes duplications in `_entry_nodes`.
The end results is 4x as many values in `_entry_nodes` and 2x in `_entry_ids`.

Theses duplications don't appear to be an issue but removing the duplicate call and refacting reduces the total time spent in `set_nodes` from 0.006 seconds to 0.001 for my specific test case.

Thanks a lot for your investigation of this, @jasonb5 !

Resolving patch:
diff --git a/scripts/lib/CIME/XML/namelist_definition.py b/scripts/lib/CIME/XML/namelist_definition.py
--- a/scripts/lib/CIME/XML/namelist_definition.py
+++ b/scripts/lib/CIME/XML/namelist_definition.py
@@ -89,6 +89,14 @@ def __init__(self, infile, files=None):
         self._group_names = CaseInsensitiveDict({})
         self._nodes = {}
 
+    def set_node_values(self, name, node):
+        self._entry_nodes.append(node)
+        self._entry_ids.append(name)
+        self._nodes[name] = node
+        self._entry_types[name] = self._get_type(node)
+        self._valid_values[name] = self._get_valid_values(node)
+        self._group_names[name] = self.get_group_name(node)
+
     def set_nodes(self, skip_groups=None):
         """
         populates the object data types for all nodes that are not part of the skip_groups array
@@ -99,26 +107,21 @@ def set_nodes(self, skip_groups=None):
             name = self.get(node, "id")
             skip_default_entry = self.get(node, "skip_default_entry") == "true"
             per_stream_entry = self.get(node, "per_stream_entry") == "true"
-            set_node_values = False
+
             if skip_groups:
                 group_name = self.get_group_name(node)
+
                 if not group_name in skip_groups:
-                    self._entry_nodes.append(node)
-                    set_node_values = True
+                    self.set_node_values(name, node)
+
                     if not skip_default_entry and not per_stream_entry:
                         default_nodes.append(node)
             else:
-                self._entry_nodes.append(node)
-                set_node_values = True
+                self.set_node_values(name, node)
+
                 if not skip_default_entry and not per_stream_entry:
                     default_nodes.append(node)
-            if set_node_values:
-                self._entry_nodes.append(node)
-                self._entry_ids.append(name)
-                self._nodes[name] = node
-                self._entry_types[name] = self._get_type(node)
-                self._valid_values[name] = self._get_valid_values(node)
-                self._group_names[name] = self.get_group_name(node)
+
         return default_nodes
 
     def get_group_name(self, node=None):
diff --git a/scripts/lib/CIME/nmlgen.py b/scripts/lib/CIME/nmlgen.py
--- a/scripts/lib/CIME/nmlgen.py
+++ b/scripts/lib/CIME/nmlgen.py
@@ -122,8 +122,6 @@ def init_defaults(self, infiles, config, skip_groups=None, skip_entry_loop=False
         # first clean out any settings left over from previous calls
         self.new_instance()
 
-        self._definition.set_nodes(skip_groups=skip_groups)
-
         # Determine the array of entry nodes that will be acted upon
         self._default_nodes = self._definition.set_nodes(skip_groups=skip_groups)
 


Test patch:
diff --git a/scripts/lib/CIME/tests/test_nmlgen.py b/scripts/lib/CIME/tests/test_nmlgen.py
new file mode 100644
--- /dev/null
+++ b/scripts/lib/CIME/tests/test_nmlgen.py
@@ -0,0 +1,63 @@
+from collections import OrderedDict
+import tempfile
+import unittest
+from unittest import mock
+
+from CIME.nmlgen import NamelistGenerator
+
+# pylint: disable=protected-access
+class TestNamelistGenerator(unittest.TestCase):
+
+    def test_init_defaults(self):
+        test_nml_infile =  b"""&test
+test1 = 'test1_updated'
+/"""
+
+        test_data = """<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="http://www.cgd.ucar.edu/~cam/namelist/namelist_definition.xsl"?>
+
+<entry_id version="2.0">
+    <entry id="test1">
+        <type>char</type>
+        <category>test</category>
+        <group>test_nml</group>
+        <valid_values>test1_value,test1_updated</valid_values>
+        <values>
+            <value>test1_value</value>
+        </values>
+    </entry>
+    <entry id="test2">
+        <type>char</type>
+        <category>test</category>
+        <group>test_nml</group>
+        <values>
+            <value>test2_value</value>
+        </values>
+    </entry>
+</entry_id>"""
+
+        with tempfile.NamedTemporaryFile() as temp, \
+                tempfile.NamedTemporaryFile() as temp2:
+            temp.write(test_data.encode())
+            temp.flush()
+
+            temp2.write(test_nml_infile)
+            temp2.flush()
+
+            case = mock.MagicMock()
+
+            nmlgen = NamelistGenerator(case, [temp.name])
+
+            nmlgen.init_defaults([temp2.name], None)
+
+            expected_groups = OrderedDict({
+                "test_nml": {
+                    "test1": ["'test1_updated'"],
+                    "test2": ['"test2_value"']
+                }
+            })
+
+            assert nmlgen._namelist._groups == expected_groups
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/scripts/lib/CIME/tests/test_xml_namelist_definition.py b/scripts/lib/CIME/tests/test_xml_namelist_definition.py
new file mode 100644
--- /dev/null
+++ b/scripts/lib/CIME/tests/test_xml_namelist_definition.py
@@ -0,0 +1,41 @@
+import tempfile
+import unittest
+
+from CIME.XML.namelist_definition import NamelistDefinition
+
+# pylint: disable=protected-access
+
+class TestXMLNamelistDefinition(unittest.TestCase):
+
+    def test_set_nodes(self):
+        test_data = """<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="http://www.cgd.ucar.edu/~cam/namelist/namelist_definition.xsl"?>
+
+<entry_id version="2.0">
+    <entry id="test1">
+        <type>char</type>
+        <category>test</category>
+    </entry>
+    <entry id="test2">
+        <type>char</type>
+        <category>test</category>
+    </entry>
+</entry_id>"""
+
+        with tempfile.NamedTemporaryFile() as temp:
+            temp.write(test_data.encode())
+            temp.flush()
+
+            nmldef = NamelistDefinition(temp.name)
+
+            nmldef.set_nodes()
+
+            assert len(nmldef._entry_nodes) == 2
+            assert nmldef._entry_ids == ["test1", "test2"]
+            assert len(nmldef._nodes) == 2
+            assert nmldef._entry_types == {"test1": "char", "test2": "char"}
+            assert nmldef._valid_values == {"test1": None, "test2": None}
+            assert nmldef._group_names == {"test1": None, "test2": None}
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py
--- a/scripts/tests/scripts_regression_tests.py
+++ b/scripts/tests/scripts_regression_tests.py
@@ -362,7 +362,8 @@ def test_a_createnewcase(self):
         testdir = os.path.join(cls._testroot, 'testcreatenewcase')
         if os.path.exists(testdir):
             shutil.rmtree(testdir)
-        args =  " --case %s --compset X --output-root %s --handle-preexisting-dirs=r " % (testdir, cls._testroot)
+        args = " --case %s --compset X --output-root %s --handle-preexisting-dirs=r" % (testdir, cls._testroot)
+        args += " --run-unsupported"
         if TEST_COMPILER is not None:
             args = args +  " --compiler %s"%TEST_COMPILER
         if TEST_MPILIB is not None:
@@ -452,6 +453,7 @@ def test_b_user_mods(self):
 
         user_mods_dir = os.path.join(CIME.utils.get_python_libs_root(), "..", "tests", "user_mods_test1")
         args = " --case %s --compset X --user-mods-dir %s --output-root %s --handle-preexisting-dirs=r"% (testdir, user_mods_dir, cls._testroot)
+        args += " --run-unsupported"
         if TEST_COMPILER is not None:
             args = args + " --compiler %s"%TEST_COMPILER
         if TEST_MPILIB is not None:
@@ -645,6 +647,7 @@ def test_h_primary_component(self):
 
         cls._testdirs.append(testdir)
         args = " --case CreateNewcaseTest --script-root %s --compset X --output-root %s --handle-preexisting-dirs u" % (testdir, cls._testroot)
+        args += " --run-unsupported"
         if TEST_COMPILER is not None:
             args += " --compiler %s"%TEST_COMPILER
         if TEST_MPILIB is not None:
@@ -832,7 +835,8 @@ def test_ka_createnewcase_extra_machines_dir(self):
         # config_machines_template.xml
         args = (" --case {testdir} --compset X --mach mymachine"
                 " --output-root {testroot} --non-local"
-                " --extra-machines-dir {extra_machines_dir}".format(
+                " --extra-machines-dir {extra_machines_dir}"
+                " --run-unsupported".format(
                     testdir=testdir, testroot=cls._testroot,
                     extra_machines_dir=extra_machines_dir))
         if CIME.utils.get_cime_default_driver() == "nuopc":