Repository: ESMCI/cime

Issue:
Be consistent when accessing environment
config_compilers uses

```
<env>VARNAME</env>
```

everything else uses

```
$ENV{VARNAME}
```

config_compilers should be changed to be consistent with the other files.


Discussion:
I was able to add a call to the genericxml get_resolved_value in the compilerblock.py
method _handle_references - it works but there is a consequence.   
with the ```<env>``` method the resulting macros file has the unresolved value ```($(NETCDF))``` while with the ```$ENV{}``` method the fully resolved value is in the file when it's written.   

I think that for this file we want to preserve the ```<env>``` style result and so this will need more work.

Resolving patch:
diff --git a/utils/python/CIME/XML/compilerblock.py b/utils/python/CIME/XML/compilerblock.py
--- a/utils/python/CIME/XML/compilerblock.py
+++ b/utils/python/CIME/XML/compilerblock.py
@@ -67,6 +67,8 @@
 from CIME.BuildTools.valuesetting import ValueSetting
 from CIME.BuildTools.possiblevalues import PossibleValues
 
+logger = logging.getLogger(__name__)
+
 class CompilerBlock(object):
 
     """Data used to translate a single <compiler> element.
@@ -115,6 +117,43 @@ def _handle_references(self, elem, set_up, tear_down, depends):
         output = elem.text
         if output is None:
             output = ""
+        logger.debug("Initial output=%s"%output)
+        reference_re = re.compile(r'\${?(\w+)}?')
+        env_ref_re   = re.compile(r'\$ENV\{(\w+)\}')
+        shell_ref_re = re.compile(r'\$SHELL\{([^}]+)\}')
+
+        for m in reference_re.finditer(output):
+            var_name = m.groups()[0]
+            if var_name not in ("SHELL","ENV"):
+                output = output.replace(m.group(), writer.variable_string(var_name))
+                depends.add(var_name)
+
+        logger.debug("preenv pass output=%s"%output)
+
+        for m in env_ref_re.finditer(output):
+            logger.debug("look for %s in env %s" % (output,writer.environment_variable_string(m.groups()[0])))
+            output = output.replace(m.group(),
+                                    writer.environment_variable_string(m.groups()[0]))
+            logger.debug("and output %s"%output)
+
+        logger.debug("postenv pass output=%s"%output)
+
+        for s in shell_ref_re.finditer(output):
+            command = s.groups()[0]
+            if "{" in command and not "}" in command:
+                expect(False, "Nesting not allowed in this syntax, use the xml form <shell> and <env> for nesting")
+            logger.debug("execute %s in shell, command %s" % (output, command))
+            new_set_up, inline, new_tear_down = \
+                writer.shell_command_strings(command)
+            output = output.replace(s.group(), inline)
+            if new_set_up is not None:
+                set_up.append(new_set_up)
+            if new_tear_down is not None:
+                tear_down.append(new_tear_down)
+            logger.debug("set_up %s inline %s tear_down %s"%(new_set_up,inline,new_tear_down))
+
+        logger.debug("First pass output=%s"%output)
+
         for child in elem:
             if child.tag == "env":
                 # <env> tags just need to be expanded by the writer.
@@ -130,6 +169,7 @@ def _handle_references(self, elem, set_up, tear_down, depends):
                     set_up.append(new_set_up)
                 if new_tear_down is not None:
                     tear_down.append(new_tear_down)
+                logger.debug("set_up %s inline %s tear_down %s"%(new_set_up,inline,new_tear_down))
             elif child.tag == "var":
                 # <var> commands also need expansion by the writer, and can
                 # add dependencies.
@@ -143,6 +183,9 @@ def _handle_references(self, elem, set_up, tear_down, depends):
                        "according to the schema.")
             if child.tail is not None:
                 output += child.tail
+
+        logger.debug("Second pass output=%s"%output)
+
         return output
 
     def _elem_to_setting(self, elem):
diff --git a/utils/python/CIME/XML/compilers.py b/utils/python/CIME/XML/compilers.py
--- a/utils/python/CIME/XML/compilers.py
+++ b/utils/python/CIME/XML/compilers.py
@@ -208,9 +208,9 @@ def _write_macros_file_v2(self, build_system, output, xml=None):
                 if value_lists[var_name].depends <= vars_written
             ]
             expect(len(ready_variables) > 0,
-                   "The config_build XML has bad <var> references. "
+                   "The file %s has bad <var> references. "
                    "Check for circular references or variables that "
-                   "are in a <var> tag but not actually defined.")
+                   "are in a <var> tag but not actually defined."%self.filename)
             big_normal_tree = None
             big_append_tree = None
             for var_name in ready_variables:


Test patch:
diff --git a/utils/python/tests/scripts_regression_tests.py b/utils/python/tests/scripts_regression_tests.py
--- a/utils/python/tests/scripts_regression_tests.py
+++ b/utils/python/tests/scripts_regression_tests.py
@@ -1795,24 +1795,39 @@ def test_environment_variable_insertion(self):
         tester = self.xml_to_tester(xml1)
         tester.assert_variable_equals("LDFLAGS", "-L/path/to/netcdf -lnetcdf",
                                       env={"NETCDF": "/path/to/netcdf"})
+        # DO it again with $ENV{} style
+        xml1 = """<compiler><LDFLAGS><append>-L$ENV{NETCDF} -lnetcdf</append></LDFLAGS></compiler>"""
+        tester = self.xml_to_tester(xml1)
+        tester.assert_variable_equals("LDFLAGS", "-L/path/to/netcdf -lnetcdf",
+                                      env={"NETCDF": "/path/to/netcdf"})
 
     def test_shell_command_insertion(self):
         """Test that <shell> elements insert shell command output."""
         xml1 = """<compiler><FFLAGS><base>-O<shell>echo 2</shell> -fast</base></FFLAGS></compiler>"""
         tester = self.xml_to_tester(xml1)
         tester.assert_variable_equals("FFLAGS", "-O2 -fast")
+        xml1 = """<compiler><FFLAGS><base>-O$SHELL{echo 2} -fast</base></FFLAGS></compiler>"""
+        tester = self.xml_to_tester(xml1)
+        tester.assert_variable_equals("FFLAGS", "-O2 -fast")
 
     def test_multiple_shell_commands(self):
         """Test that more than one <shell> element can be used."""
         xml1 = """<compiler><FFLAGS><base>-O<shell>echo 2</shell> -<shell>echo fast</shell></base></FFLAGS></compiler>"""
         tester = self.xml_to_tester(xml1)
         tester.assert_variable_equals("FFLAGS", "-O2 -fast")
+        xml1 = """<compiler><FFLAGS><base>-O$SHELL{echo 2} -$SHELL{echo fast}</base></FFLAGS></compiler>"""
+        tester = self.xml_to_tester(xml1)
+        tester.assert_variable_equals("FFLAGS", "-O2 -fast")
 
     def test_env_and_shell_command(self):
         """Test that <env> elements work inside <shell> elements."""
         xml1 = """<compiler><FFLAGS><base>-O<shell>echo <env>OPT_LEVEL</env></shell> -fast</base></FFLAGS></compiler>"""
         tester = self.xml_to_tester(xml1)
         tester.assert_variable_equals("FFLAGS", "-O2 -fast", env={"OPT_LEVEL": "2"})
+        xml1 = """<compiler><FFLAGS><base>-O$SHELL{echo $ENV{OPT_LEVEL}} -fast</base></FFLAGS></compiler>"""
+        err_msg = "Nesting not allowed.*"
+        with self.assertRaisesRegexp(SystemExit, err_msg):
+             self.xml_to_tester(xml1)
 
     def test_config_variable_insertion(self):
         """Test that <var> elements insert variables from config_build."""
@@ -1827,12 +1842,25 @@ def test_config_variable_insertion(self):
         tester = self.xml_to_tester("<compiler>"+xml1+xml2+xml3+xml4+xml5+"</compiler>")
         tester.assert_variable_equals("MPI_LIB_NAME", "stuff-mpicc-stuff")
 
+        xml1 = """<MPI_LIB_NAME>stuff-${MPI_PATH}-stuff</MPI_LIB_NAME>"""
+        xml2 = """<MPI_PATH>${MPICC}</MPI_PATH>"""
+        xml3 = """<MPICC>${MPICXX}</MPICC>"""
+        xml4 = """<MPICXX>${MPIFC}</MPICXX>"""
+        xml5 = """<MPIFC>mpicc</MPIFC>"""
+        tester = self.xml_to_tester("<compiler>"+xml1+xml2+xml3+xml4+xml5+"</compiler>")
+        tester.assert_variable_equals("MPI_LIB_NAME", "stuff-mpicc-stuff")
+
     def test_config_reject_self_references(self):
         """Test that <var> self-references are rejected."""
         # This is a special case of the next test, which also checks circular
         # references.
         xml1 = """<MPI_LIB_NAME><var>MPI_LIB_NAME</var></MPI_LIB_NAME>"""
-        err_msg = "The config_build XML has bad <var> references."
+        err_msg = ".* has bad <var> references."
+        with self.assertRaisesRegexp(SystemExit, err_msg):
+            self.xml_to_tester("<compiler>"+xml1+"</compiler>")
+
+        xml1 = """<MPI_LIB_NAME>${MPI_LIB_NAME}</MPI_LIB_NAME>"""
+        err_msg = ".* has bad <var> references."
         with self.assertRaisesRegexp(SystemExit, err_msg):
             self.xml_to_tester("<compiler>"+xml1+"</compiler>")
 
@@ -1840,7 +1868,12 @@ def test_config_reject_cyclical_references(self):
         """Test that cyclical <var> references are rejected."""
         xml1 = """<MPI_LIB_NAME><var>MPI_PATH</var></MPI_LIB_NAME>"""
         xml2 = """<MPI_PATH><var>MPI_LIB_NAME</var></MPI_PATH>"""
-        err_msg = "The config_build XML has bad <var> references."
+        err_msg = ".* has bad <var> references."
+        with self.assertRaisesRegexp(SystemExit, err_msg):
+            self.xml_to_tester("<compiler>"+xml1+xml2+"</compiler>")
+        xml1 = """<MPI_LIB_NAME>${MPI_PATH}</MPI_LIB_NAME>"""
+        xml2 = """<MPI_PATH>${MPI_LIB_NAME}</MPI_PATH>"""
+        err_msg = ".* has bad <var> references."
         with self.assertRaisesRegexp(SystemExit, err_msg):
             self.xml_to_tester("<compiler>"+xml1+xml2+"</compiler>")
 
@@ -1849,7 +1882,14 @@ def test_variable_insertion_with_machine_specific_setting(self):
         xml1 = """<compiler><MPI_LIB_NAME>something</MPI_LIB_NAME></compiler>"""
         xml2 = """<compiler MACH="{}"><MPI_LIB_NAME><var>MPI_PATH</var></MPI_LIB_NAME></compiler>""".format(self.test_machine)
         xml3 = """<compiler><MPI_PATH><var>MPI_LIB_NAME</var></MPI_PATH></compiler>"""
-        err_msg = "The config_build XML has bad <var> references."
+        err_msg = ".* has bad <var> references."
+        with self.assertRaisesRegexp(SystemExit, err_msg):
+            self.xml_to_tester(xml1+xml2+xml3)
+
+        xml1 = """<compiler><MPI_LIB_NAME>something</MPI_LIB_NAME></compiler>"""
+        xml2 = """<compiler MACH="{}"><MPI_LIB_NAME><var>MPI_PATH</var></MPI_LIB_NAME></compiler>""".format(self.test_machine)
+        xml3 = """<compiler><MPI_PATH>${MPI_LIB_NAME}</MPI_PATH></compiler>"""
+        err_msg = ".* has bad <var> references."
         with self.assertRaisesRegexp(SystemExit, err_msg):
             self.xml_to_tester(xml1+xml2+xml3)