Repository: ESMCI/cime

Issue:
Have cime abort if xmllint is not found as it fails anyway
Jackie ran into the following problem on izumi, where because she didn't have xmllint in her path her case failed to be created.

This is with branch_tags/cime5.8.32_a02 but it looks like the same problem would be on master as well.


> job is case.run USER_REQUESTED_WALLTIME None USER_REQUESTED_QUEUE None WALLTIME_FORMAT %H:%M:%S
> 
> job is case.st_archive USER_REQUESTED_WALLTIME None USER_REQUESTED_QUEUE None WALLTIME_FORMAT %H:%M:%S
> 
>  Creating Case directory /home/jkshuman/FATES_cases/C4Gr_TropTrees/BCI_ens_test/fire_test_main_4x5_2a90c206_64c557e0
> 
> 
> Setting Environment OMP_STACKSIZE=64M
> 
> Setting Environment PATH=/usr/lib64/qt-3.3/bin:/usr/local/bin:/usr/local/sbin:/usr/sbin:/cluster/torque/bin
> 
> Setting Environment ESMFMKFILE=/home/dunlap/ESMF-INSTALL/8.0.0bs16/lib/libg/Linux.intel.64.mvapich2.default/esmf.mk
> 
> Setting resource.RLIMIT_STACK to -1 from (8388608, -1)
> 
> xmllint not found, could not validate file /home/jkshuman/git/fates_main_api/cime/config/cesm/config_files.xml
> 
> xmllint not found, could not validate file /home/jkshuman/git/fates_main_api/cime/config/cesm/machines/config_machines.xml
> 
> xmllint not found, could not validate file /home/jkshuman/git/fates_main_api/cime/config/cesm/machines/config_compilers.xml
> 
> /home/jkshuman/FATES_cases/C4Gr_TropTrees/BCI_ens_test/fire_test_main_4x5_2a90c206_64c557e0/env_mach_specific.xml already exists, delete to replace
> 
> /home/jkshuman/FATES_cases/C4Gr_TropTrees/BCI_ens_test/fire_test_main_4x5_2a90c206_64c557e0/env_mach_specific.xml already exists, delete to replace
> 
> ERROR: File /home/jkshuman/FATES_cases/C4Gr_TropTrees/BCI_ens_test/fire_test_main_4x5_2a90c206_64c557e0/env_mach_pes.xml appears to have changed without a corresponding invalidation.
> 
> Traceback (most recent call last):
> 
>   File "./xmlchange", line 225, in <module>


Discussion:
Also shouldn't it be "validation" rather than "invalidation"?

scripts/lib/CIME/XML/generic_xml.py:               "File {} appears to have changed without a corresponding invalidation.".format(self.filename))
xmllint is ubiquitous in Linux, if it's not in your path you have much bigger problems.  I don't think that we should consider this change.  
@jedwards4b hmmm, but isn't that exactly a reason TO do this? If it can't find something ubiquitous -- shouldn't it go and abort as soon as possible? I'm reporting on an experience of an actual user, that I'm actually still having a hard time figuring out what's wrong with her environment. The sooner we have cime report on and abort when it identifies a problem the easier it will be for users to fix their own problems, or for us to help them. That's why I'm asking here, I think having this in place might've made it easier both for her to find her own problem, and for me to help her. Both of which I think are good. 

You are right that we can't protect against every crazy environment setup that a user *might* have. But, it seems to me to be good to protect against ones that we find actual users running into.
sure okay, I understand now - xmllint should be required and cime should fail with a reasonable error if it is not found. 

Resolving patch:
diff --git a/scripts/lib/CIME/XML/generic_xml.py b/scripts/lib/CIME/XML/generic_xml.py
--- a/scripts/lib/CIME/XML/generic_xml.py
+++ b/scripts/lib/CIME/XML/generic_xml.py
@@ -363,6 +363,7 @@ def write(self, outfile=None, force_write=False):
 
         # xmllint provides a better format option for the output file
         xmllint = find_executable("xmllint")
+
         if xmllint is not None:
             if isinstance(outfile, six.string_types):
                 run_cmd_no_fail("{} --format --output {} -".format(xmllint, outfile), input_str=xmlstr)
@@ -549,11 +550,10 @@ def validate_xml_file(self, filename, schema):
         expect(os.path.isfile(filename),"xml file not found {}".format(filename))
         expect(os.path.isfile(schema),"schema file not found {}".format(schema))
         xmllint = find_executable("xmllint")
-        if xmllint is not None:
-            logger.debug("Checking file {} against schema {}".format(filename, schema))
-            run_cmd_no_fail("{} --xinclude --noout --schema {} {}".format(xmllint, schema, filename))
-        else:
-            logger.warning("xmllint not found, could not validate file {}".format(filename))
+        expect(os.path.isfile(xmllint), " xmllint not found in PATH, xmllint is required for cime.  PATH={}".format(os.environ["PATH"]))
+
+        logger.debug("Checking file {} against schema {}".format(filename, schema))
+        run_cmd_no_fail("{} --xinclude --noout --schema {} {}".format(xmllint, schema, filename))
 
     def get_raw_record(self, root=None):
         logger.debug("writing file {}".format(self.filename))


Test patch:
diff --git a/scripts/lib/CIME/test_scheduler.py b/scripts/lib/CIME/test_scheduler.py
--- a/scripts/lib/CIME/test_scheduler.py
+++ b/scripts/lib/CIME/test_scheduler.py
@@ -472,32 +472,6 @@ def _create_newcase_phase(self, test):
         if self._pesfile is not None:
             create_newcase_cmd += " --pesfile {} ".format(self._pesfile)
 
-        if test_mods is not None:
-            create_newcase_cmd += " --user-mods-dir "
-
-            for one_test_mod in test_mods: # pylint: disable=not-an-iterable
-                if one_test_mod.find('/') != -1:
-                    (component, modspath) = one_test_mod.split('/', 1)
-                else:
-                    error = "Missing testmod component. Testmods are specified as '${component}-${testmod}'"
-                    self._log_output(test, error)
-                    return False, error
-
-                files = Files(comp_interface=self._cime_driver)
-                testmods_dir = files.get_value("TESTS_MODS_DIR", {"component": component})
-                test_mod_file = os.path.join(testmods_dir, component, modspath)
-                # if no testmod is found check if a usermod of the same name exists and
-                # use it if it does.
-                if not os.path.exists(test_mod_file):
-                    usermods_dir = files.get_value("USER_MODS_DIR", {"component": component})
-                    test_mod_file = os.path.join(usermods_dir, modspath)
-                    if not os.path.exists(test_mod_file):
-                        error = "Missing testmod file '{}', checked {} and {}".format(modspath, testmods_dir, usermods_dir)
-                        self._log_output(test, error)
-                        return False, error
-
-                create_newcase_cmd += "{} ".format(test_mod_file)
-
         mpilib = None
         ninst = 1
         ncpl = 1
@@ -527,6 +501,32 @@ def _create_newcase_phase(self, test):
                     self._cime_driver = case_opt[1:]
                     create_newcase_cmd += " --driver {}".format(self._cime_driver)
 
+        if test_mods is not None:
+            create_newcase_cmd += " --user-mods-dir "
+
+            for one_test_mod in test_mods: # pylint: disable=not-an-iterable
+                if one_test_mod.find('/') != -1:
+                    (component, modspath) = one_test_mod.split('/', 1)
+                else:
+                    error = "Missing testmod component. Testmods are specified as '${component}-${testmod}'"
+                    self._log_output(test, error)
+                    return False, error
+
+                files = Files(comp_interface=self._cime_driver)
+                testmods_dir = files.get_value("TESTS_MODS_DIR", {"component": component})
+                test_mod_file = os.path.join(testmods_dir, component, modspath)
+                # if no testmod is found check if a usermod of the same name exists and
+                # use it if it does.
+                if not os.path.exists(test_mod_file):
+                    usermods_dir = files.get_value("USER_MODS_DIR", {"component": component})
+                    test_mod_file = os.path.join(usermods_dir, modspath)
+                    if not os.path.exists(test_mod_file):
+                        error = "Missing testmod file '{}', checked {} and {}".format(modspath, testmods_dir, usermods_dir)
+                        self._log_output(test, error)
+                        return False, error
+
+                create_newcase_cmd += "{} ".format(test_mod_file)
+
         # create_test mpilib option overrides default but not explicitly set case_opt mpilib
         if mpilib is None and self._mpilib is not None:
             create_newcase_cmd += " --mpilib {}".format(self._mpilib)