Repository: ESMCI/cime

Issue:
Enable custom run_exe for a machine
It can be useful to specify a different run_exe for a machine: e.g. in a slurm env
```diff
diff --git a/cime/config/e3sm/machines/config_machines.xml b/cime/config/e3sm/machines/config_machines.xml
index 0260d44..170fc68 100644
--- a/cime/config/e3sm/machines/config_machines.xml
+++ b/cime/config/e3sm/machines/config_machines.xml
@@ -1147,7 +1147,7 @@
     <MAX_MPITASKS_PER_NODE>36</MAX_MPITASKS_PER_NODE>
     <PROJECT_REQUIRED>FALSE</PROJECT_REQUIRED>
     <mpirun mpilib="default">
-      <executable>srun</executable>
+      <executable>sbcast ${EXEROOT}/e3sm.exe /tmp/e3sm.exe;srun</executable>
       <arguments>
         <arg name="num_tasks"> -l -n {{ total_tasks }} </arg>
       </arguments>
@@ -1155,6 +1155,7 @@
     <mpirun mpilib="mpi-serial">
       <executable/>
     </mpirun>
+    <run_exe> /tmp/e3sm.exe </run_exe>
     <module_system type="module">
```

This is currently not possible in E3SM cime due to hard-wired default:
```xml
  <default_run_suffix>
    <default_run_exe>${EXEROOT}/e3sm.exe </default_run_exe>
    <default_run_misc_suffix> &gt;&gt; e3sm.log.$LID 2&gt;&amp;1 </default_run_misc_suffix>
  </default_run_suffix>
```


Discussion:
I got the custom run_exe to work locally after this patch:
```diff
diff --git a/cime/config/xml_schemas/config_machines.xsd b/cime/config/xml_schemas/config_machines.xsd
index c4d189c..2eccdbc 100644
--- a/cime/config/xml_schemas/config_machines.xsd
+++ b/cime/config/xml_schemas/config_machines.xsd
@@ -56,6 +56,8 @@
   <xs:element name="executable" type="xs:string"/>
   <xs:element name="default_run_exe" type="xs:string"/>
   <xs:element name="default_run_misc_suffix" type="xs:string"/>
+  <xs:element name="run_exe" type="xs:string"/>
+  <xs:element name="run_misc_suffix" type="xs:string"/>
   <xs:element name="RUNDIR" type="xs:string"/>
   <xs:element name="EXEROOT" type="xs:string"/>
   <xs:element name="TEST_TPUT_TOLERANCE" type="xs:string"/>
@@ -158,6 +160,8 @@
         <!-- mpirun: The mpi exec to start a job on this machine
              see detail below-->
         <xs:element ref="mpirun" minOccurs="1" maxOccurs="unbounded"/>
+        <xs:element ref="run_exe" minOccurs="0" maxOccurs="1"/>
+        <xs:element ref="run_misc_suffix" minOccurs="0" maxOccurs="1"/>
         <!-- module_system: how and what modules to load on this system ,
              see detail below -->
         <xs:element ref="module_system" minOccurs="1" maxOccurs="1"/>
diff --git a/cime/scripts/lib/CIME/XML/env_mach_specific.py b/cime/scripts/lib/CIME/XML/env_mach_specific.py
index 124ecff..8a723f3 100644
--- a/cime/scripts/lib/CIME/XML/env_mach_specific.py
+++ b/cime/scripts/lib/CIME/XML/env_mach_specific.py
@@ -39,7 +39,7 @@ class EnvMachSpecific(EnvBase):
                 if len(nodes) == 0:
                     value = self.text(default_run_exe_node) if item == "run_exe" else self.text(default_run_misc_suffix_node)
                 else:
-                    value = nodes[0].text
+                    value = self.text(nodes[0])

                 entity_node = self.make_child("entry", {"id":item, "value":value}, root=group_node)

```
I'm not sure why you wouldn't just change:
```xml
<executable>sbcast ${EXEROOT}/e3sm.exe /tmp/e3sm.exe;srun</executable>
<default_run_exe> /tmp/e3sm.exe </default_run_exe>
```
then you don't need to change the schema at all?

Did that work for you?
I'm asking you that

See above, it didn't work.
@amametjanov  -- you added `run_misc_suffix` to the `config_machines` schema, but you don't appear to be using it. Do you need that?

Not including it insures the log files are always the same
Also would this make more sense to be able to set within the `mpirun` element? I assume if you make this change on `anvil`, for example, you wouldn't want mpi-serial to be executed like:

```sh
/tmp/e3sm.exe >> e3sm.log.$LID 2>&1
```
as would happen setting it at the machine level. 

Resolving patch:
diff --git a/scripts/lib/CIME/XML/env_mach_specific.py b/scripts/lib/CIME/XML/env_mach_specific.py
--- a/scripts/lib/CIME/XML/env_mach_specific.py
+++ b/scripts/lib/CIME/XML/env_mach_specific.py
@@ -39,7 +39,7 @@ def populate(self, machobj):
                 if len(nodes) == 0:
                     value = self.text(default_run_exe_node) if item == "run_exe" else self.text(default_run_misc_suffix_node)
                 else:
-                    value = nodes[0].text
+                    value = self.text(nodes[0])
 
                 entity_node = self.make_child("entry", {"id":item, "value":value}, root=group_node)
 
@@ -475,7 +475,13 @@ def get_mpirun(self, case, attribs, job, exe_only=False):
         expect(exec_node is not None,"No executable found")
         executable = self.text(exec_node)
 
-        return executable, args
+        run_exe_node = self.get_optional_child('run_exe', root=the_match)
+        run_exe = self.text(run_exe_node)
+
+        run_misc_suffix_node = self.get_optional_child('run_misc_suffix', root=the_match)
+        run_misc_suffix = self.text(run_misc_suffix_node)
+
+        return executable, args, run_exe, run_misc_suffix
 
     def get_type_info(self, vid):
         return "char"
diff --git a/scripts/lib/CIME/case/case.py b/scripts/lib/CIME/case/case.py
--- a/scripts/lib/CIME/case/case.py
+++ b/scripts/lib/CIME/case/case.py
@@ -1364,7 +1364,6 @@ def get_mpirun_cmd(self, job=None, allow_unresolved_envvars=True):
         run_exe = env_mach_specific.get_value("run_exe")
         run_misc_suffix = env_mach_specific.get_value("run_misc_suffix")
         run_misc_suffix = "" if run_misc_suffix is None else run_misc_suffix
-        run_suffix = run_exe + run_misc_suffix
 
         mpirun_cmd_override = self.get_value("MPI_RUN_COMMAND")
         if mpirun_cmd_override not in ["", None, "UNSET"]:
@@ -1379,7 +1378,14 @@ def get_mpirun_cmd(self, job=None, allow_unresolved_envvars=True):
             "unit_testing" : False
             }
 
-        executable, mpi_arg_list = env_mach_specific.get_mpirun(self, mpi_attribs, job)
+        executable, mpi_arg_list, custom_run_exe, custom_run_misc_suffix = env_mach_specific.get_mpirun(self, mpi_attribs, job)
+        if custom_run_exe:
+            logger.info('Using a custom run_exe {}'.format(custom_run_exe))
+            run_exe = custom_run_exe
+        if custom_run_misc_suffix:
+            logger.info('Using a custom run_misc_suffix {}'.format(custom_run_misc_suffix))
+            run_misc_suffix = custom_run_misc_suffix
+
 
         # special case for aprun
         if executable is not None and "aprun" in executable and not "theta" in self.get_value("MACH"):
@@ -1393,7 +1399,7 @@ def get_mpirun_cmd(self, job=None, allow_unresolved_envvars=True):
         if self.get_value("BATCH_SYSTEM") == "cobalt":
             mpi_arg_string += " : "
 
-        return self.get_resolved_value("{} {} {}".format(executable if executable is not None else "", mpi_arg_string, run_suffix), allow_unresolved_envvars=allow_unresolved_envvars)
+        return self.get_resolved_value("{} {} {} {}".format(executable if executable is not None else "", mpi_arg_string, run_exe, run_misc_suffix), allow_unresolved_envvars=allow_unresolved_envvars)
 
     def set_model_version(self, model):
         version = "unknown"


Test patch:
diff --git a/scripts/fortran_unit_testing/run_tests.py b/scripts/fortran_unit_testing/run_tests.py
--- a/scripts/fortran_unit_testing/run_tests.py
+++ b/scripts/fortran_unit_testing/run_tests.py
@@ -360,7 +360,7 @@ def _main():
         }
 
         # We can get away with specifying case=None since we're using exe_only=True
-        mpirun_command, _ = machspecific.get_mpirun(None, mpi_attribs, None, exe_only=True)
+        mpirun_command, _, _, _ = machspecific.get_mpirun(None, mpi_attribs, None, exe_only=True)
         mpirun_command = machspecific.get_resolved_value(mpirun_command)
         logger.info("mpirun command is '{}'".format(mpirun_command))