Repository: ESMCI/cime

Issue:
Using PRE and POSTRUN scripts
When in the flow do the pre and post scripts execute? Do they happen on the compute node? Would they would be just before/after the "srun" command?

And where should the pre and postrun scripts be?  Apparently they need to be in bash?  I see `/bin/sh` executing them.

If I put them in case directory, and then do:

```
./xmlchange PRERUN_SCRIPT="../prerun_script.sh"
./xmlchange POSTRUN_SCRIPT="../postrun_script.sh"
```

I get:
```
cori10% cat SMS.ne4_ne4.m71-mar12.intel.mpt.n001p0048t48x1.BB.run.10937713 
ERROR: External script ../prerun_script.sh not found
```

If instead I try 

```
./xmlchange PRERUN_SCRIPT="./prerun_script.sh"
./xmlchange POSTRUN_SCRIPT="./postrun_script.sh"
```
That seems to go further, however, the contents of 
` prerun.external.log.10937953.180313-170053` (which is written in the run directory) does not have what I would expect, so I'm no longer sure when the scripts are executed.

Also, is it really necessary to make the script executable if it's being called with a `/bin/sh`?
```
 failed with error '/bin/sh: ./prerun_script.sh: Permission denied' 
```
I had to change the permissions.



Discussion:
Looking at the output, I see `   Running ./prerun_script.sh ` as well postrun.  And it looks like the prerun script is done before a lot of other things happen.  What I need is something that happens just before srun.

`PREMPIRUN_SCRIPT` and `POSTMPIRUN_SCRIPT` ?
They do happen on the compute nodes and they do happen pretty close to just before and after srun, but we need to re-read xml files after running each of the scripts since they could have changed values in those files which would cause them to be out of sync with the case object in memory.
I suggest that @ndkeen should use my suggestion of a custom script - see https://github.com/ESMCI/cime/issues/2342#issuecomment-372858197

Resolving patch:
diff --git a/scripts/lib/CIME/XML/env_batch.py b/scripts/lib/CIME/XML/env_batch.py
--- a/scripts/lib/CIME/XML/env_batch.py
+++ b/scripts/lib/CIME/XML/env_batch.py
@@ -189,6 +189,7 @@ def make_batch_script(self, input_template, job, case):
             overrides["job_id"] = overrides["job_id"][:15]
 
         overrides["batchdirectives"] = self.get_batch_directives(case, job, overrides=overrides)
+        overrides["mpirun"] = case.get_mpirun_cmd(job=job)
         output_text = transform_vars(open(input_template,"r").read(), case=case, subgroup=job, overrides=overrides)
         output_name = get_batch_script_for_job(job)
 
diff --git a/scripts/lib/CIME/case/case.py b/scripts/lib/CIME/case/case.py
--- a/scripts/lib/CIME/case/case.py
+++ b/scripts/lib/CIME/case/case.py
@@ -155,8 +155,6 @@ def initialize_derived_attributes(self):
             "threaded" : self.get_build_threaded(),
             }
 
-        os.environ["OMP_NUM_THREADS"] = str(self.thread_count)
-
         executable = env_mach_spec.get_mpirun(self, mpi_attribs, job="case.run", exe_only=True)[0]
         if executable is not None and "aprun" in executable:
             _, self.num_nodes, self.total_tasks, self.tasks_per_node, self.thread_count = get_aprun_cmd_for_case(self, "e3sm.exe")
@@ -1263,7 +1261,7 @@ def get_mpirun_cmd(self, job="case.run"):
         if self.get_value("BATCH_SYSTEM") == "cobalt":
             mpi_arg_string += " : "
 
-        return "{} {} {}".format(executable if executable is not None else "", mpi_arg_string, run_suffix)
+        return self.get_resolved_value("{} {} {}".format(executable if executable is not None else "", mpi_arg_string, run_suffix))
 
     def set_model_version(self, model):
         version = "unknown"
diff --git a/scripts/lib/CIME/case/case_run.py b/scripts/lib/CIME/case/case_run.py
--- a/scripts/lib/CIME/case/case_run.py
+++ b/scripts/lib/CIME/case/case_run.py
@@ -53,9 +53,6 @@ def _pre_run_check(case, lid, skip_pnl=False, da_cycle=0):
         os.environ["LBQUERY"] = "TRUE"
 
     # create the timing directories, optionally cleaning them if needed.
-    if not os.path.isdir(rundir):
-        os.mkdir(rundir)
-
     if os.path.isdir(os.path.join(rundir, "timing")):
         shutil.rmtree(os.path.join(rundir, "timing"))
 
@@ -84,16 +81,12 @@ def _run_model_impl(case, lid, skip_pnl=False, da_cycle=0):
     model = case.get_value("MODEL")
 
     # Set OMP_NUM_THREADS
-    env_mach_pes = case.get_env("mach_pes")
-    comp_classes = case.get_values("COMP_CLASSES")
-    thread_count = env_mach_pes.get_max_thread_count(comp_classes)
-    os.environ["OMP_NUM_THREADS"] = str(thread_count)
+    os.environ["OMP_NUM_THREADS"] = str(case.thread_count)
 
     # Run the model
     logger.info("{} MODEL EXECUTION BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S")))
 
     cmd = case.get_mpirun_cmd(job="case.run")
-    cmd = case.get_resolved_value(cmd)
     logger.info("run command is {} ".format(cmd))
 
     rundir = case.get_value("RUNDIR")


Test patch: