Repository: ESMCI/cime

Issue:
Record number of nodes for given job
Presently, CIME computes the number of nodes required for a job submission but this information is not recorded anywhere AFAIK. Can we store it for archiving purposes?


Discussion:
@sarats , so you want to add this to the E3SM provenance?
Yes eventually.  Ideally it would be recorded in an xml variable.  But if it was output as part of ./preview_run (on its own line) that would help.
Note, there is no reason to make this ONLY apply to E3SM. It should apply to all models, there's no reason to make it work only for one. And actually making it only work for one adds more work, so you might as well make it easier on yourself and have it work for all.
We weren't implying this should only be done for E3SM.  It should be general CIME CCS feature.  But E3SM's provenance recording system will process it a certain way.
I think that this is already recorded for a given run in several places.  It's in the log files,
and it's in the timing files.  I think introducing another xml variable for this is unwarranted. 
If you really want this information for provenance then you should save the env_mach_pes.xml
Edit/Update: Never mind, had to refresh to get latest comments. We already capture env_mach_pes.xml etc. The timing summary (e3sm_timing.<case>) & related files do not record num_nodes. 

_Yes, but probably CESM would like it too?_ 

_In fact, it might be capture 'preview_run.log' as well if not already? That could contain the mpirun/jsrun/srun invocation as well which could be handy along with the `replay.sh` script._ 

Here's the output of preview_run from a recent case I ran:
```
[jacob@chrlogin1 Ftestrh]$  ./preview_run 
CASE INFO:
  nodes: 3
  total tasks: 96
  tasks per node: 32
  thread count: 2

```
The value of 3 for "nodes" doesn't appear in any normal XML files or logs.    The hidden .case.run files has "#SBATCH  --nodes=3" at the top but that's an odd place to look.
To add, the directives in .case.run would be dependent on the batch system on a given platform, so it would prudent to have a more generic output. 

Additionally, our mpirun/srun/jsrun invocations often use runtime logic (inline shell commands) to construct the final command-line. So, we should also capture preview_run.log while we're at this.
@rljacob @sarats @jedwards4b 
Would there be any benefit to store this in a machine readable format? Or do we want to stick with output similar to `./preview_run`?

Lastly, just want confirm that details in `./preview_run` are sufficient?

Resolving patch:
diff --git a/scripts/lib/CIME/XML/env_mach_specific.py b/scripts/lib/CIME/XML/env_mach_specific.py
--- a/scripts/lib/CIME/XML/env_mach_specific.py
+++ b/scripts/lib/CIME/XML/env_mach_specific.py
@@ -115,6 +115,8 @@ def load_env(self, case, force_method=None, job=None, verbose=False):
 
         self._get_resources_for_case(case)
 
+        return [] if envs_to_set is None else envs_to_set
+
     def _get_resources_for_case(self, case):
         resource_nodes = self.get_children("resource_limits")
         if resource_nodes is not None:
@@ -254,7 +256,8 @@ def _load_envs(self, envs_to_set, verbose=False):
                     shell, cmd = env_value.split(" ",1)
                     self._source_shell_file("source "+cmd, shell, verbose=verbose)
                 else:
-                    print("Setting Environment {}={}".format(env_name, env_value))
+                    if verbose:
+                        print("Setting Environment {}={}".format(env_name, env_value))
                     logger_func("Setting Environment {}={}".format(env_name, env_value))
                     os.environ[env_name] = env_value
 
diff --git a/scripts/lib/CIME/case/case.py b/scripts/lib/CIME/case/case.py
--- a/scripts/lib/CIME/case/case.py
+++ b/scripts/lib/CIME/case/case.py
@@ -117,6 +117,7 @@ def __init__(self, case_root=None, read_only=True, record=False):
         self._component_classes = []
         self._component_description = {}
         self._is_env_loaded = False
+        self._loaded_envs = None
 
         # these are user_mods as defined in the compset
         # Command Line user_mods are handled seperately
@@ -1508,9 +1509,13 @@ def load_env(self, reset=False, job=None, verbose=False):
                 job = self.get_primary_job()
             os.environ["OMP_NUM_THREADS"] = str(self.thread_count)
             env_module = self.get_env("mach_specific")
-            env_module.load_env(self, job=job, verbose=verbose)
+            self._loaded_envs = env_module.load_env(self, job=job, verbose=verbose)
+            self._loaded_envs.append(("OMP_NUM_THREADS",
+                                      os.environ["OMP_NUM_THREADS"]))
             self._is_env_loaded = True
 
+        return self._loaded_envs
+
     def get_build_threaded(self):
         """
         Returns True if current settings require a threaded build/run.
@@ -1766,3 +1771,37 @@ def get_first_job(self):
         env_workflow = self.get_env("workflow")
         jobs = env_workflow.get_jobs()
         return jobs[0]
+
+    def preview_run(self, write, job):
+        write("CASE INFO:")
+        write("  nodes: {}".format(self.num_nodes))
+        write("  total tasks: {}".format(self.total_tasks))
+        write("  tasks per node: {}".format(self.tasks_per_node))
+        write("  thread count: {}".format(self.thread_count))
+        write("")
+
+        write("BATCH INFO:")
+        if not job:
+            job = self.get_first_job()
+
+        job_id_to_cmd = self.submit_jobs(dry_run=True, job=job)
+        env_batch = self.get_env('batch')
+        for job_id, cmd in job_id_to_cmd:
+            write("  FOR JOB: {}".format(job_id))
+            write("    ENV:")
+            loaded_envs = self.load_env(job=job_id, reset=True, verbose=False)
+
+            for name, value in iter(sorted(loaded_envs, key=lambda x: x[0])):
+                write("      Setting Environment {}={}".format(name, value))
+
+            write("")
+            write("    SUBMIT CMD:")
+            write("      {}".format(self.get_resolved_value(cmd)))
+            write("")
+            if job_id in ("case.run", "case.test"):
+                # get_job_overrides must come after the case.load_env since the cmd may use
+                # env vars.
+                overrides = env_batch.get_job_overrides(job_id, self)
+                write("    MPIRUN (job={}):".format(job_id))
+                write ("      {}".format(self.get_resolved_value(overrides["mpirun"])))
+                write("")
diff --git a/scripts/lib/CIME/provenance.py b/scripts/lib/CIME/provenance.py
--- a/scripts/lib/CIME/provenance.py
+++ b/scripts/lib/CIME/provenance.py
@@ -342,6 +342,16 @@ def _save_prerun_provenance_e3sm(case, lid):
 def _save_prerun_provenance_cesm(case, lid): # pylint: disable=unused-argument
     pass
 
+def _save_prerun_provenance_common(case, lid):
+    """ Saves common prerun provenance.
+    """
+    run_dir = case.get_value("RUNDIR")
+
+    preview_log = os.path.join(run_dir, "preview_run.log.{}".format(lid))
+
+    with open(preview_log, "w") as fd:
+        case.preview_run(lambda x: fd.write("{}\n".format(x)), None)
+
 def save_prerun_provenance(case, lid=None):
     with SharedArea():
         # Always save env
@@ -352,6 +362,8 @@ def save_prerun_provenance(case, lid=None):
             os.makedirs(logdir)
         env_module.save_all_env_info(os.path.join(logdir, "run_environment.txt.{}".format(lid)))
 
+        _save_prerun_provenance_common(case, lid)
+
         model = case.get_value("MODEL")
         if model == "e3sm":
             _save_prerun_provenance_e3sm(case, lid)


Test patch:
diff --git a/scripts/lib/CIME/tests/test_case.py b/scripts/lib/CIME/tests/test_case.py
--- a/scripts/lib/CIME/tests/test_case.py
+++ b/scripts/lib/CIME/tests/test_case.py
@@ -120,11 +120,11 @@ def test_copy(self, getuser, getfqdn, configure, create_caseroot, # pylint: disa
                 srcroot = os.path.abspath(os.path.join(
                     os.path.dirname(__file__), "../../../../../"))
                 case.create("test1", srcroot, "A", "f19_g16_rx1",
-                            machine_name="ubuntu-latest")
+                            machine_name="cori-haswell")
 
                 # Check that they're all called
                 configure.assert_called_with("A", "f19_g16_rx1",
-                                             machine_name="ubuntu-latest",
+                                             machine_name="cori-haswell",
                                              project=None, pecount=None,
                                              compiler=None, mpilib=None,
                                              pesfile=None, gridfile=None,
@@ -177,11 +177,11 @@ def test_create(self, get_user, getfqdn, configure, create_caseroot, # pylint: d
                 srcroot = os.path.abspath(os.path.join(
                     os.path.dirname(__file__), "../../../../../"))
                 case.create("test1", srcroot, "A", "f19_g16_rx1",
-                            machine_name="ubuntu-latest")
+                            machine_name="cori-haswell")
 
                 # Check that they're all called
                 configure.assert_called_with("A", "f19_g16_rx1",
-                                             machine_name="ubuntu-latest",
+                                             machine_name="cori-haswell",
                                              project=None, pecount=None,
                                              compiler=None, mpilib=None,
                                              pesfile=None, gridfile=None,
diff --git a/scripts/lib/CIME/tests/test_test_scheduler.py b/scripts/lib/CIME/tests/test_test_scheduler.py
--- a/scripts/lib/CIME/tests/test_test_scheduler.py
+++ b/scripts/lib/CIME/tests/test_test_scheduler.py
@@ -10,18 +10,18 @@ class TestTestScheduler(unittest.TestCase):
     @mock.patch("time.strftime", return_value="00:00:00")
     def test_chksum(self, strftime): # pylint: disable=unused-argument
         ts = TestScheduler(
-            ["SEQ_Ln9.f19_g16_rx1.A.ubuntu-latest_gnu"],
-            machine_name="ubuntu-latest",
+            ["SEQ_Ln9.f19_g16_rx1.A.cori-haswell_gnu"],
+            machine_name="cori-haswell",
             chksum=True,
             test_root="/tests",
         )
 
         with mock.patch.object(ts, "_shell_cmd_for_phase") as _shell_cmd_for_phase:
-            ts._run_phase("SEQ_Ln9.f19_g16_rx1.A.ubuntu-latest_gnu") # pylint: disable=protected-access
+            ts._run_phase("SEQ_Ln9.f19_g16_rx1.A.cori-haswell_gnu") # pylint: disable=protected-access
 
             _shell_cmd_for_phase.assert_called_with(
-                "SEQ_Ln9.f19_g16_rx1.A.ubuntu-latest_gnu",
-                "./case.submit --skip-preview-namelist --no-batch --chksum",
+                "SEQ_Ln9.f19_g16_rx1.A.cori-haswell_gnu",
+                "./case.submit --skip-preview-namelist --chksum",
                 "RUN",
-                from_dir="/tests/SEQ_Ln9.f19_g16_rx1.A.ubuntu-latest_gnu.00:00:00",
+                from_dir="/tests/SEQ_Ln9.f19_g16_rx1.A.cori-haswell_gnu.00:00:00",
             )