Repository: ESMCI/cime

Issue:
Need to create software_environment.txt after cime.setup
The file "software_environment.txt" lists all the modules loaded when building a CIME model.

The file was created by a command in the old cime2 env_mach_specifc.machine file such as:
"module list >& software_environment.txt"

Example on edison from ACME's CIME2 and an X-case.

edison09 scripts/tryXtiming> more software_environment.txt 
Currently Loaded Modulefiles:
  1) modules/3.2.10.4
  2) nsg/1.2.0
  3) eswrap/1.1.0-1.020200.1130.0
  4) switch/1.0-1.0502.57058.1.58.ari
  5) cray-shmem/7.4.1
  6) slurm/edison
  7) altd/2.0
  8) darshan/2.3.1
  9) craype-network-aries
 10) craype-ivybridge
 11) craype/2.5.0
 12) intel/15.0.1.133
 13) cray-libsci/16.07.1
 14) udreg/2.3.2-1.0502.9889.2.20.ari
 15) ugni/6.0-1.0502.10245.9.9.ari
 16) pmi/5.0.10-1.0000.11050.0.0.ari
 17) dmapp/7.0.1-1.0502.10246.8.47.ari
 18) gni-headers/4.0-1.0502.10317.9.2.ari
 19) xpmem/0.1-2.0502.57015.1.15.ari
 20) dvs/2.5_0.9.0-1.0502.1958.2.55.ari
 21) alps/5.2.3-2.0502.9295.14.14.ari
 22) rca/1.0.0-2.0502.57212.2.56.ari
 23) atp/2.0.2
 24) PrgEnv-intel/5.2.56
 25) cray-mpich/7.3.0
 26) papi/5.4.1.3
 27) cray-hdf5-parallel/1.8.14
 28) cray-netcdf-hdf5parallel/4.3.3.1
 29) cray-parallel-netcdf/1.6.1
 30) cmake/2.8.11.2
 31) cray-petsc/3.5.3.0



Discussion:
Note that at the ALCF we use

printenv >& software_environment.txt

At NERSC @ndkeen has found that capturing both the module list and printenv is useful. 

Yeah, i'll make it so it always does both.

@worleyph remind me:  is that file made all the time or only when SAVE_TIMING is true?

@rljacob, the generation of software_environment.txt is platform specific, via env_mach_specific.XXX. For the platforms where it is generated at all, it is always generated. When SAVE_TIMING is true, it is then copied to the performance_archive (by the cesm_postbuild script). Otherwise it only exists in the case directory, and is overwritten every time env_mach_specific is executed. The copy put into the performance archive is the one generated during the build, not the job submission or the run.

So from greping ACME's env_mach_specific files, we only need to generate this file for cori, edison, cetus, mira and titan.

Shouldn't we just do it always?

I just want to quickly add that I'd like to see software_environment.txt include not only the currently loaded modules, but also the output of `env`.  I do this now in env_mach_specific.corip1

Yes, i'm setting it up to always do both.


Resolving patch:
diff --git a/utils/python/CIME/XML/env_mach_specific.py b/utils/python/CIME/XML/env_mach_specific.py
--- a/utils/python/CIME/XML/env_mach_specific.py
+++ b/utils/python/CIME/XML/env_mach_specific.py
@@ -19,8 +19,6 @@ def __init__(self, caseroot, infile="env_mach_specific.xml"):
         """
         fullpath = infile if os.path.isabs(infile) else os.path.join(caseroot, infile)
         EnvBase.__init__(self, caseroot, fullpath)
-        self._cshscript = []
-        self._shscript = []
 
     def get_values(self, item, attribute=None, resolved=True, subgroup=None):
         """Returns the value as a string of the first xml element with item as attribute value.
@@ -59,27 +57,32 @@ def get_values(self, item, attribute=None, resolved=True, subgroup=None):
 
         return results
 
-    def load_env_for_case(self, compiler, debug, mpilib):
+    def _get_env_for_case(self, compiler, debug, mpilib):
         module_nodes = self.get_nodes("modules")
         env_nodes    = self.get_nodes("environment_variables")
 
-        if (module_nodes is not None):
+        modules_to_load = None
+        if module_nodes is not None:
             modules_to_load = self._compute_module_actions(module_nodes, compiler, debug, mpilib)
-            self.load_modules(modules_to_load)
-        if (env_nodes is not None):
+
+        envs_to_set = None
+        if env_nodes is not None:
             envs_to_set = self._compute_env_actions(env_nodes, compiler, debug, mpilib)
+
+        return modules_to_load, envs_to_set
+
+    def load_env_for_case(self, compiler, debug, mpilib):
+        modules_to_load, envs_to_set = self._get_env_for_case(compiler, debug, mpilib)
+
+        if (modules_to_load is not None):
+            self.load_modules(modules_to_load)
+        if (envs_to_set is not None):
             self.load_envs(envs_to_set)
-        with open(".env_mach_specific.csh",'w') as f:
-            f.write("\n".join(self._cshscript))
-        with open(".env_mach_specific.sh",'w') as f:
-            f.write("\n".join(self._shscript))
 
     def load_modules(self, modules_to_load):
         module_system = self.get_module_system_type()
         if (module_system == "module"):
             self._load_module_modules(modules_to_load)
-            self._cshscript += self._get_module_module_commands(modules_to_load,"csh")
-            self._shscript += self._get_module_module_commands(modules_to_load,"sh")
         elif (module_system == "soft"):
             self._load_soft_modules(modules_to_load)
         elif (module_system == "dotkit"):
@@ -89,12 +92,44 @@ def load_modules(self, modules_to_load):
         else:
             expect(False, "Unhandled module system '%s'" % module_system)
 
+    def list_modules(self):
+        module_system = self.get_module_system_type()
+        if (module_system == "module"):
+            return run_cmd_no_fail("module list 2>&1")
+        elif (module_system == "soft"):
+            return run_cmd_no_fail("softenv")
+        elif (module_system == "dotkit"):
+            return run_cmd_no_fail("use -lv")
+        elif (module_system == "none"):
+            return ""
+        else:
+            expect(False, "Unhandled module system '%s'" % module_system)
+
+    def make_env_mach_specific_file(self, compiler, debug, mpilib, shell):
+        modules_to_load, envs_to_set = self._get_env_for_case(compiler, debug, mpilib)
+
+        filename = ".env_mach_specific.%s" % shell
+        lines = []
+        if modules_to_load is not None:
+            lines.extend(self._get_module_commands(modules_to_load, shell))
+
+        if envs_to_set is not None:
+            for env_name, env_value in envs_to_set:
+                # Let bash do the work on evaluating and resolving env_value
+                if shell == "sh":
+                    lines.append("export %s=%s" % (env_name, env_value))
+                elif shell == "csh":
+                    lines.append("setenv %s %s" % (env_name, env_value))
+                else:
+                    expect(False, "Unknown shell type: '%s'" % shell)
+
+        with open(filename, "w") as fd:
+            fd.write("\n".join(lines))
+
     def load_envs(self, envs_to_set):
         for env_name, env_value in envs_to_set:
             # Let bash do the work on evaluating and resolving env_value
             os.environ[env_name] = run_cmd_no_fail("echo %s" % env_value)
-            self._cshscript.append("setenv %s %s"%(env_name,env_value))
-            self._shscript.append("export %s=%s"%(env_name,env_value))
 
     # Private API
 
@@ -135,7 +170,8 @@ def _match(self, my_value, xml_value):
         else:
             return my_value == xml_value
 
-    def _get_module_module_commands(self, modules_to_load, shell):
+    def _get_module_commands(self, modules_to_load, shell):
+        # Note this is independent of module system type
         mod_cmd = self.get_module_system_cmd_path(shell)
         cmds = []
         for action, argument in modules_to_load:
@@ -145,7 +181,7 @@ def _get_module_module_commands(self, modules_to_load, shell):
         return cmds
 
     def _load_module_modules(self, modules_to_load):
-        for cmd in self._get_module_module_commands(modules_to_load, "python"):
+        for cmd in self._get_module_commands(modules_to_load, "python"):
             py_module_code = run_cmd_no_fail(cmd)
             exec(py_module_code)
 
diff --git a/utils/python/CIME/case_setup.py b/utils/python/CIME/case_setup.py
--- a/utils/python/CIME/case_setup.py
+++ b/utils/python/CIME/case_setup.py
@@ -141,19 +141,20 @@ def case_setup(case, clean=False, test_mode=False, reset=False):
         models = drv_comp.get_valid_model_components()
         models.remove("DRV")
 
-        mach = case.get_value("MACH")
+        mach, compiler, debug, mpilib = \
+            case.get_value("MACH"), case.get_value("COMPILER"), case.get_value("DEBUG"), case.get_value("MPILIB")
         expect(mach is not None, "xml variable MACH is not set")
 
         # Create Macros file only if it does not exist
         if not os.path.exists("Macros"):
             logger.debug("Creating Macros file for %s" % mach)
-            compilers = Compilers(compiler=case.get_value("COMPILER"), machine=mach, os_=case.get_value("OS"), mpilib=case.get_value("MPILIB"))
+            compilers = Compilers(compiler=compiler, machine=mach, os_=case.get_value("OS"), mpilib=mpilib)
             compilers.write_macros_file()
         else:
             logger.debug("Macros script already created ...skipping")
 
         # Set tasks to 1 if mpi-serial library
-        if case.get_value("MPILIB") == "mpi-serial":
+        if mpilib == "mpi-serial":
             for vid, value in case:
                 if vid.startswith("NTASKS_") and value != 1:
                     case.set_value(vid, 1)
@@ -172,7 +173,7 @@ def case_setup(case, clean=False, test_mode=False, reset=False):
                 else:
                     expect(False, "NINST_%s value %d greater than NTASKS_%s %d" % (comp, ninst[comp_model], comp, ntasks))
 
-        expect(not (case.get_value("BUILD_THREADED") and case.get_value("COMPILER") == "nag"),
+        expect(not (case.get_value("BUILD_THREADED") and compiler == "nag"),
                "it is not possible to run with OpenMP if using the NAG Fortran compiler")
 
         if os.path.exists("case.run"):
@@ -198,7 +199,7 @@ def case_setup(case, clean=False, test_mode=False, reset=False):
             pcost = 3 - pcnt / 10 # (3 is 64 with 6)
 
             # Compute cost based on DEBUG
-            dcost = 3 if case.get_value("DEBUG") else 0
+            dcost = 3 if debug else 0
 
             # Compute cost based on run length
             # For simplicity, we use a heuristic just based on STOP_OPTION (not considering
@@ -279,3 +280,11 @@ def case_setup(case, clean=False, test_mode=False, reset=False):
         msg = "case.setup complete"
         append_status(msg, caseroot=caseroot, sfile="CaseStatus")
 
+        # Record env information
+        env_module = case.get_env("mach_specific")
+        env_module.make_env_mach_specific_file(compiler, debug, mpilib, "sh")
+        env_module.make_env_mach_specific_file(compiler, debug, mpilib, "csh")
+        with open("software_environment.txt", "w") as f:
+            f.write(env_module.list_modules())
+        run_cmd_no_fail("echo -e '\n' >> software_environment.txt && \
+                         env >> software_environment.txt")


Test patch: