Repository: ESMCI/cime

Issue:
Jgfouca/branch for acme split 2018 10 16
Summary

- Add two new tests (TSC and PGN) for testing non-BFB changes to the model. These tests can be launched using create_test script. PGN has the following python dependencies:
numpy
scipy
netCDF4
math
sklearn.metrics

These python libraries should be present on the system and be available on compute nodes as well. TSC test currently does not have any post-processing built in.

- PIO FindNetcdf.cmake:  Currently, FindNetCDF.cmake only uses ${NetCDF_${NCDFcomp}PATHS} if ${MPI${NCDFcomp}_FOUND} is true. There's no need for that. We should always allow the user to pass hints to find_package for where to find the libraries (and some users may have those libraries installed in different folders).

- Fix test status:  Fix test_status overall status for certain state layout.  The internal test status log showed this layout caused an incorrect FAIL to be reported to the dashboard. Fix that bug and add a unit test for this layout.

Test suite: scripts_regression_tests
Test baseline: 
Test namelist changes: 
Test status: bit for bit

Fixes [CIME Github issue #]

User interface changes?: 

Update gh-pages html (Y/N)?:

Code review:  @jedwards4b 



Discussion:
There are some pylint issues to be fixed before merge.

Resolving patch:
diff --git a/scripts/lib/CIME/provenance.py b/scripts/lib/CIME/provenance.py
--- a/scripts/lib/CIME/provenance.py
+++ b/scripts/lib/CIME/provenance.py
@@ -17,9 +17,9 @@ def _get_batch_job_id_for_syslog(case):
     """
     mach = case.get_value("MACH")
     try:
-        if mach in ['anvil', 'titan']:
+        if mach in ['titan']:
             return os.environ["PBS_JOBID"]
-        elif mach in ['edison', 'cori-haswell', 'cori-knl']:
+        elif mach in ['anvil', 'edison', 'cori-haswell', 'cori-knl']:
             return os.environ["SLURM_JOB_ID"]
         elif mach in ['mira', 'theta']:
             return os.environ["COBALT_JOBID"]
@@ -186,17 +186,14 @@ def _save_prerun_timing_e3sm(case, lid):
                 full_cmd = cmd + " " + filename
                 run_cmd_no_fail(full_cmd + "." + lid, from_dir=full_timing_dir)
                 gzip_existing_file(os.path.join(full_timing_dir, filename + "." + lid))
-
-            # mdiag_reduce = os.path.join(full_timing_dir, "mdiag_reduce." + lid)
-            # run_cmd_no_fail("./mdiag_reduce.csh", arg_stdout=mdiag_reduce, from_dir=os.path.join(caseroot, "Tools"))
-            # gzip_existing_file(mdiag_reduce)
         elif mach == "anvil":
-            for cmd, filename in [("qstat -f -1 acme >", "qstatf"),
-                                  ("qstat -f %s >" % job_id, "qstatf_jobid"),
-                                  ("qstat -r acme >", "qstatr")]:
-                full_cmd = cmd + " " + filename
-                run_cmd_no_fail(full_cmd + "." + lid, from_dir=full_timing_dir)
-                gzip_existing_file(os.path.join(full_timing_dir, filename + "." + lid))
+            for cmd, filename in [("sinfo -l", "sinfol"), 
+                                  ("squeue -o '%all' --job {}".format(job_id), "squeueall_jobid"),
+                                  ("squeue -o '%.10i %.10P %.15u %.20a %.2t %.6D %.8C %.12M %.12l %.20S %.20V %j'", "squeuef"),
+                                  ("squeue -t R -o '%.10i %R'", "squeues")]:
+                filename = "%s.%s" % (filename, lid)
+                run_cmd_no_fail(cmd, arg_stdout=filename, from_dir=full_timing_dir)
+                gzip_existing_file(os.path.join(full_timing_dir, filename))
         elif mach == "summit":
             for cmd, filename in [("bjobs -u all >", "bjobsu_all"),
                                   ("bjobs -r -u all -o 'jobid slots exec_host' >", "bjobsru_allo"),
@@ -362,7 +359,7 @@ def _save_postrun_timing_e3sm(case, lid):
         if mach == "titan":
             globs_to_copy.append("%s*OU" % job_id)
         elif mach == "anvil":
-            globs_to_copy.append("/home/%s/%s*OU" % (getpass.getuser(), job_id))
+            globs_to_copy.append("%s*run*%s" % (case.get_value("CASE"), job_id))
         elif mach in ["mira", "theta"]:
             globs_to_copy.append("%s*error" % job_id)
             globs_to_copy.append("%s*output" % job_id)


Test patch:
diff --git a/config/e3sm/tests.py b/config/e3sm/tests.py
--- a/config/e3sm/tests.py
+++ b/config/e3sm/tests.py
@@ -113,7 +113,7 @@
             "SMS.T62_oQU120_ais20.MPAS_LISIO_TEST",
             "SMS.f09_g16_a.IGCLM45_MLI",
             "SMS_P12x2.ne4_oQU240.A_WCYCL1850.allactive-mach_mods",
-            "SMS_B.ne4_ne4.FC5AV1C-L-AQUAP.cam-hommexx",
+            "SMS_B.ne4_ne4.F-EAMv1-AQP1.cam-hommexx",
             )
         },