Repository: ESMCI/cime

Issue:
Add a data assimilation test
Currently the data assimilation capability of case.run is not tested. 



Discussion:

Resolving patch:
diff --git a/utils/python/CIME/SystemTests/dae.py b/utils/python/CIME/SystemTests/dae.py
new file mode 100644
--- /dev/null
+++ b/utils/python/CIME/SystemTests/dae.py
@@ -0,0 +1,107 @@
+"""
+Implementation of the CIME data assimilation test: Tests simple DA script
+which does not change the CAM input. Compares answers to non-DA run.
+
+"""
+
+import os.path
+import logging
+import glob
+
+import CIME.XML.standard_module_setup as sms
+from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo
+from CIME.utils import expect
+from CIME.case_setup import case_setup
+
+###############################################################################
+class DAE(SystemTestsCompareTwo):
+###############################################################################
+    """
+    Implementation of the CIME data assimilation test: Tests simple DA script
+    which does not change the CAM input. Compares answers to non-DA run.
+    Refers to a faux data assimilation script in the
+    cime/utils/data_assimilation directory
+    """
+
+    ###########################################################################
+    def __init__(self, case):
+    ###########################################################################
+        SystemTestsCompareTwo.__init__(self, case,
+                                       separate_builds=False,
+                                       run_two_suffix='da',
+                                       run_one_description='no data assimilation',
+                                       run_two_description='data assimilation')
+
+    ###########################################################################
+    def _case_one_setup(self):
+    ###########################################################################
+        case_setup(self._case, test_mode=True, reset=True)
+
+    ###########################################################################
+    def _case_two_setup(self):
+    ###########################################################################
+        # Set up data assimilation in config_tests.xml once that's ready
+        # We need to find the utils/data_assimilation directory
+        # LIB_DIR should be our parent dir
+        da_dir = os.path.join(os.path.dirname(sms.LIB_DIR), "data_assimilation")
+        expect(os.path.isdir(da_dir), "ERROR: da_dir, '%s', does not exist"%da_dir)
+        da_file = os.path.join(da_dir, "da_no_data_mod.sh")
+        expect(os.path.isfile(da_file), "ERROR: da_file, '%s', does not exist"%da_file)
+
+        # Set up two data assimilation cycles each half of the full run
+        self._case.set_value("DATA_ASSIMILATION", "TRUE")
+        self._case.set_value("DATA_ASSIMILATION_SCRIPT", da_file)
+        self._case.set_value("DATA_ASSIMILATION_CYCLES", 2)
+        stopn = self._case.get_value("STOP_N")
+        expect((stopn % 2) == 0, "ERROR: DAE test requires that STOP_N be even")
+        stopn = stopn / 2
+        self._case.set_value("STOP_N", stopn)
+
+        self._case.flush()
+
+    ###########################################################################
+    def run_phase(self):
+    ###########################################################################
+        SystemTestsCompareTwo.run_phase(self)
+        # Do some checks on the data assimilation 'output' from case2
+        self._activate_case2()
+        case_root = self._get_caseroot2()
+        da_files = glob.glob(os.path.join(case_root, 'da.log.*'))
+        if da_files is None:
+            logger = logging.getLogger(__name__)
+            logger.warning("No DA files in %s", os.path.join(case_root, 'da.log.*'))
+
+        da_cycles = self._case.get_value("DATA_ASSIMILATION_CYCLES")
+        expect((da_files is not None) and (len(da_files) == da_cycles),
+               "ERROR: There were %d DA cycles in run but %d DA files were found"%
+               (da_cycles, len(da_files) if da_files is not None else 0))
+        da_files.sort()
+        cycle_num = 0
+        for fname in da_files:
+            found_caseroot = False
+            found_cycle = False
+            found_total = False
+            with open(fname) as dfile:
+                for line in dfile:
+                    expect(line[0:5] != 'ERROR', "ERROR, error line found in %s"%fname)
+                    if line[0:8] == 'caseroot':
+                        found_caseroot = True
+                    elif line[0:5] == 'cycle':
+                        found_cycle = True
+                        expect(int(line[7:]) == cycle_num,
+                               "ERROR: Wrong cycle (%d) found in %s (expected %d)"%
+                               (int(line[7:]), fname, cycle_num))
+                    elif line[0:12] == 'total_cycles':
+                        found_total = True
+                        expect(int(line[14:]) == da_cycles,
+                               "ERROR: Wrong total cycle count (%d) found in %s (expected %d)"%
+                               (int(line[14:]), fname, da_cycles))
+                    else:
+                        expect(False, "ERROR: Unrecognized line ('%s') found in %s"%(line, fname))
+
+                # End of for loop
+                expect(found_caseroot, "ERROR: No caseroot found in %s"%fname)
+                expect(found_cycle, "ERROR: No cycle found in %s"%fname)
+                expect(found_total, "ERROR: No total_cycles found in %s"%fname)
+            # End of with
+            cycle_num = cycle_num + 1
diff --git a/utils/python/CIME/case_run.py b/utils/python/CIME/case_run.py
--- a/utils/python/CIME/case_run.py
+++ b/utils/python/CIME/case_run.py
@@ -29,8 +29,8 @@ def pre_run_check(case):
     logger.debug("check_lockedfiles OK")
 
     # check that build is done
-    expect (build_complete,
-            "BUILD_COMPLETE is not true\nPlease rebuild the model interactively")
+    expect(build_complete,
+           "BUILD_COMPLETE is not true\nPlease rebuild the model interactively")
     logger.debug("build complete is %s " %build_complete)
 
     # load the module environment...
@@ -59,23 +59,23 @@ def pre_run_check(case):
     if not os.path.isdir(rundir):
         os.mkdir(rundir)
 
-    if os.path.isdir(os.path.join(rundir,"timing")):
-        shutil.rmtree(os.path.join(rundir,"timing"))
+    if os.path.isdir(os.path.join(rundir, "timing")):
+        shutil.rmtree(os.path.join(rundir, "timing"))
 
-    os.makedirs(os.path.join(rundir,"timing","checkpoints"))
+    os.makedirs(os.path.join(rundir, "timing", "checkpoints"))
 
     # run preview namelists
     preview_namelists(case)
 
     # document process
-    append_status("Run started ",caseroot=caseroot,
-                 sfile="CaseStatus")
+    append_status("Run started ", caseroot=caseroot,
+                  sfile="CaseStatus")
 
-    logger.info( "-------------------------------------------------------------------------")
-    logger.info( " - To prestage required restarts, untar a restart.tar file into %s" %(rundir))
-    logger.info( " - Case input data directory (DIN_LOC_ROOT) is %s " %(din_loc_root))
-    logger.info( " - Checking for required input datasets in DIN_LOC_ROOT")
-    logger.info( "-------------------------------------------------------------------------")
+    logger.info("-------------------------------------------------------------------------")
+    logger.info(" - To prestage required restarts, untar a restart.tar file into %s" %(rundir))
+    logger.info(" - Case input data directory (DIN_LOC_ROOT) is %s " %(din_loc_root))
+    logger.info(" - Checking for required input datasets in DIN_LOC_ROOT")
+    logger.info("-------------------------------------------------------------------------")
 
 ###############################################################################
 def run_model(case):
@@ -96,7 +96,7 @@ def run_model(case):
     logger.info("run command is %s " %cmd)
     rundir = case.get_value("RUNDIR")
     run_cmd_no_fail(cmd, from_dir=rundir)
-    logger.info( "%s MODEL EXECUTION HAS FINISHED" %(time.strftime("%Y-%m-%d %H:%M:%S")))
+    logger.info("%s MODEL EXECUTION HAS FINISHED" %(time.strftime("%Y-%m-%d %H:%M:%S")))
 
 ###############################################################################
 def post_run_check(case, lid):
@@ -107,8 +107,8 @@ def post_run_check(case, lid):
     model = case.get_value("MODEL")
 
     # find the last model.log and cpl.log
-    model_logfile = os.path.join(rundir,model + ".log." + lid)
-    cpl_logfile   = os.path.join(rundir,"cpl" + ".log." + lid)
+    model_logfile = os.path.join(rundir, model + ".log." + lid)
+    cpl_logfile = os.path.join(rundir, "cpl" + ".log." + lid)
 
     if not os.path.isfile(model_logfile):
         msg = "Model did not complete, no %s log file "%model_logfile
@@ -121,16 +121,16 @@ def post_run_check(case, lid):
     elif os.stat(model_logfile).st_size == 0:
         msg = " Run FAILED "
         append_status(msg, caseroot=caseroot, sfile="CaseStatus")
-        expect (False, msg)
+        expect(False, msg)
     else:
         with open(cpl_logfile, 'r') as fd:
             if 'SUCCESSFUL TERMINATION' in fd.read():
                 msg = "Run SUCCESSFUL"
-                append_status(msg, caseroot=caseroot, sfile="CaseStatus" )
+                append_status(msg, caseroot=caseroot, sfile="CaseStatus")
             else:
                 msg = "Model did not complete - see %s \n " %(cpl_logfile)
                 append_status(msg, caseroot=caseroot, sfile="CaseStatus")
-                expect (False, msg)
+                expect(False, msg)
 
 ###############################################################################
 def save_logs(case, lid):
@@ -142,7 +142,7 @@ def save_logs(case, lid):
 
         caseroot = case.get_value("CASEROOT")
         rundir = case.get_value("RUNDIR")
-        logfiles = glob.glob(os.path.join(rundir,"*.log.%s"%(lid)))
+        logfiles = glob.glob(os.path.join(rundir, "*.log.%s"%(lid)))
         for logfile in logfiles:
             if os.path.isfile(logfile):
                 logfile_gz = gzip_existing_file(logfile)
@@ -171,7 +171,7 @@ def resubmit_check(case):
     elif dout_s and mach == 'mira':
         caseroot = case.get_value("CASEROOT")
         cimeroot = case.get_value("CIMEROOT")
-        cmd = "ssh cooleylogin1 'cd %s; CIMEROOT=%s ./case.submit %s --job case.st_archive' "%(caseroot, cimeroot, caseroot)
+        cmd = "ssh cooleylogin1 'cd %s; CIMEROOT=%s ./case.submit %s --job case.st_archive'"%(caseroot, cimeroot, caseroot)
         run_cmd(cmd, verbose=True)
 
     if resubmit:
@@ -182,35 +182,46 @@ def resubmit_check(case):
         submit(case, job=job, resubmit=True)
 
 ###############################################################################
-def do_data_assimilation(da_script, lid):
+def do_data_assimilation(da_script, cycle, data_assimilation_cycles, lid):
 ###############################################################################
-    cmd = da_script + "1> da.log.%s 2>&1" %(lid)
+    cmd = da_script + " 1> da.log.%s %d %d 2>&1" %(lid, cycle, data_assimilation_cycles)
     logger.debug("running %s" %da_script)
     run_cmd_no_fail(cmd)
     # disposeLog(case, 'da', lid)  THIS IS UNDEFINED!
 
+###############################################################################
+def new_lid():
+###############################################################################
+    lid = time.strftime("%y%m%d-%H%M%S")
+    os.environ["LID"] = lid
+    return lid
+
 ###############################################################################
 def case_run(case):
 ###############################################################################
     # Set up the run, run the model, do the postrun steps
     run_with_submit = case.get_value("RUN_WITH_SUBMIT")
-    expect (run_with_submit,
-            "You are not calling the run script via the submit script. "
-            "As a result, short-term archiving will not be called automatically."
-            "Please submit your run using the submit script like so:"
-            " ./case.submit")
+    expect(run_with_submit,
+           "You are not calling the run script via the submit script. "
+           "As a result, short-term archiving will not be called automatically."
+           "Please submit your run using the submit script like so:"
+           " ./case.submit")
 
     data_assimilation = case.get_value("DATA_ASSIMILATION")
     data_assimilation_cycles = case.get_value("DATA_ASSIMILATION_CYCLES")
     data_assimilation_script = case.get_value("DATA_ASSIMILATION_SCRIPT")
 
     # set up the LID
-    lid = time.strftime("%y%m%d-%H%M%S")
-    os.environ["LID"] = lid
+    lid = new_lid()
 
     save_prerun_provenance(case)
 
-    for _ in range(data_assimilation_cycles):
+    for cycle in range(data_assimilation_cycles):
+        # After the first DA cycle, runs are restart runs
+        if cycle > 0:
+            case.set_value("CONTINUE_RUN", "TRUE")
+            lid = new_lid()
+
         pre_run_check(case)
         run_model(case)
         post_run_check(case, lid)
@@ -219,7 +230,7 @@ def case_run(case):
             get_timing(case, lid)     # Run the getTiming script
 
         if data_assimilation:
-            do_data_assimilation(data_assimilation_script, lid)
+            do_data_assimilation(data_assimilation_script, cycle, data_assimilation_cycles, lid)
 
         save_postrun_provenance(case)
 


Test patch:
diff --git a/cime_config/config_tests.xml b/cime_config/config_tests.xml
--- a/cime_config/config_tests.xml
+++ b/cime_config/config_tests.xml
@@ -3,16 +3,17 @@
 <!--
 
 The following are the test functionality categories:
-  1) smoke tests
-  2) basic reproducibility tests
-  3) restart tests
-  4) threading/pe-count modification tests
-  5) sequencing (layout) modification tests
-  6) multi-instance tests
-  7) archiving (short-term and long-term) tests
-  8) performance tests
-  9) spinup tests
-  10) other component-specific tests
+   1) smoke tests
+   2) basic reproducibility tests
+   3) restart tests
+   4) threading/pe-count modification tests
+   5) sequencing (layout) modification tests
+   6) multi-instance tests
+   7) archiving (short-term and long-term) tests
+   8) performance tests
+   9) spinup tests
+  10) data assimilation tests
+  11) other component-specific tests
 
 NOTES:
 - unless otherwise noted everything is run in one executable directory
@@ -151,6 +152,12 @@ SPO    smoke spinup-ocean test
 
 LAR    long term archive test
 
+======================================================================
+    Data Assimilation Tests
+======================================================================
+
+DAE    data assimilation test: non answer changing
+
 ======================================================================
     Other component-specific tests
 ======================================================================
@@ -162,6 +169,18 @@ LII    CLM initial condition interpolation test
 
 <config_test>
 
+  <test NAME="DAE">
+    <DESC>data assimilation test, default 1 day, two DA cycles, no data modification</DESC>
+    <INFO_DBUG>1</INFO_DBUG>
+    <STOP_OPTION>ndays</STOP_OPTION>
+    <STOP_N>4</STOP_N>
+    <HIST_OPTION>ndays</HIST_OPTION>
+    <HIST_N>1</HIST_N>
+    <CCSM_TCOST>0</CCSM_TCOST>
+    <DOUT_S>FALSE</DOUT_S>
+    <BUILD_THREADED>TRUE</BUILD_THREADED>
+  </test>
+
   <test NAME="ERI">
     <DESC>hybrid/branch/exact restart test, default 3+19/10+9/5+4 days</DESC>
     <INFO_DBUG>1</INFO_DBUG>