Repository: ESMCI/cime

Issue:
problem with st_archive and short runs (< 1 month)
If the short term archiver is run prior to the end of month boundary then it errors out because it can't find the component monthly history file corresponding to the component restart history file. 

The st_archive log errors from a F-WACCM case are:

==> f.e15.FWscAMIP.f19_f19.misc08_cam5_4_81.002.st_archive.570708 <==
out=
err=ncdump: nhfil: No such variable
histfiles_savein_rundir ['f.e15.FWscAMIP.f19_f19.misc08_cam5_4_81.002.cam.h0.1979-01.nc', 'f.e15.FWscAMIP.f19_f19.misc08_cam5_4_81.002.cam.h1.1979-01-01-00000.nc', 'f.e15.FWscAMIP.f19_f19.misc08_cam5_4_81.002.cam.h2.1979-01-01-00000.nc'] 
doing short term archiving for clm (lnd)
removing interim restart file /glade/scratch/mmills/f.e15.FWscAMIP.f19_f19.misc08_cam5_4_81.002/run/f.e15.FWscAMIP.f19_f19.misc08_cam5_4_81.002.clm2.r.1979-01-06-00000.nc
 WARNING: ncdump -v locfnh /glade/scratch/mmills/f.e15.FWscAMIP.f19_f19.misc08_cam5_4_81.002/run/f.e15.FWscAMIP.f19_f19.misc08_cam5_4_81.002.clm2.rh0.1979-01-06-00000.nc  failed rc=1
out=
err=ncdump: locfnh: No such variable
removing interim restart file /glade/scratch/mmills/f.e15.FWscAMIP.f19_f19.misc08_cam5_4_81.002/run/f.e15.FWscAMIP.f19_f19.misc08_cam5_4_81.002.clm2.rh0.1979-01-06-00000.nc
ERROR: restart file /glade/scratch/mmills/f.e15.FWscAMIP.f19_f19.misc08_cam5_4_81.002/run/f.e15.FWscAMIP.f19_f19.misc08_cam5_4_81.002.clm2.h0.1979-01.nc does not exist


Discussion:
Can this be reproduced using CIME only?

I am still having this issue, and it also occurs when the run ends on an end-of-month boundary. I am using my copy of Francis’ Phase 3 sandbox, which I have here:

/glade/u/home/mmills/cesm/phase3_n03_cam5_4_90_compsets/

This is the latest case to fail:

/glade/p/work/mmills/case/f.e20.FSDW.f09_f09.phase3_n03_cam5_4_90.02

See f.e20.FSDW.f09_f09.phase3_n03_cam5_4_90.02.st_archive.259967

ERROR: restart file /glade/scratch/mmills/f.e20.FSDW.f09_f09.phase3_n03_cam5_4_90.02/run/f.e20.FSDW.f09_f09.phase3_n03_cam5_4_90.02.clm2.h0.1990-09.nc does not exist 

I am not seeing any _.clm2.h0._ files in the run directory or the short-term archive:

/glade/scratch/mmills/f.e20.FSDW.f09_f09.phase3_n03_cam5_4_90.02/run

/glade/scratch/mmills/archive/f.e20.FSDW.f09_f09.phase3_n03_cam5_4_90.02

I have located the CLM history output. They are in the short-term archive under atm/hist, instead of under lnd/hist. So are the history files for CICE and MOSART.

See:

/glade/scratch/mmills/archive/f.e15.FWscAMIP.f19_f19.misc08_cam5_4_81.002/atm/hist
/glade/scratch/mmills/archive/f.e20.FSDW.f09_f09.phase3_n03_cam5_4_90.02/atm/hist
/glade/scratch/mmills/archive/f.e20.FWAMIP.f09_f09.phase3_n03_cam5_4_90.03/atm/hist


Resolving patch:
diff --git a/utils/python/CIME/case_st_archive.py b/utils/python/CIME/case_st_archive.py
old mode 100644
new mode 100755
--- a/utils/python/CIME/case_st_archive.py
+++ b/utils/python/CIME/case_st_archive.py
@@ -120,6 +120,7 @@ def _archive_log_files(case):
         srcfile = join(rundir, os.path.basename(logfile))
         destfile = join(archive_logdir, os.path.basename(logfile))
         shutil.move(srcfile, destfile)
+        logger.info("moving \b%s to \b%s" %(srcfile, destfile))
 
 
 ###############################################################################
@@ -161,10 +162,10 @@ def _archive_history_files(case, archive, archive_entry,
                            "history file %s does not exist " %srcfile)
                     destfile = join(archive_histdir, histfile)
                     if histfile in histfiles_savein_rundir:
-                        logger.debug("copying %s to %s " %(srcfile, destfile))
+                        logger.info("copying \n%s to \n%s " %(srcfile, destfile))
                         shutil.copy(srcfile, destfile)
                     else:
-                        logger.debug("moving %s to %s " %(srcfile, destfile))
+                        logger.info("moving \n%s to \n%s " %(srcfile, destfile))
                         shutil.move(srcfile, destfile)
 
 
@@ -180,7 +181,7 @@ def get_histfiles_for_restarts(case, archive, archive_entry, restfile):
         cmd = "ncdump -v %s %s " %(rest_hist_varname, os.path.join(rundir, restfile))
         rc, out, error = run_cmd(cmd)
         if rc != 0:
-            logger.warn(" WARNING: %s failed rc=%d\nout=%s\nerr=%s" %(cmd, rc, out, error))
+            logger.debug(" WARNING: %s failed rc=%d\nout=%s\nerr=%s" %(cmd, rc, out, error))
 
         searchname = "%s =" %rest_hist_varname
         if searchname in out:
@@ -193,7 +194,9 @@ def get_histfiles_for_restarts(case, archive, archive_entry, restfile):
                 if matchobj:
                     histfile = matchobj.group(1).strip()
                     histfile = os.path.basename(histfile)
-                    histfiles.append(histfile)
+                    # append histfile to the list ONLY if it exists in rundir before the archviving
+                    if os.path.isfile(os.path.join(rundir,histfile)): 
+                        histfiles.append(histfile)
     return histfiles
 
 
@@ -261,21 +264,24 @@ def _archive_restarts(case, archive, archive_entry,
                     srcfile = os.path.join(rundir, restfile)
                     destfile = os.path.join(archive_restdir, restfile)
                     shutil.copy(srcfile, destfile)
+                    logger.info("copying \n%s to \n%s" %(srcfile, destfile))
                     for histfile in histfiles_for_restart:
                         srcfile = os.path.join(rundir, histfile)
                         destfile = os.path.join(archive_restdir, histfile)
                         expect(os.path.isfile(srcfile),
                                "restart file %s does not exist " %srcfile)
                         shutil.copy(srcfile, destfile)
+                        logger.info("copying \n%s to \n%s" %(srcfile, destfile))
                 else:
                     # Only archive intermediate restarts if requested - otherwise remove them
                     if case.get_value('DOUT_S_SAVE_INTERIM_RESTART_FILES'):
                         srcfile = os.path.join(rundir, restfile)
                         destfile = os.path.join(archive_restdir, restfile)
-                        logger.debug("moving %s to %s" %(srcfile, destfile))
+                        logger.info("moving \n%s to \n%s" %(srcfile, destfile))
                         expect(os.path.isfile(srcfile),
                                "restart file %s does not exist " %srcfile)
                         shutil.move(srcfile, destfile)
+                        logger.info("moving \n%s to \n%s" %(srcfile, destfile))
 
                         # need to copy the history files needed for interim restarts - since
                         # have not archived all of the history files yet
@@ -285,7 +291,7 @@ def _archive_restarts(case, archive, archive_entry,
                             expect(os.path.isfile(srcfile),
                                    "hist file %s does not exist " %srcfile)
                             shutil.copy(srcfile, destfile)
-                            logger.debug("copying %s to %s" %(srcfile, destfile))
+                            logger.info("copying \n%s to \n%s" %(srcfile, destfile))
                     else:
                         srcfile = os.path.join(rundir, restfile)
                         logger.info("removing interim restart file %s" %srcfile)
@@ -323,7 +329,9 @@ def _archive_process(case, archive):
             continue
 
         # archive restarts and all necessary associated fields (e.g. rpointer files)
+        logger.info('-------------------------------------------')
         logger.info('doing short term archiving for %s (%s)' % (compname, compclass))
+        logger.info('-------------------------------------------')
         datenames = _get_datenames(case)
         for datename in datenames:
             datename_is_last = False


Test patch: