Repository: ESMCI/cime

Issue:
Bug when resetting RUNDIR
From Jim Benedict:

There appears to be a bug when setting RUNDIR within env_run.xml in CESM2.  I first encountered this error in cesm2_0_beta03 (it may have existed in earlier tags), and I also see it in cesm2_0_beta05.  It occurs when I try to create an aquaplanet case, but I also see it when I try to make a simple F compset run with cesm2_0_beta05.  When RUNDIR is not modified, no errors occur.

The screen output below shows the error message when RUNDIR in env_run.xml is user-specified (and after ./preview_namelists is attempted).  The case directory for this run is:
/glade/u/home/benedict/ys/model/cesm/cesm2_0_beta05/cases/F_RUNDIRtest

~~~  Error message  ~~~
120 yslogin3% ./preview_namelists
Creating component namelists
   Calling cam buildnml
Traceback (most recent call last):
  File "./preview_namelists", line 64, in <module>
    _main_func(__doc__)
  File "./preview_namelists", line 61, in _main_func
    create_namelists(case)
  File "/glade/u/home/benedict/ys/model/cesm/cesm2_0_beta05/cime/scripts/Tools/../../utils/python/CIME/preview_namelists.py", line 82, in create_namelists
    mod.buildnml(case, caseroot, compname)
  File "/glade/u/home/benedict/ys/model/cesm/cesm2_0_beta05/components/cam/cime_config/buildnml", line 189, in buildnml
    shutil.copy(file1,file2)
  File "/glade/apps/opt/python/2.7.7/gnu-westmere/4.8.2/lib/python2.7/shutil.py", line 119, in copy
    copyfile(src, dst)
  File "/glade/apps/opt/python/2.7.7/gnu-westmere/4.8.2/lib/python2.7/shutil.py", line 83, in copyfile
    with open(dst, 'wb') as fdst:
IOError: [Errno 2] No such file or directory: '/glade/p/umia0018/F_RUNDIRtest/run/drv_flds_in’

From Jim Edwards:
 Yes, it is a bug - can you open a cime issue?   I think that the workaround is to create the directory after you run the 
xmlchange command but before anything else. 



Discussion:
Can someone provide a command sequence to reproduce this? For instance, is this before `./case.setup` (when I would expect it to work) or after (when I would expect it to fail)?

Resolving patch:
diff --git a/scripts/lib/CIME/build.py b/scripts/lib/CIME/build.py
--- a/scripts/lib/CIME/build.py
+++ b/scripts/lib/CIME/build.py
@@ -4,7 +4,7 @@
 from CIME.XML.standard_module_setup  import *
 from CIME.utils                 import get_model, append_status, analyze_build_log, stringify_bool
 from CIME.provenance            import save_build_provenance
-from CIME.preview_namelists     import create_namelists
+from CIME.preview_namelists     import create_namelists, create_dirs
 from CIME.check_lockedfiles     import check_lockedfiles, lock_file, unlock_file
 import glob, shutil, time, threading, gzip, subprocess
 
@@ -407,6 +407,9 @@ def build_checks(case, build_threaded, comp_interface, use_esmf_lib,
 
     case.set_value("BUILD_COMPLETE", False)
 
+    # User may have rm -rf their build directory
+    create_dirs(case)
+
     case.flush()
     if not model_only:
         logger.info("Generating component namelists as part of build")
diff --git a/scripts/lib/CIME/preview_namelists.py b/scripts/lib/CIME/preview_namelists.py
--- a/scripts/lib/CIME/preview_namelists.py
+++ b/scripts/lib/CIME/preview_namelists.py
@@ -12,13 +12,12 @@ def create_dirs(case):
     Make necessary directories for case
     """
     # Get data from XML
-    exeroot = case.get_value("EXEROOT")
-    libroot = case.get_value("LIBROOT")
-    incroot = case.get_value("INCROOT")
-    rundir = case.get_value("RUNDIR")
+    exeroot  = case.get_value("EXEROOT")
+    libroot  = case.get_value("LIBROOT")
+    incroot  = case.get_value("INCROOT")
+    rundir   = case.get_value("RUNDIR")
     caseroot = case.get_value("CASEROOT")
 
-
     docdir = os.path.join(caseroot, "CaseDocs")
     dirs_to_make = []
     models = case.get_values("COMP_CLASSES")
@@ -47,6 +46,8 @@ def create_namelists(case):
     """
     case.flush()
 
+    create_dirs(case)
+
     casebuild = case.get_value("CASEBUILD")
     caseroot = case.get_value("CASEROOT")
     rundir = case.get_value("RUNDIR")
@@ -76,7 +77,7 @@ def create_namelists(case):
         try:
             with open(cmd, 'r') as f:
                 first_line = f.readline()
-            if "python" in first_line:    
+            if "python" in first_line:
                 logger.info("   Calling %s buildnml"%compname)
                 mod = imp.load_source("buildnml", cmd)
                 mod.buildnml(case, caseroot, compname)
@@ -97,9 +98,9 @@ def create_namelists(case):
             case.flush()
             run_cmd_no_fail("%s %s" % (cmd, caseroot), verbose=False)
             # refresh case xml object from file
-            case.read_xml()            
-    logger.info("Finished creating component namelists")
+            case.read_xml()
 
+    logger.info("Finished creating component namelists")
 
     # Save namelists to docdir
     if (not os.path.isdir(docdir)):
@@ -110,7 +111,6 @@ def create_namelists(case):
         except (OSError, IOError) as e:
             expect(False, "Failed to write %s/README: %s" % (docdir, e))
 
-
     for cpglob in ["*_in_[0-9]*", "*modelio*", "*_in",
                    "*streams*txt*", "*stxt", "*maps.rc", "*cism.config*"]:
         for file_to_copy in glob.glob(os.path.join(rundir, cpglob)):


Test patch:
diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py
--- a/scripts/tests/scripts_regression_tests.py
+++ b/scripts/tests/scripts_regression_tests.py
@@ -201,12 +201,13 @@ def test_b_cime_f90_unit_tests(self):
 
         machine           = MACHINE.get_machine_name()
         compiler          = MACHINE.get_default_compiler()
-        test_dir = os.path.join(cls._testroot,"driver_f90_tests")
-        cls._testdirs.append(test_dir)
-        os.makedirs(test_dir)
         if (machine != "yellowstone" or compiler != "intel"):
             #TODO: get rid of this restriction
             self.skipTest("Skipping TestUnitTest - only supported on yellowstone with intel")
+
+        test_dir = os.path.join(cls._testroot,"driver_f90_tests")
+        cls._testdirs.append(test_dir)
+        os.makedirs(test_dir)
         test_spec_dir = CIME.utils.get_cime_root()
         unit_test_tool = os.path.abspath(os.path.join(test_spec_dir,"tools","unit_testing","run_tests.py"))
 
@@ -226,10 +227,9 @@ def tearDownClass(cls):
                 teardown_root = False
             elif do_teardown:
                 shutil.rmtree(tfile)
-        if teardown_root:
-            shutil.rmtree(cls._testroot)
-
 
+        if teardown_root and do_teardown:
+            shutil.rmtree(cls._testroot)
 
 ###############################################################################
 class J_TestCreateNewcase(unittest.TestCase):