Repository: ESMCI/cime

Issue:
create_clone is not working across different users
I am trying to clone a case created by a different user - using as an example the command

`./create_clone --case testclone --clone /glade/scratch/klindsay/temp_caseroots/g.e20.GECO.T62_g16.mvertens_test.001`

create_clone is picking up the username from the clone directory and trying to write into that user's directory (which generates an error)

The output is the following:

 ```
Creating Case directory /glade/p/work/mvertens/cesm2_0_beta04.klindsay_spinup/cime/scripts/testclone
 Successfully created new case testclone from clone case g.e20.GECO.T62_g16.mvertens_test.001 
/glade/p/work/mvertens/cesm2_0_beta04.klindsay_spinup/cime/scripts/testclone/env_mach_specific.xml already exists, delete to replace
Checking file env_case
Creating batch script case.run
Writing case.run script from input template /glade/p/work/mvertens/cesm2_0_beta04.klindsay_spinup/cime/cime_config/cesm/machines/template.case.run
Writing case.st_archive script from input template /glade/p/work/mvertens/cesm2_0_beta04.klindsay_spinup/cime/cime_config/cesm/machines/template.st_archive
Writing case.lt_archive script from input template /glade/p/work/mvertens/cesm2_0_beta04.klindsay_spinup/cime/cime_config/cesm/machines/template.lt_archive
Locking file env_mach_pes.xml
ERROR: Could not make directory '/glade/scratch/klindsay/testclone/bld/cpl/obj', error: [Errno 13] Permission denied: '/glade/scratch/klindsay/testclone'

```

The problem is that CIME_OUTPUT_ROOT is resolved in case.py in the following statements:

```
        # Resolve the CIME_OUTPUT_ROOT variable, other than this
        # we don't want to resolve variables until we need them
        if output_root is None:
            output_root = self.get_value("CIME_OUTPUT_ROOT")
        self.set_value("CIME_OUTPUT_ROOT", output_root)
```

But this causes create_clone to look at a value of CIME_OUTPUT_ROOT that has the wrong user_name and is causing the above error.  

So I believe what happened is that a change was made to case.py and caused problems to create_clone as a side effect. So I would suggest that create_clone tests be carried out any time changes to case.py are made in the future.  This utility is extensively used by cesm scientists in their workflow. Maybe we can add this as a possible item in the testing summary for PRs.






Discussion:
I think that you are correct.  We should not resolve $USER for this variable.   
What testing should we do to put this in a PR? I'm not sure when this was introduced - but I want to make sure that if we remove it we don't break something else.
@mvertens thanks for that comment. I think all of us too often fix a bug without adding a testing to ensure we don't regress. 

Resolving patch:
diff --git a/utils/python/CIME/case.py b/utils/python/CIME/case.py
--- a/utils/python/CIME/case.py
+++ b/utils/python/CIME/case.py
@@ -5,7 +5,7 @@
 through the Case module.
 """
 from copy   import deepcopy
-import glob, os, shutil, traceback, math
+import glob, os, shutil, traceback, math, string
 from CIME.XML.standard_module_setup import *
 
 from CIME.utils                     import expect, get_cime_root, append_status
@@ -628,12 +628,12 @@ def configure(self, compset_name, grid_name, machine_name=None,
             for component_class in self._component_classes:
                 if component_class == "DRV":
                     component_class = "CPL"
-                string = "NTASKS_" + component_class
-                pes_ntasks[string] = opti_tasks
-                string = "NTHRDS_" + component_class
-                pes_nthrds[string] = opti_thrds
-                string = "ROOTPE_" + component_class
-                pes_rootpe[string] = 0
+                string_ = "NTASKS_" + component_class
+                pes_ntasks[string_] = opti_tasks
+                string_ = "NTHRDS_" + component_class
+                pes_nthrds[string_] = opti_thrds
+                string_ = "ROOTPE_" + component_class
+                pes_rootpe[string_] = 0
         else:
             pesobj = Pes(self._pesfile)
 
@@ -908,6 +908,21 @@ def create_clone(self, newcase, keepexe=False, mach_dir=None, project=None):
         newcase = self.copy(newcasename, newcaseroot, newsrcroot=srcroot)
         newcase.set_value("CIMEROOT", newcase_cimeroot)
 
+        # if we are cloning to a different user modify the output directory
+        olduser = self.get_value("USER")
+        newuser = os.environ.get("USER")
+        if olduser != newuser:
+            outputroot = self.get_value("CIME_OUTPUT_ROOT")
+            outputroot = string.replace(outputroot, olduser, newuser)
+            # try to make the new output directory and raise an exception
+            # on any error other than directory already exists.
+            try:
+                os.makedirs(outputroot)
+            except OSError:
+                if not os.path.isdir(outputroot):
+                    raise
+            newcase.set_value("CIME_OUTPUT_ROOT", outputroot)
+            newcase.set_value("USER", newuser)
         # determine if will use clone executable or not
         if keepexe:
             orig_exeroot = self.get_value("EXEROOT")


Test patch:
diff --git a/utils/python/tests/scripts_regression_tests.py b/utils/python/tests/scripts_regression_tests.py
--- a/utils/python/tests/scripts_regression_tests.py
+++ b/utils/python/tests/scripts_regression_tests.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 import io, glob, os, re, shutil, signal, sys, tempfile, \
-    threading, time, logging, unittest, getpass
+    threading, time, logging, unittest, getpass, string
 
 from xml.etree.ElementTree import ParseError
 
@@ -170,44 +170,78 @@ def setup_proxy():
 ###############################################################################
 class J_TestCreateNewcase(unittest.TestCase):
 ###############################################################################
-    def setUp(self):
-        self._testroot = MACHINE.get_value("CIME_OUTPUT_ROOT")
+    @classmethod
+    def setUpClass(self):
         self._testdirs = []
         self._do_teardown = []
+        self._testroot = os.path.join(MACHINE.get_value("CIME_OUTPUT_ROOT"),
+                                      'TestCreateNewcase.%s'% CIME.utils.get_timestamp())
 
-    def test_createnewcase(self):
-        testdir = os.path.join(self._testroot, 'scripts_regression_tests.testcreatenewcase.%s'% CIME.utils.get_timestamp())
+    def test_a_createnewcase(self):
+        testdir = os.path.join(self._testroot, 'testcreatenewcase')
         if os.path.exists(testdir):
             shutil.rmtree(testdir)
 
         self._testdirs.append(testdir)
 
-        run_cmd_assert_result(self, "%s/create_newcase --case %s --compset X --res f19_g16" % (SCRIPT_DIR, testdir), from_dir=SCRIPT_DIR)
+        run_cmd_assert_result(self, "%s/create_newcase --case %s --compset X --res f19_g16 --output-root %s" % (SCRIPT_DIR, testdir, self._testroot), from_dir=SCRIPT_DIR)
         run_cmd_assert_result(self, "./case.setup", from_dir=testdir)
         run_cmd_assert_result(self, "./case.build", from_dir=testdir)
 
         self._do_teardown.append(testdir)
 
-    def test_user_mods(self):
-        testdir = os.path.join(self._testroot, 'scripts_regression_tests.testusermods.%s'% CIME.utils.get_timestamp())
+    def test_b_user_mods(self):
+        testdir = os.path.join(self._testroot, 'testusermods')
         if os.path.exists(testdir):
             shutil.rmtree(testdir)
 
         self._testdirs.append(testdir)
 
         user_mods_dir = os.path.join(CIME.utils.get_python_libs_root(), "tests", "user_mods_test1")
-        run_cmd_assert_result(self, "%s/create_newcase --case %s --compset X --res f19_g16 --user-mods-dir %s" % (SCRIPT_DIR, testdir, user_mods_dir),
-                              from_dir=SCRIPT_DIR)
+        run_cmd_assert_result(self, "%s/create_newcase --case %s --compset X --res f19_g16 --user-mods-dir %s --output-root %s"
+                              % (SCRIPT_DIR, testdir, user_mods_dir, self._testroot),from_dir=SCRIPT_DIR)
 
         self.assertTrue(os.path.isfile(os.path.join(testdir,"SourceMods","src.drv","somefile.F90")), msg="User_mods SourceMod missing")
         with open(os.path.join(testdir,"user_nl_cpl"),"r") as fd:
             contents = fd.read()
             self.assertTrue("a different cpl test option" in contents, msg="User_mods contents of user_nl_cpl missing")
             self.assertTrue("a cpl namelist option" in contents, msg="User_mods contents of user_nl_cpl missing")
+        self._do_teardown.append(testdir)
+
+    def test_c_create_clone_keepexe(self):
+        testdir = os.path.join(self._testroot, 'test_create_clone_keepexe')
+        if os.path.exists(testdir):
+            shutil.rmtree(testdir)
+        prevtestdir = self._testdirs[0]
+        self._testdirs.append(testdir)
+
+        run_cmd_assert_result(self, "%s/create_clone --clone %s --case %s --keepexe" %
+                              (SCRIPT_DIR, prevtestdir, testdir),from_dir=SCRIPT_DIR)
 
         self._do_teardown.append(testdir)
 
-    def tearDown(self):
+    def test_d_create_clone_new_user(self):
+        testdir = os.path.join(self._testroot, 'test_create_clone_new_user')
+        if os.path.exists(testdir):
+            shutil.rmtree(testdir)
+        prevtestdir = self._testdirs[0]
+        self._testdirs.append(testdir)
+        # change the USER and CIME_OUTPUT_ROOT to nonsense values
+        # this is intended as a test of whether create_clone is independent of user
+        run_cmd_assert_result(self, "./xmlchange USER=this_is_not_a_user",
+                              from_dir=prevtestdir)
+        fakeoutputroot = string.replace(self._testroot, os.environ.get("USER"), "this_is_not_a_user")
+        run_cmd_assert_result(self, "./xmlchange CIME_OUTPUT_ROOT=%s"%fakeoutputroot,
+                              from_dir=prevtestdir)
+
+        run_cmd_assert_result(self, "%s/create_clone --clone %s --case %s" %
+                              (SCRIPT_DIR, prevtestdir, testdir),from_dir=SCRIPT_DIR)
+
+        self._do_teardown.append(testdir)
+        self._do_teardown.append(self._testroot)
+
+    @classmethod
+    def tearDownClass(self):
         do_teardown = len(self._do_teardown) > 0 and sys.exc_info() == (None, None, None)
 
         for tfile in self._testdirs: