Repository: ESMCI/cime

Issue:
When specifying test root with ./create_test -r test_root, the run/ and bld/ directories should also change
`./create_test cime_developer -r $SCRATCH/acme_scratch/foo -t foo`

Will create `/global/cscratch1/sd/ndk/acme_scratch/foo` with the case directories as expected, however I also get the same case directory names in `/global/cscratch1/sd/ndk/acme_scratch` which contain only bld/ and run/ dirs.

In xml, we have:

``` <RUNDIR>$CESMSCRATCHROOT/$CASE/run</RUNDIR>
  <EXEROOT>$CESMSCRATCHROOT/$CASE/bld</EXEROOT>
```

Could we add $TESTROOT?  That is:
`<RUNDIR>$CESMSCRATCHROOT/$TESTROOT/$CASE/run</RUNDIR>`
Where it would normally be empty unless specified.

@jgfouca edit: Add a flag[s] to create_test to allow user to change rundir and exeroot potentially with a short cut option to have them go under the case directory.



Discussion:
This is a feature not a bug.   To change the location of build and run directories you should change $CESMSCRATCHROOT.   

OK, sounds like the suggested way to have multiple test case dirs is to change CESMSCRATCHROOT.

a) Why is this not an option to create_test?
b) What in the world, then, is the -r flag for?

I'd prefer to see acme_scratch, and then below that have named directories, each one containing the case dirs of tests I'm running.

Why is this not an option:   Noone ever requested it.
What is the -r flag for:   The -r flag moves the case directory, on many hpc systems there is a distinction between scratch space which is not backed up and workspace which is.   Typical usage has been to create the case directory in workspace so that it's backed-up and the run and bld directories in scratch space which is usually larger and possible faster IO.  

You can add the acme_scratch directory to the CESMSCRATCHROOT for your machine in config_machines.xml 

@jedwards4b is correct. When I first joined ACME, I also assumed that run/bld should always just go with the case. I got a lot of push back and had to revert that change for the reasons he described.

if i get around to changing it, should i change only for myself?

Sounds like the solution is to add the option to create_test to set CESMSCRATCHROOT.  If that is set to the same dir as -r, will it give the desired behavior?

@rljacob yes, i believe it would.

Yes that would work but you probably want to add the option to both create_test and create_newcase and implement it in create_newcase (create_test would just pass it on)

Couldn't `CESMSCRATCHROOT` be modified via `-user_mods_dir`?

No --user_mods_dir is a different functionality altogether.   You _could_ actually use it to modify CESMSCRATCHROOT but that would be a really convoluted way to go about it.  

After discussion, will add additional option to create_test to move bld and run dirs.

So @rljacob this will be a separate option and the behavior of "-r" won't change? I think that would be good, I can see myself using both options.


Resolving patch:
diff --git a/utils/python/CIME/case.py b/utils/python/CIME/case.py
--- a/utils/python/CIME/case.py
+++ b/utils/python/CIME/case.py
@@ -505,7 +505,7 @@ def configure(self, compset_name, grid_name, machine_name=None,
                   project=None, pecount=None, compiler=None, mpilib=None,
                   user_compset=False, pesfile=None,
                   user_grid=False, gridfile=None, ninst=1, test=False,
-                  walltime=None, queue=None):
+                  walltime=None, queue=None, output_root=None):
 
         #--------------------------------------------
         # compset, pesfile, and compset components
@@ -683,6 +683,12 @@ def configure(self, compset_name, grid_name, machine_name=None,
         elif machobj.get_value("PROJECT_REQUIRED"):
             expect(project is not None, "PROJECT_REQUIRED is true but no project found")
 
+        # Resolve the CIME_OUTPUT_ROOT variable, other than this
+        # we don't want to resolve variables until we need them
+        if output_root is None:
+            output_root = self.get_value("CIME_OUTPUT_ROOT")
+        self.set_value("CIME_OUTPUT_ROOT", output_root)
+
         # Overwriting an existing exeroot or rundir can cause problems
         exeroot = self.get_value("EXEROOT")
         rundir = self.get_value("RUNDIR")
@@ -708,7 +714,6 @@ def configure(self, compset_name, grid_name, machine_name=None,
         if test:
             self.set_value("TEST",True)
 
-
     def get_compset_var_settings(self):
         compset_obj = Compsets(infile=self.get_value("COMPSETS_SPEC_FILE"))
         matches = compset_obj.get_compset_var_settings(self._compsetname, self._gridname)
diff --git a/utils/python/CIME/preview_namelists.py b/utils/python/CIME/preview_namelists.py
--- a/utils/python/CIME/preview_namelists.py
+++ b/utils/python/CIME/preview_namelists.py
@@ -34,6 +34,8 @@ def create_dirs(case):
                 os.makedirs(dir_to_make)
             except OSError as e:
                 expect(False, "Could not make directory '%s', error: %s" % (dir_to_make, e))
+            with open(os.path.join(dir_to_make,"CASEROOT"),"w+") as fd:
+                fd.write(caseroot+"\n")
 
 def create_namelists(case):
 


Test patch:
diff --git a/scripts/Tools/bless_test_results b/scripts/Tools/bless_test_results
--- a/scripts/Tools/bless_test_results
+++ b/scripts/Tools/bless_test_results
@@ -50,7 +50,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter
     default_baseline_name = CIME.utils.get_current_branch(repo=CIME.utils.get_cime_root())
     default_baseline_root = _MACHINE.get_value("CCSM_BASELINE")
     default_compiler      = _MACHINE.get_default_compiler()
-    scratch_root          = _MACHINE.get_value("CESMSCRATCHROOT")
+    scratch_root          = _MACHINE.get_value("CIME_OUTPUT_ROOT")
     default_testroot      = os.path.join(scratch_root)
 
     CIME.utils.setup_standard_logging_options(parser)
diff --git a/scripts/Tools/compare_test_results b/scripts/Tools/compare_test_results
--- a/scripts/Tools/compare_test_results
+++ b/scripts/Tools/compare_test_results
@@ -48,7 +48,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter
     default_baseline_name = CIME.utils.get_current_branch(repo=CIME.utils.get_cime_root())
     default_baseline_root = _MACHINE.get_value("CCSM_BASELINE")
     default_compiler      = _MACHINE.get_default_compiler()
-    scratch_root          = _MACHINE.get_value("CESMSCRATCHROOT")
+    scratch_root          = _MACHINE.get_value("CIME_OUTPUT_ROOT")
     default_testroot      = os.path.join(scratch_root)
 
     CIME.utils.setup_standard_logging_options(parser)
diff --git a/scripts/create_test b/scripts/create_test
--- a/scripts/create_test
+++ b/scripts/create_test
@@ -99,7 +99,10 @@ OR
 
     parser.add_argument("-r", "--test-root",
                         help="Where test cases will be created."
-                        " Will default to scratch root XML machine file")
+                        " Will default to output root as defined in the config_machines file")
+
+    parser.add_argument("--output-root",
+                        help="Where the case output is written.")
 
     parser.add_argument("--baseline-root",
                         help="Specifies an root directory for baseline"
@@ -328,7 +331,8 @@ OR
     return test_names, test_extra_data, args.compiler, mach_obj.get_machine_name(), args.no_run, args.no_build, args.no_setup, args.no_batch,\
         args.test_root, args.baseline_root, args.clean, baseline_cmp_name, baseline_gen_name, \
         args.namelists_only, args.project, args.test_id, args.parallel_jobs, args.walltime, \
-        args.single_submit, args.proc_pool, args.use_existing, args.save_timing, args.queue, args.allow_baseline_overwrite
+        args.single_submit, args.proc_pool, args.use_existing, args.save_timing, args.queue, \
+        args.allow_baseline_overwrite, args.output_root
 
 ###############################################################################
 def single_submit_impl(machine_name, test_id, proc_pool, project, args, job_cost_map, wall_time, test_root):
@@ -433,7 +437,7 @@ def single_submit_impl(machine_name, test_id, proc_pool, project, args, job_cost
 # pragma pylint: disable=protected-access
 def create_test(test_names, test_data, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root,
                 baseline_root, clean, baseline_cmp_name, baseline_gen_name, namelists_only, project, test_id, parallel_jobs,
-                walltime, single_submit, proc_pool, use_existing, save_timing, queue, allow_baseline_overwrite):
+                walltime, single_submit, proc_pool, use_existing, save_timing, queue, allow_baseline_overwrite, output_root):
 ###############################################################################
     impl = TestScheduler(test_names, test_data=test_data,
                          no_run=no_run, no_build=no_build, no_setup=no_setup, no_batch=no_batch,
@@ -444,7 +448,8 @@ def create_test(test_names, test_data, compiler, machine_name, no_run, no_build,
                          namelists_only=namelists_only,
                          project=project, parallel_jobs=parallel_jobs, walltime=walltime,
                          proc_pool=proc_pool, use_existing=use_existing, save_timing=save_timing,
-                         queue=queue, allow_baseline_overwrite=allow_baseline_overwrite)
+                         queue=queue, allow_baseline_overwrite=allow_baseline_overwrite,
+                         output_root=output_root)
 
     success = impl.run_tests()
 
@@ -482,15 +487,16 @@ def _main_func(description):
                                    (libroot, libroot), arg_stdout=None, arg_stderr=None)
         return
 
-    test_names, test_data, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root, baseline_root, clean, \
-        baseline_cmp_name, baseline_gen_name, namelists_only, project, test_id, parallel_jobs, \
-        walltime, single_submit, proc_pool, use_existing, save_timing, queue, allow_baseline_overwrite \
+    test_names, test_data, compiler, machine_name, no_run, no_build, no_setup, no_batch, \
+    test_root, baseline_root, clean, baseline_cmp_name, baseline_gen_name, namelists_only, \
+    project, test_id, parallel_jobs, walltime, single_submit, proc_pool, use_existing, \
+    save_timing, queue, allow_baseline_overwrite, output_root \
         = parse_command_line(sys.argv, description)
 
     sys.exit(create_test(test_names, test_data, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root,
                          baseline_root, clean, baseline_cmp_name, baseline_gen_name, namelists_only,
                          project, test_id, parallel_jobs, walltime, single_submit, proc_pool, use_existing, save_timing,
-                         queue, allow_baseline_overwrite))
+                         queue, allow_baseline_overwrite, output_root))
 
 ###############################################################################
 
diff --git a/utils/python/CIME/test_scheduler.py b/utils/python/CIME/test_scheduler.py
--- a/utils/python/CIME/test_scheduler.py
+++ b/utils/python/CIME/test_scheduler.py
@@ -42,7 +42,8 @@ def __init__(self, test_names, test_data=None,
                  clean=False, namelists_only=False,
                  project=None, parallel_jobs=None,
                  walltime=None, proc_pool=None,
-                 use_existing=False, save_timing=False, queue=None, allow_baseline_overwrite=False):
+                 use_existing=False, save_timing=False, queue=None,
+                 allow_baseline_overwrite=False, output_root=None):
     ###########################################################################
         self._cime_root  = CIME.utils.get_cime_root()
         self._cime_model = CIME.utils.get_model()
@@ -56,7 +57,7 @@ def __init__(self, test_names, test_data=None,
         self._no_setup = no_setup
         self._no_build = no_build or no_setup or namelists_only
         self._no_run   = no_run or self._no_build
-
+        self._output_root = output_root
         # Figure out what project to use
         if project is None:
             self._project = CIME.utils.get_project()
@@ -72,7 +73,13 @@ def __init__(self, test_names, test_data=None,
                "Does not make sense to request a queue without batch system")
 
         # Determine and resolve test_root
-        self._test_root = self._machobj.get_value("CESMSCRATCHROOT") if test_root is None else test_root
+        if test_root is not None:
+            self._test_root = test_root
+        elif self._output_root is not None:
+            self._test_root = self._output_root
+        else:
+            self._test_root = self._machobj.get_value("CIME_OUTPUT_ROOT")
+
         if self._project is not None:
             self._test_root = self._test_root.replace("$PROJECT", self._project)
 
@@ -301,6 +308,9 @@ def _create_newcase_phase(self, test):
                                test_dir, grid, machine, compiler, compset)
         if self._project is not None:
             create_newcase_cmd += " --project %s " % self._project
+        if self._output_root is not None:
+            create_newcase_cmd += " --output-root %s " % self._output_root
+
 
         if test_mods is not None:
             files = Files()
diff --git a/utils/python/tests/scripts_regression_tests.py b/utils/python/tests/scripts_regression_tests.py
--- a/utils/python/tests/scripts_regression_tests.py
+++ b/utils/python/tests/scripts_regression_tests.py
@@ -170,7 +170,7 @@ def setup_proxy():
 class J_TestCreateNewcase(unittest.TestCase):
 ###############################################################################
     def setUp(self):
-        self._testroot = MACHINE.get_value("CESMSCRATCHROOT")
+        self._testroot = MACHINE.get_value("CIME_OUTPUT_ROOT")
         self._testdirs = []
         self._do_teardown = []
 
@@ -223,7 +223,7 @@ class M_TestWaitForTests(unittest.TestCase):
     ###########################################################################
     def setUp(self):
     ###########################################################################
-        self._testroot = MACHINE.get_value("CESMSCRATCHROOT")
+        self._testroot = MACHINE.get_value("CIME_OUTPUT_ROOT")
         self._testdir_all_pass    = os.path.join(self._testroot, 'scripts_regression_tests.testdir_all_pass.%s'% CIME.utils.get_timestamp())
         self._testdir_with_fail   = os.path.join(self._testroot, 'scripts_regression_tests.testdir_with_fail.%s'% CIME.utils.get_timestamp())
         self._testdir_unfinished  = os.path.join(self._testroot, 'scripts_regression_tests.testdir_unfinished.%s'% CIME.utils.get_timestamp())
@@ -431,7 +431,7 @@ def setUp(self):
         self._baseline_name     = "fake_testing_only_%s" % CIME.utils.get_timestamp()
         self._machine           = MACHINE.get_machine_name()
         self._baseline_area     = MACHINE.get_value("CCSM_BASELINE")
-        self._testroot          = MACHINE.get_value("CESMSCRATCHROOT")
+        self._testroot          = MACHINE.get_value("CIME_OUTPUT_ROOT")
         self._compiler          = MACHINE.get_default_compiler()
         self._hasbatch          = MACHINE.has_batch_system() and not NO_BATCH
         self._do_teardown       = True # Will never do teardown if test failed
@@ -1126,7 +1126,7 @@ class C_TestXMLQuery(unittest.TestCase):
 
     def setUp(self):
         # Create case directory
-        self._testroot = MACHINE.get_value("CESMSCRATCHROOT")  # "/tmp/"
+        self._testroot = MACHINE.get_value("CIME_OUTPUT_ROOT")  # "/tmp/"
         self._testdirs = []
         self._do_teardown = []