Repository: ESMCI/cime

Issue:
Appending Arguments to `BATCH_COMMAND_FLAGS` Overwrites Existing PBS Batch Arguments in CESM Scripts
Hello, 
I would like to add a new PBS argument to my CESM job submission scripts. However, when I use `./xmlchange` to add a new argument to `BATCH_COMMAND_FLAGS`, it seems to ignore the other PBS batch arguments, rather than appending the new one to them.

Here's an example of what I'm experiencing to replicate this:
```
./create_newcase --case ~/cases/b.day1.0 --res f19_g17 --compset B1850
qcmd -- ./case.build
```
Running `./preview_run` then shows the following submission command:

```
qsub -q regular -l walltime=12:00:00 -A SCSG0002 -v ARGS_FOR_SCRIPT='--resubmit --silent' .case.run
```
-------
However, when I use `./xmlchange` to add a new `BATCH_COMMAND_FLAGS`:
```
./xmlchange --subgroup=case.run BATCH_COMMAND_FLAGS="-l place=group=rack"
```
The `./preview_run` shows:

```
qsub -l place=group=rack -v ARGS_FOR_SCRIPT='--resubmit' .case.run
```

As a result, the previous flags `-q regular -l walltime=12:00:00 -A SCSG0002` are removed, causing PBS to fail when submitting the job.

I understand that it's possible to manually include all necessary PBS arguments in `BATCH_COMMAND_FLAGS`. However, this approach doesn't seem to be the most efficient or intuitive. By default, the `BATCH_COMMAND_FLAGS` is empty and CIME automatically finds the necessary PBS information (i.e. job queue, walltime, project code). 

**More importantly, it seems that once `BATCH_COMMAND_FLAGS` is modified, CIME no longer pulls queue, project values, or walltime from the case information, which can potentially lead to confusion and errors.**

Is there a different way to append a new PBS argument to the CIME job scripts? 
If not, it could be helpful to the user community to add this functionality and address the issue with `BATCH_COMMAND_FLAGS`. 


Discussion:
You are correct that BATCH_COMMAND_FLAGS replaces rather than appends the current list of qsub flags.   The approach I recommend is to edit the env_batch.xml file and add      ``` <argument> -l place=group=rack </argument>    ``` under ```  <submit_args>  ```
@jasonb5 @jgfouca  - Would it be better if BATCH_COMMAND_FLAGS append rather than replace existing flags?  Perhaps we should populate BATCH_COMMAND_FLAGS with the default set of flags and always use it instead of having it be an empty string?
@jedwards4b , I like the idea of populating it with whatever CIME computes for the batch command. This would allow the user to `./xmlchange --append XXX` if that's what they want.

Resolving patch:
diff --git a/CIME/XML/env_batch.py b/CIME/XML/env_batch.py
--- a/CIME/XML/env_batch.py
+++ b/CIME/XML/env_batch.py
@@ -555,7 +555,7 @@ def get_batch_directives(self, case, job, overrides=None, output_format="default
 
         return "\n".join(result)
 
-    def get_submit_args(self, case, job):
+    def get_submit_args(self, case, job, resolve=True):
         """
         return a list of touples (flag, name)
         """
@@ -563,7 +563,7 @@ def get_submit_args(self, case, job):
 
         submit_arg_nodes = self._get_arg_nodes(case, bs_nodes)
 
-        submitargs = self._process_args(case, submit_arg_nodes, job)
+        submitargs = self._process_args(case, submit_arg_nodes, job, resolve=resolve)
 
         return submitargs
 
@@ -597,7 +597,7 @@ def _get_arg_nodes(self, case, bs_nodes):
 
         return submit_arg_nodes
 
-    def _process_args(self, case, submit_arg_nodes, job):
+    def _process_args(self, case, submit_arg_nodes, job, resolve=True):
         submitargs = " "
 
         for arg in submit_arg_nodes:
@@ -619,7 +619,7 @@ def _process_args(self, case, submit_arg_nodes, job):
                 if " " in flag:
                     flag, name = flag.split()
                 if name:
-                    if "$" in name:
+                    if resolve and "$" in name:
                         rflag = self._resolve_argument(case, flag, name, job)
                         if len(rflag) > len(flag):
                             submitargs += " {}".format(rflag)
@@ -628,10 +628,13 @@ def _process_args(self, case, submit_arg_nodes, job):
                 else:
                     submitargs += " {}".format(flag)
             else:
-                try:
-                    submitargs += self._resolve_argument(case, flag, name, job)
-                except ValueError:
-                    continue
+                if resolve:
+                    try:
+                        submitargs += self._resolve_argument(case, flag, name, job)
+                    except ValueError:
+                        continue
+                else:
+                    submitargs += " {} {}".format(flag, name)
 
         return submitargs
 
@@ -961,10 +964,7 @@ def _submit_single_job(
 
             return
 
-        submitargs = self.get_submit_args(case, job)
-        args_override = case.get_value("BATCH_COMMAND_FLAGS", subgroup=job)
-        if args_override:
-            submitargs = args_override
+        submitargs = case.get_value("BATCH_COMMAND_FLAGS", subgroup=job)
 
         if dep_jobs is not None and len(dep_jobs) > 0:
             logger.debug("dependencies: {}".format(dep_jobs))
diff --git a/CIME/case/case.py b/CIME/case/case.py
--- a/CIME/case/case.py
+++ b/CIME/case/case.py
@@ -1586,6 +1586,13 @@ def configure(
             )
 
         env_batch.set_job_defaults(bjobs, self)
+        # Set BATCH_COMMAND_FLAGS to the default values
+
+        for job in bjobs:
+            if test and job[0] == "case.run" or not test and job[0] == "case.test":
+                continue
+            submitargs = env_batch.get_submit_args(self, job[0], resolve=False)
+            self.set_value("BATCH_COMMAND_FLAGS", submitargs, subgroup=job[0])
 
         # Make sure that parallel IO is not specified if total_tasks==1
         if self.total_tasks == 1:


Test patch:
diff --git a/CIME/tests/test_sys_create_newcase.py b/CIME/tests/test_sys_create_newcase.py
--- a/CIME/tests/test_sys_create_newcase.py
+++ b/CIME/tests/test_sys_create_newcase.py
@@ -68,12 +68,52 @@ def test_a_createnewcase(self):
         self.run_cmd_assert_result("./case.build", from_dir=testdir)
         with Case(testdir, read_only=False) as case:
             case.set_value("CHARGE_ACCOUNT", "fred")
+            # to be used in next test
+            batch_system = case.get_value("BATCH_SYSTEM")
+
+        # on systems (like github workflow) that do not have batch, set this for the next test
+        if batch_system == "none":
+            self.run_cmd_assert_result(
+                './xmlchange --subgroup case.run BATCH_COMMAND_FLAGS="-q \$JOB_QUEUE"',
+                from_dir=testdir,
+            )
 
         # this should not fail with a locked file issue
         self.run_cmd_assert_result("./case.build", from_dir=testdir)
 
         self.run_cmd_assert_result("./case.st_archive --test-all", from_dir=testdir)
 
+        with Case(testdir, read_only=False) as case:
+            batch_command = case.get_value("BATCH_COMMAND_FLAGS", subgroup="case.run")
+
+        self.run_cmd_assert_result(
+            './xmlchange --append --subgroup case.run BATCH_COMMAND_FLAGS="-l trythis"',
+            from_dir=testdir,
+        )
+        # Test that changes to BATCH_COMMAND_FLAGS work
+        with Case(testdir, read_only=False) as case:
+            new_batch_command = case.get_value(
+                "BATCH_COMMAND_FLAGS", subgroup="case.run"
+            )
+
+        self.assertTrue(
+            new_batch_command == batch_command + " -l trythis",
+            msg=f"Failed to correctly append BATCH_COMMAND_FLAGS {new_batch_command} {batch_command}#",
+        )
+
+        self.run_cmd_assert_result(
+            "./xmlchange JOB_QUEUE=fred --subgroup case.run --force", from_dir=testdir
+        )
+
+        with Case(testdir, read_only=False) as case:
+            new_batch_command = case.get_value(
+                "BATCH_COMMAND_FLAGS", subgroup="case.run"
+            )
+            self.assertTrue(
+                "fred" in new_batch_command,
+                msg="Failed to update JOB_QUEUE in BATCH_COMMAND_FLAGS",
+            )
+
         # Trying to set values outside of context manager should fail
         case = Case(testdir, read_only=False)
         with self.assertRaises(utils.CIMEError):