internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: ESMCI/cime

Issue:
Cryptic error message for typo in compset name
From a cesm2_0_alpha07 sandbox that uses cime5.3.0-alpha.33

```
[yslogin6:/glade/scratch/sacks/cesm_code/cesm2_0_alpha07/cime/scripts]$ ./create_newcase --case test_cism_0906b --compset TG --res f09_g16
Did not find a compset match for longname TG
Compset forcing is
ERROR: No variable CONFIG_ATM_FILE found in case
```

The first line of the error message is understandable, but the next two lines make no sense. For such a common error, we should have a more user-friendly error message. e.g., can the next two lines simply be removed from the output in this case?


Discussion:
That first line should say "Did not find longname compset match for alias TG"
@rljacob that would be incorrect - what is meant is "did not find an alias for TG assuming it is a compset longname"   Now I can tell that TG is not a compset longname but I haven't figured out a general rule to determine what is a compset longname.  That's why there is currently no error at this point in the code. 

Resolving patch:
diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py
--- a/scripts/lib/CIME/case.py
+++ b/scripts/lib/CIME/case.py
@@ -456,8 +456,11 @@ def _set_compset(self, compset_name, files):
                     return compset_alias, science_support
 
         if compset_alias is None:
-            logger.info("Did not find a compset match for longname {} ".format(compset_name))
+            logger.info("Did not find an alias or longname compset match for {} ".format(compset_name))
             self._compsetname = compset_name
+            # if this is a valiid compset longname there will be at least 7 components.
+            components = self.get_compset_components()
+            expect(len(components) > 6, "No compset alias {} found and this does not appear to be a compset longname.".format(compset_name))
 
         return None, science_support
 
@@ -549,7 +552,7 @@ def get_compset_components(self):
         if compset is None:
             compset = self._compsetname
         expect(compset is not None,
-               "ERROR: compset is not set")
+               "compset is not set")
         # the first element is always the date operator - skip it
         elements = compset.split('_')[1:] # pylint: disable=maybe-no-member
         for element in elements:
@@ -619,6 +622,7 @@ def _get_component_config_data(self, files):
             node_name = 'CONFIG_' + comp_class + '_FILE'
             # Add the group and elements for the config_files.xml
             comp_config_file = files.get_value(node_name, {"component":comp_name}, resolved=False)
+            expect(comp_config_file is not None,"No component {} found for class {}".format(comp_name, comp_class))
             self.set_value(node_name, comp_config_file)
             comp_config_file = self.get_resolved_value(comp_config_file)
             expect(comp_config_file is not None and os.path.isfile(comp_config_file),


Test patch:
Repository: ESMCI/cime

Issue:
idmap file errors should be picked up by scripts 
The test SMS.f45_g37.X fails at runtime because one of the mapping files it needs is pointing to idmap, we should pick this up and error out in scripts instead of at runtime. 


Discussion:

Resolving patch:
diff --git a/scripts/lib/CIME/nmlgen.py b/scripts/lib/CIME/nmlgen.py
--- a/scripts/lib/CIME/nmlgen.py
+++ b/scripts/lib/CIME/nmlgen.py
@@ -175,6 +175,14 @@ def _to_namelist_literals(self, name, value):
                 value[i] = self.quote_string(scalar)
         return compress_literal_list(value)
 
+    def get_group_variables(self, group_name):
+        group_variables = {}
+        group = self._namelist._groups[group_name]
+        for name in sorted(group.keys()):
+            value = group[name][0]
+            group_variables[name] = value
+        return group_variables
+
     def get_value(self, name):
         """Get the current value of a given namelist variable.
 


Test patch:
Repository: ESMCI/cime

Issue:
no error if ninst > ntasks in multidriver mode
is in 
   cime/scripts/lib/CIME/case_setup.py; line 139.


Discussion:

Resolving patch:
diff --git a/scripts/lib/CIME/case_setup.py b/scripts/lib/CIME/case_setup.py
--- a/scripts/lib/CIME/case_setup.py
+++ b/scripts/lib/CIME/case_setup.py
@@ -133,15 +133,15 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False):
                 continue
             ninst  = case.get_value("NINST_{}".format(comp))
             ntasks = case.get_value("NTASKS_{}".format(comp))
-            if ninst > ntasks:
-                if ntasks == 1:
-                    case.set_value("NTASKS_{}".format(comp), ninst)
-                else:
-                    expect(False, "NINST_{} value {:d} greater than NTASKS_{} {:d}".format(comp, ninst, comp, ntasks))
             # But the NINST_LAYOUT may only be concurrent in multi_driver mode
             if multi_driver:
                 expect(case.get_value("NINST_LAYOUT_{}".format(comp)) == "concurrent",
                        "If multi_driver is TRUE, NINST_LAYOUT_{} must be concurrent".format(comp))
+            elif ninst > ntasks:
+                if ntasks == 1:
+                    case.set_value("NTASKS_{}".format(comp), ninst)
+                else:
+                    expect(False, "NINST_{} value {:d} greater than NTASKS_{} {:d}".format(comp, ninst, comp, ntasks))
 
         if os.path.exists("case.run"):
             logger.info("Machine/Decomp/Pes configuration has already been done ...skipping")


Test patch:
diff --git a/config/config_tests.xml b/config/config_tests.xml
--- a/config/config_tests.xml
+++ b/config/config_tests.xml
@@ -491,7 +491,6 @@ NODEFAIL          Tests restart upon detected node failure. Generates fake failu
     <HIST_OPTION>$STOP_OPTION</HIST_OPTION>
     <HIST_N>$STOP_N</HIST_N>
     <CONTINUE_RUN>FALSE</CONTINUE_RUN>
-    <REST_OPTION>none</REST_OPTION>
   </test>
 
   <test NAME="PET">
diff --git a/scripts/lib/CIME/SystemTests/system_tests_common.py b/scripts/lib/CIME/SystemTests/system_tests_common.py
--- a/scripts/lib/CIME/SystemTests/system_tests_common.py
+++ b/scripts/lib/CIME/SystemTests/system_tests_common.py
@@ -388,7 +388,8 @@ def _compare_baseline(self):
 
             # compare memory usage to baseline
             newestcpllogfiles = self._get_latest_cpl_logs()
-            memlist = self._get_mem_usage(newestcpllogfiles[0])
+            if len(newestcpllogfiles) > 0:
+                memlist = self._get_mem_usage(newestcpllogfiles[0])
             for cpllog in newestcpllogfiles:
                 m = re.search(r"/(cpl.*.log).*.gz",cpllog)
                 if m is not None:
Repository: ESMCI/cime

Issue:
--skip-preview-namelist on batch systems 
The syntax of --skip-preview-namelist option needs to be modified to work on pbs batch systems. 


Discussion:
@jgfouca I'd like your thoughts on this, it doesn't work on pbs systems and I don't see a way to make a command line argument to case.run work in general.
Just to fill-in others, the Jims discussed this and discovered:

```
qsub -v args_for_caserun='--skip-preview-namelist' ...
```

then, in case.run, we'd need to pass os.environ["args_for_caserun"] into parse_args instead of sys.argv

We'll probably need a new config_batch entry for all batch types that will tell us how to pass environment variables to the submit script.
@jedwards4b I think I have a rough idea how to do this if you want to reassign to me.

Resolving patch:
diff --git a/scripts/lib/CIME/XML/env_batch.py b/scripts/lib/CIME/XML/env_batch.py
--- a/scripts/lib/CIME/XML/env_batch.py
+++ b/scripts/lib/CIME/XML/env_batch.py
@@ -442,7 +442,11 @@ def _submit_single_job(self, case, job, depid=None, no_batch=False,
                 submitcmd += string + " "
 
         if job == 'case.run' and skip_pnl:
-            submitcmd += " --skip-preview-namelist"
+            batch_env_flag = self.get_value("batch_env", subgroup=None)
+            if not batch_env_flag:
+                submitcmd += " --skip-preview-namelist"
+            else:
+                submitcmd += " {} ARGS_FOR_SCRIPT='--skip-preview-namelist'".format(batch_env_flag)
 
         if dry_run:
             return submitcmd


Test patch:
diff --git a/config/acme/machines/template.case.test b/config/acme/machines/template.case.test
--- a/config/acme/machines/template.case.test
+++ b/config/acme/machines/template.case.test
@@ -63,6 +63,8 @@ def _main_func(description):
         test_results = doctest.testmod(verbose=True)
         sys.exit(1 if test_results.failed > 0 else 0)
 
+    sys.argv.extend([] if "ARGS_FOR_SCRIPT" not in os.environ else os.environ["ARGS_FOR_SCRIPT"].split())
+
     caseroot, testname, reset = parse_command_line(sys.argv, description)
     with Case(caseroot, read_only=False) as case:
         success = case_test(case, testname, reset)
diff --git a/config/cesm/machines/template.case.test b/config/cesm/machines/template.case.test
--- a/config/cesm/machines/template.case.test
+++ b/config/cesm/machines/template.case.test
@@ -63,6 +63,8 @@ def _main_func(description):
         test_results = doctest.testmod(verbose=True)
         sys.exit(1 if test_results.failed > 0 else 0)
 
+    sys.argv.extend([] if "ARGS_FOR_SCRIPT" not in os.environ else os.environ["ARGS_FOR_SCRIPT"].split())
+
     caseroot, testname, reset = parse_command_line(sys.argv, description)
     with Case(caseroot, read_only=False) as case:
         success = case_test(case, testname, reset)
Repository: ESMCI/cime

Issue:
type conversion issues
Some type conversion issues are appearing after the merge of #1949 


Discussion:

Resolving patch:
diff --git a/scripts/lib/CIME/SystemTests/erp.py b/scripts/lib/CIME/SystemTests/erp.py
--- a/scripts/lib/CIME/SystemTests/erp.py
+++ b/scripts/lib/CIME/SystemTests/erp.py
@@ -37,10 +37,10 @@ def _case_two_setup(self):
             nthreads  = self._case1.get_value("NTHRDS_{}".format(comp))
             rootpe    = self._case1.get_value("ROOTPE_{}".format(comp))
             if ( nthreads > 1 ):
-                self._case.set_value("NTHRDS_{}".format(comp), nthreads/2)
+                self._case.set_value("NTHRDS_{}".format(comp), int(nthreads/2))
             if ( ntasks > 1 ):
-                self._case.set_value("NTASKS_{}".format(comp), ntasks/2)
-                self._case.set_value("ROOTPE_{}".format(comp), rootpe/2)
+                self._case.set_value("NTASKS_{}".format(comp), int(ntasks/2))
+                self._case.set_value("ROOTPE_{}".format(comp), int(rootpe/2))
 
         RestartTest._case_two_setup(self)
         # Note, some components, like CESM-CICE, have
diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py
--- a/scripts/lib/CIME/utils.py
+++ b/scripts/lib/CIME/utils.py
@@ -296,7 +296,7 @@ def run_cmd(cmd, input_str=None, from_dir=None, verbose=None,
             arg_stdout.close() # pylint: disable=no-member
         if isinstance(arg_stderr, io.IOBase) and arg_stderr is not arg_stdout:
             arg_stderr.close() # pylint: disable=no-member
-        
+
 
     if (verbose != False and (verbose or logger.isEnabledFor(logging.DEBUG))):
         if stat != 0:
@@ -342,7 +342,7 @@ def check_minimum_python_version(major, minor):
     >>> check_minimum_python_version(sys.version_info[0], sys.version_info[1])
     >>>
     """
-    expect(sys.version_info[0] > major or 
+    expect(sys.version_info[0] > major or
            (sys.version_info[0] == major and sys.version_info[1] >= minor),
            "Python {:d}, minor version {:d}+ is required, you have {:d}.{:d}".format(major, minor, sys.version_info[0], sys.version_info[1]))
 
@@ -868,8 +868,16 @@ def convert_to_string(value, type_str=None, vid=""):
     """
     Convert value back to string.
     vid is only for generating better error messages.
+    >>> convert_to_string(6, type_str="integer") == '6'
+    True
+    >>> convert_to_string('6', type_str="integer") == '6'
+    True
+    >>> convert_to_string('6.0', type_str="real") == '6.0'
+    True
+    >>> convert_to_string(6.01, type_str="real") == '6.01'
+    True
     """
-    if value is not None and type(value) is not str:
+    if value is not None and not isinstance(value, six.string_types):
         if type_str == "char":
             expect(isinstance(value, six.string_types), "Wrong type for entry id '{}'".format(vid))
         elif type_str == "integer":
@@ -1302,7 +1310,7 @@ def is_python_executable(filepath):
                 first_line = f.readline()
             except:
                 pass
-        
+
         return first_line is not None and first_line.startswith("#!") and "python" in first_line
     return False
 


Test patch:
Repository: ESMCI/cime

Issue:
Unhelpful error if python less then 2.7
I somehow had python 2.6 in my path and saw this when trying to make a case.
```
blogin3[126]: ./create_newcase --case Xtry --compset X --res f19_g16
Traceback (most recent call last):
  File "./create_newcase", line 3, in <module>
    from Tools.standard_script_setup import *
  File "/lcrc/group/earthscience/jacob/cimeESMCI2/scripts/Tools/standard_script_setup.py", line 17, in <module>
    CIME.utils.check_minimum_python_version(2, 7)
  File "/lcrc/group/earthscience/jacob/cimeESMCI2/scripts/Tools/../../scripts/lib/CIME/utils.py", line 347, in check_minimum_python_version
    "Python {:d}, minor version {:d}+ is required, you have {:d}.{:d}".format(major, minor, sys.version_info[0], sys.version_info[1]))
ValueError: zero length field name in format
Exit 1
```
The error message is buried in traceback output.


Discussion:

Resolving patch:
diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py
--- a/scripts/lib/CIME/utils.py
+++ b/scripts/lib/CIME/utils.py
@@ -49,7 +49,7 @@ def expect(condition, error_msg, exc_type=SystemExit, error_prefix="ERROR:"):
         if logger.isEnabledFor(logging.DEBUG):
             import pdb
             pdb.set_trace()
-        raise exc_type("{} {}".format(error_prefix, error_msg))
+        raise exc_type(error_prefix + " " + error_msg)
 
 def id_generator(size=6, chars=string.ascii_lowercase + string.digits):
     return ''.join(random.choice(chars) for _ in range(size))
@@ -342,9 +342,9 @@ def check_minimum_python_version(major, minor):
     >>> check_minimum_python_version(sys.version_info[0], sys.version_info[1])
     >>>
     """
+    msg = "Python " + str(major) + ", minor version " + str(minor) + " is required, you have " + str(sys.version_info[0]) + "." + str(sys.version_info[1])
     expect(sys.version_info[0] > major or
-           (sys.version_info[0] == major and sys.version_info[1] >= minor),
-           "Python {:d}, minor version {:d}+ is required, you have {:d}.{:d}".format(major, minor, sys.version_info[0], sys.version_info[1]))
+           (sys.version_info[0] == major and sys.version_info[1] >= minor), msg)
 
 def normalize_case_id(case_id):
     """


Test patch:
Repository: ESMCI/cime

Issue:
Figure out how to handle COST*
Concerns have been raised about the accuracy of the current COST_PES case setting. There are proposals to use a COST_PER_NODE concept instead.


Discussion:

Resolving patch:
diff --git a/scripts/lib/CIME/get_timing.py b/scripts/lib/CIME/get_timing.py
--- a/scripts/lib/CIME/get_timing.py
+++ b/scripts/lib/CIME/get_timing.py
@@ -138,13 +138,18 @@ def _getTiming(self, inst=0):
         run_type = self.case.get_value("RUN_TYPE")
         stop_option = self.case.get_value("STOP_OPTION")
         stop_n = self.case.get_value("STOP_N")
+
         cost_pes = self.case.get_value("COST_PES")
+        costpes_per_node = self.case.get_value("COSTPES_PER_NODE")
+
         totalpes = self.case.get_value("TOTALPES")
         max_mpitasks_per_node = self.case.get_value("MAX_MPITASKS_PER_NODE")
         smt_factor = max(1,int(self.case.get_value("MAX_TASKS_PER_NODE") / max_mpitasks_per_node))
 
         if cost_pes > 0:
             pecost = cost_pes
+        elif costpes_per_node:
+            pecost = self.case.num_nodes * costpes_per_node
         else:
             pecost = totalpes
 
@@ -309,7 +314,7 @@ def _getTiming(self, inst=0):
 
         self.write("\n")
         self.write("  total pes active           : {} \n".format(totalpes*maxthrds*smt_factor ))
-        self.write("  pes per node               : {} \n".format(max_mpitasks_per_node))
+        self.write("  mpi tasks per node               : {} \n".format(max_mpitasks_per_node))
         self.write("  pe count for cost estimate : {} \n".format(pecost))
         self.write("\n")
 


Test patch:
Repository: ESMCI/cime

Issue:
Error check compset name
Improve error checking on compset name, if an alias isn't matched make sure that the compset name (now assumed to be a longname) lists 7 components. 

Test suite: scripts_regression_tests.py hand tests of create_newcase
Test baseline: 
Test namelist changes: 
Test status: bit for bit

Fixes #1871 

User interface changes?: 

Update gh-pages html (Y/N)?:

Code review: 



Discussion:
try again
Looks good now, thanks!

Resolving patch:
diff --git a/scripts/lib/acme_cime_mgmt.py b/scripts/lib/acme_cime_mgmt.py
--- a/scripts/lib/acme_cime_mgmt.py
+++ b/scripts/lib/acme_cime_mgmt.py
@@ -18,8 +18,8 @@ def setup():
     if ESMCI_REMOTE_NAME not in remotes:
         run_cmd_no_fail("git remote add {} {}".format(ESMCI_REMOTE_NAME, ESMCI_URL))
 
-    run_cmd_no_fail("git fetch {}".format(ESMCI_REMOTE_NAME))
-    run_cmd_no_fail("git fetch {} --tags".format(ESMCI_REMOTE_NAME))
+    run_cmd_no_fail("git fetch --prune {}".format(ESMCI_REMOTE_NAME))
+    run_cmd_no_fail("git fetch --prune {} --tags".format(ESMCI_REMOTE_NAME))
 
 ###############################################################################
 def get_tag(prefix, expected_num=1):


Test patch:
Repository: ESMCI/cime

Issue:
understand and better use python logging facility
Currently a script called as a subroutine from run_sub_or_cmd needs to add the following lines to
have logger output redirect to it's output file:
```
    ch = logging.StreamHandler(stream=sys.stdout)                               
    ch.setLevel(logging.DEBUG)                                                  
    logger.addHandler(ch)       
```
However this does not redirect exception output from the expect routine.    
This means that the error message ends up in the batch system output log and
not in the output stream set by the subroutine - how do we get this log output in the correct location?


Discussion:

Resolving patch:
diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py
--- a/scripts/lib/CIME/utils.py
+++ b/scripts/lib/CIME/utils.py
@@ -2,9 +2,8 @@
 Common functions used by cime python scripts
 Warning: you cannot use CIME Classes in this module as it causes circular dependencies
 """
-import io, logging, gzip, sys, os, time, re, shutil, glob, string, random, imp, errno, signal
+import io, logging, gzip, sys, os, time, re, shutil, glob, string, random, imp, errno, signal, traceback, warnings
 import stat as statlib
-import warnings
 import six
 from contextlib import contextmanager
 #pylint: disable=import-error
@@ -31,6 +30,13 @@ def redirect_stderr(new_target):
     finally:
         sys.stderr = old_target # restore to the previous value
 
+@contextmanager
+def redirect_stdout_stderr(new_target):
+    old_stdout, old_stderr = sys.stdout, sys.stderr
+    try:
+        yield new_target
+    finally:
+        sys.stdout, sys.stderr = old_stdout, old_stderr
 
 def expect(condition, error_msg, exc_type=SystemExit, error_prefix="ERROR:"):
     """
@@ -213,7 +219,7 @@ def run_sub_or_cmd(cmd, cmdargs, subname, subargs, logfile=None, case=None,
         mod = imp.load_source(subname, cmd)
         logger.info("   Calling {}".format(cmd))
         if logfile:
-            with redirect_stdout(open(logfile,"w")):
+            with redirect_stdout_stderr(open(logfile,"w")):
                 getattr(mod, subname)(*subargs)
         else:
             getattr(mod, subname)(*subargs)
@@ -221,7 +227,12 @@ def run_sub_or_cmd(cmd, cmdargs, subname, subargs, logfile=None, case=None,
         do_run_cmd = True
     except AttributeError:
         do_run_cmd = True
-    except:
+    except BaseException as e:
+        if logfile:
+            msg = e.__str__()
+            excmsg = "Exception during build:\n{}\n{}".format(msg, traceback.format_exc())
+            with open(logfile, "a") as fd:
+                fd.write(excmsg)
         raise
 
     if do_run_cmd:


Test patch:
Repository: ESMCI/cime

Issue:
Rewrite buildlib for pio/cesm_share/gptl/mct/mpi-serial in python



Discussion:

Resolving patch:
diff --git a/scripts/lib/CIME/build.py b/scripts/lib/CIME/build.py
--- a/scripts/lib/CIME/build.py
+++ b/scripts/lib/CIME/build.py
@@ -238,7 +238,7 @@ def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid
         file_build = os.path.join(exeroot, "{}.bldlog.{}".format(lib, lid))
         my_file = os.path.join(cimeroot, "src", "build_scripts", "buildlib.{}".format(lib))
         logger.info("Building {} with output to file {}".format(lib,file_build))
-        
+
         stat,_,_ = run_sub_or_cmd(my_file, [full_lib_path, os.path.join(exeroot, sharedpath), caseroot], 'buildlib',
                               [full_lib_path, os.path.join(exeroot, sharedpath), caseroot], logfile=file_build, combine_output=True)
 
diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py
--- a/scripts/lib/CIME/case.py
+++ b/scripts/lib/CIME/case.py
@@ -1030,9 +1030,9 @@ def _create_caseroot_tools(self):
         toolfiles = [os.path.join(toolsdir, "check_lockedfiles"),
                      os.path.join(toolsdir, "getTiming"),
                      os.path.join(toolsdir, "save_provenance"),
-                     os.path.join(machines_dir,"Makefile"),
-                     os.path.join(machines_dir,"mkSrcfiles"),
-                     os.path.join(machines_dir,"mkDepends")]
+                     os.path.join(toolsdir,"Makefile"),
+                     os.path.join(toolsdir,"mkSrcfiles"),
+                     os.path.join(toolsdir,"mkDepends")]
 
         # used on Titan
         if os.path.isfile( os.path.join(toolsdir,"mdiag_reduce.csh") ):
diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py
--- a/scripts/lib/CIME/utils.py
+++ b/scripts/lib/CIME/utils.py
@@ -2,7 +2,8 @@
 Common functions used by cime python scripts
 Warning: you cannot use CIME Classes in this module as it causes circular dependencies
 """
-import io, logging, gzip, sys, os, time, re, shutil, glob, string, random, imp, errno, signal, traceback, warnings
+import io, logging, gzip, sys, os, time, re, shutil, glob, string, random, imp
+import errno, signal, traceback, warnings, filecmp
 import stat as statlib
 import six
 from contextlib import contextmanager
@@ -41,7 +42,6 @@ def redirect_stdout_stderr(new_target):
 
 @contextmanager
 def redirect_logger(new_target, logger_name):
-
     ch = logging.StreamHandler(stream=new_target)
     ch.setLevel(logging.DEBUG)
     log = logging.getLogger(logger_name)
@@ -1461,6 +1461,11 @@ def resolve_mail_type_args(args):
 
         args.mail_type = resolved_mail_types
 
+def copyifnewer(src, dest):
+    """ if dest does not exist or is older than src copy src to dest """
+    if not os.path.isfile(dest) or not filecmp.cmp(src, dest):
+        shutil.copy2(src, dest)
+
 class SharedArea(object):
     """
     Enable 0002 umask within this manager


Test patch: