Repository: ESMCI/cime

Issue:
config_pes_converter is broken 
I'm not sure how it ever could have worked - I am getting an error:
 No such file or directory: '../config/cesm/machines/userdefined_laptop_template/config_pes_new.xml'

But it should be opening this file to write, not to read.   


Discussion:
The error probably means the directory doesn't exist, perhaps because these files and scripts have moved around a bit. I'll take a look.

There is another error in the file you'll need to fix
 
-    CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)
+    args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)


Resolving patch:
diff --git a/scripts/Tools/xmlconvertors/config_pes_converter.py b/scripts/Tools/xmlconvertors/config_pes_converter.py
--- a/scripts/Tools/xmlconvertors/config_pes_converter.py
+++ b/scripts/Tools/xmlconvertors/config_pes_converter.py
@@ -1,7 +1,8 @@
 #!/usr/bin/env python
 """
-config_pes_converter.py -- convert (or verify) config_pesfrom CIME2 format
-    to CIME5
+config_pes_converter.py -- convert (or verify) config_pes elements from CIME2
+format to CIME5. This tool will compare the two versions and suggest updates to
+the CIME5 file.
 
 The location of these files are needed by the script:
     CIME2: cime/machines-acme/config_pes.xml
@@ -25,14 +26,15 @@ def parse_command_line(args):
     CIME.utils.setup_standard_logging_options(parser)
 
     # Set command line options
-    parser.add_argument("-cime2file", "--cime2file", help="location of config_grid.xml file in CIME2 repository")
-    parser.add_argument("-cime5file", "--cime5file", help="location of config_grids.xml file in CIME5 repository")
+    parser.add_argument("-cime2file", "--cime2file",
+                        help="location of config_grid.xml file in CIME2 format",
+                        required=True)
+    parser.add_argument("-cime5file", "--cime5file",
+                        help="location of config_grids.xml file in CIME5 format",
+                        required=True)
 
-    CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)
-
-    if args.cime2file is None or args.cime5file is None:
-        parser.print_help()
-        exit()
+    args = CIME.utils.parse_args_and_handle_standard_logging_options(args,
+                                                                     parser)
 
     return args.cime2file, args.cime5file
 
@@ -176,18 +178,23 @@ class PesTree(grid_xml_converter.DataTree):
     def __init__(self, xmlfilename):
         # original xml file has bad comments
         import re, StringIO
-        with open(xmlfilename,'r') as xmlfile:
-            t1 = xmlfile.read()
-            t2 = re.sub(r'(?<=<!--)([ -]+)',
-                        lambda x: x.group(0).replace('-',' '), t1)
-            t3 = re.sub(r'([ -]+)(?=-->)',
-                        lambda x: x.group(0).replace('-',' '), t2)
-            tempxml = StringIO.StringIO(t3)
-            super(PesTree, self).__init__(tempxml)
-            tempxml.close()
+        if os.access(xmlfilename, os.R_OK):
+            with open(xmlfilename, 'r') as xmlfile:
+                t1 = xmlfile.read()
+                t2 = re.sub(r'(?<=<!--)([ -]+)',
+                            lambda x: x.group(0).replace('-', ' '), t1)
+                t3 = re.sub(r'([ -]+)(?=-->)',
+                            lambda x: x.group(0).replace('-', ' '), t2)
+                tempxml = StringIO.StringIO(t3)
+                super(PesTree, self).__init__(tempxml)
+                tempxml.close()
 
+        else:
+            super(PesTree, self).__init__(xmlfilename)
 
     def populate(self):
+        if self.root is None:
+            return
         xmlnodes = self.root.findall('grid')
         nodeclass = Cime5PesNode
 
@@ -256,7 +263,7 @@ def diff_tree(atree, btree):
     LOGGER.info("Number of wrong nodes: %d" % len(fixlist))
     LOGGER.info("Number of missing nodes: %d" % len(addlist))
     for miss in addlist:
-        LOGGER.info(miss[0].keyvalue())
+        LOGGER.debug(miss[0].keyvalue())
     LOGGER.info("Number of duplicate nodes: %d" % len(duplist))
     for dup in duplist:
         LOGGER.info(dup)
diff --git a/scripts/Tools/xmlconvertors/grid_xml_converter.py b/scripts/Tools/xmlconvertors/grid_xml_converter.py
--- a/scripts/Tools/xmlconvertors/grid_xml_converter.py
+++ b/scripts/Tools/xmlconvertors/grid_xml_converter.py
@@ -1,6 +1,9 @@
 #!/usr/bin/env python
 """
-grid_xml_converter.py -- convert (or verify) grid xml from CIME2 format to CIME5
+grid_xml_converter.py -- convert (or verify) grid elements from CIME2 format
+to CIME5. This tool will compare the two versions and suggest updates
+to the CIME5 file.
+
 The location of these files are needed by the script:
     CIME2: cime/scripts/Tools/config_grid.xml
     CIME5: config/acme/config_grids.xml
@@ -30,14 +33,14 @@ def parse_command_line(args):
     CIME.utils.setup_standard_logging_options(parser)
 
     # Set command line options
-    parser.add_argument("-cime2file", "--cime2file", help="location of config_grid.xml file in CIME2 repository")
-    parser.add_argument("-cime5file", "--cime5file", help="location of config_grids.xml file in CIME5 repository")
-
-    CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)
+    parser.add_argument("-cime2file", "--cime2file",
+                        help="location of config_grid.xml file in CIME2 format",
+                        required=True)
+    parser.add_argument("-cime5file", "--cime5file",
+                        help="location of config_grids.xml file in CIME5 format",
+                        required=True)
 
-    if args.cime2file is None or args.cime5file is None:
-        parser.print_help()
-        exit()
+    args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)
 
     return args.cime2file, args.cime5file
 
@@ -65,18 +68,18 @@ def __str__(self):
     def to_cime5(self):
         node = ET.Element('grid')
         if 'compset' in self.data and self.data['compset'] is not None:
-            node.set('compset',self.data['compset'])
+            node.set('compset', self.data['compset'])
 
-        for k in ['sname','lname','alias','support']:
+        for k in ['sname', 'lname', 'alias', 'support']:
             if k in self.data and self.data[k] is not None:
-                ET.SubElement(node,k).text=self.data[k]
+                ET.SubElement(node, k).text = self.data[k]
 
         return node
 
 
 
     def __eq__(self, other):
-        for k in ['lname','sname','compset','alias']:
+        for k in ['lname', 'sname', 'compset', 'alias']:
             if k not in self.data and k not in other.data:
                 continue
             if k not in self.data or k not in other.data:
@@ -90,7 +93,7 @@ def set_data(self, xmlnode):
         self.xmlnode = xmlnode
         if xmlnode.text is not None:
             self.data['lname'] = xmlnode.text
-        for k in ['lname','sname','alias','compset']:
+        for k in ['lname', 'sname', 'alias', 'compset']:
             tmpval = xmlnode.get(k)
             if tmpval is not None:
                 self.data[k] = tmpval.strip()
@@ -101,7 +104,7 @@ def set_data(self, xmlnode):
 class Cime5GridNode(GridNode):
     def set_data(self, xmlnode):
         self.xmlnode = xmlnode
-        for k in ['sname','lname','support','alias']:
+        for k in ['sname', 'lname', 'support', 'alias']:
             if xmlnode.find(k) is not None:
                 self.data[k] = xmlnode.find(k).text.strip()
         if xmlnode.get('compset') is not None:
@@ -109,36 +112,37 @@ def set_data(self, xmlnode):
 
 class GridmapNode(DataNode):
     def set_data(self, xmlnode):
-        self.keys=[]
-        self.data['maps']={}
+        self.keys = []
+        self.data['maps'] = {}
         self.xmlnode = xmlnode
-        for k in ['atm_grid','lnd_grid','ocn_grid','rof_grid','glc_grid','wav_grid','ice_grid']:
+        for k in ['atm_grid', 'lnd_grid', 'ocn_grid', 'rof_grid', 'glc_grid',
+                  'wav_grid', 'ice_grid']:
             att = xmlnode.get(k)
             if att is not None:
-                self.data[k]=att.strip()
+                self.data[k] = att.strip()
                 self.keys.append(k)
         self.sort()
         for child in xmlnode.getchildren():
-            self.data['maps'][child.tag]=child.text.strip()
+            self.data['maps'][child.tag] = child.text.strip()
     def sort(self):
         newlist = sorted(self.keys, key=operator.itemgetter(0))
         self.keys = newlist
     def to_cime5(self):
         node = ET.Element('gridmap')
-        for k in ['atm_grid','lnd_grid','ocn_grid','rof_grid','glc_grid']:
+        for k in ['atm_grid', 'lnd_grid', 'ocn_grid', 'rof_grid', 'glc_grid']:
             if k in self.data:
-                node.set(k,self.data[k])
-        for key,value in self.data['maps'].iteritems():
-            ET.SubElement(node,key).text=value
+                node.set(k, self.data[k])
+        for key, value in self.data['maps'].iteritems():
+            ET.SubElement(node, key).text = value
         return node
     def __str__(self):
         return str(self.keyvalue()) + str(self.data)
-    def __eq__(self,other):
+    def __eq__(self, other):
         if self.keyvalue() != other.keyvalue():
             return False
         if len(self.data['maps']) != len(other.data['maps']):
             return False
-        for key,value in self.data['maps'].iteritems():
+        for key, value in self.data['maps'].iteritems():
             if key not in other.data['maps'] or value != other.data['maps'][key]:
                 return False
         return True
@@ -154,27 +158,27 @@ class DomainNode(DataNode):
 
     def to_cime5(self):
         node = ET.Element('domain')
-        node.set('name',self.data['name'])
-        for tag in ['nx','ny','desc','support']:
+        node.set('name', self.data['name'])
+        for tag in ['nx', 'ny', 'desc', 'support']:
             if tag in self.data:
-                ET.SubElement(node,tag).text=self.data[tag]
-        for fop in ['file','path']:
+                ET.SubElement(node, tag).text = self.data[tag]
+        for fop in ['file', 'path']:
             if fop in self.data:
                 for comp, mask, filename in self.data[fop]:
                     attribs = {'%s_mask' % comp:mask}
-                    ET.SubElement(node,fop,attribs).text=filename
+                    ET.SubElement(node, fop, attribs).text = filename
         return node
 
 
 
     def sort(self):
-        for fop in ['file','path']:
-            newlist = sorted(self.data[fop],key=operator.itemgetter(0))
+        for fop in ['file', 'path']:
+            newlist = sorted(self.data[fop], key=operator.itemgetter(0))
             self.data[fop] = newlist
 
     def __eq__(self, other):
         # Check for different name, nx, or ny values
-        for k in ['name','nx','ny']:
+        for k in ['name', 'nx', 'ny']:
             if k not in self.data and k not in other.data:
                 continue
             if k not in self.data or k not in other.data:
@@ -182,7 +186,7 @@ def __eq__(self, other):
             if self.data[k] != other.data[k]:
                 return False
         # Compare (sorted) file, path lists for equality
-        for fop in ['file','path']:
+        for fop in ['file', 'path']:
             if fop not in self.data and fop not in other.data:
                 contine
             if fop not in self.data or fop not in other.data:
@@ -190,8 +194,8 @@ def __eq__(self, other):
             if len(self.data[fop]) != len(other.data[fop]):
                 return False
 
-            for i in range(0,len(self.data[fop])):
-                for j in range(0,2):
+            for i in range(0, len(self.data[fop])):
+                for j in range(0, 2):
                     if self.data[fop][i][j] != other.data[fop][i][j]:
                         return False
 
@@ -209,21 +213,23 @@ def set_data(self, xmlnode):
         self.data['name'] = xmlnode.get('name').strip()
         self.data['file'] = []
         self.data['path'] = []
-        for tag in ['nx','ny','desc']:
+        for tag in ['nx', 'ny', 'desc']:
             child = xmlnode.find(tag)
             if child is not None:
                 self.data[tag] = child.text
 
         # Find any griddom nodes that match this name
-        griddoms = self.xmlroot.findall('.griddom[@grid="%s"]' % self.data['name'])
+        griddoms = self.xmlroot.findall('.griddom[@grid="%s"]'
+                                        % self.data['name'])
         for gd in griddoms:
             mask = gd.get('mask')
-            for comp in ['ATM','LND','OCN','ICE']:
-                for fop in ['FILE','PATH']:
-                    tag = '%s_DOMAIN_%s' % (comp,fop)
+            for comp in ['ATM', 'LND', 'OCN', 'ICE']:
+                for fop in ['FILE', 'PATH']:
+                    tag = '%s_DOMAIN_%s' % (comp, fop)
                     n = gd.find(tag)
                     if n is not None:
-                        self.data[fop.lower()].append([comp.lower(),mask,n.text])
+                        self.data[fop.lower()].append([comp.lower(), mask,
+                                                       n.text])
         # sort the file and path entries
         self.sort()
 
@@ -236,43 +242,40 @@ def set_data(self, xmlnode):
         self.data['name'] = xmlnode.get('name')
         self.data['file'] = []
         self.data['path'] = []
-        for tag in ['nx','ny','desc','support']:
+        for tag in ['nx', 'ny', 'desc', 'support']:
             child = xmlnode.find(tag)
             if child is not None:
                 self.data[tag] = child.text
-        for comp in ['lnd','atm','ocn','ice']:
+        for comp in ['lnd', 'atm', 'ocn', 'ice']:
             masktag = '%s_mask' % comp
-            for fop in ['file','path']:
-                fopnodes = xmlnode.findall('%s[@%s]' % (fop,masktag))
+            for fop in ['file', 'path']:
+                fopnodes = xmlnode.findall('%s[@%s]' % (fop, masktag))
                 for n in fopnodes:
                     mask = n.get(masktag)
                     filename = n.text.strip()
-                    self.data[fop].append([comp,mask,filename])
+                    self.data[fop].append([comp, mask, filename])
 
         # sort the file and path entries
         self.sort()
 
-
 class DataTree(object):
     def __init__(self, xmlfilename):
         self.xmlfilename = xmlfilename
-        self.doc = ET.parse(xmlfilename)
+
+        if hasattr(xmlfilename, 'read') or os.access(xmlfilename, os.R_OK):
+            self.doc = ET.parse(xmlfilename)
+        else:
+            self.doc = ET.ElementTree()
+
         self.root = self.doc.getroot()
         self.index = 0
         self.n = 0
         self.nodes = []
         self.populate()
 
-    def populate(self,expression,classname):
-        xmlnodes = self.root.findall(expression)
-        for xmlnode in xmlnodes:
-            datanode = classname(self.root)
-            datanode.set_data(xmlnode)
-            self.nodes.append(datanode)
-
     def next(self):
         if self.index >= len(self.nodes):
-            self.index=0
+            self.index = 0
             raise StopIteration
         if self.index < len(self.nodes):
             self.index += 1
@@ -287,8 +290,8 @@ def postprocess(self, fixlist, addlist, newxmlfile, currentxmlfile,
             logger.info("\n\nWriting suggested nodes to %s" % newxmlfile)
             logger.info("Copy 'grid' nodes into corresponding location in")
             logger.info(currentxmlfile)
-            self.writexml(addlist,newxmlfile)
-            self.writexml(fixlist,badxmlfile)
+            self.writexml(addlist, newxmlfile)
+            self.writexml(fixlist, badxmlfile)
             if len(fixlist) > 0:
                 logger.info("Some nodes should be removed from")
                 logger.info("config/acme/config_grids.xml. These nodes")
@@ -296,20 +299,22 @@ def postprocess(self, fixlist, addlist, newxmlfile, currentxmlfile,
 
 class GridTree(DataTree):
     def populate(self):
+        if self.root is None:
+            return
         xmlnodes = self.root.findall('GRID')
-        nodeclass=Cime2GridNode
+        nodeclass = Cime2GridNode
         if len(xmlnodes) == 0:
             xmlnodes = self.root.findall('./grids/grid')
-            nodeclass=Cime5GridNode
+            nodeclass = Cime5GridNode
 
         for xmlnode in xmlnodes:
             datanode = nodeclass(self.root)
             datanode.set_data(xmlnode)
             self.nodes.append(datanode)
 
-    def writexml(self,addlist,newfilename):
+    def writexml(self, addlist, newfilename):
         root = ET.Element('grid_data')
-        grids = ET.SubElement(root,'grids')
+        grids = ET.SubElement(root, 'grids')
         for a, b in addlist:
             if b is not None:
                 grids.append(ET.Element('REPLACE'))
@@ -320,26 +325,29 @@ def writexml(self,addlist,newfilename):
                 grids.append(a.to_cime5())
         xmllint = find_executable("xmllint")
         if xmllint is not None:
-            run_cmd_no_fail("%s --format --output %s -"%(xmllint,newfilename),
+            run_cmd_no_fail("%s --format --output %s -"%(xmllint, newfilename),
                     input_str=ET.tostring(root))
 
 
 class DomainTree(DataTree):
     def populate(self):
+        if self.root is None:
+            return
+
         xmlnodes = self.root.findall('gridhorz')
-        nodeclass=Cime2DomainNode
+        nodeclass = Cime2DomainNode
         if len(xmlnodes) == 0:
             xmlnodes = self.root.findall('./domains/domain')
-            nodeclass=Cime5DomainNode
+            nodeclass = Cime5DomainNode
 
         for node in xmlnodes:
             datanode = nodeclass(self.root)
             datanode.set_data(node)
             self.nodes.append(datanode)
 
-    def writexml(self,addlist,newfilename):
+    def writexml(self, addlist, newfilename):
         root = ET.Element('grid_data')
-        domains = ET.SubElement(root,'domains')
+        domains = ET.SubElement(root, 'domains')
         for a, b in addlist:
             if b is not None:
                 domains.append(ET.Element('REPLACE'))
@@ -349,11 +357,13 @@ def writexml(self,addlist,newfilename):
                 domains.append(a.to_cime5())
         xmllint = find_executable("xmllint")
         if xmllint is not None:
-            run_cmd_no_fail("%s --format --output %s -"%(xmllint,newfilename),
-                    input_str=ET.tostring(root))
+            run_cmd_no_fail("%s --format --output %s -" % (xmllint, newfilename),
+                            input_str=ET.tostring(root))
 
 class GridmapTree(DataTree):
     def populate(self):
+        if self.root is None:
+            return
         xmlnodes = self.root.findall('gridmap')
         if len(xmlnodes) == 0:
             xmlnodes = self.root.findall('./gridmaps/gridmap')
@@ -362,9 +372,9 @@ def populate(self):
             datanode.set_data(xmlnode)
             self.nodes.append(datanode)
 
-    def writexml(self,addlist,newfilename):
+    def writexml(self, addlist, newfilename):
         root = ET.Element('gridmaps')
-        gridmaps = ET.SubElement(root,'gridmap')
+        gridmaps = ET.SubElement(root, 'gridmap')
         for a, b in addlist:
             if b is not None:
                 gridmaps.append(ET.Element('REPLACE'))
@@ -374,8 +384,8 @@ def writexml(self,addlist,newfilename):
                 gridmaps.append(a.to_cime5())
         xmllint = find_executable("xmllint")
         if xmllint is not None:
-            run_cmd_no_fail("%s --format --output %s -"%(xmllint,newfilename),
-                    input_str=ET.tostring(root))
+            run_cmd_no_fail("%s --format --output %s -" % (xmllint, newfilename),
+                            input_str=ET.tostring(root))
 
 def diff_tree(atree, btree):
     afound = []
@@ -401,15 +411,15 @@ def diff_tree(atree, btree):
                 bfound.append(bnode)
 
                 if anode == bnode:
-                    oklist.append([anode,bnode])
+                    oklist.append([anode, bnode])
                 else:
-                    fixlist.append([anode,bnode])
+                    fixlist.append([anode, bnode])
                 break
 
         if anode in afound:
             continue
 
-        addlist.append([anode,None])
+        addlist.append([anode, None])
 
 
 
@@ -419,7 +429,7 @@ def diff_tree(atree, btree):
     logger.info("Number of duplicate nodes: %d" % len(duplist))
     for dup in duplist:
         logger.info(dup)
-    return [oklist,fixlist,addlist]
+    return [oklist, fixlist, addlist]
 
 
 
@@ -447,5 +457,5 @@ def grid_compare():
     cime5gridmaptree.postprocess(fixlist, addlist, "tempgridmap.xml",
                                  cime5file, "badgridmap.xml")
 
-if __name__=="__main__":
+if __name__ == "__main__":
     grid_compare()


Test patch: