Repository: ESMCI/cime

Issue:
bug when requesting multiple variables from xmlquery 
There is a slight problem with this in that it seems to be a little buggy:
```#!/usr/bin/env perl
use strict;

my ($a, $b, $c);

$a = `./xmlquery NTASKS_ATM --value`;
$b = `./xmlquery NTASKS_LND --value`;
$c = `./xmlquery NTASKS_OCN --value`;

print "1: $a<>$b<>$c\n";

$a, $b, $c = `./xmlquery NTASKS_ATM,NTASKS_LND,NTASKS_OCN --value`;

print "2: $a<>$b<>$c\n"
```
 ```
perl /glade/scratch/jedwards/tmp.pl
1: 1152<>864<>36
2: 1152<>864<>
Results in group mach_pes
115286436```



Discussion:

Resolving patch:
diff --git a/doc/tools_autodoc.py b/doc/tools_autodoc.py
--- a/doc/tools_autodoc.py
+++ b/doc/tools_autodoc.py
@@ -18,19 +18,72 @@
 
 import argparse
 import os
-import traceback
+import re
+import stat
 import shutil
 from string import Template
+import traceback
 
 if sys.version_info[0] == 2:
     from ConfigParser import SafeConfigParser as config_parser
 else:
     from configparser import ConfigParser as config_parser
 
+# define rst templates
+_tool_template = Template('''
+.. _$tool_name:
+
+####################################################
+$tool_name 
+####################################################
+
+**$tool_name** is a script in CIMEROOT/scripts/Tools.
+
+.. toctree::
+   :maxdepth: 1
+
+.. command-output:: ./$tool_name --help
+   :cwd: ../../$tools_dir
+''')
+
+_script_template = Template('''
+.. _$script_name:
+
+####################################################
+$script_name 
+####################################################
+
+**$script_name** is a script in CIMEROOT/scripts.
+
+.. toctree::
+   :maxdepth: 1
+
+.. command-output:: ./$script_name --help
+   :cwd: ../../$scripts_dir
+''')
+
+_tmpl_template = Template('''
+.. _$tmpl_name:
+
+####################################################
+$tmpl_name 
+####################################################
+
+**$tmpl_name** is a script template in CIMEROOT/config/cesm/machines
+that is dynamically created in the CASEROOT. Note that scripts
+**.case.run** and **.case.test** are hidden files and should never
+be run manually.
+
+.. toctree::
+   :maxdepth: 1
+
+.. command-output:: ./$tmpl_name --help
+   :cwd: ./temp_files
+''')
+
+
 # -------------------------------------------------------------------------------
-#
 # User input
-#
 # -------------------------------------------------------------------------------
 
 def commandline_options():
@@ -54,7 +107,9 @@ def commandline_options():
     return options
 
 
-
+# -------------------------------------------------------------------------------
+# read the tools_autodoc.cfg configuration file
+# -------------------------------------------------------------------------------
 def read_config_file(filename):
     """Read the configuration file and process
 
@@ -70,19 +125,15 @@ def read_config_file(filename):
 
     return config
 
-
 # -------------------------------------------------------------------------------
-#
-# main
-#
+# create the rst files for the Tools configuration settings
 # -------------------------------------------------------------------------------
+def get_tools(config, doc_dir):
 
-def main(options):
-    config = read_config_file(options.config[0])
-
+    # get the input tools dir
     tools_dir = config.get('tools','tools_dir')
     tools_dir = os.path.abspath(tools_dir)
-
+    
     # get list of files to exclude
     exclude_files = config.get('tools','exclude_files').split()
 
@@ -112,6 +163,157 @@ def main(options):
         if include:
             tools_files.append(f)
 
+    tools_dir = config.get('tools','tools_dir')
+    for f in tools_files:
+        tool_file = os.path.join(doc_dir, '{0}.rst'.format(f))
+        with open(tool_file,'w') as tf:
+            contents = _tool_template.substitute(tool_name=f, tools_dir=tools_dir)
+            tf.write(contents)
+
+    return tools_files
+
+
+# -------------------------------------------------------------------------------
+# create the rst files for the scripts configuration settings
+# -------------------------------------------------------------------------------
+def get_scripts(config, doc_dir):
+
+    # get the input scripts dir
+    scripts_dir = config.get('scripts','scripts_dir')
+    scripts_dir = os.path.abspath(scripts_dir)
+
+    # get list of files to exclude
+    exclude_files = config.get('scripts','exclude_files').split()
+
+    # get list of files to exclude
+    exclude_ext = config.get('scripts','exclude_ext').split()
+
+    # get list of files to exclude
+    exclude_prefix = config.get('scripts','exclude_prefix').split()
+
+    # get a list of all files in the scripts_dir
+    all_files = next(os.walk(scripts_dir))[2]
+
+    scripts_files = list()
+    # exclude files 
+    for f in all_files:
+        f = f.strip()
+        include = True
+        for e in exclude_files:
+            if f == e.strip():
+                include = False
+        for e in exclude_ext:
+            if f.endswith(e.strip()):
+                include = False
+        for e in exclude_prefix:
+            if f.startswith(e.strip()):
+                include = False
+        if include:
+            scripts_files.append(f)
+
+    scripts_dir = config.get('scripts','scripts_dir')
+    for f in scripts_files:
+        script_file = os.path.join(doc_dir, '{0}.rst'.format(f))
+        with open(script_file,'w') as tf:
+            contents = _script_template.substitute(script_name=f, scripts_dir=scripts_dir)
+            tf.write(contents)
+
+    return scripts_files
+
+# -------------------------------------------------------------------------------
+# get the template files and substitute the {{...}} patterns so they can be
+# run with the --help command
+# -------------------------------------------------------------------------------
+def get_templates(config, doc_dir):
+
+    # get the input template dir 
+    templates_dir = config.get('templates','templates_dir')
+    templates_dir = os.path.abspath(templates_dir)
+
+    # get list of files to exclude
+    exclude_files = config.get('templates','exclude_files').split()
+
+    # get list of files to exclude
+    exclude_ext = config.get('templates','exclude_ext').split()
+
+    # get list of files to exclude
+    exclude_prefix = config.get('templates','exclude_prefix').split()
+
+    # get a list of all files in the templates_dir
+    all_files = next(os.walk(templates_dir))[2]
+
+    template_files = list()
+    # exclude files 
+    for f in all_files:
+        f = f.strip()
+        include = True
+        for e in exclude_files:
+            if f == e.strip():
+                include = False
+        for e in exclude_ext:
+            if f.endswith(e.strip()):
+                include = False
+        for e in exclude_prefix:
+            if f.startswith(e.strip()):
+                include = False
+        if include:
+            template_files.append(f)
+
+    # create temporary files with the {{..}} stripped out
+    temp_files = list()
+    temp_dir = '{0}/temp_files'.format(doc_dir)
+    if not os.path.exists(temp_dir):
+        os.makedirs(temp_dir)
+
+    for fname in template_files:
+        with open(os.path.join(templates_dir,fname),'r') as f:
+            content = f.read()
+        content = content.replace("{{ batchdirectives }}", "# {{ batchdirective }}", 1)
+        content = content.replace("os.chdir( '{{ caseroot }}')", "# os.chdir( '{{ caseroot }}')", 1)
+        content = content.replace('os.path.join("{{ cimeroot }}", "scripts", "Tools")', 'os.path.join("../../../..","scripts", "Tools")',1)
+        # create a temporary file
+        tf = fname.split('.')
+        tfname = '.'.join(tf[1:])
+        if tfname == 'st_archive':
+            tfname = 'case.st_archive'
+        tfile = os.path.join(temp_dir, tfname)
+        with open(tfile, 'w') as tf:
+            tf.write(content)
+        
+        temp_files.append(tfname)
+
+    for f in temp_files:
+        tmpl_file = os.path.join(doc_dir, '{0}.rst'.format(f))
+        with open(tmpl_file,'w') as tf:
+            contents = _tmpl_template.substitute(tmpl_name=f, temp_dir=temp_dir)
+            tf.write(contents)
+            tf.close()
+        exefile = os.path.join(doc_dir, 'temp_files/{0}'.format(f))
+        st = os.stat(exefile)
+        os.chmod(exefile, st.st_mode | stat.S_IEXEC)
+
+    return temp_files
+
+# -------------------------------------------------------------------------------
+# main
+# -------------------------------------------------------------------------------
+
+def main(options):
+    all_files = list()
+    config = read_config_file(options.config[0])
+
+    # get the output doc dir
+    doc_dir = config.get('doc','doc_dir')
+    doc_dir = os.path.abspath(doc_dir)
+
+    # gather the files from different locations in the CIMEROOT
+    tools_files = get_tools(config, doc_dir)
+    scripts_files = get_scripts(config, doc_dir)
+    template_files = get_templates(config, doc_dir)
+
+    all_files = tools_files + scripts_files + template_files
+    all_files.sort()
+
     # copy the index.rst.template to index.rst
     doc_dir = config.get('doc','doc_dir')
     doc_dir = os.path.abspath(doc_dir)
@@ -126,32 +328,9 @@ def main(options):
 
     # open index_rst_file in append mode
     with open(index_rst_file,'a') as index_rst:
-        for f in tools_files:
+        for f in all_files:
             index_rst.write('   {0}\n'.format(f))
 
-    tool_template = Template('''
-.. _$tool_name:
-
-####################################################
-$tool_name 
-####################################################
-
-**$tool_name** is a script in CIMEROOT/scripts/Tools.
-
-.. toctree::
-   :maxdepth: 1
-
-.. command-output:: ./$tool_name --help
-   :cwd: ../../$tools_dir
-''')
-
-    tools_dir = config.get('tools','tools_dir')
-    for f in tools_files:
-        tool_file = os.path.join(doc_dir, '{0}.rst'.format(f))
-        with open(tool_file,'w') as tf:
-            contents = tool_template.substitute(tool_name=f, tools_dir=tools_dir)
-            tf.write(contents)
-
     return 0
 
 


Test patch:
diff --git a/config/cesm/machines/template.case.test b/config/cesm/machines/template.case.test
--- a/config/cesm/machines/template.case.test
+++ b/config/cesm/machines/template.case.test
@@ -22,20 +22,8 @@ import argparse
 ###############################################################################
 def parse_command_line(args, description):
 ###############################################################################
-    parser = argparse.ArgumentParser(
-        usage="""\n{0} [<testname>] [--verbose]
-OR
-{0} --help
-
-\033[1mEXAMPLES:\033[0m
-    \033[1;32m# case.test SMS\033[0m
-    > {0}
-""".format(os.path.basename(args[0])),
-
-description=description,
-
-formatter_class=argparse.ArgumentDefaultsHelpFormatter
-)
+    parser = argparse.ArgumentParser(description=description,
+                                     formatter_class=argparse.RawTextHelpFormatter)
 
     CIME.utils.setup_standard_logging_options(parser)