Repository: ESMCI/cime

Issue:
Adding user_datm.streams.txt.* files erases the stream files from CASEROOT/Buildconf/datm.input_data_list after build
To me this seams the same or very similar to issue #916. This occurs in cesm release 2.1.1, so I created a new issue.
Briefly, when adding user_datm.stream files under CaseRoot/, those streams are erased from the Buildconf/ after build, and not replaced. Checking/downloading input data then does not work.

Steps to reproduce:
```
cd $CESMDIR
git clone https://github.com/ESCOMP/cesm.git cesm-test
cd cesm-test
git checkout release-cesm2.1.1
./manage_externals/checkout_externals
./cime/scripts/create_newcase --case "$CESMDIR"/cases/test1 --res f09_g17 --compset IHistClm50Sp --run-unsupported
cd "$CESMDIR"/cases/test1
./case.setup
./preview_namelists
cp CaseDocs/datm.streams.txt.CLMGSWP3v1.* ./
rename datm user_datm datm.stream*
./case.build
cat Buildconf/datm.input_data_list # all datm streams are missing
```



Discussion:
@mvertens can we discuss this issue when you have time?
After a conversation between @mvertens and @jedwards4b - the following has been suggested:
- created a temporary xml file in memory that is a copy of the user stream file - but that has the right header and footer to be recognized as xml. Then this could be parsed to generated the inputdata list.
In fact - this approach could be used for all of the stream data files - not just the user stream files.

Resolving patch:
diff --git a/scripts/lib/CIME/XML/stream.py b/scripts/lib/CIME/XML/stream.py
new file mode 100644
--- /dev/null
+++ b/scripts/lib/CIME/XML/stream.py
@@ -0,0 +1,49 @@
+"""
+Interface to the streams.xml style files.  This class inherits from GenericXML.py
+
+stream files predate cime and so do not conform to entry id format
+"""
+from CIME.XML.standard_module_setup import *
+from CIME.XML.generic_xml import GenericXML
+from CIME.XML.files import Files
+from CIME.utils import expect
+
+logger = logging.getLogger(__name__)
+
+class Stream(GenericXML):
+
+    def __init__(self, infile=None, files=None):
+        """
+        initialize an object
+        """
+        if files is None:
+            files = Files()
+        schema = None
+        GenericXML.__init__(self, infile, schema=schema)
+
+    def get_value(self, item, attribute=None, resolved=True, subgroup=None):
+        """
+        Get Value of fields in a stream.xml file
+        """
+        expect(subgroup is None, "This class does not support subgroups")
+        value = None
+        node = None
+        names = item.split('/')
+        node = None
+        for name in names:
+            node = self.scan_child(name, root=node)
+        if node is not None:
+            value = self.text(node).strip()
+
+        if value is None:
+            # if all else fails
+            #pylint: disable=assignment-from-none
+            value = GenericXML.get_value(self, item, attribute, resolved, subgroup)
+
+        if resolved:
+            if value is not None:
+                value = self.get_resolved_value(value)
+            elif item in os.environ:
+                value = os.environ[item]
+
+        return value
diff --git a/scripts/lib/CIME/case/case_setup.py b/scripts/lib/CIME/case/case_setup.py
--- a/scripts/lib/CIME/case/case_setup.py
+++ b/scripts/lib/CIME/case/case_setup.py
@@ -71,7 +71,7 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False):
     din_loc_root = case.get_value("DIN_LOC_ROOT")
     testcase     = case.get_value("TESTCASE")
     expect(not (not os.path.isdir(din_loc_root) and testcase != "SBN"),
-           "inputdata root is not a directory: {}".format(din_loc_root))
+           "inputdata root is not a directory or is not readable: {}".format(din_loc_root))
 
     # Remove batch scripts
     if reset or clean:
diff --git a/scripts/lib/CIME/nmlgen.py b/scripts/lib/CIME/nmlgen.py
--- a/scripts/lib/CIME/nmlgen.py
+++ b/scripts/lib/CIME/nmlgen.py
@@ -15,7 +15,8 @@
     character_literal_to_string, string_to_character_literal, \
     expand_literal_list, compress_literal_list, merge_literal_lists
 from CIME.XML.namelist_definition import NamelistDefinition
-from CIME.utils import expect
+from CIME.utils import expect, safe_copy
+from CIME.XML.stream import Stream
 
 logger = logging.getLogger(__name__)
 
@@ -23,7 +24,8 @@
 
 _ymd_re = re.compile(r"%(?P<digits>[1-9][0-9]*)?y(?P<month>m(?P<day>d)?)?")
 
-_stream_file_template = """
+_stream_file_template = """<?xml version="1.0"?>
+<file id="stream" version="1.0">
 <dataSource>
    GENERIC
 </dataSource>
@@ -52,6 +54,7 @@
       {offset}
    </offset>
 </fieldInfo>
+</file>
 """
 
 class NamelistGenerator(object):
@@ -412,7 +415,7 @@ def _sub_paths(self, filenames, year_start, year_end):
                     new_lines.append(new_line)
         return "\n".join(new_lines)
 
-    def create_stream_file_and_update_shr_strdata_nml(self, config, #pylint:disable=too-many-locals
+    def create_stream_file_and_update_shr_strdata_nml(self, config, caseroot, #pylint:disable=too-many-locals
                            stream, stream_path, data_list_path):
         """Write the pseudo-XML file corresponding to a given stream.
 
@@ -426,48 +429,62 @@ def create_stream_file_and_update_shr_strdata_nml(self, config, #pylint:disable=
         `data_list_path` - Path of file to append input data information to.
         """
 
-        # Stream-specific configuration.
+        if os.path.exists(stream_path):
+            os.unlink(stream_path)
+        user_stream_path = os.path.join(caseroot, "user_"+os.path.basename(stream_path))
+
+        # Use the user's stream file, or create one if necessary.
         config = config.copy()
         config["stream"] = stream
 
-        # Figure out the details of this stream.
-        if stream in ("prescribed", "copyall"):
-            # Assume only one file for prescribed mode!
-            grid_file = self.get_default("strm_grid_file", config)
-            domain_filepath, domain_filenames = os.path.split(grid_file)
-            data_file = self.get_default("strm_data_file", config)
-            data_filepath, data_filenames = os.path.split(data_file)
+
+        # Stream-specific configuration.
+        if os.path.exists(user_stream_path):
+            safe_copy(user_stream_path, stream_path)
+            strmobj = Stream(infile=stream_path)
+            domain_filepath = strmobj.get_value("domainInfo/filePath")
+            data_filepath = strmobj.get_value("fieldInfo/filePath")
+            domain_filenames = strmobj.get_value("domainInfo/fileNames")
+            data_filenames = strmobj.get_value("fieldInfo/fileNames")
         else:
-            domain_filepath = self.get_default("strm_domdir", config)
-            domain_filenames = self.get_default("strm_domfil", config)
-            data_filepath = self.get_default("strm_datdir", config)
-            data_filenames = self.get_default("strm_datfil", config)
-
-        domain_varnames = self._sub_fields(self.get_default("strm_domvar", config))
-        data_varnames = self._sub_fields(self.get_default("strm_datvar", config))
-        offset = self.get_default("strm_offset", config)
-        year_start = int(self.get_default("strm_year_start", config))
-        year_end = int(self.get_default("strm_year_end", config))
-        data_filenames = self._sub_paths(data_filenames, year_start, year_end)
-        domain_filenames = self._sub_paths(domain_filenames, year_start, year_end)
-
-        # Overwrite domain_file if should be set from stream data
-        if domain_filenames == 'null':
-            domain_filepath = data_filepath
-            domain_filenames = data_filenames.splitlines()[0]
-
-        stream_file_text = _stream_file_template.format(
-            domain_varnames=domain_varnames,
-            domain_filepath=domain_filepath,
-            domain_filenames=domain_filenames,
-            data_varnames=data_varnames,
-            data_filepath=data_filepath,
-            data_filenames=data_filenames,
-            offset=offset,
-        )
-
-        with open(stream_path, 'w') as stream_file:
-            stream_file.write(stream_file_text)
+            # Figure out the details of this stream.
+            if stream in ("prescribed", "copyall"):
+                # Assume only one file for prescribed mode!
+                grid_file = self.get_default("strm_grid_file", config)
+                domain_filepath, domain_filenames = os.path.split(grid_file)
+                data_file = self.get_default("strm_data_file", config)
+                data_filepath, data_filenames = os.path.split(data_file)
+            else:
+                domain_filepath = self.get_default("strm_domdir", config)
+                domain_filenames = self.get_default("strm_domfil", config)
+                data_filepath = self.get_default("strm_datdir", config)
+                data_filenames = self.get_default("strm_datfil", config)
+
+            domain_varnames = self._sub_fields(self.get_default("strm_domvar", config))
+            data_varnames = self._sub_fields(self.get_default("strm_datvar", config))
+            offset = self.get_default("strm_offset", config)
+            year_start = int(self.get_default("strm_year_start", config))
+            year_end = int(self.get_default("strm_year_end", config))
+            data_filenames = self._sub_paths(data_filenames, year_start, year_end)
+            domain_filenames = self._sub_paths(domain_filenames, year_start, year_end)
+
+            # Overwrite domain_file if should be set from stream data
+            if domain_filenames == 'null':
+                domain_filepath = data_filepath
+                domain_filenames = data_filenames.splitlines()[0]
+
+            stream_file_text = _stream_file_template.format(
+                domain_varnames=domain_varnames,
+                domain_filepath=domain_filepath,
+                domain_filenames=domain_filenames,
+                data_varnames=data_varnames,
+                data_filepath=data_filepath,
+                data_filenames=data_filenames,
+                offset=offset,
+            )
+
+            with open(stream_path, 'w') as stream_file:
+                stream_file.write(stream_file_text)
 
         lines_hash = self._get_input_file_hash(data_list_path)
         with open(data_list_path, 'a') as input_data_list:


Test patch: