Repository: ESMCI/cime

Issue:
Automated input data download does not work for multi-instance runs
I recently found out that scripts to download missing input data do not work for multi-instance runs. In my multi-instance run, I had a different `ncdata` file for each instance. Launching `check_input_data` downloads only the `ncdata` file for the last instance. I looked into the script and it looks like `check_input_data` script download files mentioned in  `*.input_data_list` (e.g. `cam.input_data_list`) and `*.input_data_list`  file captures only the file from the last instance. 

I am not sure what would be the best way to solve this issue. One solution that I can think of is to add a "ninst" (total number of instances) loop which creates `*.input_data_list` files for each instance and then `check_input_data` scripts go through all these files to download missing data.


Discussion:

Resolving patch:
diff --git a/scripts/lib/CIME/nmlgen.py b/scripts/lib/CIME/nmlgen.py
--- a/scripts/lib/CIME/nmlgen.py
+++ b/scripts/lib/CIME/nmlgen.py
@@ -8,6 +8,7 @@
 
 import datetime
 import re
+import hashlib
 
 from CIME.XML.standard_module_setup import *
 from CIME.namelist import Namelist, parse, \
@@ -467,17 +468,25 @@ def create_stream_file_and_update_shr_strdata_nml(self, config, #pylint:disable=
 
         with open(stream_path, 'w') as stream_file:
             stream_file.write(stream_file_text)
+
+        lines_hash = self._get_input_file_hash(data_list_path)
         with open(data_list_path, 'a') as input_data_list:
             for i, filename in enumerate(domain_filenames.split("\n")):
                 if filename.strip() == '':
                     continue
                 filepath = os.path.join(domain_filepath, filename.strip())
-                input_data_list.write("domain{:d} = {}\n".format(i+1, filepath))
+                string = "domain{:d} = {}\n".format(i+1, filepath)
+                hashValue = hashlib.md5(string.rstrip().encode('utf-8')).hexdigest()
+                if hashValue not in lines_hash:
+                    input_data_list.write(string)
             for i, filename in enumerate(data_filenames.split("\n")):
                 if filename.strip() == '':
                     continue
                 filepath = os.path.join(data_filepath, filename.strip())
-                input_data_list.write("file{:d} = {}\n".format(i+1, filepath))
+                string = "file{:d} = {}\n".format(i+1, filepath)
+                hashValue = hashlib.md5(string.rstrip().encode('utf-8')).hexdigest()
+                if hashValue not in lines_hash:
+                    input_data_list.write(string)
         self.update_shr_strdata_nml(config, stream, stream_path)
 
     def update_shr_strdata_nml(self, config, stream, stream_path):
@@ -589,39 +598,57 @@ def create_shr_strdata_nml(self):
     def get_group_variables(self, group_name):
         return self._namelist.get_group_variables(group_name)
 
-
-    def _write_input_files(self, input_data_list):
+    def _get_input_file_hash(self, data_list_path):
+        lines_hash = set()
+        if os.path.isfile(data_list_path):
+            with open(data_list_path, "r") as input_data_list:
+                for line in input_data_list:
+                    hashValue = hashlib.md5(line.rstrip().encode('utf-8')).hexdigest()
+                    logger.debug( "Found line {} with hash {}".format(line,hashValue))
+                    lines_hash.add(hashValue)
+        return lines_hash
+
+    def _write_input_files(self, data_list_path):
         """Write input data files to list."""
-        for group_name in self._namelist.get_group_names():
-            for variable_name in self._namelist.get_variable_names(group_name):
-                input_pathname = self._definition.get_node_element_info(variable_name, "input_pathname")
-                if input_pathname is not None:
-                    # This is where we end up for all variables that are paths
-                    # to input data files.
-                    literals = self._namelist.get_variable_value(group_name, variable_name)
-                    for literal in literals:
-                        file_path = character_literal_to_string(literal)
-                        # NOTE - these are hard-coded here and a better way is to make these extensible
-                        if file_path == 'UNSET' or file_path == 'idmap':
-                            continue
-                        if input_pathname == 'abs':
-                            # No further mangling needed for absolute paths.
-                            # At this point, there are overwrites that should be ignored
-                            if not os.path.isabs(file_path):
+        # append to input_data_list file
+        lines_hash = self._get_input_file_hash(data_list_path)
+        with open(data_list_path, "a") as input_data_list:
+            for group_name in self._namelist.get_group_names():
+                for variable_name in self._namelist.get_variable_names(group_name):
+                    input_pathname = self._definition.get_node_element_info(variable_name, "input_pathname")
+                    if input_pathname is not None:
+                        # This is where we end up for all variables that are paths
+                        # to input data files.
+                        literals = self._namelist.get_variable_value(group_name, variable_name)
+                        for literal in literals:
+                            file_path = character_literal_to_string(literal)
+                            # NOTE - these are hard-coded here and a better way is to make these extensible
+                            if file_path == 'UNSET' or file_path == 'idmap':
                                 continue
+                            if input_pathname == 'abs':
+                                # No further mangling needed for absolute paths.
+                                # At this point, there are overwrites that should be ignored
+                                if not os.path.isabs(file_path):
+                                    continue
+                                else:
+                                    pass
+                            elif input_pathname.startswith('rel:'):
+                                # The part past "rel" is the name of a variable that
+                                # this variable specifies its path relative to.
+                                root_var = input_pathname[4:]
+                                root_dir = self.get_value(root_var)
+                                file_path = os.path.join(root_dir, file_path)
+                            else:
+                                expect(False,
+                                       "Bad input_pathname value: {}.".format(input_pathname))
+                            # Write to the input data list.
+                            string = "{} = {}".format(variable_name, file_path)
+                            hashValue = hashlib.md5(string.rstrip().encode('utf-8')).hexdigest()
+                            if hashValue not in lines_hash:
+                                logger.debug("Adding line {} with hash {}".format(string,hashValue))
+                                input_data_list.write(string+"\n")
                             else:
-                                pass
-                        elif input_pathname.startswith('rel:'):
-                            # The part past "rel" is the name of a variable that
-                            # this variable specifies its path relative to.
-                            root_var = input_pathname[4:]
-                            root_dir = self.get_value(root_var)
-                            file_path = os.path.join(root_dir, file_path)
-                        else:
-                            expect(False,
-                                   "Bad input_pathname value: {}.".format(input_pathname))
-                        # Write to the input data list.
-                        input_data_list.write("{} = {}\n".format(variable_name, file_path))
+                                logger.debug("Line already in file {}".format(string))
 
     def write_output_file(self, namelist_file, data_list_path=None, groups=None, sorted_groups=True):
         """Write out the namelists and input data files.
@@ -645,9 +672,7 @@ def write_output_file(self, namelist_file, data_list_path=None, groups=None, sor
         self._namelist.write(namelist_file, groups=groups, sorted_groups=sorted_groups)
 
         if data_list_path is not None:
-            # append to input_data_list file
-            with open(data_list_path, "a") as input_data_list:
-                self._write_input_files(input_data_list)
+            self._write_input_files(data_list_path)
 
     def add_nmlcontents(self, filename, group, append=True, format_="nmlcontents", sorted_groups=True):
         """ Write only contents of nml group """


Test patch: