Repository: ESMCI/cime

Issue:
Issue with namelist compare test
compare_namelist.py does not like the following legal cam namelist setting:

 csw_specifier          = 'DMS      -> 1.0 * /glade/p/cesmdata/cseg/inputdata/atm/cam/chem/ocnexch/Csw_DMS_HiResML_f09f09_1990_2019_20190616a.nc',
         'CH3COCH3 -> 0.75* /glade/p/cesmdata/cseg/inputdata/atm/cam/chem/ocnexch/Csw_Acetone_HiResML_f09f09_1990_2019_20190814a.nc'

Seems to be an issue with the scaling factor.
This is the traceback from my test:
---------------------------------------------------
2020-06-10 14:18:29: Exception during namelist operations:
ERROR: In file '/glade/scratch/fvitt/SMS_D_Ln9.f09_f09_mg17.FCHIST.cheyenne_intel.cam-outfrq9s_ocnemis.C.mytests_test/CaseDocs/atm_in', Dict entry ''DMS      -> 1.0 * /glade/p/cesmdata/cse
g/inputdata/atm/cam/chem/ocnexch/Csw_DMS_HiResML_f09f09_1990_2019_20190616a.nc'' does not match expected format
Traceback (most recent call last):
  File "/glade/work/fvitt/camdev/chem_cam6_2_032/cime/scripts/Tools/../../scripts/lib/CIME/case/case_cmpgen_namelists.py", line 123, in case_cmpgen_namelists
    success, output = _do_full_nl_comp(self, test_name, compare_name, baseline_root)
  File "/glade/work/fvitt/camdev/chem_cam6_2_032/cime/scripts/Tools/../../scripts/lib/CIME/case/case_cmpgen_namelists.py", line 46, in _do_full_nl_comp
    elif is_namelist_file(item):
  File "/glade/work/fvitt/camdev/chem_cam6_2_032/cime/scripts/Tools/../../scripts/lib/CIME/compare_namelists.py", line 548, in is_namelist_file
    assert "does not appear to be a namelist file" in str(e), str(e)
AssertionError: ERROR: In file '/glade/scratch/fvitt/SMS_D_Ln9.f09_f09_mg17.FCHIST.cheyenne_intel.cam-outfrq9s_ocnemis.C.mytests_test/CaseDocs/atm_in', Dict entry ''DMS      -> 1.0 * /glade/p/cesmdata/cseg/inputdata/atm/cam/chem/ocnexch/Csw_DMS_HiResML_f09f09_1990_2019_20190616a.nc'' does not match expected format

 ---------------------------------------------------



Discussion:
@fvitt If you remove the whitespaces between the scaling factor and path this will work e.g. `DMS -> 1.0*/glade/p/cesmdata/cseg/inputdata/atm/cam/chem/ocnexch/Csw_DMS_HiResML_f09f09_1990_2019_20190616a.nc`

If those whitespaces are required then I can put in a fix otherwise it might just be worth making the error message more verbose.
In my opinion the compare should not fail if there are white spaces surrounding the * character.

That makes sense, I'll check to see if there are any other value formats we missed.

Resolving patch:
diff --git a/scripts/lib/CIME/compare_namelists.py b/scripts/lib/CIME/compare_namelists.py
--- a/scripts/lib/CIME/compare_namelists.py
+++ b/scripts/lib/CIME/compare_namelists.py
@@ -20,7 +20,19 @@ def _normalize_lists(value_str):
     "'one two'"
     >>> _normalize_lists("1 2  3, 4 ,  5")
     '1,2,3,4,5'
+    >>> _normalize_lists("2, 2*13")
+    '2,2*13'
+    >>> _normalize_lists("'DMS -> 1.0 * value.nc'")
+    "'DMS -> 1.0 * value.nc'"
+    >>> _normalize_lists("1.0* value.nc")
+    '1.0*value.nc'
+    >>> _normalize_lists("1.0*value.nc")
+    '1.0*value.nc'
     """
+    # Handle special case "value * value" which should not be treated as list
+    parsed = re.match(r"^([^*=->\s]*)\s*(\*)\s*(.*)$", value_str)
+    if parsed is not None:
+        value_str = "".join(parsed.groups())
     result = ""
     inside_quotes = False
     idx = 0
@@ -51,8 +63,22 @@ def _normalize_lists(value_str):
 ###############################################################################
 def _interpret_value(value_str, filename):
 ###############################################################################
+    """
+    >>> _interpret_value("one", "foo")
+    'one'
+    >>> _interpret_value("one, two", "foo")
+    ['one', 'two']
+    >>> _interpret_value("3*1.0", "foo")
+    ['1.0', '1.0', '1.0']
+    >>> _interpret_value("'DMS -> value.nc'", "foo")
+    OrderedDict([('DMS', 'value.nc')])
+    >>> _interpret_value("'DMS -> 1.0 * value.nc'", "foo")
+    OrderedDict([('DMS', '1.0*value.nc')])
+    >>> _interpret_value("'DMS -> 1.0* value.nc'", "foo")
+    OrderedDict([('DMS', '1.0*value.nc')])
+    """
     comma_re = re.compile(r'\s*,\s*')
-    dict_re = re.compile(r"^'(\S+)\s*->\s*(\S+)\s*'")
+    dict_re = re.compile(r"^'(\S+)\s*->\s*(\S+|(?:\S+\s*\*\s*\S+))\s*'")
 
     value_str = _normalize_lists(value_str)
 
@@ -311,6 +337,13 @@ def _normalize_string_value(name, value, case):
         items = [_normalize_string_value(name, item, case) for item in items]
         return ":".join(items)
     elif ("/" in value):
+        # Handle special format scale*path, normalize the path and reconstruct
+        parsed = re.match(r"^([^*]+\*)(/[^/]+)*", value)
+        if parsed is not None and len(parsed.groups()) == 2:
+            items = list(parsed.groups())
+            items[1] = os.path.basename(items[1])
+            return "".join(items)
+
         # File path, just return the basename unless its a seq_maps.rc mapping
         # mapname or maptype
         if "mapname" not in name and "maptype" not in name:
@@ -487,6 +520,30 @@ def _compare_namelists(gold_namelists, comp_namelists, case):
     ... /'''
     >>> _compare_namelists(_parse_namelists(teststr1.splitlines(), 'foo'), _parse_namelists(teststr2.splitlines(), 'bar'), 'ERB.f19_g16.B1850C5.sandiatoss3_intel')
     ''
+    >>> teststr1 = '''&nml
+    ... csw_specifier = 'DMS -> 1.0 * value.nc'
+    ... /'''
+    >>> _compare_namelists(_parse_namelists(teststr1.splitlines(), 'foo'),\
+    _parse_namelists(teststr1.splitlines(), 'foo'), "case")
+    ''
+    >>> teststr2 = '''&nml
+    ... csw_specifier = 'DMS -> 2.0 * value.nc'
+    ... /'''
+    >>> comments = _compare_namelists(_parse_namelists(teststr1.splitlines(), 'foo'),\
+    _parse_namelists(teststr2.splitlines(), 'foo'), "case")
+    >>> print(comments)
+      BASE: csw_specifier dict item DMS = 1.0*value.nc
+      COMP: csw_specifier dict item DMS = 2.0*value.nc
+    <BLANKLINE>
+    >>> teststr2 = '''&nml
+    ... csw_specifier = 'DMS -> 1.0 * other.nc'
+    ... /'''
+    >>> comments = _compare_namelists(_parse_namelists(teststr1.splitlines(), 'foo'),\
+    _parse_namelists(teststr2.splitlines(), 'foo'), "case")
+    >>> print(comments)
+      BASE: csw_specifier dict item DMS = 1.0*value.nc
+      COMP: csw_specifier dict item DMS = 1.0*other.nc
+    <BLANKLINE>
     """
     different_namelists = OrderedDict()
     for namelist, gold_names in gold_namelists.items():


Test patch: