Repository: ioos/compliance-checker

Issue:
The variable “aps” is referred to in a formula_terms attribute but not defined
Stems from this issue: https://github.com/ioos/compliance-checker/issues/501

https://drive.google.com/open?id=0B6AWgiTrQ0xFQ1J0bFVDNlpyV1k
(2) The variable “aps” is referred to in a formula_terms attribute but not defined: IOOS misses this;


Discussion:
This appears to be occurring because the `standard_name` value of "hybrid_sigma_is not contained in Appendix D:  http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/cf-conventions.html#parametric-v-coord
I also don't see it in the standard name table:
http://cfconventions.org/Data/cf-standard-names/46/build/cf-standard-name-table.html

There is logic present to check that variables in `formula_terms` are checked, but this is only active when the `standard_name` itself is valid.

This appears to be a formula not present in Appendix D.  I also don't see the `formula` attribute in the current CF-1.7 standard.

Resolving patch:
diff --git a/compliance_checker/cf/appendix_d.py b/compliance_checker/cf/appendix_d.py
--- a/compliance_checker/cf/appendix_d.py
+++ b/compliance_checker/cf/appendix_d.py
@@ -22,42 +22,41 @@
 is omitted from the formula_terms attribute should be assumed to be zero.
 '''
 
-# a list of tuples
-# Each tuple contains the standard name followed by a regex for the formula
-# terms
-dimless_vertical_coordinates = [
-    ("atmosphere_ln_pressure_coordinate",
-     # "p0: var1 lev: var2"
-     r'(p0): ([A-Za-z][A-Za-z0-9_]*) (lev): ([A-Za-z][A-Za-z0-9_]*)'),
-    ("atmosphere_sigma_coordinate",
-     # "sigma: var1 ps: var2 ptop: var3"
-     r'(sigma): ([A-Za-z][A-Za-z0-9_]*) (ps): ([A-Za-z][A-Za-z0-9_]*) (ptop): ([A-Za-z][A-Za-z0-9_]*)'),
-    ("atmosphere_hybrid_sigma_pressure_coordinate",
-     # "a: var1 b: var2 ps: var3 p0: var4"
-     r'(a): ([A-Za-z][A-Za-z0-9_]*) (b): ([A-Za-z][A-Za-z0-9_]*) (ps): ([A-Za-z][A-Za-z0-9_]*) (p0): ([A-Za-z][A-Za-z0-9_]*)'),
-    ("atmosphere_hybrid_height_coordinate",
-     # "a: var1 b: var2 orog: var3"
-     r'(a): ([A-Za-z][A-Za-z0-9_]*) (b): ([A-Za-z][A-Za-z0-9_]*) (orog): ([A-Za-z][A-Za-z0-9_]*)'),
-    ("atmosphere_sleve_coordinate",
-     # "a: var1 b1: var2 b2: var3 ztop: var4 zsurf1: var5 zsurf2: var6"
-     r'(a): ([A-Za-z][A-Za-z0-9_]*) (b1): ([A-Za-z][A-Za-z0-9_]*) (b2): ([A-Za-z][A-Za-z0-9_]*) (ztop): ([A-Za-z][A-Za-z0-9_]*) (zsurf1): ([A-Za-z][A-Za-z0-9_]*) (zsurf2): ([A-Za-z][A-Za-z0-9_]*)'),
-    ("ocean_sigma_coordinate",
-     # "sigma: var1 eta: var2 depth: var3"
-     r'(sigma): ([A-Za-z][A-Za-z0-9_]*) (eta): ([A-Za-z][A-Za-z0-9_]*) (depth): ([A-Za-z][A-Za-z0-9_]*)'),
-    ("ocean_s_coordinate",
-     # "s: var1 eta: var2 depth: var3 a: var4 b: var5 depth_c: var6"
-     r'(s): ([A-Za-z][A-Za-z0-9_]*) (eta): ([A-Za-z][A-Za-z0-9_]*) (depth): ([A-Za-z][A-Za-z0-9_]*) (a): ([A-Za-z][A-Za-z0-9_]*) (b): ([A-Za-z][A-Za-z0-9_]*) (depth_c): ([A-Za-z][A-Za-z0-9_]*)'),
-    ("ocean_sigma_z_coordinate",
-     # "sigma: var1 eta: var2 depth: var3 depth_c: var4 nsigma: var5 zlev: var6"
-     r'(sigma): ([A-Za-z][A-Za-z0-9_]*) (eta): ([A-Za-z][A-Za-z0-9_]*) (depth): ([A-Za-z][A-Za-z0-9_]*) (depth_c): ([A-Za-z][A-Za-z0-9_]*) (nsigma): ([A-Za-z][A-Za-z0-9_]*) (zlev): ([A-Za-z][A-Za-z0-9_]*)'),
-    ("ocean_double_sigma_coordinate",
-     # "sigma: var1 depth: var2 z1: var3 z2: var4 a: var5 href: var6 k_c: var7"
-     r'(sigma): ([A-Za-z][A-Za-z0-9_]*) (depth): ([A-Za-z][A-Za-z0-9_]*) (z1): ([A-Za-z][A-Za-z0-9_]*) (z2): ([A-Za-z][A-Za-z0-9_]*) (a): ([A-Za-z][A-Za-z0-9_]*) (href): ([A-Za-z][A-Za-z0-9_]*) (k_c): ([A-Za-z][A-Za-z0-9_]*)'),
+# Contains the standard name followed by the set of expected formula terms
+dimless_vertical_coordinates = {
+    "atmosphere_ln_pressure_coordinate": {'p0', 'lev'},
+    "atmosphere_sigma_coordinate": {'sigma', 'ps', 'ptop'},
+    "atmosphere_hybrid_sigma_pressure_coordinate":
+       ({'a', 'b', 'ps'}, {'ap', 'b', 'ps'}),
+    "atmosphere_hybrid_height_coordinate": {'a', 'b', 'orog'},
+    "atmosphere_sleve_coordinate":
+       {'a', 'b1', 'b2', 'ztop', 'zsurf1', 'zsurf2'},
+    "ocean_sigma_coordinate": {'sigma', 'eta', 'depth'},
+    "ocean_s_coordinate": {'s', 'eta', 'depth', 'a', 'b', 'depth_c'},
+    "ocean_sigma_z_coordinate":
+       {'sigma', 'eta', 'depth', 'depth_c', 'nsigma', 'zlev'},
+    "ocean_double_sigma_coordinate":
+       {'sigma', 'depth', 'z1', 'z2', 'a', 'href', 'k_c'},
     # This comes from CF 1.7 but is used in circulation so we include it
-    ("ocean_s_coordinate_g1",
-     # "s: var1 C: var2 eta: var3 depth: var4 depth_c: var5"
-     r'(s): ([A-Za-z][A-Za-z0-9_]*) (C): ([A-Za-z][A-Za-z0-9_]*) (eta): ([A-Za-z][A-Za-z0-9_]*) (depth): ([A-Za-z][A-Za-z0-9_]*) (depth_c): ([A-Za-z][A-Za-z0-9_]*)'),
-    ("ocean_s_coordinate_g2",
-     # "s: var1 C: var2 eta: var3 depth: var4 depth_c: var5"
-     r'(s): ([A-Za-z][A-Za-z0-9_]*) (C): ([A-Za-z][A-Za-z0-9_]*) (eta): ([A-Za-z][A-Za-z0-9_]*) (depth): ([A-Za-z][A-Za-z0-9_]*) (depth_c): ([A-Za-z][A-Za-z0-9_]*)'),
-]
+    # TODO (badams): include these *only* with the CF 1.7 checker once we get
+    # around to writing it
+    "ocean_s_coordinate_g1": {'s', 'C', 'eta', 'depth', 'depth_c'},
+    "ocean_s_coordinate_g2": {'s', 'C', 'eta', 'depth', 'depth_c'}
+ }
+
+def no_missing_terms(formula_name, term_set):
+    """
+    Returns true if the set is not missing terms corresponding to the
+    entries in Appendix D, False otherwise.  The set of terms should be exactly
+    equal, and not contain more or less terms than expected.
+    """
+    reqd_terms = dimless_vertical_coordinates[formula_name]
+    def has_all_terms(reqd_termset):
+        return len(reqd_termset ^ term_set) == 0
+
+    if isinstance(reqd_terms, set):
+        return has_all_terms(reqd_terms)
+    # if it's not a set, it's likely some other form of iterable with multiple
+    # possible definitions i.e. a/ap are interchangeable in
+    else:
+        return any(has_all_terms(req) for req in reqd_terms)
diff --git a/compliance_checker/cf/cf.py b/compliance_checker/cf/cf.py
--- a/compliance_checker/cf/cf.py
+++ b/compliance_checker/cf/cf.py
@@ -2,7 +2,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals, division, print_function
 from compliance_checker.base import BaseCheck, BaseNCCheck, Result, TestCtx
-from compliance_checker.cf.appendix_d import dimless_vertical_coordinates
+from compliance_checker.cf.appendix_d import (dimless_vertical_coordinates,
+                                              no_missing_terms)
 from compliance_checker.cf.appendix_f import grid_mapping_dict
 from compliance_checker.cf import util
 from compliance_checker import cfutil
@@ -1689,16 +1690,15 @@ def check_dimensional_vertical_coordinate(self, ds):
         '''
         ret_val = []
         z_variables = cfutil.get_z_variables(ds)
-        dimless_standard_names = [name for name, regx in dimless_vertical_coordinates]
+        #dimless_standard_names = [name for name, regx in dimless_vertical_coordinates]
         for name in z_variables:
             variable = ds.variables[name]
             standard_name = getattr(variable, 'standard_name', None)
             units = getattr(variable, 'units', None)
             positive = getattr(variable, 'positive', None)
             # Skip the variable if it's dimensionless
-            if hasattr(variable, 'formula_terms'):
-                continue
-            if standard_name in dimless_standard_names:
+            if (hasattr(variable, 'formula_terms') or
+                standard_name in dimless_vertical_coordinates):
                 continue
 
             valid_vertical_coord = TestCtx(BaseCheck.HIGH,
@@ -1743,7 +1743,6 @@ def check_dimensionless_vertical_coordinate(self, ds):
         '''
         ret_val = []
 
-        dimless = dict(dimless_vertical_coordinates)
         z_variables = cfutil.get_z_variables(ds)
         deprecated_units = [
             'level',
@@ -1756,7 +1755,8 @@ def check_dimensionless_vertical_coordinate(self, ds):
             units = getattr(variable, 'units', None)
             formula_terms = getattr(variable, 'formula_terms', None)
             # Skip the variable if it's dimensional
-            if formula_terms is None and standard_name not in dimless:
+            if (formula_terms is None and
+                standard_name not in dimless_vertical_coordinates):
                 continue
 
             is_not_deprecated = TestCtx(BaseCheck.LOW,
@@ -1783,7 +1783,6 @@ def _check_formula_terms(self, ds, coord):
         :rtype: compliance_checker.base.Result
         '''
         variable = ds.variables[coord]
-        dimless = dict(dimless_vertical_coordinates)
         standard_name = getattr(variable, 'standard_name', None)
         formula_terms = getattr(variable, 'formula_terms', None)
         valid_formula_terms = TestCtx(BaseCheck.HIGH,
@@ -1796,31 +1795,37 @@ def _check_formula_terms(self, ds, coord):
         if not formula_terms:
             return valid_formula_terms.to_result()
 
-        valid_formula_terms.assert_true(standard_name in dimless,
+        # check that the formula_terms are well formed and are present
+        # The pattern for formula terms is always component: variable_name
+        # the regex grouping always has component names in even positions and
+        # the corresponding variable name in even positions.
+        matches = regex.findall(r'([A-Za-z][A-Za-z0-9_]*: )([A-Za-z][A-Za-z0-9_]*)',
+                                variable.formula_terms)
+        terms = set(m[0][:-2] for m in matches)
+        # get the variables named in the formula terms and check if any
+        # are not present in the dataset
+        missing_vars = sorted(set(m[1] for m in matches) - set(ds.variables))
+        missing_fmt = "The following variable(s) referenced in formula_terms are not present in the dataset variables: {}"
+        valid_formula_terms.assert_true(len(missing_vars) == 0,
+                                    missing_fmt.format(', '.join(missing_vars)))
+        # try to reconstruct formula_terms by adding space in between the regex
+        # matches.  If it doesn't exactly match the original, the formatting
+        # of the attribute is incorrect
+        reconstructed_formula = ' '.join(m[0] + m[1] for m in matches)
+        valid_formula_terms.assert_true(reconstructed_formula == formula_terms,
+                                        "Attribute formula_terms is not well-formed")
+
+        valid_formula_terms.assert_true(standard_name in
+                                        dimless_vertical_coordinates,
                                         "unknown standard_name for dimensionless vertical coordinate: {}"
                                         "".format(standard_name))
-        if standard_name not in dimless:
+        if standard_name not in dimless_vertical_coordinates:
             return valid_formula_terms.to_result()
 
-        regx_match = regex.match(dimless[standard_name], formula_terms)
-        valid_formula_terms.assert_true(regx_match is not None,
+        valid_formula_terms.assert_true(no_missing_terms(standard_name, terms),
                                         "formula_terms are invalid for {}, please see appendix D of CF 1.6"
                                         "".format(standard_name))
 
-        if regx_match is None:
-            return valid_formula_terms.to_result()
-
-        # The pattern for formula terms is always component: variable_name
-        # the regex grouping always has component names in even positions and
-        # the corresponding variable name in even positions.
-        match_groups = regx_match.groups()
-
-        for i in range(int(len(match_groups) / 2)):
-            variable_name = match_groups[i * 2 + 1]
-            valid_formula_terms.assert_true(variable_name in ds.variables,
-                                            "variable {} referenced by formula_terms does not exist"
-                                            "".format(variable_name))
-
         return valid_formula_terms.to_result()
 
     def check_time_coordinate(self, ds):


Test patch:
diff --git a/compliance_checker/tests/helpers.py b/compliance_checker/tests/helpers.py
--- a/compliance_checker/tests/helpers.py
+++ b/compliance_checker/tests/helpers.py
@@ -37,5 +37,7 @@ class MockVariable(object):
 
     def __init__(self, copy_var=None):
         if copy_var is not None:
+            self.name = copy_var.name
+            self.dimensions = copy_var.dimensions
             for att in copy_var.ncattrs():
                 setattr(self, att, getattr(copy_var, att))
diff --git a/compliance_checker/tests/test_cf.py b/compliance_checker/tests/test_cf.py
--- a/compliance_checker/tests/test_cf.py
+++ b/compliance_checker/tests/test_cf.py
@@ -9,6 +9,7 @@
 from compliance_checker.tests.resources import STATIC_FILES
 from compliance_checker.tests import BaseTestCase
 from compliance_checker.tests.helpers import MockTimeSeries, MockVariable
+from compliance_checker.cf.appendix_d import no_missing_terms
 
 import os
 import re
@@ -696,32 +697,43 @@ def test_appendix_d(self):
         for that term. The order of elements is not significant.
         '''
 
-        dimless = dict(dimless_vertical_coordinates)
-
-        def verify(std_name, test_str):
-            regex_matches = re.match(dimless[std_name], test_str)
-            self.assertIsNotNone(regex_matches)
-
         # For each of the listed dimensionless vertical coordinates,
-        # verify that the formula_terms match the provided regex
-        verify('atmosphere_ln_pressure_coordinate',
-               "p0: var1 lev: var2")
-        verify('atmosphere_sigma_coordinate',
-               "sigma: var1 ps: var2 ptop: var3")
-        verify('atmosphere_hybrid_sigma_pressure_coordinate',
-               "a: var1 b: var2 ps: var3 p0: var4")
-        verify('atmosphere_hybrid_height_coordinate',
-               "a: var1 b: var2 orog: var3")
-        verify('atmosphere_sleve_coordinate',
-               "a: var1 b1: var2 b2: var3 ztop: var4 zsurf1: var5 zsurf2: var6")
-        verify('ocean_sigma_coordinate',
-               "sigma: var1 eta: var2 depth: var3")
-        verify('ocean_s_coordinate',
-               "s: var1 eta: var2 depth: var3 a: var4 b: var5 depth_c: var6")
-        verify('ocean_sigma_z_coordinate',
-               "sigma: var1 eta: var2 depth: var3 depth_c: var4 nsigma: var5 zlev: var6")
-        verify('ocean_double_sigma_coordinate',
-               "sigma: var1 depth: var2 z1: var3 z2: var4 a: var5 href: var6 k_c: var7")
+        # verify that the formula_terms match the provided set of terms
+        self.assertTrue(no_missing_terms('atmosphere_ln_pressure_coordinate',
+                                         {"p0", "lev"}))
+        self.assertTrue(no_missing_terms('atmosphere_sigma_coordinate',
+                                         {"sigma", "ps", "ptop"}))
+        self.assertTrue(no_missing_terms('atmosphere_hybrid_sigma_pressure_coordinate',
+                                         {'a', 'b', 'ps'}))
+        # test alternative terms for
+        # 'atmosphere_hybrid_sigma_pressure_coordinate'
+        self.assertTrue(no_missing_terms('atmosphere_hybrid_sigma_pressure_coordinate',
+                                         {'ap', 'b', 'ps'}))
+        # check that an invalid set of terms fails
+        self.assertFalse(no_missing_terms('atmosphere_hybrid_sigma_pressure_coordinate',
+                                          {'a', 'b', 'p'}))
+        self.assertTrue(no_missing_terms('atmosphere_hybrid_height_coordinate',
+                                          {"a", "b", "orog"}))
+        # missing terms should cause failure
+        self.assertFalse(no_missing_terms('atmosphere_hybrid_height_coordinate',
+                                          {"a", "b"}))
+        # excess terms should cause failure
+        self.assertFalse(no_missing_terms('atmosphere_hybrid_height_coordinate',
+                                         {"a", "b", "c", "orog"}))
+        self.assertTrue(no_missing_terms('atmosphere_sleve_coordinate',
+                                         {"a", "b1", "b2", "ztop", "zsurf1",
+                                          "zsurf2"}))
+        self.assertTrue(no_missing_terms('ocean_sigma_coordinate',
+                                         {"sigma", "eta", "depth"}))
+        self.assertTrue(no_missing_terms('ocean_s_coordinate',
+                                         {"s", "eta", "depth", "a", "b",
+                                          "depth_c"}))
+        self.assertTrue(no_missing_terms('ocean_sigma_z_coordinate',
+                                         {"sigma", "eta", "depth", "depth_c",
+                                          "nsigma", "zlev"}))
+        self.assertTrue(no_missing_terms('ocean_double_sigma_coordinate',
+                                         {"sigma", "depth", "z1", "z2", "a",
+                                          "href", "k_c"}))
 
     def test_dimensionless_vertical(self):
         '''
@@ -733,9 +745,9 @@ def test_dimensionless_vertical(self):
 
         result_dict = {result.name: result for result in results}
         result = result_dict[u'§4.3.2 lev does not contain deprecated units']
-        assert result.value == (1, 1)
+        assert result.value[0] == result.value[1]
         result = result_dict[u'§4.3.2 lev has valid formula_terms']
-        assert result.value == (6, 6)
+        assert result.value[0] == result.value[1]
 
         # Check negative compliance
         dataset = self.load_dataset(STATIC_FILES['bad'])
@@ -749,10 +761,22 @@ def test_dimensionless_vertical(self):
         assert result.msgs[0] == u'formula_terms is a required attribute and must be a non-empty string'
 
         result = result_dict[u'§4.3.2 lev2 has valid formula_terms']
-        assert result.value == (3, 6)
-        assert result.msgs[0] == 'variable var1 referenced by formula_terms does not exist'
-        assert result.msgs[1] == 'variable var2 referenced by formula_terms does not exist'
-        assert result.msgs[2] == 'variable var3 referenced by formula_terms does not exist'
+        assert result.value == (4, 5)
+        err_str = "The following variable(s) referenced in formula_terms are not present in the dataset variables: var1, var2, var3"
+        self.assertTrue(err_str in result.msgs)
+
+        # test with an invalid formula_terms
+        dataset.variables['lev2'] = MockVariable(dataset.variables['lev2'])
+        lev2 = dataset.variables['lev2']
+        lev2.formula_terms = 'a: var1 b:var2 orog:'
+        # create a malformed formula_terms attribute and check that it fails
+        results = self.cf.check_dimensionless_vertical_coordinate(dataset)
+        result_dict = {result.name: result for result in results}
+        result = result_dict[u'§4.3.2 lev2 has valid formula_terms']
+        self.assertTrue('Attribute formula_terms is not well-formed'
+                        in result.msgs)
+
+
 
     def test_is_time_variable(self):
         var1 = MockVariable()