Repository: ioos/compliance-checker

Issue:
Clarify when metadata is read from global attributes vice inferred from file.
There has recently been a long discussion on the [ACDD](http://wiki.esipfed.org/index.php/Category:Attribute_Conventions_Dataset_Discovery) list discussing the relative merits of including metadata as global attributes when that metadata could (or should) be computed from the actual data in the file.  The classic example is including elements describing spatio-temporal bounds in global attributes in netCDF files served via THREDDS.  Since THREDDS inherits the global attributes from only one file (first, last, or random) and not from the entire aggregation, attributes such as start_time, end_time often do not describe the actual time axis in the aggregated data set time.  

For attributes like these, the checker should compare the global attribute with the computed quantity and note any inconsistencies.  It's going to be up to us to determine which attributes fit in to this category.   The [ESIP wiki](http://wiki.esipfed.org/index.php/NetCDF_Utilities_Metadata_Handling) describes this problem and is starting to characterize the behavior of some common applications.  Basically, beware of inherited global attributes.



Discussion:

Resolving patch:
diff --git a/compliance_checker/acdd.py b/compliance_checker/acdd.py
--- a/compliance_checker/acdd.py
+++ b/compliance_checker/acdd.py
@@ -1,8 +1,15 @@
 import json
 import itertools
-from compliance_checker.base import BaseCheck, BaseNCCheck, check_has, score_group, Result
+import numpy as np
 from pkgutil import get_data
 
+from dateutil.parser import parse as parse_dt
+from udunitspy import Unit
+
+from compliance_checker.base import BaseCheck, BaseNCCheck, check_has, score_group, Result
+from compliance_checker.cf.cf import _possiblexunits, _possibleyunits
+from compliance_checker.cf.util import is_time_variable, is_vertical_coordinate
+
 class ACDDBaseCheck(BaseCheck):
 
     ###############################################################################
@@ -125,6 +132,216 @@ def check_var_coverage_content_type(self, ds):
         retval = [Result(BaseCheck.HIGH, v[0] is not None and v[0] in allowed, (v[1], "var_coverage_content_type")) for v in vars]
         return retval
 
+    ###############################################################################
+    #
+    # DATA EXTENTS MATCHING ATTRIBUTES
+    #
+    ###############################################################################
+
+    def check_lat_extents(self, ds):
+        """
+        Check that the values of geospatial_lat_min/geospatial_lat_max approximately match the data.
+        """
+        if not (hasattr(ds.dataset, 'geospatial_lat_min') and hasattr(ds.dataset, 'geospatial_lat_max')):
+            return
+
+        lat_min = ds.dataset.geospatial_lat_min
+        lat_max = ds.dataset.geospatial_lat_max
+
+        # identify lat var(s) as per CF 4.1
+        lat_vars = {}       # var -> number of criteria passed
+        for name, var in ds.dataset.variables.iteritems():
+
+            # must have units
+            if not hasattr(var, 'units'):
+                continue
+
+            lat_vars[var] = 0
+
+            # units in this set
+            if var.units in _possibleyunits:
+                lat_vars[var] += 1
+
+            # standard name of "latitude"
+            if hasattr(var, 'standard_name') and var.standard_name == 'latitude':
+                lat_vars[var] += 1
+
+            # axis of "Y"
+            if hasattr(var, 'axis') and var.axis == 'Y':
+                lat_vars[var] += 1
+
+        # trim out any zeros
+        lat_vars = {k:v for k, v in lat_vars.iteritems() if v > 0}
+
+        if len(lat_vars) == 0:
+            return Result(BaseCheck.MEDIUM,
+                          False,
+                          'geospatial_lat_extents_match',
+                          'Could not find lat variable to test extent of geospatial_lat_min/max, see CF-1.6 spec chapter 4.1')
+
+        # sort by criteria passed
+        final_lats = sorted(lat_vars, key=lambda x: lat_vars[x], reverse=True)
+
+        obs_mins = {var._name:np.nanmin(var) for var in final_lats if not np.isnan(var).all()}
+        obs_maxs = {var._name:np.nanmax(var) for var in final_lats if not np.isnan(var).all()}
+
+        min_pass = any((np.isclose(lat_min, min_val) for min_val in obs_mins.itervalues()))
+        max_pass = any((np.isclose(lat_max, max_val) for max_val in obs_maxs.itervalues()))
+
+        allpass = sum((min_pass, max_pass))
+
+        msgs = []
+        if not min_pass:
+            msgs.append("Data for possible latitude variables (%s) did not match geospatial_lat_min value (%s)" % (obs_mins, lat_min))
+        if not max_pass:
+            msgs.append("Data for possible latitude variables (%s) did not match geospatial_lat_max value (%s)" % (obs_maxs, lat_max))
+
+        return Result(BaseCheck.MEDIUM,
+                      (allpass, 2),
+                      'geospatial_lat_extents_match',
+                      msgs)
+
+    def check_lon_extents(self, ds):
+        """
+        Check that the values of geospatial_lon_min/geospatial_lon_max approximately match the data.
+        """
+        if not (hasattr(ds.dataset, 'geospatial_lon_min') and hasattr(ds.dataset, 'geospatial_lon_max')):
+            return
+
+        lon_min = ds.dataset.geospatial_lon_min
+        lon_max = ds.dataset.geospatial_lon_max
+
+        # identify lon var(s) as per CF 4.2
+        lon_vars = {}       # var -> number of criteria passed
+        for name, var in ds.dataset.variables.iteritems():
+
+            # must have units
+            if not hasattr(var, 'units'):
+                continue
+
+            lon_vars[var] = 0
+
+            # units in this set
+            if var.units in _possiblexunits:
+                lon_vars[var] += 1
+
+            # standard name of "longitude"
+            if hasattr(var, 'standard_name') and var.standard_name == 'longitude':
+                lon_vars[var] += 1
+
+            # axis of "Y"
+            if hasattr(var, 'axis') and var.axis == 'X':
+                lon_vars[var] += 1
+
+        # trim out any zeros
+        lon_vars = {k:v for k, v in lon_vars.iteritems() if v > 0}
+
+        if len(lon_vars) == 0:
+            return Result(BaseCheck.MEDIUM,
+                          False,
+                          'geospatial_lon_extents_match',
+                          'Could not find lon variable to test extent of geospatial_lon_min/max, see CF-1.6 spec chapter 4.2')
+
+        # sort by criteria passed
+        final_lons = sorted(lon_vars, key=lambda x: lon_vars[x], reverse=True)
+
+        obs_mins = {var._name:np.nanmin(var) for var in final_lons if not np.isnan(var).all()}
+        obs_maxs = {var._name:np.nanmax(var) for var in final_lons if not np.isnan(var).all()}
+
+        min_pass = any((np.isclose(lon_min, min_val) for min_val in obs_mins.itervalues()))
+        max_pass = any((np.isclose(lon_max, max_val) for max_val in obs_maxs.itervalues()))
+
+        allpass = sum((min_pass, max_pass))
+
+        msgs = []
+        if not min_pass:
+            msgs.append("Data for possible longitude variables (%s) did not match geospatial_lon_min value (%s)" % (obs_mins, lon_min))
+        if not max_pass:
+            msgs.append("Data for possible longitude variables (%s) did not match geospatial_lon_max value (%s)" % (obs_maxs, lon_max))
+
+        return Result(BaseCheck.MEDIUM,
+                      (allpass, 2),
+                      'geospatial_lon_extents_match',
+                      msgs)
+
+    def check_vertical_extents(self, ds):
+        """
+        Check that the values of geospatial_vertical_min/geospatial_vertical_max approximately match the data.
+        """
+        if not (hasattr(ds.dataset, 'geospatial_vertical_min') and hasattr(ds.dataset, 'geospatial_vertical_max')):
+            return
+
+        vert_min = ds.dataset.geospatial_vertical_min
+        vert_max = ds.dataset.geospatial_vertical_max
+
+        # identify vertical vars as per CF 4.3
+        v_vars = [var for name, var in ds.dataset.variables.iteritems() if is_vertical_coordinate(name, var)]
+
+        if len(v_vars) == 0:
+            return Result(BaseCheck.MEDIUM,
+                          False,
+                          'geospatial_vertical_extents_match',
+                          'Could not find vertical variable to test extent of geospatial_vertical_min/geospatial_vertical_max, see CF-1.6 spec chapter 4.3')
+
+        obs_mins = {var._name:np.nanmin(var) for var in v_vars if not np.isnan(var).all()}
+        obs_maxs = {var._name:np.nanmax(var) for var in v_vars if not np.isnan(var).all()}
+
+        min_pass = any((np.isclose(vert_min, min_val) for min_val in obs_mins.itervalues()))
+        max_pass = any((np.isclose(vert_max, max_val) for max_val in obs_maxs.itervalues()))
+
+        allpass = sum((min_pass, max_pass))
+
+        msgs = []
+        if not min_pass:
+            msgs.append("Data for possible vertical variables (%s) did not match geospatial_vertical_min value (%s)" % (obs_mins, vert_min))
+        if not max_pass:
+            msgs.append("Data for possible vertical variables (%s) did not match geospatial_vertical_max value (%s)" % (obs_maxs, vert_max))
+
+        return Result(BaseCheck.MEDIUM,
+                      (allpass, 2),
+                      'geospatial_vertical_extents_match',
+                      msgs)
+
+
+    def check_time_extents(self, ds):
+        """
+        Check that the values of time_coverage_start/time_coverage_end approximately match the data.
+        """
+        if not (hasattr(ds.dataset, 'time_coverage_start') and hasattr(ds.dataset, 'time_coverage_end')):
+            return
+
+        epoch = parse_dt("1970-01-01 00:00:00 UTC")
+        t_min = (parse_dt(ds.dataset.time_coverage_start) - epoch).total_seconds()
+        t_max = (parse_dt(ds.dataset.time_coverage_end) - epoch).total_seconds()
+
+        # identify t vars as per CF 4.4
+        t_vars = [var for name, var in ds.dataset.variables.iteritems() if is_time_variable(name, var)]
+
+        if len(t_vars) == 0:
+            return Result(BaseCheck.MEDIUM,
+                          False,
+                          'time_coverage_extents_match',
+                          'Could not find time variable to test extent of time_coverage_start/time_coverage_end, see CF-1.6 spec chapter 4.4')
+
+        obs_mins = {var._name:Unit(str(var.units)).get_converter("seconds since 1970-01-01").evaluate(np.nanmin(var)) for var in t_vars}
+        obs_maxs = {var._name:Unit(str(var.units)).get_converter("seconds since 1970-01-01").evaluate(np.nanmax(var)) for var in t_vars}
+
+        min_pass = any((np.isclose(t_min, min_val) for min_val in obs_mins.itervalues()))
+        max_pass = any((np.isclose(t_max, max_val) for max_val in obs_maxs.itervalues()))
+
+        allpass = sum((min_pass, max_pass))
+
+        msgs = []
+        if not min_pass:
+            msgs.append("Data for possible time variables (%s) did not match time_coverage_start value (%s)" % (obs_mins, t_min))
+        if not max_pass:
+            msgs.append("Data for possible time variables (%s) did not match time_coverage_end value (%s)" % (obs_maxs, t_max))
+
+        return Result(BaseCheck.MEDIUM,
+                      (allpass, 2),
+                      'time_coverage_extents_match',
+                      msgs)
+
 class ACDDNCCheck(BaseNCCheck, ACDDBaseCheck):
     @classmethod
     def beliefs(cls):
diff --git a/compliance_checker/cf/cf.py b/compliance_checker/cf/cf.py
--- a/compliance_checker/cf/cf.py
+++ b/compliance_checker/cf/cf.py
@@ -5,107 +5,10 @@
 
 from compliance_checker.base import BaseCheck, BaseNCCheck, check_has, score_group, Result
 from compliance_checker.cf.appendix_d import dimless_vertical_coordinates
-from compliance_checker.cf.util import NCGraph, StandardNameTable, units_known, units_convertible, units_temporal
+from compliance_checker.cf.util import NCGraph, StandardNameTable, units_known, units_convertible, units_temporal, map_axes, find_coord_vars, is_time_variable, is_vertical_coordinate, _possiblet, _possiblez, _possiblex, _possibley, _possibleaxis, _possiblexunits, _possibleyunits, _possibletunits, _possibleaxisunits
 
 from netCDF4 import Dimension, Variable
 
-
-# copied from paegan
-# paegan may depend on these later
-_possiblet = ["time", "TIME", "Time",
-           "t", "T",
-           "ocean_time", "OCEAN_TIME",
-           "jd", "JD",
-           "dn", "DN",
-           "times", "TIMES", "Times",
-           "mt", "MT",
-           "dt", "DT",
-          ]
-_possiblez = ["depth", "DEPTH",
-           "depths", "DEPTHS",
-           "height", "HEIGHT",
-           "altitude", "ALTITUDE",
-           "alt", "ALT", 
-           "Alt", "Altitude",
-           "h", "H",
-           "s_rho", "S_RHO",
-           "s_w", "S_W",
-           "z", "Z",
-           "siglay", "SIGLAY",
-           "siglev", "SIGLEV",
-           "sigma", "SIGMA",
-           "vertical", "VERTICAL", "lev", "LEV", "level", "LEVEL"
-          ]
-_possiblex = ["x", "X",
-           "lon", "LON",
-           "xlon", "XLON",
-           "lonx", "lonx",
-           "lon_u", "LON_U",
-           "lon_v", "LON_V",
-           "lonc", "LONC",
-           "Lon", "Longitude",
-           "longitude", "LONGITUDE",
-           "lon_rho", "LON_RHO",
-           "lon_psi", "LON_PSI",
-
-          ]
-_possibley = ["y", "Y",
-           "lat", "LAT",
-           "ylat", "YLAT",
-           "laty", "laty",
-           "lat_u", "LAT_U",
-           "lat_v", "LAT_V",
-           "latc", "LATC",
-           "Lat", "Latitude",
-           "latitude", "LATITUDE",
-           "lat_rho", "LAT_RHO",
-           "lat_psi", "LAT_PSI",
-
-          ]
-
-_possibleaxis = _possiblet + _possiblez + _possiblex + _possibley
-
-
-_possiblexunits = ['degrees_east',
-                    'degree_east',
-                    'degrees_E',
-                    'degree_E',
-                    'degreesE',
-                    'degreeE'
-                    ]
-
-_possibleyunits = ['degrees_north',
-                'degree_north',
-                'degrees_N',
-                'degree_N',
-                'degreesN',
-                'degreeN'
-                    ]
-
-_possibletunits = ['day', 
-                'days', 
-                'd', 
-                'hour', 
-                'hours', 
-                'hr', 
-                'hrs', 
-                'h', 
-                'year', 
-                'years', 
-                'minute', 
-                'minutes', 
-                'm', 
-                'min', 
-                'mins', 
-                'second', 
-                'seconds', 
-                's', 
-                'sec', 
-                'secs'
-                ]
-
-_possibleaxisunits =  _possiblexunits + _possibleyunits +_possibletunits
-
 def guess_dim_type(dimension):
     """
     Guesses the type of dimension of a variable X/Y/Z/T
@@ -168,28 +71,6 @@ def is_variable(name, var):
     return True
 
 
-def map_axes(dim_vars, reverse_map=False):
-    """
-    axis name       -> [dimension names]
-    dimension name  -> [axis_name], length 0 if reverse_map
-    """
-    ret_val = defaultdict(list)
-    axes = ['X', 'Y', 'Z', 'T']
-
-    for k, v in dim_vars.iteritems():
-        axis = getattr(v, 'axis', '')
-        if not axis:
-            continue
-
-        axis = axis.upper()
-        if axis in axes:
-            if reverse_map:
-                ret_val[k].append(axis)
-            else:
-                ret_val[axis].append(k)
-
-    return dict(ret_val)
-
 # helper to see if we should do DSG tests
 def is_likely_dsg(func):
     @wraps(func)
@@ -246,9 +127,7 @@ def _find_coord_vars(self, ds, refresh=False):
         if ds in self._coord_vars and not refresh:
             return self._coord_vars[ds]
 
-        for d in ds.dataset.dimensions:
-            if d in ds.dataset.variables and ds.dataset.variables[d].dimensions == (d,):
-                self._coord_vars[ds].append(ds.dataset.variables[d])
+        self._coord_vars[ds] = find_coord_vars(ds.dataset)
 
         return self._coord_vars[ds]
 
@@ -1078,30 +957,6 @@ def check_longitude(self, ds):
 
         return ret_val
 
-    def _is_vertical_coordinate(self, var_name, var):
-        '''
-        Determines if a variable is a vertical coordinate variable
-        
-        4.3
-        A vertical coordinate will be identifiable by: units of pressure; or the presence of the positive attribute with a
-        value of up or down (case insensitive).  Optionally, the vertical type may be indicated additionally by providing
-        the standard_name attribute with an appropriate value, and/or the axis attribute with the value Z.
-        '''
-        # Known name
-        satisfied = var_name.lower() in _possiblez 
-        satisfied |= getattr(var, 'standard_name', '') in _possiblez
-        # Is the axis set to Z?
-        satisfied |= getattr(var, 'axis', '').lower() == 'z'
-        is_pressure = units_convertible(getattr(var, 'units', '1'), 'dbar')
-        # Pressure defined or positive defined
-        satisfied |= is_pressure
-        if not is_pressure:
-            satisfied |= getattr(var,'positive', '').lower() in ('up', 'down')
-        return satisfied
-
-
-
-
     def check_vertical_coordinate(self, ds):
         """
         4.3 Variables representing dimensional height or depth axes must always
@@ -1117,7 +972,7 @@ def check_vertical_coordinate(self, ds):
         
         ret_val = [] 
         for k,v in ds.dataset.variables.iteritems(): 
-            if self._is_vertical_coordinate(k,v):
+            if is_vertical_coordinate(k,v):
                 # Vertical variables MUST have units
                 has_units = hasattr(v, 'units') 
                 result = Result(BaseCheck.HIGH, \
@@ -1177,7 +1032,7 @@ def check_dimensional_vertical_coordinate(self, ds):
         ret_val = []
         for k,v in ds.dataset.variables.iteritems():
             # If this is not a vertical coordinate
-            if not self._is_vertical_coordinate(k,v):
+            if not is_vertical_coordinate(k,v):
                 continue
 
             # If this is not height or depth
@@ -1277,16 +1132,6 @@ def check_dimensionless_vertical_coordinate(self, ds):
 
         return ret_val
 
-    def _is_time_variable(self, varname, var):
-        '''
-        Identifies if a variable is represents time
-        '''
-        satisfied = varname.lower() == 'time'
-        satisfied |= getattr(var, 'standard_name', '') == 'time'
-        satisfied |= getattr(var, 'axis', '') == 'T'
-        satisfied |= units_convertible('seconds since 1900-01-01', getattr(var, 'units', ''))
-        return satisfied
-
     def check_time_coordinate(self, ds):
         """
         4.4 Variables representing time must always explicitly include the units
@@ -1315,7 +1160,7 @@ def check_time_coordinate(self, ds):
 
         ret_val = []
         for k,v in ds.dataset.variables.iteritems():
-            if not self._is_time_variable(k,v):
+            if not is_time_variable(k,v):
                 continue
             # Has units
             has_units = hasattr(v, 'units')
diff --git a/compliance_checker/cf/util.py b/compliance_checker/cf/util.py
--- a/compliance_checker/cf/util.py
+++ b/compliance_checker/cf/util.py
@@ -1,11 +1,108 @@
 import os
 import os.path
 import itertools
+from collections import defaultdict
 from lxml import etree
 from udunitspy import Unit, UdunitsError, Converter
 from netCDF4 import Dimension, Variable
 from pkgutil import get_data
 
+# copied from paegan
+# paegan may depend on these later
+_possiblet = ["time", "TIME", "Time",
+           "t", "T",
+           "ocean_time", "OCEAN_TIME",
+           "jd", "JD",
+           "dn", "DN",
+           "times", "TIMES", "Times",
+           "mt", "MT",
+           "dt", "DT",
+          ]
+_possiblez = ["depth", "DEPTH",
+           "depths", "DEPTHS",
+           "height", "HEIGHT",
+           "altitude", "ALTITUDE",
+           "alt", "ALT", 
+           "Alt", "Altitude",
+           "h", "H",
+           "s_rho", "S_RHO",
+           "s_w", "S_W",
+           "z", "Z",
+           "siglay", "SIGLAY",
+           "siglev", "SIGLEV",
+           "sigma", "SIGMA",
+           "vertical", "VERTICAL", "lev", "LEV", "level", "LEVEL"
+          ]
+_possiblex = ["x", "X",
+           "lon", "LON",
+           "xlon", "XLON",
+           "lonx", "lonx",
+           "lon_u", "LON_U",
+           "lon_v", "LON_V",
+           "lonc", "LONC",
+           "Lon", "Longitude",
+           "longitude", "LONGITUDE",
+           "lon_rho", "LON_RHO",
+           "lon_psi", "LON_PSI",
+
+          ]
+_possibley = ["y", "Y",
+           "lat", "LAT",
+           "ylat", "YLAT",
+           "laty", "laty",
+           "lat_u", "LAT_U",
+           "lat_v", "LAT_V",
+           "latc", "LATC",
+           "Lat", "Latitude",
+           "latitude", "LATITUDE",
+           "lat_rho", "LAT_RHO",
+           "lat_psi", "LAT_PSI",
+
+          ]
+
+_possibleaxis = _possiblet + _possiblez + _possiblex + _possibley
+
+
+_possiblexunits = ['degrees_east',
+                    'degree_east',
+                    'degrees_E',
+                    'degree_E',
+                    'degreesE',
+                    'degreeE'
+                    ]
+
+_possibleyunits = ['degrees_north',
+                'degree_north',
+                'degrees_N',
+                'degree_N',
+                'degreesN',
+                'degreeN'
+                    ]
+
+_possibletunits = ['day', 
+                'days', 
+                'd', 
+                'hour', 
+                'hours', 
+                'hr', 
+                'hrs', 
+                'h', 
+                'year', 
+                'years', 
+                'minute', 
+                'minutes', 
+                'm', 
+                'min', 
+                'mins', 
+                'second', 
+                'seconds', 
+                's', 
+                'sec', 
+                'secs'
+                ]
+
+_possibleaxisunits =  _possiblexunits + _possibleyunits +_possibletunits
+
 class DotDict(dict):
     """
     Subclass of dict that will recursively look up attributes with dot notation.
@@ -169,3 +266,70 @@ def units_temporal(units):
         return False
     return r
 
+def map_axes(dim_vars, reverse_map=False):
+    """
+    axis name       -> [dimension names]
+    dimension name  -> [axis_name], length 0 if reverse_map
+    """
+    ret_val = defaultdict(list)
+    axes = ['X', 'Y', 'Z', 'T']
+
+    for k, v in dim_vars.iteritems():
+        axis = getattr(v, 'axis', '')
+        if not axis:
+            continue
+
+        axis = axis.upper()
+        if axis in axes:
+            if reverse_map:
+                ret_val[k].append(axis)
+            else:
+                ret_val[axis].append(k)
+
+    return dict(ret_val)
+
+def find_coord_vars(ncds):
+    """
+    Finds all coordinate variables in a dataset.
+
+    A variable with the same name as a dimension is called a coordinate variable.
+    """
+    coord_vars = []
+
+    for d in ncds.dimensions:
+        if d in ncds.variables and ncds.variables[d].dimensions == (d,):
+            coord_vars.append(ncds.variables[d])
+
+    return coord_vars
+
+def is_time_variable(varname, var):
+    """
+    Identifies if a variable is represents time
+    """
+    satisfied = varname.lower() == 'time'
+    satisfied |= getattr(var, 'standard_name', '') == 'time'
+    satisfied |= getattr(var, 'axis', '') == 'T'
+    satisfied |= units_convertible('seconds since 1900-01-01', getattr(var, 'units', ''))
+    return satisfied
+
+def is_vertical_coordinate(var_name, var):
+    """
+    Determines if a variable is a vertical coordinate variable
+    
+    4.3
+    A vertical coordinate will be identifiable by: units of pressure; or the presence of the positive attribute with a
+    value of up or down (case insensitive).  Optionally, the vertical type may be indicated additionally by providing
+    the standard_name attribute with an appropriate value, and/or the axis attribute with the value Z.
+    """
+    # Known name
+    satisfied = var_name.lower() in _possiblez 
+    satisfied |= getattr(var, 'standard_name', '') in _possiblez
+    # Is the axis set to Z?
+    satisfied |= getattr(var, 'axis', '').lower() == 'z'
+    is_pressure = units_convertible(getattr(var, 'units', '1'), 'dbar')
+    # Pressure defined or positive defined
+    satisfied |= is_pressure
+    if not is_pressure:
+        satisfied |= getattr(var,'positive', '').lower() in ('up', 'down')
+    return satisfied
+


Test patch: