Repository: ioos/compliance-checker
Pull request #795 (2020-04-10 19:01:04)

Adds ability to fetch latest CF standard name table
Adds an option to the `-d` flag to download the latest CF standard name
table without needing to explicitly specify a version.  This is expected
to be useful in cases with non-interactive runs () of the compliance
checker (e.g. scheduled checker runs) where an up to date standard name
table is desired.

Supersedes #793 and eliminates the need for BeautifulSoup4 library.


Patch:
diff --git a/README.md b/README.md
index d138650..86d66a8 100644
--- a/README.md
+++ b/README.md
@@ -190,7 +190,9 @@ optional arguments:
   -l, --list-tests      List the available tests
   -d DOWNLOAD_STANDARD_NAMES, --download-standard-names DOWNLOAD_STANDARD_NAMES
                         Specify a version of the cf standard name table to
-                        download as packaged version
+                        download as packaged version. Either specify a version
+                        number (e.g. "72") to fetch a specific version or
+                        "latest" to get the latest CF standard name table.
 ```
 
 ## Examples
diff --git a/cchecker.py b/cchecker.py
index de653e4..812dc22 100755
--- a/cchecker.py
+++ b/cchecker.py
@@ -128,7 +128,10 @@ def main():
 
     parser.add_argument('-d', '--download-standard-names',
                         help=("Specify a version of the cf standard name table"
-                              " to download as packaged version"))
+                              " to download as packaged version. Either specify"
+                              " a version number (e.g. \"72\") to fetch a "
+                              "specific version or \"latest\" to get the "
+                              "latest CF standard name table."))
 
     # Add command line args from generator plugins
     check_suite.add_plugin_args(parser)
diff --git a/compliance_checker/cf/util.py b/compliance_checker/cf/util.py
index 143634f..a2581f6 100644
--- a/compliance_checker/cf/util.py
+++ b/compliance_checker/cf/util.py
@@ -6,7 +6,9 @@ import sys
 from copy import deepcopy
 from collections import defaultdict
 from lxml import etree
+import lxml.html
 from cf_units import Unit
+from urllib.parse import urljoin
 from netCDF4 import Dimension, Variable
 from pkgutil import get_data
 from pkg_resources import resource_filename
@@ -264,7 +266,8 @@ class StandardNameTable(object):
             with io.open(cached_location, 'r', encoding='utf-8') as fp:
                 resource_text = fp.read()
         elif os.environ.get('CF_STANDARD_NAME_TABLE') and os.path.exists(os.environ['CF_STANDARD_NAME_TABLE']):
-            with io.open(os.environ['CF_STANDARD_NAME_TABLE'], 'r', encoding='utf-8') as fp:
+            with io.open(os.environ['CF_STANDARD_NAME_TABLE'], 'r',
+                         encoding='utf-8') as fp:
                 resource_text = fp.read()
         else:
             resource_text = get_data("compliance_checker", "data/cf-standard-name-table.xml")
@@ -327,15 +330,25 @@ def download_cf_standard_name_table(version, location=None):
     if location is None:  # This case occurs when updating the packaged version from command line
         location = resource_filename('compliance_checker', 'data/cf-standard-name-table.xml')
 
-    url = "http://cfconventions.org/Data/cf-standard-names/{0}/src/cf-standard-name-table.xml".format(version)
-    r = requests.get(url, allow_redirects=True)
-    if r.status_code == 200:
-        print("Downloading cf-standard-names table version {0} from: {1}".format(version, url), file=sys.stderr)
-        with open(location, 'wb') as f:
-            f.write(r.content)
+    if version == 'latest':
+        tables_tree = lxml.html.parse("http://cfconventions.org/documents.html")
+        end_str = "cf-standard-name-table.xml"
+        xpath_expr = ("//a[substring(@href, string-length(@href) - "
+                      "string-length('{0}') +1) "
+                      " = '{0}'][1]".format(end_str))
+        latest_vers = tables_tree.xpath(xpath_expr)[0]
+
+        url = urljoin("http://cfconventions.org", latest_vers.attrib['href'])
     else:
-        r.raise_for_status()
-    return
+        url = "http://cfconventions.org/Data/cf-standard-names/{0}/src/cf-standard-name-table.xml".format(version)
+
+    r = requests.get(url, allow_redirects=True)
+    r.raise_for_status()
+
+    print("Downloading cf-standard-names table version {0} from: {1}".format(version, url),
+            file=sys.stderr)
+    with open(location, 'wb') as f:
+        f.write(r.content)
 
 
 def create_cached_data_dir():
diff --git a/compliance_checker/ioos.py b/compliance_checker/ioos.py
index 568c8d1..3a615ca 100644
--- a/compliance_checker/ioos.py
+++ b/compliance_checker/ioos.py
@@ -440,7 +440,7 @@ class IOOS1_2Check(IOOSNCCheck):
             ('infoUrl', base.UrlValidator()),
             'license',
             ('naming_authority', NamingAuthorityValidator()),
-            #'platform', # checked in check_platform_global
+            'platform',
             'platform_name',
             'publisher_country',
             ('publisher_email', base.EmailValidator()),
@@ -785,9 +785,17 @@ class IOOS1_2Check(IOOSNCCheck):
 
     def check_single_platform(self, ds):
         """
-        Verify that a dataset only has a single platform attribute, and thus
-        a single platform variable. Gridded model datasets are not required
-        to declare a platform or platform variables.
+        Verify that a dataset only has a single platform attribute. If one exists,
+        examine the featureType of the dataset. If the featureType is
+        [point, timeSeries, profile, trajectory] and cf_role in [timeseries_id,
+        profile_id, trajectory_id] dimensionality of the variable containing
+        cf_role must be 1 as "we only want a single glider/auv/ship"; if
+        featureType in [timeseries_id, trajectory_id], dimensionality of the
+        variable must also be one as  "we only want a single timeSeries aka buoy".
+        If cf_role==profile_id, it can have whatever dimension.
+
+        Gridded model datasets are not required to declare a platform
+        or platform variables.
 
         Args:
             ds (netCDF-4 Dataset): open Dataset object
@@ -796,6 +804,7 @@ class IOOS1_2Check(IOOSNCCheck):
             Result
         """
 
+        results = []
         glb_platform = getattr(ds, "platform", None)
 
         platform_set = set()
@@ -806,114 +815,62 @@ class IOOS1_2Check(IOOSNCCheck):
         if num_platforms > 1 and glb_platform:
             msg = "A dataset may only have one platform; {} found".format(len(platform_set))
             val = False
+            results.append(Result(BaseCheck.HIGH, val, "platform", None if val else [msg]))
 
         elif ((not glb_platform) and num_platforms > 0):
-            msg = "If a platform variable exists, a global attribute \"platform\" must also exist"
+            msg = "If platform variables exist, a global attribute \"platform\" must also exist"
             val = False
+            results.append(Result(BaseCheck.HIGH, val, "platform", None if val else [msg]))
 
         elif num_platforms == 0 and glb_platform:
             msg = "A dataset with a global \"platform\" attribute must platform have variables"
             val = False
+            results.append(Result(BaseCheck.HIGH, val, "platform", None if val else [msg]))
 
         elif num_platforms == 0 and (not glb_platform):
             msg = "Gridded model datasets are not required to declare a platform"
             val = True
+            results.append(Result(BaseCheck.HIGH, val, "platform", None if val else [msg]))
+
+        else: # num_platforms==1 and glb_platform, test the dimensionality
+
+            num_plat_val = True if num_platforms == 1 else False
+
+            feature_type = getattr(ds, "featureType", "").lower()
+            if not feature_type:
+                return results
+
+            # filter out cf_role exists
+            cf_role_vars = ds.get_variables_by_attributes(cf_role=lambda x: x is not None)
+            num_cf_role_vars = len(cf_role_vars)
+            msg = "With a single platform provided, the dimension of the cf_role " +\
+                  "variable {cf_role_var} (cf_role=={cf_role}) should also " +\
+                  "be equal to 1 (it is {dim})"
+
+            for var in cf_role_vars:
+                cf_role = getattr(var, "cf_role")
+                shp = var.shape[0] if len(var.shape) > 0 else 1
+                if (
+                       feature_type in ["point", "timeseries", "profile", "trajectory", "timeseriesprofile", "trajectoryprofile"]
+                       and
+                       cf_role in ["timeseries_id", "profile_id", "trajectory_id"]
+                   ):
+                    if (num_cf_role_vars==1) or (num_cf_role_vars>1 and cf_role!="profile_id"):
+                        # shape must be 1 (or if no length, that's okay too)
+                        _val = shp==1
+                    elif (num_cf_role_vars>1) and (cf_role=="profile_id"):
+                        # can have any dimension if there are more than one cf_role?
+                        _val = True
 
-        else:
-            val = True
-
-        return Result(BaseCheck.HIGH, val, "platform", None if val else [msg])
-
-    def check_cf_dsg(self, ds):
-        """
-        Check that the dataset follows the restrictions for CF Discrete
-        Sampling Geometries set by the IOOS Metadata Profile.
-
-        Examine each variable with a cf_role. For each (featureType, cf_role):
-            - (timeSeries, timeseries_id)
-            - (timeSeriesProfile, timeseries_id)
-            - (trajectory, trajectory_id)
-            - (trajectoryProfile, trajectory_id)
-            - (profile, profile_id)
-        the dimension of the variable should be 1. For datasets with
-        the "point" featureType, do nothing.
-
-
-        https://github.com/ioos/compliance-checker/issues/748#issuecomment-606659685
-
-        Parameters
-        ----------
-        ds: netCDF4.Dataset (open)
-
-        Returns
-        -------
-        list of Result objects
-        """
-
-        results = []
-
-        feature_type = getattr(ds, "featureType", "").lower()
-        if not feature_type:
-            return results
- 
-        # loop through all variables with cf_role
-        cf_role_vars = ds.get_variables_by_attributes(cf_role=lambda x: x is not None)
-        num_cf_role_vars = len(cf_role_vars)
-
-        for var in cf_role_vars:
-            cf_role = getattr(var, "cf_role")
-            shp = var.shape[0] if len(var.shape) > 0 else 1
- 
-            if feature_type == "timeseries" and cf_role == "timeseries_id":
-                msg = (
-                    "Dimension length of the variable with "
-                    "cf_role='timeseries_id (the 'station' dimension) "
-                    "is {dim}. Note that the IOOS profile restricts "
-                    "timeSeries datasets with multiple features to share "
-                    "the same lat/lon position (ie. to exist on the same "
-                    "platform). Datasets that include multiple platforms "
-                    "are not valid and will cause harvesting errors."
-                ).format(dim=shp)
- 
-                _val = shp <= 1 # fails if > 1
- 
-            elif ( # even though tested condition is the same, different msg
-                     feature_type == "profile" and cf_role == "profile_id" or
-                     feature_type == "trajectory" and cf_role == "trajectory_id" or
-                     feature_type == "timeseriesprofile" and cf_role == "timeseries_id" or
-                     feature_type == "trajectoryprofile" and cf_role == "trajectory_id"
-                 ):
-
-                msg = (
-                    "Dimension length of the variable `{cf_role_var}` with "
-                    "cf_role=`{cf_role}` (the 'station/trajectory/profile' "
-                    "dimension) must be equal to 1 (it is {dim}). The IOOS "
-                    "profile restricts {feature_type} datasets to a "
-                    "single platform (ie. station/trajectory/profile) per dataset."
-                ).format(
-                    cf_role_var=var.name, cf_role=cf_role,
-                    dim=shp, feature_type=feature_type
-                )
- 
-                _val = shp == 1
- 
-            elif feature_type == "point": # do nothing
-                _val = True
-
-            else: # featureType and cf_role don't match up to our restrictions
-                _val = True
- 
-            if not _val:
-                results.append(
-                    Result(
-                        BaseCheck.HIGH,
-                        _val,
-                        "CF Discrete Sampling Geometry Compliance",
-                        [msg]
+                    results.append(
+                       Result(
+                           BaseCheck.HIGH,
+                           _val,
+                           "platform variables",
+                           None if _val else [msg.format(cf_role_var=var.name, cf_role=cf_role, dim=shp)]
+                       )
                     )
-                )
- 
- 
+
         return results
 
     def check_platform_vocabulary(self, ds):


Test patch:
diff --git a/compliance_checker/tests/test_ioos_profile.py b/compliance_checker/tests/test_ioos_profile.py
index 23883de..ed3bac4 100644
--- a/compliance_checker/tests/test_ioos_profile.py
+++ b/compliance_checker/tests/test_ioos_profile.py
@@ -685,44 +685,16 @@ class TestIOOS1_2(BaseTestCase):
         ds = MockTimeSeries() # time, lat, lon, depth
 
         # no global attr but also no platform variables, should pass
-        result = self.ioos.check_single_platform(ds)
-        self.assertTrue(result.value)
-        self.assertEqual(result.msgs, [])
+        results = self.ioos.check_single_platform(ds)
+        self.assertTrue(results[0].value)
+        self.assertEqual(results[0].msgs, [])
 
         # give platform global, no variables, fail
         ds.setncattr("platform", "buoy")
-        result = self.ioos.check_single_platform(ds)
-        self.assertFalse(result.value)
-
-        # global platform, one platform variable, pass
-        temp = ds.createVariable("temp", "d", ("time"))
-        temp.setncattr("platform", "platform_var")
-        plat = ds.createVariable("platform_var", np.byte)
-        result = self.ioos.check_single_platform(ds)
-        self.assertTrue(result.value)
-        self.assertEqual(result.msgs, [])
-
-        # two platform variables, fail
-        temp2 = ds.createVariable("temp2", "d", ("time"))
-        temp2.setncattr("platform", "platform_var2")
-        plat = ds.createVariable("platform_var2", np.byte)
-        result = self.ioos.check_single_platform(ds)
-        self.assertFalse(result.value)
-
-        # no global attr, one variable, fail
-        ds = MockTimeSeries() # time, lat, lon, depth
-        temp = ds.createVariable("temp", "d", ("time"))
-        temp.setncattr("platform", "platform_var")
-        plat = ds.createVariable("platform_var", np.byte)
-        result = self.ioos.check_single_platform(ds)
-        self.assertFalse(result.value)
- 
-    def test_check_cf_dsg(self):
-
-        ds = MockTimeSeries() # time, lat, lon, depth
-        ds.setncattr("platform", "single_string")
+        results = self.ioos.check_single_platform(ds)
+        self.assertFalse(results[0].value)
 
-        # correct cf_role & featureType, pass
+        # global attribute, one platform variable, correct cf_role & featureType, pass
         ds.setncattr("featureType", "profile")
         ds.createDimension("profile", 1)
         temp = ds.createVariable("temp", "d", ("time"))
@@ -730,145 +702,33 @@ class TestIOOS1_2(BaseTestCase):
         plat = ds.createVariable("platform_var", np.byte)
         cf_role_var = ds.createVariable("cf_role_var", np.byte, ("profile",))
         cf_role_var.setncattr("cf_role", "timeseries_id")
-        results = self.ioos.check_cf_dsg(ds)
+        results = self.ioos.check_single_platform(ds)
         self.assertTrue(all(r.value for r in results))
         self.assertTrue(all(r.msgs==[] for r in results))
 
-        # correct featureType, incorrect cf_role var dimension
-        ds = MockTimeSeries() # time, lat, lon, depth
-        ds.setncattr("featureType", "trajectoryprofile")
-        ds.createDimension("trajectory", 2) # should only be 1
-        temp = ds.createVariable("temp", "d", ("time"))
-        temp.setncattr("platform", "platform_var")
-        plat = ds.createVariable("platform_var", np.byte)
-        cf_role_var = ds.createVariable("cf_role_var", np.byte, ("trajectory",))
-        cf_role_var.setncattr("cf_role", "trajectory_id")
-        results = self.ioos.check_cf_dsg(ds)
+        # global attr, multiple platform variables, correct cf_role & featureType, fail
+        plat2 = ds.createVariable("platform_var_2", np.byte)
+        temp2 = ds.createVariable("temp2", "d", ("time"))
+        temp2.setncattr("platform", "platform_var2")
+        results = self.ioos.check_single_platform(ds)
         self.assertFalse(results[0].value)
 
-        # featureType==timeSeries, cf_role=timeseries_id
-        ds = MockTimeSeries()
-        ds.setncattr("featureType", "timeSeries")
-        ds.createDimension("station", 1)
-        temp = ds.createVariable("temp", "d", ("time"))
-        temp.setncattr("platform", "platform_var")
-        plat = ds.createVariable("platform_var", np.byte)
-        cf_role_var = ds.createVariable("cf_role_var", np.byte, ("station",))
-        cf_role_var.setncattr("cf_role", "timeseries_id")
-        results = self.ioos.check_cf_dsg(ds)
-
-        # check should pass with no results
-        self.assertEqual(results, [])
-
-        # featureType==timeSeriesProfile, cf_role==timeseries_id, dim 1, pass
-        ds = MockTimeSeries()
-        ds.setncattr("featureType", "timeSeriesProfile")
-        ds.createDimension("station", 1)
-        temp = ds.createVariable("temp", "d", ("time"))
-        temp.setncattr("platform", "platform_var")
-        plat = ds.createVariable("platform_var", np.byte)
-        cf_role_var = ds.createVariable("cf_role_var", np.byte, ("station",))
-        cf_role_var.setncattr("cf_role", "timeseries_id")
-        results = self.ioos.check_cf_dsg(ds)
-        self.assertEqual(results, [])
-
-        # featureType==timeSeriesProfile, cf_role==timeseries_id, dim 2, fail
-        ds = MockTimeSeries()
-        ds.setncattr("platform", "platform")
-        ds.setncattr("featureType", "timeSeriesProfile")
-        ds.createDimension("station", 2)
-        temp = ds.createVariable("temp", "d", ("time"))
-        temp.setncattr("platform", "platform_var")
-        plat = ds.createVariable("platform_var", np.byte)
-        cf_role_var = ds.createVariable("cf_role_var", np.byte, ("station",))
-        cf_role_var.setncattr("cf_role", "timeseries_id")
-        results = self.ioos.check_cf_dsg(ds)
+        # no global attr, one platform var, correct cf_role & featureType, fail
+        ds.delncattr("platform")
         self.assertFalse(results[0].value)
 
-        # featureType==trajectory, cf_role==trajectory_id, dim 1, pass
-        ds = MockTimeSeries()
-        ds.setncattr("featureType", "trajectory")
-        ds.createDimension("trajectory", 1)
-        temp = ds.createVariable("temp", "d", ("time"))
-        temp.setncattr("platform", "platform_var")
-        plat = ds.createVariable("platform_var", np.byte)
-        cf_role_var = ds.createVariable("cf_role_var", np.byte, ("trajectory",))
-        cf_role_var.setncattr("cf_role", "trajectory_id")
-        results = self.ioos.check_cf_dsg(ds)
-        self.assertEqual(results, [])
-
-        # featureType==trajectory, cf_role==trajectory, dim 2, fail
-        ds = MockTimeSeries()
-        ds.setncattr("featureType", "trajectory")
-        ds.createDimension("trajectory", 2)
+        # global attr, one platform var, correct featureType, incorrect cf_role var dimension
+        ds = MockTimeSeries() # time, lat, lon, depth
+        ds.setncattr("featureType", "trajectoryprofile")
+        ds.createDimension("trajectory", 2) # should only be 1
         temp = ds.createVariable("temp", "d", ("time"))
         temp.setncattr("platform", "platform_var")
         plat = ds.createVariable("platform_var", np.byte)
         cf_role_var = ds.createVariable("cf_role_var", np.byte, ("trajectory",))
         cf_role_var.setncattr("cf_role", "trajectory_id")
-        results = self.ioos.check_cf_dsg(ds)
+        results = self.ioos.check_single_platform(ds)
         self.assertFalse(results[0].value)
 
-        # featureType==trajectoryProfile, cf_role==trajectory_id, dim 1, pass
-        ds = MockTimeSeries()
-        ds.setncattr("featureType", "trajectoryProfile")
-        ds.createDimension("trajectoryprof", 1)
-        temp = ds.createVariable("temp", "d", ("time"))
-        temp.setncattr("platform", "platform_var")
-        plat = ds.createVariable("platform_var", np.byte)
-        cf_role_var = ds.createVariable("cf_role_var", np.byte, ("trajectoryprof",))
-        cf_role_var.setncattr("cf_role", "trajectory_id")
-        results = self.ioos.check_cf_dsg(ds)
-        self.assertEqual(results, [])
-
-        # featureType==trajectoryProfile, cf_role==trajectory_id, dim 2, fail
-        ds = MockTimeSeries()
-        ds.setncattr("featureType", "trajectoryProfile")
-        ds.createDimension("trajectoryprof", 2)
-        temp = ds.createVariable("temp", "d", ("time"))
-        temp.setncattr("platform", "platform_var")
-        plat = ds.createVariable("platform_var", np.byte)
-        cf_role_var = ds.createVariable("cf_role_var", np.byte, ("trajectoryprof",))
-        cf_role_var.setncattr("cf_role", "trajectory_id")
-        results = self.ioos.check_cf_dsg(ds)
-        self.assertFalse(results[0].value)
-
-        # featureType==profile, cf_role==profile_id, dim 1, pass
-        ds = MockTimeSeries()
-        ds.setncattr("featureType", "profile")
-        ds.createDimension("prof", 1)
-        temp = ds.createVariable("temp", "d", ("time"))
-        temp.setncattr("platform", "platform_var")
-        plat = ds.createVariable("platform_var", np.byte)
-        cf_role_var = ds.createVariable("cf_role_var", np.byte, ("prof",))
-        cf_role_var.setncattr("cf_role", "profile_id")
-        results = self.ioos.check_cf_dsg(ds)
-        self.assertEqual(results, [])
-
-        # featureType==profile, cf_role==profile_id, dim 2, fail
-        ds = MockTimeSeries()
-        ds.setncattr("featureType", "profile")
-        ds.createDimension("prof", 2)
-        temp = ds.createVariable("temp", "d", ("time"))
-        temp.setncattr("platform", "platform_var")
-        plat = ds.createVariable("platform_var", np.byte)
-        cf_role_var = ds.createVariable("cf_role_var", np.byte, ("prof",))
-        cf_role_var.setncattr("cf_role", "profile_id")
-        results = self.ioos.check_cf_dsg(ds)
-        self.assertFalse(results[0].value)
-
-        # featureType==point -- do nothing
-        ds = MockTimeSeries()
-        ds.setncattr("featureType", "point")
-        ds.createDimension("blah", 2)
-        temp = ds.createVariable("temp", "d", ("time"))
-        temp.setncattr("platform", "platform_var")
-        plat = ds.createVariable("platform_var", np.byte)
-        cf_role_var = ds.createVariable("cf_role_var", np.byte, ("blah",))
-        cf_role_var.setncattr("cf_role", "profile_id")
-        results = self.ioos.check_cf_dsg(ds)
-        self.assertEqual(results, [])
-
     def test_check_platform_vocabulary(self):
         ds = MockTimeSeries() # time, lat, lon, depth
         ds.setncattr("platform_vocabulary", "http://google.com")