Repository: ioos/compliance-checker
Pull request #791 (2020-04-08 14:57:44)

IOOS  1.2  platform  DSG
Separate platform checks from the CF DSG checks and add corresponding unit tests, per #748.


Patch:
diff --git a/compliance_checker/cf/cf.py b/compliance_checker/cf/cf.py
index a013185..1c3d206 100644
--- a/compliance_checker/cf/cf.py
+++ b/compliance_checker/cf/cf.py
@@ -2006,7 +2006,7 @@ class CF1_6Check(CFNCCheck):
         valid_masks = TestCtx(BaseCheck.HIGH, self.section_titles['3.5'])
 
         valid_masks.assert_true(isinstance(flag_masks, np.ndarray),
-                                "{}'s flag_masks must be an array of values not {}".format(name, type(flag_masks).__name__))
+                                "{}'s flag_masks must be an array of values not {}".format(name, type(flag_masks)))
 
         if not isinstance(flag_masks, np.ndarray):
             return valid_masks.to_result()
diff --git a/compliance_checker/data/templates/ccheck.html.j2 b/compliance_checker/data/templates/ccheck.html.j2
index 02d7c8a..b398821 100644
--- a/compliance_checker/data/templates/ccheck.html.j2
+++ b/compliance_checker/data/templates/ccheck.html.j2
@@ -45,18 +45,18 @@ li{
   <div class="col-md-12">
     <div class="page-header">
       <h2 align="center">IOOS Compliance Checker Report</h2>
-      <p align="center">For dataset {{source_name|e}}</p>
+      <p align="center">For dataset {{source_name}}</p>
     </div>
   </div>
   <div class="col-md-12">
     <h3 align="center">
       {% if cc_url %}
-        <a href={{cc_url|e}} target="_blank">{{testname|e}}</a>
+        <a href={{cc_url}} target="_blank">{{testname}}</a>
       {% else %}
-        {{testname|e}}
+        {{testname}}
       {% endif %}
     </h3>
-    <!--<a href={{cc_url|e}}>Reference Page</a>-->
+    <!--<a href={{cc_url}}>Reference Page</a>-->
   </div>
 
   <!-- -------------- -->
@@ -71,7 +71,7 @@ li{
       <!-- High Priority -->
 
       <div class="table-collapse">
-        <a data-target="high-priority-table" href="#"> {{scoreheader.get(3)|e}} <i class="glyphicon glyphicon-collapse-up"></i></a>
+        <a data-target="high-priority-table" href="#"> {{scoreheader.get(3)}} <i class="glyphicon glyphicon-collapse-up"></i></a>
       </div>
       {% if high_count -%}
       <div class="failures">
@@ -93,13 +93,13 @@ li{
 
                   <tr>
                     {% if result["msgs"]|length > 0 -%}
-                      <td>{{result['name']|e}}</td>
+                      <td>{{result['name']}}</td>
                       <td>
                           {% for msg in result['msgs'] -%}
 
                           <!-- bulleted list inside the table row -->
                         <ul>
-                          <li>{{msg|e}}</li>
+                          <li>{{msg}}</li>
                         </ul>
 
                         {% endfor -%}
@@ -136,7 +136,7 @@ li{
     <div class="col-md-12">
       {% if medium_count -%}
       <div class="table-collapse">
-        <a data-target="medium-priority-table" href="#"> {{scoreheader.get(2)|e}} <i class="glyphicon glyphicon-collapse-up"></i></a>
+        <a data-target="medium-priority-table" href="#"> {{scoreheader.get(2)}} <i class="glyphicon glyphicon-collapse-up"></i></a>
       </div>
       <div class="failures">
         | <span class="label label-danger label-as-badge">{{ medium_count }}</span>
@@ -156,13 +156,13 @@ li{
 
                   <tr>
                     {% if result["msgs"]|length -%}
-                      <td>{{result['name']|e}}</td>
+                      <td>{{result['name']}}</td>
                       <td>
                           {% for msg in result['msgs'] -%}
 
                           <!-- bulleted list inside the table row -->
                         <ul>
-                          <li>{{msg|e}}</li>
+                          <li>{{msg}}</li>
                         </ul>
 
                         {% endfor -%}
@@ -185,7 +185,7 @@ li{
     <div class="col-md-12">
       {% if low_count -%}
       <div class="table-collapse">
-        <a data-target="low-priority-table" href="#"> {{scoreheader.get(1)|e}} <i class="glyphicon glyphicon-collapse-up"></i></a>
+        <a data-target="low-priority-table" href="#"> {{scoreheader.get(1)}} <i class="glyphicon glyphicon-collapse-up"></i></a>
       </div>
       <div class="failures">
         | <span class="label label-danger label-as-badge">{{ low_count }}</span>
@@ -205,13 +205,13 @@ li{
 
                   <tr>
                     {% if result["msgs"]|length -%}
-                      <td>{{result['name']|e}}</td>
+                      <td>{{result['name']}}</td>
                       <td>
                           {% for msg in result['msgs'] -%}
 
                           <!-- bulleted list inside the table row -->
                         <ul>
-                          <li>{{msg|e}}</li>
+                          <li>{{msg}}</li>
                         </ul>
 
                         {% endfor -%}
diff --git a/compliance_checker/ioos.py b/compliance_checker/ioos.py
index 3a615ca..568c8d1 100644
--- a/compliance_checker/ioos.py
+++ b/compliance_checker/ioos.py
@@ -440,7 +440,7 @@ class IOOS1_2Check(IOOSNCCheck):
             ('infoUrl', base.UrlValidator()),
             'license',
             ('naming_authority', NamingAuthorityValidator()),
-            'platform',
+            #'platform', # checked in check_platform_global
             'platform_name',
             'publisher_country',
             ('publisher_email', base.EmailValidator()),
@@ -785,17 +785,9 @@ class IOOS1_2Check(IOOSNCCheck):
 
     def check_single_platform(self, ds):
         """
-        Verify that a dataset only has a single platform attribute. If one exists,
-        examine the featureType of the dataset. If the featureType is
-        [point, timeSeries, profile, trajectory] and cf_role in [timeseries_id,
-        profile_id, trajectory_id] dimensionality of the variable containing
-        cf_role must be 1 as "we only want a single glider/auv/ship"; if
-        featureType in [timeseries_id, trajectory_id], dimensionality of the
-        variable must also be one as  "we only want a single timeSeries aka buoy".
-        If cf_role==profile_id, it can have whatever dimension.
-
-        Gridded model datasets are not required to declare a platform
-        or platform variables.
+        Verify that a dataset only has a single platform attribute, and thus
+        a single platform variable. Gridded model datasets are not required
+        to declare a platform or platform variables.
 
         Args:
             ds (netCDF-4 Dataset): open Dataset object
@@ -804,7 +796,6 @@ class IOOS1_2Check(IOOSNCCheck):
             Result
         """
 
-        results = []
         glb_platform = getattr(ds, "platform", None)
 
         platform_set = set()
@@ -815,62 +806,114 @@ class IOOS1_2Check(IOOSNCCheck):
         if num_platforms > 1 and glb_platform:
             msg = "A dataset may only have one platform; {} found".format(len(platform_set))
             val = False
-            results.append(Result(BaseCheck.HIGH, val, "platform", None if val else [msg]))
 
         elif ((not glb_platform) and num_platforms > 0):
-            msg = "If platform variables exist, a global attribute \"platform\" must also exist"
+            msg = "If a platform variable exists, a global attribute \"platform\" must also exist"
             val = False
-            results.append(Result(BaseCheck.HIGH, val, "platform", None if val else [msg]))
 
         elif num_platforms == 0 and glb_platform:
             msg = "A dataset with a global \"platform\" attribute must platform have variables"
             val = False
-            results.append(Result(BaseCheck.HIGH, val, "platform", None if val else [msg]))
 
         elif num_platforms == 0 and (not glb_platform):
             msg = "Gridded model datasets are not required to declare a platform"
             val = True
-            results.append(Result(BaseCheck.HIGH, val, "platform", None if val else [msg]))
-
-        else: # num_platforms==1 and glb_platform, test the dimensionality
-
-            num_plat_val = True if num_platforms == 1 else False
-
-            feature_type = getattr(ds, "featureType", "").lower()
-            if not feature_type:
-                return results
-
-            # filter out cf_role exists
-            cf_role_vars = ds.get_variables_by_attributes(cf_role=lambda x: x is not None)
-            num_cf_role_vars = len(cf_role_vars)
-            msg = "With a single platform provided, the dimension of the cf_role " +\
-                  "variable {cf_role_var} (cf_role=={cf_role}) should also " +\
-                  "be equal to 1 (it is {dim})"
-
-            for var in cf_role_vars:
-                cf_role = getattr(var, "cf_role")
-                shp = var.shape[0] if len(var.shape) > 0 else 1
-                if (
-                       feature_type in ["point", "timeseries", "profile", "trajectory", "timeseriesprofile", "trajectoryprofile"]
-                       and
-                       cf_role in ["timeseries_id", "profile_id", "trajectory_id"]
-                   ):
-                    if (num_cf_role_vars==1) or (num_cf_role_vars>1 and cf_role!="profile_id"):
-                        # shape must be 1 (or if no length, that's okay too)
-                        _val = shp==1
-                    elif (num_cf_role_vars>1) and (cf_role=="profile_id"):
-                        # can have any dimension if there are more than one cf_role?
-                        _val = True
 
-                    results.append(
-                       Result(
-                           BaseCheck.HIGH,
-                           _val,
-                           "platform variables",
-                           None if _val else [msg.format(cf_role_var=var.name, cf_role=cf_role, dim=shp)]
-                       )
-                    )
+        else:
+            val = True
+
+        return Result(BaseCheck.HIGH, val, "platform", None if val else [msg])
+
+    def check_cf_dsg(self, ds):
+        """
+        Check that the dataset follows the restrictions for CF Discrete
+        Sampling Geometries set by the IOOS Metadata Profile.
+
+        Examine each variable with a cf_role. For each (featureType, cf_role):
+            - (timeSeries, timeseries_id)
+            - (timeSeriesProfile, timeseries_id)
+            - (trajectory, trajectory_id)
+            - (trajectoryProfile, trajectory_id)
+            - (profile, profile_id)
+        the dimension of the variable should be 1. For datasets with
+        the "point" featureType, do nothing.
+
+
+        https://github.com/ioos/compliance-checker/issues/748#issuecomment-606659685
+
+        Parameters
+        ----------
+        ds: netCDF4.Dataset (open)
+
+        Returns
+        -------
+        list of Result objects
+        """
+
+        results = []
 
+        feature_type = getattr(ds, "featureType", "").lower()
+        if not feature_type:
+            return results
+ 
+        # loop through all variables with cf_role
+        cf_role_vars = ds.get_variables_by_attributes(cf_role=lambda x: x is not None)
+        num_cf_role_vars = len(cf_role_vars)
+
+        for var in cf_role_vars:
+            cf_role = getattr(var, "cf_role")
+            shp = var.shape[0] if len(var.shape) > 0 else 1
+ 
+            if feature_type == "timeseries" and cf_role == "timeseries_id":
+                msg = (
+                    "Dimension length of the variable with "
+                    "cf_role='timeseries_id (the 'station' dimension) "
+                    "is {dim}. Note that the IOOS profile restricts "
+                    "timeSeries datasets with multiple features to share "
+                    "the same lat/lon position (ie. to exist on the same "
+                    "platform). Datasets that include multiple platforms "
+                    "are not valid and will cause harvesting errors."
+                ).format(dim=shp)
+ 
+                _val = shp <= 1 # fails if > 1
+ 
+            elif ( # even though tested condition is the same, different msg
+                     feature_type == "profile" and cf_role == "profile_id" or
+                     feature_type == "trajectory" and cf_role == "trajectory_id" or
+                     feature_type == "timeseriesprofile" and cf_role == "timeseries_id" or
+                     feature_type == "trajectoryprofile" and cf_role == "trajectory_id"
+                 ):
+
+                msg = (
+                    "Dimension length of the variable `{cf_role_var}` with "
+                    "cf_role=`{cf_role}` (the 'station/trajectory/profile' "
+                    "dimension) must be equal to 1 (it is {dim}). The IOOS "
+                    "profile restricts {feature_type} datasets to a "
+                    "single platform (ie. station/trajectory/profile) per dataset."
+                ).format(
+                    cf_role_var=var.name, cf_role=cf_role,
+                    dim=shp, feature_type=feature_type
+                )
+ 
+                _val = shp == 1
+ 
+            elif feature_type == "point": # do nothing
+                _val = True
+
+            else: # featureType and cf_role don't match up to our restrictions
+                _val = True
+ 
+            if not _val:
+                results.append(
+                    Result(
+                        BaseCheck.HIGH,
+                        _val,
+                        "CF Discrete Sampling Geometry Compliance",
+                        [msg]
+                    )
+                )
+ 
+ 
         return results
 
     def check_platform_vocabulary(self, ds):


Test patch:
diff --git a/compliance_checker/tests/test_ioos_profile.py b/compliance_checker/tests/test_ioos_profile.py
index ed3bac4..23883de 100644
--- a/compliance_checker/tests/test_ioos_profile.py
+++ b/compliance_checker/tests/test_ioos_profile.py
@@ -685,16 +685,44 @@ class TestIOOS1_2(BaseTestCase):
         ds = MockTimeSeries() # time, lat, lon, depth
 
         # no global attr but also no platform variables, should pass
-        results = self.ioos.check_single_platform(ds)
-        self.assertTrue(results[0].value)
-        self.assertEqual(results[0].msgs, [])
+        result = self.ioos.check_single_platform(ds)
+        self.assertTrue(result.value)
+        self.assertEqual(result.msgs, [])
 
         # give platform global, no variables, fail
         ds.setncattr("platform", "buoy")
-        results = self.ioos.check_single_platform(ds)
-        self.assertFalse(results[0].value)
+        result = self.ioos.check_single_platform(ds)
+        self.assertFalse(result.value)
+
+        # global platform, one platform variable, pass
+        temp = ds.createVariable("temp", "d", ("time"))
+        temp.setncattr("platform", "platform_var")
+        plat = ds.createVariable("platform_var", np.byte)
+        result = self.ioos.check_single_platform(ds)
+        self.assertTrue(result.value)
+        self.assertEqual(result.msgs, [])
+
+        # two platform variables, fail
+        temp2 = ds.createVariable("temp2", "d", ("time"))
+        temp2.setncattr("platform", "platform_var2")
+        plat = ds.createVariable("platform_var2", np.byte)
+        result = self.ioos.check_single_platform(ds)
+        self.assertFalse(result.value)
+
+        # no global attr, one variable, fail
+        ds = MockTimeSeries() # time, lat, lon, depth
+        temp = ds.createVariable("temp", "d", ("time"))
+        temp.setncattr("platform", "platform_var")
+        plat = ds.createVariable("platform_var", np.byte)
+        result = self.ioos.check_single_platform(ds)
+        self.assertFalse(result.value)
+ 
+    def test_check_cf_dsg(self):
+
+        ds = MockTimeSeries() # time, lat, lon, depth
+        ds.setncattr("platform", "single_string")
 
-        # global attribute, one platform variable, correct cf_role & featureType, pass
+        # correct cf_role & featureType, pass
         ds.setncattr("featureType", "profile")
         ds.createDimension("profile", 1)
         temp = ds.createVariable("temp", "d", ("time"))
@@ -702,33 +730,145 @@ class TestIOOS1_2(BaseTestCase):
         plat = ds.createVariable("platform_var", np.byte)
         cf_role_var = ds.createVariable("cf_role_var", np.byte, ("profile",))
         cf_role_var.setncattr("cf_role", "timeseries_id")
-        results = self.ioos.check_single_platform(ds)
+        results = self.ioos.check_cf_dsg(ds)
         self.assertTrue(all(r.value for r in results))
         self.assertTrue(all(r.msgs==[] for r in results))
 
-        # global attr, multiple platform variables, correct cf_role & featureType, fail
-        plat2 = ds.createVariable("platform_var_2", np.byte)
-        temp2 = ds.createVariable("temp2", "d", ("time"))
-        temp2.setncattr("platform", "platform_var2")
-        results = self.ioos.check_single_platform(ds)
+        # correct featureType, incorrect cf_role var dimension
+        ds = MockTimeSeries() # time, lat, lon, depth
+        ds.setncattr("featureType", "trajectoryprofile")
+        ds.createDimension("trajectory", 2) # should only be 1
+        temp = ds.createVariable("temp", "d", ("time"))
+        temp.setncattr("platform", "platform_var")
+        plat = ds.createVariable("platform_var", np.byte)
+        cf_role_var = ds.createVariable("cf_role_var", np.byte, ("trajectory",))
+        cf_role_var.setncattr("cf_role", "trajectory_id")
+        results = self.ioos.check_cf_dsg(ds)
         self.assertFalse(results[0].value)
 
-        # no global attr, one platform var, correct cf_role & featureType, fail
-        ds.delncattr("platform")
+        # featureType==timeSeries, cf_role=timeseries_id
+        ds = MockTimeSeries()
+        ds.setncattr("featureType", "timeSeries")
+        ds.createDimension("station", 1)
+        temp = ds.createVariable("temp", "d", ("time"))
+        temp.setncattr("platform", "platform_var")
+        plat = ds.createVariable("platform_var", np.byte)
+        cf_role_var = ds.createVariable("cf_role_var", np.byte, ("station",))
+        cf_role_var.setncattr("cf_role", "timeseries_id")
+        results = self.ioos.check_cf_dsg(ds)
+
+        # check should pass with no results
+        self.assertEqual(results, [])
+
+        # featureType==timeSeriesProfile, cf_role==timeseries_id, dim 1, pass
+        ds = MockTimeSeries()
+        ds.setncattr("featureType", "timeSeriesProfile")
+        ds.createDimension("station", 1)
+        temp = ds.createVariable("temp", "d", ("time"))
+        temp.setncattr("platform", "platform_var")
+        plat = ds.createVariable("platform_var", np.byte)
+        cf_role_var = ds.createVariable("cf_role_var", np.byte, ("station",))
+        cf_role_var.setncattr("cf_role", "timeseries_id")
+        results = self.ioos.check_cf_dsg(ds)
+        self.assertEqual(results, [])
+
+        # featureType==timeSeriesProfile, cf_role==timeseries_id, dim 2, fail
+        ds = MockTimeSeries()
+        ds.setncattr("platform", "platform")
+        ds.setncattr("featureType", "timeSeriesProfile")
+        ds.createDimension("station", 2)
+        temp = ds.createVariable("temp", "d", ("time"))
+        temp.setncattr("platform", "platform_var")
+        plat = ds.createVariable("platform_var", np.byte)
+        cf_role_var = ds.createVariable("cf_role_var", np.byte, ("station",))
+        cf_role_var.setncattr("cf_role", "timeseries_id")
+        results = self.ioos.check_cf_dsg(ds)
         self.assertFalse(results[0].value)
 
-        # global attr, one platform var, correct featureType, incorrect cf_role var dimension
-        ds = MockTimeSeries() # time, lat, lon, depth
-        ds.setncattr("featureType", "trajectoryprofile")
-        ds.createDimension("trajectory", 2) # should only be 1
+        # featureType==trajectory, cf_role==trajectory_id, dim 1, pass
+        ds = MockTimeSeries()
+        ds.setncattr("featureType", "trajectory")
+        ds.createDimension("trajectory", 1)
+        temp = ds.createVariable("temp", "d", ("time"))
+        temp.setncattr("platform", "platform_var")
+        plat = ds.createVariable("platform_var", np.byte)
+        cf_role_var = ds.createVariable("cf_role_var", np.byte, ("trajectory",))
+        cf_role_var.setncattr("cf_role", "trajectory_id")
+        results = self.ioos.check_cf_dsg(ds)
+        self.assertEqual(results, [])
+
+        # featureType==trajectory, cf_role==trajectory, dim 2, fail
+        ds = MockTimeSeries()
+        ds.setncattr("featureType", "trajectory")
+        ds.createDimension("trajectory", 2)
         temp = ds.createVariable("temp", "d", ("time"))
         temp.setncattr("platform", "platform_var")
         plat = ds.createVariable("platform_var", np.byte)
         cf_role_var = ds.createVariable("cf_role_var", np.byte, ("trajectory",))
         cf_role_var.setncattr("cf_role", "trajectory_id")
-        results = self.ioos.check_single_platform(ds)
+        results = self.ioos.check_cf_dsg(ds)
         self.assertFalse(results[0].value)
 
+        # featureType==trajectoryProfile, cf_role==trajectory_id, dim 1, pass
+        ds = MockTimeSeries()
+        ds.setncattr("featureType", "trajectoryProfile")
+        ds.createDimension("trajectoryprof", 1)
+        temp = ds.createVariable("temp", "d", ("time"))
+        temp.setncattr("platform", "platform_var")
+        plat = ds.createVariable("platform_var", np.byte)
+        cf_role_var = ds.createVariable("cf_role_var", np.byte, ("trajectoryprof",))
+        cf_role_var.setncattr("cf_role", "trajectory_id")
+        results = self.ioos.check_cf_dsg(ds)
+        self.assertEqual(results, [])
+
+        # featureType==trajectoryProfile, cf_role==trajectory_id, dim 2, fail
+        ds = MockTimeSeries()
+        ds.setncattr("featureType", "trajectoryProfile")
+        ds.createDimension("trajectoryprof", 2)
+        temp = ds.createVariable("temp", "d", ("time"))
+        temp.setncattr("platform", "platform_var")
+        plat = ds.createVariable("platform_var", np.byte)
+        cf_role_var = ds.createVariable("cf_role_var", np.byte, ("trajectoryprof",))
+        cf_role_var.setncattr("cf_role", "trajectory_id")
+        results = self.ioos.check_cf_dsg(ds)
+        self.assertFalse(results[0].value)
+
+        # featureType==profile, cf_role==profile_id, dim 1, pass
+        ds = MockTimeSeries()
+        ds.setncattr("featureType", "profile")
+        ds.createDimension("prof", 1)
+        temp = ds.createVariable("temp", "d", ("time"))
+        temp.setncattr("platform", "platform_var")
+        plat = ds.createVariable("platform_var", np.byte)
+        cf_role_var = ds.createVariable("cf_role_var", np.byte, ("prof",))
+        cf_role_var.setncattr("cf_role", "profile_id")
+        results = self.ioos.check_cf_dsg(ds)
+        self.assertEqual(results, [])
+
+        # featureType==profile, cf_role==profile_id, dim 2, fail
+        ds = MockTimeSeries()
+        ds.setncattr("featureType", "profile")
+        ds.createDimension("prof", 2)
+        temp = ds.createVariable("temp", "d", ("time"))
+        temp.setncattr("platform", "platform_var")
+        plat = ds.createVariable("platform_var", np.byte)
+        cf_role_var = ds.createVariable("cf_role_var", np.byte, ("prof",))
+        cf_role_var.setncattr("cf_role", "profile_id")
+        results = self.ioos.check_cf_dsg(ds)
+        self.assertFalse(results[0].value)
+
+        # featureType==point -- do nothing
+        ds = MockTimeSeries()
+        ds.setncattr("featureType", "point")
+        ds.createDimension("blah", 2)
+        temp = ds.createVariable("temp", "d", ("time"))
+        temp.setncattr("platform", "platform_var")
+        plat = ds.createVariable("platform_var", np.byte)
+        cf_role_var = ds.createVariable("cf_role_var", np.byte, ("blah",))
+        cf_role_var.setncattr("cf_role", "profile_id")
+        results = self.ioos.check_cf_dsg(ds)
+        self.assertEqual(results, [])
+
     def test_check_platform_vocabulary(self):
         ds = MockTimeSeries() # time, lat, lon, depth
         ds.setncattr("platform_vocabulary", "http://google.com")