Repository: ioos/compliance-checker

Issue:
check_data_types exception during cf test: type object 'str' has no attribute 'kind'
Using version 4.3.3rc2+54.gb086b57, I am getting the following exception for all of our netcdf files under cf:1.6 and cf:1.7 test... an example netcdf file: 
http://gyre.umeoce.maine.edu/ncei/neracoos/A01/A0141.ocean.001m.merged.nc

WARNING: The following exceptions occured during the cf:1.7 checker (possibly in
dicate compliance checker issues):
cf:1.7.check_data_types: type object 'str' has no attribute 'kind'
  File "c:\apps\anaconda3\envs\cc-test-dev\lib\site-packages\compliance_checker-
4.3.3rc2+54.gb086b57-py3.8.egg\compliance_checker\cf\cf.py", line 1326, in check
_data_types
    if v.dtype.kind != "S" and all(

Any thoughts appreciated.


Discussion:
NetCDF4-Python casts string-like attributes to `str` type, which is does not have `numpy`'s kind attribute.   There is code in the checker to address this discrepancy, but it looks like not all the edge cases are handled.  I'll take a look at what's occurring with this particular dataset.
@benjwadams : What about checking in `cf.check_data_types` for type of `str` this way (the `if v.dtype is not str:` is new):

``` python
    def check_data_types(self, ds):
        """
        Checks the data type of all netCDF variables to ensure they are valid
        data types under CF.

        CF §2.2 The netCDF data types char, byte, short, int, float or real, and
        double are all acceptable

        :param netCDF4.Dataset ds: An open netCDF dataset
        :rtype: compliance_checker.base.Result
        """
        fails = []
        total = len(ds.variables)

        for k, v in ds.variables.items():
            # NEW: 
            if v.dtype is not str:
                if v.dtype.kind != "S" and all(
                    v.dtype.type != t
                    for t in (
                        np.character,
                        np.dtype("|S1"),
                        np.dtype("b"),
                        np.dtype("i2"),
                        np.dtype("i4"),
                        np.float32,
                        np.double,
                    )
                ):
                    fails.append(
                        "The variable {} failed because the datatype is {}".format(
                            k, v.datatype
                        )
                    )
        return Result(
            BaseCheck.HIGH,
            (total - len(fails), total),
            self.section_titles["2.2"],
            msgs=fails,
        )
```

**Note:** We need to check for `v.dtype is not str` because `not isinstance(v, str)` does not work (because `v` is of type `netCDF4._netCDF4.Variable`.

Resolving patch:
diff --git a/compliance_checker/cf/cf.py b/compliance_checker/cf/cf.py
--- a/compliance_checker/cf/cf.py
+++ b/compliance_checker/cf/cf.py
@@ -506,7 +506,7 @@ def _parent_var_attr_type_check(self, attr_name, var, ctx):
         attr_val = var.getncattr(attr_name)
 
         if isinstance(attr_val, (str, bytes)):
-            type_match = var.dtype.kind == "S"
+            type_match = (var.dtype is str) or (var.dtype.kind == "S")
             val_type = type(attr_val)
         else:
             val_type = attr_val.dtype.type
@@ -871,7 +871,9 @@ def _get_coord_axis_map(self, ds):
         for coord_name in self._find_aux_coord_vars(ds):
             coord_var = ds.variables[coord_name]
             # Skip label auxiliary coordinates
-            if coord_var.dtype.char == "S":
+            if hasattr(coord_var.dtype, "char") and coord_var.dtype.char == "S":
+                continue
+            elif coord_var.dtype == str:
                 continue
             for dimension in coord_var.dimensions:
                 if dimension not in coord_axis_map:
@@ -1323,16 +1325,20 @@ def check_data_types(self, ds):
         total = len(ds.variables)
 
         for k, v in ds.variables.items():
-            if v.dtype.kind != "S" and all(
-                v.dtype.type != t
-                for t in (
-                    np.character,
-                    np.dtype("|S1"),
-                    np.dtype("b"),
-                    np.dtype("i2"),
-                    np.dtype("i4"),
-                    np.float32,
-                    np.double,
+            if (
+                v.dtype is not str
+                and v.dtype.kind != "S"
+                and all(
+                    v.dtype.type != t
+                    for t in (
+                        np.character,
+                        np.dtype("|S1"),
+                        np.dtype("b"),
+                        np.dtype("i2"),
+                        np.dtype("i4"),
+                        np.float32,
+                        np.double,
+                    )
                 )
             ):
                 fails.append(
@@ -1752,7 +1758,9 @@ def check_units(self, ds):
                 continue
 
             # Skip labels
-            if variable.dtype.char == "S":
+            if hasattr(variable.dtype, "char") and variable.dtype.char == "S":
+                continue
+            elif variable.dtype == str:
                 continue
 
             standard_name = getattr(variable, "standard_name", None)
@@ -1810,7 +1818,8 @@ def _check_valid_cf_units(self, ds, variable_name):
         # Is this even in the database? also, if there is no standard_name,
         # there's no way to know if it is dimensionless.
         should_be_dimensionless = (
-            variable.dtype.char == "S"
+            variable.dtype is str
+            or (hasattr(variable.dtype, "char") and variable.dtype.char == "S")
             or std_name_units_dimensionless
             or standard_name is None
         )
@@ -1859,7 +1868,9 @@ def _check_valid_udunits(self, ds, variable_name):
 
         # If the variable is supposed to be dimensionless, it automatically passes
         should_be_dimensionless = (
-            variable.dtype.char == "S" or std_name_units_dimensionless
+            variable.dtype is str
+            or (hasattr(variable.dtype, "char") and variable.dtype.char == "S")
+            or std_name_units_dimensionless
         )
 
         valid_udunits = TestCtx(BaseCheck.HIGH, self.section_titles["3.1"])
@@ -1991,7 +2002,9 @@ def check_standard_name(self, ds):
 
             # Unfortunately, §6.1 allows for string types to be listed as
             # coordinates.
-            if ncvar.dtype.char == "S":
+            if hasattr(ncvar.dtype, "char") and ncvar.dtype.char == "S":
+                continue
+            elif ncvar.dtype == str:
                 continue
 
             standard_name = getattr(ncvar, "standard_name", None)
@@ -2386,7 +2399,9 @@ def check_coordinate_types(self, ds):
             # §6.1 allows for labels to be referenced as auxiliary coordinate
             # variables, which should not be checked like the rest of the
             # coordinates.
-            if variable.dtype.char == "S":
+            if hasattr(variable.dtype, "char") and variable.dtype.char == "S":
+                continue
+            elif variable.dtype == str:
                 continue
 
             axis = getattr(variable, "axis", None)
@@ -2958,7 +2973,12 @@ def check_aux_coordinates(self, ds):
                     continue
 
                 # §6.1 Allows for "labels" to be referenced as coordinates
-                if ds.variables[aux_coord].dtype.char == "S":
+                if (
+                    hasattr(ds.variables[aux_coord].dtype, "char")
+                    and ds.variables[aux_coord].dtype.char == "S"
+                ):
+                    continue
+                elif ds.variables[aux_coord].dtype == str:
                     continue
 
                 aux_coord_dims = set(ds.variables[aux_coord].dimensions)
@@ -4180,7 +4200,9 @@ def check_compression_gathering(self, ds):
                 )
             # ensure compression variable is a proper index, and thus is an
             # signed or unsigned integer type of some sort
-            if compress_var.dtype.kind not in {"i", "u"}:
+            if (compress_var.dtype is str) or (
+                compress_var.dtype.kind not in {"i", "u"}
+            ):
                 valid = False
                 reasoning.append(
                     "Compression variable {} must be an integer type to form a proper array index".format(
diff --git a/compliance_checker/cfutil.py b/compliance_checker/cfutil.py
--- a/compliance_checker/cfutil.py
+++ b/compliance_checker/cfutil.py
@@ -227,7 +227,9 @@ def is_geophysical(ds, variable):
         return False
 
     # Is it a string but with no defined units?
-    if ncvar.dtype.char == "S":
+    if hasattr(ncvar.dtype, "char") and ncvar.dtype.char == "S":
+        return False
+    elif ncvar.dtype is str:
         return False
 
     # Is it an instrument descriptor?


Test patch:
diff --git a/compliance_checker/tests/data/string_type_variable.cdl b/compliance_checker/tests/data/string_type_variable.cdl
new file mode 100644
--- /dev/null
+++ b/compliance_checker/tests/data/string_type_variable.cdl
@@ -0,0 +1,18 @@
+netcdf string_type_variable {
+dimensions:
+        lon = 10 ;
+variables:
+        double lon(lon) ;
+		lon:long_name = "location of something a specific longitudinal coordinates" ;
+		lon:axis = "X" ;
+		lon:units = "degrees_east" ;
+		lon:standard_name = "longitude" ;
+        string j(lon) ;
+		j:long_name = "name of each measurement location" ;
+
+// global attributes:
+	:Conventions = "CF-1.7" ;
+	:title = "file to test proper processing of string-type variables" ;
+	:history = "2020-07-22T17:04: hand-made" ;
+}
+
diff --git a/compliance_checker/tests/resources.py b/compliance_checker/tests/resources.py
--- a/compliance_checker/tests/resources.py
+++ b/compliance_checker/tests/resources.py
@@ -107,6 +107,7 @@ def generate_dataset(cdl_path, nc_path):
     "self-referencing-var": get_filename("tests/data/self-referencing-var.cdl"),
     "self_referencing": get_filename("tests/data/non-comp/self_referencing.cdl"),
     "sldmb_43093_agg": get_filename("tests/data/examples/sldmb_43093_agg.cdl"),
+    "string": get_filename("tests/data/string_type_variable.cdl"),
     "swan": get_filename("tests/data/examples/swan.cdl"),
     "sp041": get_filename("tests/data/examples/sp041.cdl"),
     "timeseries": get_filename("tests/data/timeseries.cdl"),
diff --git a/compliance_checker/tests/test_cf.py b/compliance_checker/tests/test_cf.py
--- a/compliance_checker/tests/test_cf.py
+++ b/compliance_checker/tests/test_cf.py
@@ -123,14 +123,26 @@ def load_dataset(self, nc_dataset):
     def test_check_data_types(self):
         """
         Invoke check_data_types() and loop through all variables to check data
-        types. Pertains to 2.2 The netCDF data types char, byte, short, int,
-        float or real, and double are all acceptable.
+        types. Pertains to 2.2. The netCDF data types char, byte, short, int,
+        float or real, and double are all acceptable. NetCDF4 allows string as
+        data type, which is also acceptable.
         """
 
+        # check default netCDF data types
         dataset = self.load_dataset(STATIC_FILES["rutgers"])
         result = self.cf.check_data_types(dataset)
         assert result.value[0] == result.value[1]
 
+        # check if variables of type `string` is properly processed
+        dataset = self.load_dataset(STATIC_FILES["string"])
+        if dataset.file_format != "NETCDF4":
+            raise RuntimeError(
+                "netCDF file of wrong format (not netCDF4) was created for checking"
+            )
+        result = self.cf.check_data_types(dataset)
+        assert result.value[0] == result.value[1]
+
+        # check bad data types
         dataset = self.load_dataset(STATIC_FILES["bad_data_type"])
         result = self.cf.check_data_types(dataset)
 
@@ -1078,6 +1090,26 @@ def test_check_grid_mapping(self):
         )
         assert all(r.name == expected_name for r in results.values())
 
+    def test_is_geophysical(self):
+
+        # check whether string type variable, which are not `cf_role`, are
+        # properly processed
+        dataset = self.load_dataset(STATIC_FILES["string"])
+        if dataset.file_format != "NETCDF4":
+            raise RuntimeError(
+                "netCDF file of wrong format (not netCDF4) was created for checking"
+            )
+        try:
+            result = cfutil.is_geophysical(dataset, "j")
+        except AttributeError:
+            pytest.fail(
+                "Test probably fails because var.dtype.kind or var.dtype.char "
+                "was tested on string-type variable. Consider checking for "
+                "`var.dtype is str`"
+            )
+        assert not result
+        # assert False
+
     # TODO: overhaul to use netCDF global attributes or mocks and variable
     #       attributes
     def test_check_attr_type(self):