Repository: ioos/compliance-checker

Issue:
Unnecessary data shuffling in CF check_packed_data
While investigating the causes of issue #70 (slow-running checker for large files) @fxmzb123 found that the `check_packed_data` method (in CF suite) was responsible for a lot of the running time. The current implementation calls `numpy.reshape` on each variable 5 times, which can be pretty slow for large multi-dimensional arrays. This could certainly be optimised by doing the reshape only once for a variable.

Going further, I think that reshaping data arrays is actually unnecessary here. The CF convention only refers to the data types of the variable itself, and some of its attributes. Surely these can be determined without manipulating the entire array.



Discussion:
To be a bit more specific, for a NetCDF object, each variable has an attribute `var.dtype` (the type of the values stored in the file), and this needs to be compared to `var.add_offset.dtype` and `var.scale_factor.dtype`.

The [CF convention](http://cfconventions.org/Data/cf-conventions/cf-conventions-1.6/build/cf-conventions.html#packed-data) also specifies that the types of the variable attributes _FillValue, valid_min, valid_max, valid_range (when they exist) should also be the same as `var.dtype`. This is currently not implemented.


Resolving patch:
diff --git a/compliance_checker/cf/cf.py b/compliance_checker/cf/cf.py
--- a/compliance_checker/cf/cf.py
+++ b/compliance_checker/cf/cf.py
@@ -2360,66 +2360,77 @@ def check_packed_data(self, ds):
         When data to be packed contains missing values the attributes that indicate missing values (_FillValue, valid_min,
         valid_max, valid_range) must be of the same data type as the packed data.
         """
-        
         ret_val = []
-        reasoning = []
-        names = list(ds.dataset.variables.iterkeys())
         for name, var in ds.dataset.variables.iteritems():
-            if hasattr(var, 'add_offset') or hasattr(var, 'scale_factor'):
+
+            add_offset = getattr(var, 'add_offset', None)
+            scale_factor = getattr(var, 'scale_factor', None)
+            if not (add_offset or scale_factor):
+                continue
+
+            valid = True
+            reasoning = []
+
+            # if only one of these attributes is defined, assume they
+            # are the same type (value doesn't matter here)
+            if not add_offset:
+                add_offset = scale_factor
+            if not scale_factor:
+                scale_factor = add_offset
+
+            if type(add_offset) != type(scale_factor):
                 valid = False
-                if hasattr(var, 'add_offset') and hasattr(var, 'scale_factor'):
-                    offset = getattr(var, 'add_offset', '')
-                    scale = getattr(var, 'scale_factor', '')
-
-                    data_size = np.s_[ds.dataset.variables[name].size]
-                    data_type_check = np.reshape(ds.dataset.variables[name],data_size)[0]
-                    #, type(ds.dataset.variables[name][0,0,0]) ,type(scale), type(offset)
-                    for x in xrange(data_size):
-                        if type(np.reshape(ds.dataset.variables[name],data_size)[x]) != 'numpy.ma.core.MaskedConstant':
-                            data_type_check = np.reshape(ds.dataset.variables[name],data_size)[x]
-                            break
-                    if type(data_type_check) == type(scale) == type(offset):
-                        valid = True
-                        result = Result(BaseCheck.MEDIUM,                            \
-                                valid,                                       \
-                                ('var', name, 'packed_data'), \
-                                reasoning)
-                        ret_val.append(result)
-                        reasoning = []
-                    elif type(scale) == type(offset) != type(np.reshape(ds.dataset.variables[name],data_size)[x]):
-                        if type(scale) in [float(), type(np.float32(1.)), int(), type(np.int16(1))]  and (type(np.reshape(ds.dataset.variables[name],data_size)[x])) in [type(np.int8(1)), type(np.int16(1)), int()]:
-                           valid = True
-                           result = Result(BaseCheck.MEDIUM,                            \
-                                valid,                                       \
-                                ('var', name, 'packed_data'), \
-                                reasoning)
-                           ret_val.append(result)
-                           reasoning = []
-                        else: 
-                           valid = False
-
-                           reasoning.append("'add_offset' and 'scale_factor' are not of type float or int, or the data variable is not of type byte, short, or int.")
-                           result = Result(BaseCheck.MEDIUM,                            \
-                                valid,                                       \
-                                ('var', name, 'packed_data'), \
-                                reasoning)
-                           ret_val.append(result) 
-                           reasoning = []    
-                    else:
+                reasoning.append("Attributes add_offset and scale_factor have different data type.")
+            elif type(scale_factor) != var.dtype:
+                # Check both attributes are type float or double
+                if not type(scale_factor) in [np.float, np.float16, np.float32, np.float64, np.float128]:
+                    valid = False
+                    reasoning.append("Attributes add_offset and scale_factor are not of type float or double.")
+                else:
+                    # Check variable type is byte, short or int
+                    if not var.dtype in [np.int, np.int8, np.int16, np.int32, np.int64]:
                         valid = False
-                        reasoning.append("'add_offset' and 'scale_factor' are not both of type float or int or the data variable is not of type byte, short, or int.")
-                        result = Result(BaseCheck.MEDIUM,                            \
-                                valid,                                       \
-                                ('var', name, 'packed_data'), \
-                                reasoning)
-                        ret_val.append(result) 
-                        reasoning = []
+                        reasoning.append("Variable is not of type byte, short, or int.")
 
+            result = Result(BaseCheck.MEDIUM, valid, ('var', name, 'packed_data'), reasoning)
+            ret_val.append(result)
+            reasoning = []
 
+            valid = True
+            # test further with  _FillValue , valid_min , valid_max , valid_range
+            if hasattr(var, "_FillValue"):
+                if var._FillValue.dtype !=  var.dtype:
+                    valid = False
+                    reasoning.append("Type of _FillValue attribute (%s) does not match variable type (%s)" %\
+                                     (var._FillValue.dtype, var.dtype))
+            if hasattr(var, "valid_min"):
+                if var.valid_min.dtype != var.dtype:
+                    valid = False
+                    reasoning.append("Type of valid_min attribute (%s) does not match variable type (%s)" %\
+                                     (var.valid_min.dtype, var.dtype))
+            if hasattr(var, "valid_max"):
+                if var.valid_max.dtype != var.dtype:
+                    valid = False
+                    reasoning.append("Type of valid_max attribute (%s) does not match variable type (%s)" %\
+                                     (var.valid_max.dtype, var.dtype))
+            if hasattr(var, "valid_range"):
+                if var.valid_range.dtype != var.dtype:
+                    valid = False
+                    reasoning.append("Type of valid_range attribute (%s) does not match variable type (%s)" %\
+                                     (var.valid_range.dtype, var.dtype))
+
+            result = Result(BaseCheck.MEDIUM, valid, ('var', name, 'fillvalue_valid_range_attributes'), reasoning)
+            ret_val.append(result)
 
         return ret_val
 
-        
+
+
+
+
+
+
+
 
     def check_compression(self, ds):
         """


Test patch:
diff --git a/compliance_checker/tests/test_cf.py b/compliance_checker/tests/test_cf.py
--- a/compliance_checker/tests/test_cf.py
+++ b/compliance_checker/tests/test_cf.py
@@ -648,8 +648,11 @@ def test_check_alternative_coordinates(self):
     def test_check_packed_data(self):
         dataset = self.get_pair(static_files['bad_data_type'])
         results = self.cf.check_packed_data(dataset)
+        self.assertEqual(len(results), 4)
         self.assertFalse(results[0].value)
         self.assertTrue(results[1].value)
+        self.assertTrue(results[2].value)
+        self.assertFalse(results[3].value)
 
 
     def test_check_compression(self):