Repository: ARM-DOE/ACT

Issue:
Error when clean routine tries to remove unused bits
When running the script below on the ARM ADC systems, an error occurs when trying to add a DQR to the QC which stems from trying to remove unused bits from those QC variables

```
  File “/home/kehoe/dev/dq/lib/python/ACT/act/qc/qcfilter.py”, line 1065, in unset_bit
    array = array & ~(1 << bit_number - 1)
TypeError: ufunc ‘bitwise_and’ not supported for the input types, and the inputs could not be safely coerced to any supported types according to the casting rule ‘’safe’'
```

```
import act
import glob
import numpy as np

ds = 'sgpmetE13.b1'
site = ds[0:3]
files = glob.glob('/data/archive/'+site+'/'+ds+'/'+ds+'.2010*cdf')
years = [f.split('.')[-3][0:4] for f in files]
years = np.unique(years)

for y in years:
    files = glob.glob('/data/archive/'+site+'/'+ds+'/'+ds+'.'+y+'*cdf')
    files.sort()
    obj = act.io.armfiles.read_netcdf(files)
    obj = act.qc.arm.add_dqr_to_qc(obj, variable='temp_mean')
    obj = obj.where(obj['qc_temp_mean'] == 0)
    obj = obj.resample(time='M').mean()
    for i in range(len(obj['time'].values)):
        print(','.join([str(obj['time'].values[i]), str(obj['temp_mean'].values[i])]))
    obj.close()
```




Discussion:
@mgrover1 I'm not able to assign this to @kenkehoe for some reason but other Ken's from ARM show up that aren't on the ACT project board.  Do you know why by chance?

---Disregard, I added the ARM-DOE/ACT team to the Triage category.  Please adjust if needed

Resolving patch:
diff --git a/act/qc/qcfilter.py b/act/qc/qcfilter.py
--- a/act/qc/qcfilter.py
+++ b/act/qc/qcfilter.py
@@ -478,6 +478,11 @@ def set_test(self, var_name, index=None, test_number=None, flag_value=False):
 
         qc_variable = np.array(self._obj[qc_var_name].values)
 
+        # Ensure the qc_variable data type is integer. This ensures bitwise comparison
+        # will not cause an error.
+        if qc_variable.dtype.kind not in np.typecodes['AllInteger']:
+            qc_variable = qc_variable.astype(int)
+
         # Determine if test number is too large for current data type. If so
         # up convert data type.
         dtype = qc_variable.dtype
@@ -546,7 +551,14 @@ def unset_test(
         if var_name is not None:
             qc_var_name = self._obj.qcfilter.check_for_ancillary_qc(var_name)
 
+        # Get QC variable
         qc_variable = self._obj[qc_var_name].values
+
+        # Ensure the qc_variable data type is integer. This ensures bitwise comparison
+        # will not cause an error.
+        if qc_variable.dtype.kind not in np.typecodes['AllInteger']:
+            qc_variable = qc_variable.astype(int)
+
         if flag_value:
             qc_variable[index] = flag_values_reset_value
         else:


Test patch:
diff --git a/act/tests/baseline/test_assessment_overplot_multi.png b/act/tests/baseline/test_assessment_overplot_multi.png
Binary files a/act/tests/baseline/test_assessment_overplot_multi.png and b/act/tests/baseline/test_assessment_overplot_multi.png differ
diff --git a/act/tests/baseline/test_qc_flag_block_plot.png b/act/tests/baseline/test_qc_flag_block_plot.png
Binary files a/act/tests/baseline/test_qc_flag_block_plot.png and b/act/tests/baseline/test_qc_flag_block_plot.png differ
diff --git a/act/tests/test_plotting.py b/act/tests/test_plotting.py
--- a/act/tests/test_plotting.py
+++ b/act/tests/test_plotting.py
@@ -857,14 +857,10 @@ def test_assessment_overplot_multi():
 
     # Plot data
     display = TimeSeriesDisplay(ds, subplot_shape=(1,), figsize=(10, 6))
-    display.plot(var_name1, label=var_name1, assessment_overplot=True, overplot_behind=True)
-    display.plot(
-        var_name2,
-        day_night_background=True,
-        color='green',
-        label=var_name2,
-        assessment_overplot=True,
-    )
+    display.plot(var_name1, label=var_name1, assessment_overplot=True,
+                 overplot_behind=True, linestyle='')
+    display.plot(var_name2, day_night_background=True, color='green',
+                 label=var_name2, assessment_overplot=True, linestyle='')
 
     ds.close()
     try:
diff --git a/act/tests/test_qc.py b/act/tests/test_qc.py
--- a/act/tests/test_qc.py
+++ b/act/tests/test_qc.py
@@ -345,6 +345,16 @@ def test_qcfilter3():
     )
     assert np.sum(result) == 4950
 
+    # Test where QC variables are not integer type
+    ds_object = ds_object.resample(time='5min').mean(keep_attrs=True)
+    ds_object.qcfilter.add_test(var_name, index=range(0, ds_object.time.size),
+                                test_meaning='Testing float')
+    assert np.sum(ds_object[qc_var_name].values) == 582
+
+    ds_object[qc_var_name].values = ds_object[qc_var_name].values.astype(np.float32)
+    ds_object.qcfilter.remove_test(var_name, test_number=2)
+    assert np.sum(ds_object[qc_var_name].values) == 6
+
 
 def test_qctests():
     ds_object = read_netcdf(EXAMPLE_IRT25m20s)