Repository: opendatacube/datacube-core

Issue:
Procedures in `datacube.storage.masking` do not respect `NaN` nodata
### Expected behaviour
Suppose I have a `DataArray`:
```
<xarray.DataArray (x: 2, y: 2)>
array([[ 0.,  1.],
       [ 2., nan]])
Coordinates:
  * x        (x) float64 0.0 1.0
  * y        (y) float64 0.0 1.0
Attributes:
    nodata:   nan
```
I expect `valid_data_mask` to exclude the `nan` entry.

### Actual behaviour
It does not:
```
<xarray.DataArray (x: 2, y: 2)>
array([[ True,  True],
       [ True,  True]])
Coordinates:
  * x        (x) float64 0.0 1.0
  * y        (y) float64 0.0 1.0
```

### Steps to reproduce the behaviour
As above.

### Environment information

* Which ``datacube --version`` are you using?
ODC version 1.7

* What datacube deployment/enviornment are you running against?
Does not matter.

* Related issues:
    - #750 


Discussion:

Resolving patch:
diff --git a/datacube/storage/_load.py b/datacube/storage/_load.py
--- a/datacube/storage/_load.py
+++ b/datacube/storage/_load.py
@@ -15,6 +15,7 @@
 )
 
 from datacube.utils import ignore_exceptions_if
+from datacube.utils.math import dtype_is_float
 from datacube.utils.geometry import GeoBox, Coordinate, roi_is_empty
 from datacube.model import Measurement
 from datacube.drivers._types import ReaderDriver
@@ -26,13 +27,23 @@
 ProgressFunction = Callable[[int, int], Any]  # pylint: disable=invalid-name
 
 
-def _default_fuser(dst: np.ndarray, src: np.ndarray, dst_nodata: float) -> None:
+def _default_fuser(dst: np.ndarray, src: np.ndarray, dst_nodata) -> None:
     """ Overwrite only those pixels in `dst` with `src` that are "not valid"
 
         For every pixel in dst that equals to dst_nodata replace it with pixel
         from src.
     """
-    where_nodata = (dst == dst_nodata) if not np.isnan(dst_nodata) else np.isnan(dst)
+    if dtype_is_float(dst.dtype):
+        if dst_nodata is None or np.isnan(dst_nodata):
+            where_nodata = np.isnan(dst)
+        else:
+            where_nodata = np.isnan(dst) | (dst == dst_nodata)
+    else:
+        if dst_nodata is None:
+            where_nodata = np.full_like(dst, False, dtype=np.bool)
+        else:
+            where_nodata = dst == dst_nodata
+
     np.copyto(dst, src, where=where_nodata)
 
 
diff --git a/datacube/storage/masking.py b/datacube/storage/masking.py
--- a/datacube/storage/masking.py
+++ b/datacube/storage/masking.py
@@ -6,10 +6,15 @@
 
 import collections
 import warnings
-import pandas
 
+import pandas
+import numpy
+import xarray
 from xarray import DataArray, Dataset
 
+from datacube.utils.math import dtype_is_float
+
+
 FLAGS_ATTR_NAME = 'flags_definition'
 
 
@@ -109,12 +114,20 @@ def valid_data_mask(data):
     if isinstance(data, Dataset):
         return data.apply(valid_data_mask)
 
-    if isinstance(data, DataArray):
-        if 'nodata' not in data.attrs:
-            return True
-        return data != data.nodata
+    if not isinstance(data, DataArray):
+        raise TypeError('valid_data_mask not supported for type {}'.format(type(data)))
+
+    nodata = data.attrs.get('nodata', None)
+
+    if dtype_is_float(data.dtype):
+        if nodata is None or numpy.isnan(nodata):
+            return ~xarray.ufuncs.isnan(data)
+        return (data != nodata) & ~xarray.ufuncs.isnan(data)
 
-    raise TypeError('valid_data_mask not supported for type {}'.format(type(data)))
+    # not float
+    if nodata is None:
+        return xarray.full_like(data, True, dtype=numpy.bool)
+    return data != nodata
 
 
 def mask_valid_data(data, keep_attrs=True):
diff --git a/datacube/utils/math.py b/datacube/utils/math.py
--- a/datacube/utils/math.py
+++ b/datacube/utils/math.py
@@ -56,6 +56,13 @@ def is_almost_int(x: float, tol: float):
     return x < tol
 
 
+def dtype_is_float(dtype) -> bool:
+    """
+    Check if `dtype` is floating-point.
+    """
+    return numpy.dtype(dtype).kind == 'f'
+
+
 def valid_mask(xx, nodata):
     """
     Compute mask such that xx[mask] contains valid pixels.


Test patch:
diff --git a/tests/api/test_masking.py b/tests/api/test_masking.py
--- a/tests/api/test_masking.py
+++ b/tests/api/test_masking.py
@@ -361,3 +361,38 @@ def test_valid_data_mask():
 
     output_da = valid_data_mask(data_array)
     assert output_da.equals(expected_data_array)
+
+    expected_data_array = DataArray(np.array([[True, True, True], [True, True, True], [True, True, True]],
+                                             dtype='bool'))
+    data_array = DataArray([[1, -999, -999], [2, 3, -999], [-999, -999, -999]])
+    dataset = Dataset(data_vars={'var_one': data_array})
+
+    output_ds = valid_data_mask(dataset)
+    assert output_ds.data_vars['var_one'].equals(expected_data_array)
+
+    output_da = valid_data_mask(data_array)
+    assert output_da.equals(expected_data_array)
+
+    expected_data_array = DataArray(np.array([[True, False, False], [True, True, False], [False, False, False]],
+                                             dtype='bool'))
+
+    data_array = DataArray([[1, -999, -999], [2, 3, -999], [-999, -999, float('nan')]], attrs=attrs)
+    dataset = Dataset(data_vars={'var_one': data_array})
+
+    output_ds = valid_data_mask(dataset)
+    assert output_ds.data_vars['var_one'].equals(expected_data_array)
+
+    output_da = valid_data_mask(data_array)
+    assert output_da.equals(expected_data_array)
+
+    expected_data_array = DataArray(np.array([[True, True, True], [True, True, True], [True, True, False]],
+                                             dtype='bool'))
+
+    data_array = DataArray([[1, -999, -999], [2, 3, -999], [-999, -999, float('nan')]])
+    dataset = Dataset(data_vars={'var_one': data_array})
+
+    output_ds = valid_data_mask(dataset)
+    assert output_ds.data_vars['var_one'].equals(expected_data_array)
+
+    output_da = valid_data_mask(data_array)
+    assert output_da.equals(expected_data_array)
diff --git a/tests/storage/test_storage_load.py b/tests/storage/test_storage_load.py
--- a/tests/storage/test_storage_load.py
+++ b/tests/storage/test_storage_load.py
@@ -4,7 +4,7 @@
 import numpy as np
 
 from datacube.storage._load import (
-    xr_load,
+    xr_load, _default_fuser
 )
 
 from datacube.api.core import Datacube
@@ -13,6 +13,32 @@
 from datacube.testutils.iodriver import mk_rio_driver, tee_new_load_context
 
 
+def test_default_fuser():
+    dest = np.full((2, 2), -1.0)
+    src1 = np.array([[0.0, -1.0], [-1.0, 6.0]])
+
+    _default_fuser(dest, src1, -1.0)
+    assert np.all(dest == src1)
+
+    src2 = np.array([[9.0, np.nan], [-1.0, 3.0]])
+    _default_fuser(dest, src2, -1.0)
+    assert np.allclose(dest, np.array([[0.0, np.nan], [-1.0, 6.0]]), equal_nan=True)
+
+    src3 = np.full((2, 2), 55.0)
+    _default_fuser(dest, src3, -1.0)
+    assert np.all(dest == np.array([[0.0, 55.0], [55.0, 6.0]]))
+
+    dest = np.full((2, 2), -1)
+    src1 = np.array([[0, -1], [-1, 6]])
+
+    _default_fuser(dest, src1, -1)
+    assert np.all(dest == src1)
+
+    src2 = np.array([[9, 4], [-1, 3]])
+    _default_fuser(dest, src1, None)
+    assert np.all(dest == src1)
+
+
 def test_new_xr_load(data_folder):
     base = "file://" + str(data_folder) + "/metadata.yml"