Repository: opendatacube/datacube-core

Issue:
Dataset consistency check fails in presence of `NaN`
When indexing datasets that reference lineage datasets which contain `nan` and with `verify_lineage=True` un-modified lineage datasets are reported as inconsistent preventing indexing.

**Reason:**

values like `nan` and `inf` get converted to strings before serialising to db, the inverse process is never applied during load, and maybe doesn't exist? 

**Solution:**

apply the same process to documents loaded from the file before comparison.

Longer term solution -- have a proper strategy for dealing with full range of floating point values.

related to #524 






Discussion:

Resolving patch:
diff --git a/datacube/index/_datasets.py b/datacube/index/_datasets.py
--- a/datacube/index/_datasets.py
+++ b/datacube/index/_datasets.py
@@ -118,7 +118,8 @@ def bulk_has(self, ids_):
         with self._db.connect() as connection:
             existing = set(connection.datasets_intersection(ids_))
 
-        return [x in existing for x in ids_]
+        return [x in existing for x in
+                map((lambda x: UUID(x) if isinstance(x, str) else x), ids_)]
 
     def add(self, dataset, with_lineage=None, **kwargs):
         """
diff --git a/datacube/index/hl.py b/datacube/index/hl.py
--- a/datacube/index/hl.py
+++ b/datacube/index/hl.py
@@ -5,7 +5,7 @@
 from types import SimpleNamespace
 
 from datacube.model import Dataset
-from datacube.utils import changes, InvalidDocException, SimpleDocNav
+from datacube.utils import changes, InvalidDocException, SimpleDocNav, jsonify_document
 from datacube.model.utils import dedup_lineage, remap_lineage_doc, flatten_datasets
 from datacube.utils.changes import get_doc_changes
 
@@ -152,7 +152,7 @@ def resolve(main_ds, uri):
 
             for uuid in lineage_uuids:
                 if uuid in db_dss:
-                    ok, err = check_consistent(ds_by_uuid[uuid].doc_without_lineage_sources,
+                    ok, err = check_consistent(jsonify_document(ds_by_uuid[uuid].doc_without_lineage_sources),
                                                db_dss[uuid].metadata_doc)
                     if not ok:
                         bad_lineage.append((uuid, err))


Test patch:
diff --git a/integration_tests/index/test_index_data.py b/integration_tests/index/test_index_data.py
--- a/integration_tests/index/test_index_data.py
+++ b/integration_tests/index/test_index_data.py
@@ -163,6 +163,9 @@ def test_has_dataset(index, telemetry_dataset):
     assert not index.datasets.has(UUID('f226a278-e422-11e6-b501-185e0f80a5c0'))
     assert not index.datasets.has('f226a278-e422-11e6-b501-185e0f80a5c0')
 
+    assert index.datasets.bulk_has([_telemetry_uuid, UUID('f226a278-e422-11e6-b501-185e0f80a5c0')]) == [True, False]
+    assert index.datasets.bulk_has([str(_telemetry_uuid), 'f226a278-e422-11e6-b501-185e0f80a5c0']) == [True, False]
+
 
 def test_get_dataset(index, telemetry_dataset):
     # type: (Index, Dataset) -> None
diff --git a/integration_tests/test_dataset_add.py b/integration_tests/test_dataset_add.py
--- a/integration_tests/test_dataset_add.py
+++ b/integration_tests/test_dataset_add.py
@@ -1,3 +1,4 @@
+import math
 import yaml
 import toolz
 
@@ -305,6 +306,45 @@ def test_dataset_add_ambgious_products(dataset_add_configs, index_empty, clirunn
     assert index.datasets.has(ds.id) is True
 
 
+def test_dataset_add_with_nans(dataset_add_configs, index_empty, clirunner):
+    p = dataset_add_configs
+    index = index_empty
+
+    clirunner(['metadata_type', 'add', p.metadata])
+    clirunner(['product', 'add', p.products])
+
+    mk = dataset_maker(0)
+
+    c = mk('C', product_type='C',
+           val_is_nan=math.nan,
+           val_is_inf=math.inf,
+           val_is_neginf=-math.inf)
+
+    b = mk('B', sources={'bc': c}, product_type='B')
+    a = mk('A', sources={'ac': c}, product_type='A')
+
+    prefix = write_files({
+        'dataset.yml': yaml.safe_dump_all([a, b]),
+    })
+
+    r = clirunner(['dataset', 'add',
+                   '--auto-add-lineage',
+                   '--verify-lineage',
+                   str(prefix/'dataset.yml')])
+
+    assert "ERROR" not in r.output
+
+    a, b, c = [SimpleDocNav(v) for v in (a, b, c)]
+
+    assert index.datasets.bulk_has([a.id, b.id, c.id]) == [True, True, True]
+
+    c_doc = index.datasets.get(c.id).metadata_doc
+
+    assert c_doc['val_is_nan'] == 'NaN'
+    assert c_doc['val_is_inf'] == 'Infinity'
+    assert c_doc['val_is_neginf'] == '-Infinity'
+
+
 def test_dataset_add_inconsistent_measurements(dataset_add_configs, index_empty, clirunner):
     p = dataset_add_configs
     index = index_empty