Repository: opendatacube/datacube-core

Issue:
issue in find_less_mature when indexing datasets without region_code
### Expected behaviour
Need a better solution of identifying the same dataset with less maturity, hard matching `region_code` is less ideal.

### Actual behaviour
couldn't index any dataset without `region_code`, e.g., summary products and tide data (https://data.dea.ga.gov.au/?prefix=hltc/v2.0.0/)

code here does some "hard-coded" match to the `region_code` without checking the item existence.
 https://github.com/opendatacube/datacube-core/blob/24b046e737ea94bc088ee0f0566e839370152be9/datacube/index/abstract.py#L928-L930



### Steps to reproduce the behaviour
run `s3-to-dc` (`odc-apps-dc-tools==0.2.13`) to index the products mentioned above  
```
  File "/tmp/odc-tools/apps/dc_tools/odc/apps/dc_tools/s3_to_dc.py", line 75, in dump_to_odc
    index_update_dataset(
  File "/tmp/odc-tools/apps/dc_tools/odc/apps/dc_tools/utils.py", line 262, in index_update_dataset
    dc.index.datasets.add(ds, archive_less_mature=archive_less_mature)
  File "/env/lib/python3.10/site-packages/datacube/index/postgres/_datasets.py", line 195, in add
    self.archive_less_mature(dataset, archive_less_mature)
  File "/env/lib/python3.10/site-packages/datacube/index/abstract.py", line 908, in archive_less_mature
    less_mature = self.find_less_mature(ds, delta)
  File "/env/lib/python3.10/site-packages/datacube/index/abstract.py", line 929, in find_less_mature
    region_code=ds.metadata.region_code,
  File "/env/lib/python3.10/site-packages/datacube/utils/documents.py", line 498, in __getattr__
    raise AttributeError(
AttributeError: Unknown field 'region_code'. Expected one of ['id', 'label', 'format', 'sources', 'creation_dt', 'grid_spatial', 'measurements', 'creation_time', 'format', 'label', 'lat', 'lon', 'time', 'platform', 'instrument', 'product_type']
```

### Environment information

* 1.8.15
* any 





Discussion:
Do we generate summary products with non-final maturity?  (i.e. is the solution to simply not use "archive-less-mature" behaviour for these products?)

If the user could specify a custom metadata fields to identify matching datasets, what field(s) would be used for these products?
It doesn't seem to me like these products even have a `dataset_maturity` value? The error thrown could be improved, though.
It sounds that archiving is optional and supposed to on certain datasets with both dataset_maturity and region_code in the metadata. Can we check if the datasets satisfy the conditions then? If not, no archiving.

Resolving patch:
diff --git a/datacube/index/abstract.py b/datacube/index/abstract.py
--- a/datacube/index/abstract.py
+++ b/datacube/index/abstract.py
@@ -22,7 +22,7 @@
 from datacube.index.exceptions import TransactionException
 from datacube.index.fields import Field
 from datacube.model import Dataset, MetadataType, Range
-from datacube.model import DatasetType as Product
+from datacube.model import Product
 from datacube.utils import cached_property, jsonify_document, read_documents, InvalidDocException
 from datacube.utils.changes import AllowPolicy, Change, Offset, DocumentMismatchError, check_doc_unchanged
 from datacube.utils.generic import thread_local_cache
@@ -922,21 +922,41 @@ def find_less_mature(self, ds: Dataset, delta: int = 500) -> Iterable[Dataset]:
         """
         less_mature = []
         assert delta >= 0
+
+        def check_maturity_information(dataset, props):
+            # check that the dataset metadata includes all maturity-related properties
+            # passing in the required props to enable greater extensibility should it be needed
+            for prop in props:
+                if hasattr(dataset.metadata, prop) and (getattr(dataset.metadata, prop) is not None):
+                    return
+                raise ValueError(
+                    f"Dataset {dataset.id} is missing property {prop} required for maturity check"
+                )
+
+        check_maturity_information(ds, ["region_code", "time", "dataset_maturity"])
+
         # 'expand' the date range by `delta` milliseconds to give a bit more leniency in datetime comparison
         expanded_time_range = Range(ds.metadata.time.begin - timedelta(milliseconds=delta),
                                     ds.metadata.time.end + timedelta(milliseconds=delta))
         dupes = self.search(product=ds.product.name,
                             region_code=ds.metadata.region_code,
                             time=expanded_time_range)
+
         for dupe in dupes:
             if dupe.id == ds.id:
                 continue
+
+            # only need to check that dupe has dataset maturity, missing/null region_code and time
+            # would already have been filtered out during the search query
+            check_maturity_information(dupe, ["dataset_maturity"])
+
             if dupe.metadata.dataset_maturity == ds.metadata.dataset_maturity:
                 # Duplicate has the same maturity, which one should be archived is unclear
                 raise ValueError(
                     f"A dataset with the same maturity as dataset {ds.id} already exists, "
                     f"with id: {dupe.id}"
                 )
+
             if dupe.metadata.dataset_maturity < ds.metadata.dataset_maturity:
                 # Duplicate is more mature than dataset
                 # Note that "final" < "nrt"
@@ -944,6 +964,7 @@ def find_less_mature(self, ds: Dataset, delta: int = 500) -> Iterable[Dataset]:
                     f"A more mature version of dataset {ds.id} already exists, with id: "
                     f"{dupe.id} and maturity: {dupe.metadata.dataset_maturity}"
                 )
+
             less_mature.append(dupe)
         return less_mature
 


Test patch:
diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py
--- a/integration_tests/conftest.py
+++ b/integration_tests/conftest.py
@@ -363,6 +363,16 @@ def final_dataset(index, extended_eo3_metadata_type, ls8_eo3_product, final_data
         *final_dataset_doc)
 
 
+@pytest.fixture
+def ds_no_region(index, extended_eo3_metadata_type, ls8_eo3_product, final_dataset_doc):
+    doc_no_region = deepcopy(final_dataset_doc)
+    doc_no_region[0]["properties"]["odc:region_code"] = None
+    return doc_to_ds_no_add(
+        index,
+        ls8_eo3_product.name,
+        *doc_no_region)
+
+
 @pytest.fixture
 def ga_s2am_ard3_final(index, eo3_sentinel_metadata_type, ga_s2am_ard_3_product, ga_s2am_ard_3_final_doc):
     return doc_to_ds_no_add(
diff --git a/integration_tests/index/test_index_data.py b/integration_tests/index/test_index_data.py
--- a/integration_tests/index/test_index_data.py
+++ b/integration_tests/index/test_index_data.py
@@ -94,7 +94,7 @@ def test_archive_datasets(index, local_config, ls8_eo3_dataset):
     assert not indexed_dataset.is_archived
 
 
-def test_archive_less_mature(index, final_dataset, nrt_dataset):
+def test_archive_less_mature(index, final_dataset, nrt_dataset, ds_no_region):
     # case 1: add nrt then final; nrt should get archived
     index.datasets.add(nrt_dataset, with_lineage=False, archive_less_mature=0)
     index.datasets.get(nrt_dataset.id).is_active
@@ -110,6 +110,16 @@ def test_archive_less_mature(index, final_dataset, nrt_dataset):
         index.datasets.add(nrt_dataset, with_lineage=False, archive_less_mature=0)
 
 
+def test_cannot_search_for_less_mature(index, nrt_dataset, ds_no_region):
+    # if a dataset is missing a property required for finding less mature datasets,
+    # it should error
+    index.datasets.add(nrt_dataset, with_lineage=False, archive_less_mature=0)
+    index.datasets.get(nrt_dataset.id).is_active
+    assert ds_no_region.metadata.region_code is None
+    with pytest.raises(ValueError, match="region_code"):
+        index.datasets.add(ds_no_region, with_lineage=False, archive_less_mature=0)
+
+
 def test_archive_less_mature_approx_timestamp(index, ga_s2am_ard3_final, ga_s2am_ard3_interim):
     # test archive_less_mature where there's a slight difference in timestamps
     index.datasets.add(ga_s2am_ard3_interim, with_lineage=False)