Repository: opendatacube/datacube-core

Issue:
Problem: Why grid_spatial is not a search field
### Problem: Why `grid_spatial` is not a search field
The `search_fields` used by `index.datasets.search` functions include `native fields` (those that are native to table schema), so called `fixed fields (not dynamic in metadata type schema)`, and other search fields defined in `metadata_type`. None of these include `grid_spatial`, which is puzzling. Except `telemetry`, all other `metadata types` we know of has `grid_spatial`. The `fixed fields` don't seem to correlate with `required fields` (id, creation_dt, label, sources) in metadata type schema either (but seem to be present in all metadata types we have listed). The problem with this is that, while we can still access `grid_spatial` selectively through database layer, we loose query processing capabilities of various search functions (particularly `search_returning` )



Discussion:

Resolving patch:
diff --git a/datacube/drivers/postgres/_api.py b/datacube/drivers/postgres/_api.py
--- a/datacube/drivers/postgres/_api.py
+++ b/datacube/drivers/postgres/_api.py
@@ -1,7 +1,7 @@
 # coding=utf-8
 
 # We often have one-arg-per column, so these checks aren't so useful.
-# pylint: disable=too-many-arguments,too-many-public-methods
+# pylint: disable=too-many-arguments,too-many-public-methods,too-many-lines
 
 # SQLAlchemy queries require "column == None", not "column is None" due to operator overloading:
 # pylint: disable=singleton-comparison
@@ -446,6 +446,7 @@ def search_datasets_query(expressions, source_exprs=None,
         :type limit: int
         :rtype: sqlalchemy.Expression
         """
+
         if select_fields:
             select_columns = tuple(
                 f.alchemy_expression.label(f.name)
@@ -535,6 +536,84 @@ def search_datasets(self, expressions,
                                                   select_fields, with_source_ids, limit)
         return self._connection.execute(select_query)
 
+    @staticmethod
+    def search_unique_datasets_query(expressions, select_fields, limit):
+        """
+        'unique' here refer to that the query results do not contain datasets
+        having the same 'id' more than once.
+
+        We are not dealing with dataset_source table here and we are not joining
+        dataset table with dataset_location table. We are aggregating stuff
+        in dataset_location per dataset basis if required. It returns the construted
+        query.
+        """
+
+        # expressions involving DATASET_SOURCE cannot not done for now
+        for expression in expressions:
+            assert expression.field.required_alchemy_table != DATASET_SOURCE,\
+                'Joins with dataset_source cannot be done for this query'
+
+        # expressions involving 'uri' and 'uris' will be handled different
+        expressions = [expression for expression in expressions
+                       if expression.field.required_alchemy_table != DATASET_LOCATION]
+
+        if select_fields:
+            select_columns = []
+            for field in select_fields:
+                if field.name in {'uri', 'uris'}:
+                    # All active URIs, from newest to oldest
+                    uris_field = func.array(
+                        select([
+                            _dataset_uri_field(SELECTED_DATASET_LOCATION)
+                        ]).where(
+                            and_(
+                                SELECTED_DATASET_LOCATION.c.dataset_ref == DATASET.c.id,
+                                SELECTED_DATASET_LOCATION.c.archived == None
+                            )
+                        ).order_by(
+                            SELECTED_DATASET_LOCATION.c.added.desc(),
+                            SELECTED_DATASET_LOCATION.c.id.desc()
+                        ).label('uris')
+                    ).label('uris')
+                    select_columns.append(uris_field)
+                else:
+                    select_columns.append(field.alchemy_expression.label(field.name))
+        else:
+            select_columns = _DATASET_SELECT_FIELDS
+
+        raw_expressions = PostgresDbAPI._alchemify_expressions(expressions)
+
+        # We don't need 'DATASET_LOCATION table in the from expression
+        select_fields_ = [field for field in select_fields if field.name not in {'uri', 'uris'}]
+
+        from_expression = PostgresDbAPI._from_expression(DATASET, expressions, select_fields_)
+        where_expr = and_(DATASET.c.archived == None, *raw_expressions)
+
+        return(
+            select(
+                select_columns
+            ).select_from(
+                from_expression
+            ).where(
+                where_expr
+            ).limit(
+                limit
+            )
+        )
+
+    def search_unique_datasets(self, expressions, select_fields=None, limit=None):
+        """
+        Processes a search query without duplicating datasets. 'unique' here refer to
+        that the results do not contain datasets having the same 'id' more than once.
+        we achieve this by not allowing dataset table to join with dataset_location or
+        dataset_source tables. Joining with other tables would not result in multiple
+        records per dataset due to the direction of cardinality.
+        """
+
+        select_query = self.search_unique_datasets_query(expressions, select_fields, limit)
+
+        return self._connection.execute(select_query)
+
     def get_duplicates(self, match_fields, expressions):
         # type: (Tuple[PgField], Tuple[PgExpression]) -> Iterable[tuple]
         group_expressions = tuple(f.alchemy_expression for f in match_fields)
diff --git a/datacube/index/_datasets.py b/datacube/index/_datasets.py
--- a/datacube/index/_datasets.py
+++ b/datacube/index/_datasets.py
@@ -10,16 +10,45 @@
 
 from datacube.model import Dataset, DatasetType
 from datacube.model.utils import flatten_datasets
-from datacube.utils import jsonify_document, changes
+from datacube.utils import jsonify_document, changes, cached_property
 from datacube.utils.changes import get_doc_changes
 from . import fields
 
+import json
+from datacube.drivers.postgres._fields import SimpleDocField, DateDocField
+from datacube.drivers.postgres._schema import DATASET
+from sqlalchemy import select, func
+from datacube.model.fields import Field
+
 _LOG = logging.getLogger(__name__)
 
 
 # It's a public api, so we can't reorganise old methods.
 # pylint: disable=too-many-public-methods, too-many-lines
 
+class DatasetSpatialMixin(object):
+    __slots__ = ()
+
+    @property
+    def _gs(self):
+        return self.grid_spatial
+
+    @property
+    def crs(self):
+        return Dataset.crs.__get__(self)
+
+    @cached_property
+    def extent(self):
+        return Dataset.extent.func(self)
+
+    @property
+    def transform(self):
+        return Dataset.transform.__get__(self)
+
+    @property
+    def bounds(self):
+        return Dataset.bounds.__get__(self)
+
 
 class DatasetResource(object):
     """
@@ -703,3 +732,180 @@ def search_eager(self, **query):
         :rtype: list[Dataset]
         """
         return list(self.search(**query))
+
+    def get_product_time_bounds(self, product: str):
+        """
+        Returns the minimum and maximum acquisition time of the product.
+        """
+
+        # Get the offsets from dataset doc
+        product = self.types.get_by_name(product)
+        dataset_section = product.metadata_type.definition['dataset']
+        min_offset = dataset_section['search_fields']['time']['min_offset']
+        max_offset = dataset_section['search_fields']['time']['max_offset']
+
+        time_min = DateDocField('aquisition_time_min',
+                                'Min of time when dataset was acquired',
+                                DATASET.c.metadata,
+                                False,  # is it indexed
+                                offset=min_offset,
+                                selection='least')
+
+        time_max = DateDocField('aquisition_time_max',
+                                'Max of time when dataset was acquired',
+                                DATASET.c.metadata,
+                                False,  # is it indexed
+                                offset=max_offset,
+                                selection='greatest')
+
+        with self._db.connect() as connection:
+            result = connection.execute(
+                select(
+                    [func.min(time_min.alchemy_expression), func.max(time_max.alchemy_expression)]
+                ).where(
+                    DATASET.c.dataset_type_ref == product.id
+                )
+            ).first()
+
+        return result
+
+    # pylint: disable=redefined-outer-name
+    def search_returning_datasets_light(self, field_names: tuple, custom_offsets=None, limit=None, **query):
+        """
+        This is a dataset search function that returns the results as objects of a dynamically
+        generated Dataset class that is a subclass of tuple.
+
+        Only the requested fields will be returned together with related derived attributes as property functions
+        similer to the datacube.model.Dataset class. For example, if 'extent'is requested all of
+        'crs', 'extent', 'transform', and 'bounds' are available as property functions.
+
+        The field_names can be custom fields in addition to those specified in metadata_type, fixed fields, or
+        native fields. The field_names can also be derived fields like 'extent', 'crs', 'transform',
+        and 'bounds'. The custom fields require custom offsets of the metadata doc be provided.
+
+        The datasets can be selected based on values of custom fields as long as relevant custom
+        offsets are provided. However custom field values are not transformed so must match what is
+        stored in the database.
+
+        :param field_names: A tuple of field names that would be returned including derived fields
+                            such as extent, crs
+        :param custom_offsets: A dictionary of offsets in the metadata doc for custom fields
+        :param limit: Number of datasets returned per product.
+        :param query: key, value mappings of query that will be processed against metadata_types,
+                      product definitions and/or dataset table.
+        :return: A Dynamically generated DatasetLight (a subclass of namedtuple and possibly with
+        property functions).
+        """
+
+        assert field_names
+
+        for product, query_exprs in self.make_query_expr(query, custom_offsets):
+
+            select_fields = self.make_select_fields(product, field_names, custom_offsets)
+            select_field_names = tuple(field.name for field in select_fields)
+            result_type = namedtuple('DatasetLight', select_field_names)
+
+            if 'grid_spatial' in select_field_names:
+                class DatasetLight(result_type, DatasetSpatialMixin):
+                    pass
+            else:
+                class DatasetLight(result_type):
+                    __slots__ = ()
+
+            with self._db.connect() as connection:
+                results = connection.search_unique_datasets(
+                    query_exprs,
+                    select_fields=select_fields,
+                    limit=limit
+                )
+
+            for result in results:
+                field_values = dict()
+                for i_, field in enumerate(select_fields):
+                    # We need to load the simple doc fields
+                    if isinstance(field, SimpleDocField):
+                        field_values[field.name] = json.loads(result[i_])
+                    else:
+                        field_values[field.name] = result[i_]
+
+                yield DatasetLight(**field_values)
+
+    def make_select_fields(self, product, field_names, custom_offsets):
+        """
+        Parse and generate the list of select fields to be passed to the database API.
+        """
+
+        assert product and field_names
+
+        dataset_fields = product.metadata_type.dataset_fields
+        dataset_section = product.metadata_type.definition['dataset']
+
+        select_fields = []
+        for field_name in field_names:
+            if dataset_fields.get(field_name):
+                select_fields.append(dataset_fields[field_name])
+            else:
+                # try to construct the field
+                if field_name in {'transform', 'extent', 'crs', 'bounds'}:
+                    grid_spatial = dataset_section.get('grid_spatial')
+                    if grid_spatial:
+                        select_fields.append(SimpleDocField(
+                            'grid_spatial', 'grid_spatial', DATASET.c.metadata,
+                            False,
+                            offset=grid_spatial
+                        ))
+                elif custom_offsets and field_name in custom_offsets:
+                    select_fields.append(SimpleDocField(
+                        field_name, field_name, DATASET.c.metadata,
+                        False,
+                        offset=custom_offsets[field_name]
+                    ))
+                elif field_name == 'uris':
+                    select_fields.append(Field('uris', 'uris'))
+
+        return select_fields
+
+    def make_query_expr(self, query, custom_offsets):
+        """
+        Generate query expressions including queries based on custom fields
+        """
+
+        product_queries = list(self._get_product_queries(query))
+        custom_query = dict()
+        if not product_queries:
+            # The key, values in query that are un-machable with info
+            # in metadata types and product definitions, perhaps there are custom
+            # fields, will need to handle custom fields separately
+
+            canonical_query = query.copy()
+            custom_query = {key: canonical_query.pop(key) for key in custom_offsets
+                            if key in canonical_query}
+            product_queries = list(self._get_product_queries(canonical_query))
+
+            if not product_queries:
+                raise ValueError('No products match search terms: %r' % query)
+
+        for q, product in product_queries:
+            dataset_fields = product.metadata_type.dataset_fields
+            query_exprs = tuple(fields.to_expressions(dataset_fields.get, **q))
+            custom_query_exprs = tuple(self.get_custom_query_expressions(custom_query, custom_offsets))
+
+            yield product, query_exprs + custom_query_exprs
+
+    def get_custom_query_expressions(self, custom_query, custom_offsets):
+        """
+        Generate query expressions for custom fields. it is assumed that custom fields are to be found
+        in metadata doc and their offsets are provided. custom_query is a dict of key fields involving
+        custom fields.
+        """
+
+        custom_exprs = []
+        for key in custom_query:
+            # for now we assume all custom query fields are SimpleDocFields
+            custom_field = SimpleDocField(
+                custom_query[key], custom_query[key], DATASET.c.metadata,
+                False, offset=custom_offsets[key]
+            )
+            custom_exprs.append(fields.as_expression(custom_field, custom_query[key]))
+
+        return custom_exprs


Test patch:
diff --git a/integration_tests/test_index_datasets_search.py b/integration_tests/test_index_datasets_search.py
new file mode 100644
--- /dev/null
+++ b/integration_tests/test_index_datasets_search.py
@@ -0,0 +1,130 @@
+import pytest
+from pathlib import PosixPath
+
+from integration_tests.test_full_ingestion import ensure_datasets_are_indexed
+
+
+@pytest.mark.parametrize('datacube_env_name', ('datacube',), indirect=True)
+@pytest.mark.usefixtures('default_metadata_type',
+                         'indexed_ls5_scene_products')
+def test_index_datasets_search_light(index, tmpdir, clirunner,
+                                     example_ls5_dataset_paths):
+
+    def index_dataset(path):
+        return clirunner(['dataset', 'add', str(path)])
+
+    def index_products():
+        valid_uuids = []
+        for uuid, ls5_dataset_path in example_ls5_dataset_paths.items():
+            valid_uuids.append(uuid)
+            index_dataset(ls5_dataset_path)
+
+        # Ensure that datasets are actually indexed
+        ensure_datasets_are_indexed(index, valid_uuids)
+
+        return valid_uuids
+
+    valid_uuids = index_products()
+
+    # Test derived properties such as 'extent'
+    results = list(index.datasets.search_returning_datasets_light(field_names=('id', 'extent', 'time'),
+                                                                  product='ls5_nbar_scene'))
+    for dataset in results:
+        assert dataset.id in valid_uuids
+        # Assume projection is defined as
+        #         datum: GDA94
+        #         ellipsoid: GRS80
+        #         zone: -55
+        # for all datasets. This should give us epsg 28355
+        assert dataset.extent.crs.epsg == 28355
+
+    # test custom fields
+    results = list(index.datasets.search_returning_datasets_light(field_names=('id', 'zone'),
+                                                                  custom_offsets={'zone': ['grid_spatial',
+                                                                                           'projection', 'zone']},
+                                                                  product='ls5_nbar_scene'))
+    for dataset in results:
+        assert dataset.zone == -55
+
+    # Test conditional queries involving custom fields
+    results = list(index.datasets.search_returning_datasets_light(field_names=('id', 'zone'),
+                                                                  custom_offsets={'zone': ['grid_spatial',
+                                                                                           'projection', 'zone']},
+                                                                  product='ls5_nbar_scene',
+                                                                  zone='-55'))
+    assert len(results) > 0
+
+    results = list(index.datasets.search_returning_datasets_light(field_names=('id', 'zone'),
+                                                                  custom_offsets={'zone': ['grid_spatial',
+                                                                                           'projection', 'zone']},
+                                                                  product='ls5_nbar_scene',
+                                                                  zone='-65'))
+    assert len(results) == 0
+
+    # Test uris
+
+    # Test datasets with just one uri location
+    results_no_uri = list(index.datasets.search_returning_datasets_light(field_names=('id'),
+                                                                         product='ls5_nbar_scene'))
+    results_with_uri = list(index.datasets.search_returning_datasets_light(field_names=('id', 'uris'),
+                                                                           product='ls5_nbar_scene'))
+    assert len(results_no_uri) == len(results_with_uri)
+    for result in results_with_uri:
+        assert len(result.uris) == 1
+
+    # 'uri' field bahave same as 'uris' ('uri' could be deprecated!)
+    results_with_uri = list(index.datasets.search_returning_datasets_light(field_names=('id', 'uri'),
+                                                                           product='ls5_nbar_scene'))
+    assert len(results_no_uri) == len(results_with_uri)
+    for result in results_with_uri:
+        assert len(result.uri) == 1
+
+    # Add a new uri to a dataset
+    new_loc = PosixPath(tmpdir.strpath) / 'temp_location' / 'agdc-metadata.yaml'
+    index.datasets.add_location(valid_uuids[0], new_loc.as_uri())
+
+    results_with_uri = list(index.datasets.search_returning_datasets_light(field_names=('id', 'uris'),
+                                                                           product='ls5_nbar_scene',
+                                                                           id=valid_uuids[0]))
+    assert len(results_with_uri) == 1
+    assert len(results_with_uri[0].uris) == 2
+
+    results_with_uri = list(index.datasets.search_returning_datasets_light(field_names=('id', 'uri'),
+                                                                           product='ls5_nbar_scene',
+                                                                           id=valid_uuids[0]))
+    assert len(results_with_uri) == 1
+    assert len(results_with_uri[0].uri) == 2
+
+
+@pytest.mark.parametrize('datacube_env_name', ('datacube',), indirect=True)
+@pytest.mark.usefixtures('default_metadata_type',
+                         'indexed_ls5_scene_products')
+def test_index_get_product_time_bounds(index, clirunner, example_ls5_dataset_paths):
+
+    def index_dataset(path):
+        return clirunner(['dataset', 'add', str(path)])
+
+    def index_products():
+        valid_uuids = []
+        for uuid, ls5_dataset_path in example_ls5_dataset_paths.items():
+            valid_uuids.append(uuid)
+            index_dataset(ls5_dataset_path)
+
+        # Ensure that datasets are actually indexed
+        ensure_datasets_are_indexed(index, valid_uuids)
+
+        return valid_uuids
+
+    valid_uuids = index_products()
+
+    # lets get time values
+    dataset_times = list(index.datasets.search_returning_datasets_light(field_names=('time',),
+                                                                        product='ls5_nbar_scene'))
+
+    # get time bounds
+    time_bounds = index.datasets.get_product_time_bounds(product='ls5_nbar_scene')
+    left = sorted(dataset_times, key=lambda dataset: dataset.time.lower)[0].time.lower
+    right = sorted(dataset_times, key=lambda dataset: dataset.time.upper)[-1].time.upper
+
+    assert left == time_bounds[0]
+    assert right == time_bounds[1]