Repository: CS-SI/eodag
Pull request #917 (2023-11-08 16:28:06)

feat: implement new core method get_queryables
Implements a new core method `def get_queryables(provider=None, product_type=None) -> List[str]`:

- If no `provider` and no `product_type` are passed, then returns default search method queryable parameters: `["productType", "start", "end", "geom", "locations", "id"]`
- If a `provider` is passed (but no `product_type`), then returns its queryable parameters from provider's config
- If a `product_type` is passed (but no `provider`), then return the intersection of the queryables of all the providers supporting it
- If both a `product_type` and a `provider` are passed, then returns the queryable parameters from provider's config and the queryable parameters for that product type

The following exceptions are raised:
- `UnsupportedProvider()` if the given `provider` is not found
- `UnsupportedProductType()` if the given `product_type` is not found


github-actions[bot]: ## Test Results
       4 files  ±0         4 suites  ±0   5m 41s [:stopwatch:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "duration of all tests") +30s
   431 tests +1     428 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "passed tests") +1    3 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "skipped / disabled tests") ±0  0 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "failed tests") ±0 
1 724 runs  +4  1 648 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "passed tests") +4  76 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "skipped / disabled tests") ±0  0 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "failed tests") ±0 

Results for commit 92070626. ± Comparison against base commit f0257dff.

[test-results]:data:application/gzip;base64,H4sIABS4S2UC/03Myw7CIBCF4VdpWLsAZgrUlzGUS0LsxVBYGd9dqC12+X+TOW/iw+Q2cu/w1pEth9TC5qhTWJeSgKxAOaX9CK0eWzamEld/eoZX/WngdZgK0AYuxjUeEvNSN5nkeNS5yQSqRr9NKU64bO59nTTrPIdUggycSiq4sDhSwEErMBy41HYEx6TH3gnFegXk8wUuBfw7CAEAAA==

github-actions[bot]: <strong>Code Coverage (Ubuntu)</strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `82%` | :white_check_mark: |

_Minimum allowed coverage is `70%`_

<p align="right">Generated by :monkey: cobertura-action against 92070626d4b0349a83c2327adb3e17f45e681583 </p>
github-actions[bot]: <strong>Code Coverage (Windows)</strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `76%` | :white_check_mark: |

_Minimum allowed coverage is `70%`_

<p align="right">Generated by :monkey: cobertura-action against 92070626d4b0349a83c2327adb3e17f45e681583 </p>
dalpasso: Changed the implementation:

- added a default list of queryables to all the providers
- when `product_type` is given, but no `provider`, then return the intersection of the queryables of all the providers supporting it


Patch:
diff --git a/docs/api_reference/core.rst b/docs/api_reference/core.rst
index 6a5f8551..3aed3132 100644
--- a/docs/api_reference/core.rst
+++ b/docs/api_reference/core.rst
@@ -80,6 +80,7 @@ Misc
 
    EODataAccessGateway.group_by_extent
    EODataAccessGateway.guess_product_type
+   EODataAccessGateway.get_queryables
 
 .. autoclass:: eodag.api.core.EODataAccessGateway
    :members: set_preferred_provider, get_preferred_provider, update_providers_config, list_product_types,
diff --git a/eodag/api/core.py b/eodag/api/core.py
index 4ebb10de..30dfb600 100644
--- a/eodag/api/core.py
+++ b/eodag/api/core.py
@@ -22,7 +22,7 @@ import os
 import re
 import shutil
 from operator import itemgetter
-from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Set, Tuple, Union
 
 import geojson
 import pkg_resources
@@ -71,6 +71,7 @@ from eodag.utils.exceptions import (
     NoMatchingProductType,
     PluginImplementationError,
     RequestError,
+    UnsupportedProductType,
     UnsupportedProvider,
 )
 from eodag.utils.stac_reader import fetch_stac_items
@@ -1997,3 +1998,58 @@ class EODataAccessGateway:
         plugin_conf = {"name": name}
         plugin_conf.update({key.replace("-", "_"): val for key, val in options.items()})
         return self._plugins_manager.get_crunch_plugin(name, **plugin_conf)
+
+    def get_queryables(
+        self, provider: Optional[str] = None, product_type: Optional[str] = None
+    ) -> Set[str]:
+        """Fetch the queryable properties for a given product type and/or provider.
+
+        :param product_type: (optional) The EODAG product type.
+        :type product_type: str
+        :param provider: (optional) The provider.
+        :type provider: str
+        :returns: A set containing the EODAG queryable properties.
+        :rtype: set
+        """
+        default_queryables = {"productType", "start", "end", "geom", "locations", "id"}
+        if provider is None and product_type is None:
+            return default_queryables
+
+        plugins = self._plugins_manager.get_search_plugins(product_type, provider)
+
+        # dictionary of the queryable properties of the providers supporting the given product type
+        all_queryable_properties = dict()
+        for plugin in plugins:
+            if (
+                product_type
+                and product_type not in plugin.config.products.keys()
+                and provider is None
+            ):
+                raise UnsupportedProductType(product_type)
+            elif product_type and product_type not in plugin.config.products.keys():
+                raise UnsupportedProductType(
+                    f"{product_type} is not available for provider {provider}"
+                )
+
+            provider_queryables = set(default_queryables)
+
+            metadata_mapping = deepcopy(getattr(plugin.config, "metadata_mapping", {}))
+
+            # product_type-specific metadata-mapping
+            metadata_mapping.update(
+                getattr(plugin.config, "products", {})
+                .get(product_type, {})
+                .get("metadata_mapping", {})
+            )
+
+            for key, value in metadata_mapping.items():
+                if isinstance(value, list) and "TimeFromAscendingNode" not in key:
+                    provider_queryables.add(key)
+
+            all_queryable_properties[plugin.provider] = provider_queryables
+
+        if provider is None:
+            # intersection of the queryables among the providers
+            return set.intersection(*all_queryable_properties.values())
+        else:
+            return all_queryable_properties[provider]
diff --git a/eodag/rest/server.py b/eodag/rest/server.py
index a50b92bf..4c9c928f 100755
--- a/eodag/rest/server.py
+++ b/eodag/rest/server.py
@@ -467,8 +467,8 @@ def list_collection_queryables(
         fetch_collection_queryable_properties(*conf_args)
     )
 
-    for prop in provider_properties:
-        titled_name = re.sub(CAMEL_TO_SPACE_TITLED, " ", prop).title()
+    for prop in sorted(provider_properties):
+        titled_name = re.sub(CAMEL_TO_SPACE_TITLED, " ", prop.split(":")[-1]).title()
         queryables[prop] = QueryableProperty(description=titled_name)
 
     return queryables
diff --git a/eodag/rest/utils.py b/eodag/rest/utils.py
index 4cee4d85..770c4284 100644
--- a/eodag/rest/utils.py
+++ b/eodag/rest/utils.py
@@ -1068,7 +1068,6 @@ class Queryables(BaseModel):
                 description="Datetime",
                 ref="https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/datetime",
             ),
-            "ids": QueryableProperty(description="IDs"),
         }
     )
     additional_properties: bool = Field(
@@ -1103,8 +1102,14 @@ def rename_to_stac_standard(key: str) -> str:
     stac_config_properties: Dict[str, Any] = stac_config["item"]["properties"]
 
     for stac_property, value in stac_config_properties.items():
+        if isinstance(value, list):
+            value = value[0]
         if str(value).endswith(key):
             return stac_property
+
+    if key in OSEO_METADATA_MAPPING:
+        return "oseo:" + key
+
     return key
 
 
@@ -1121,15 +1126,17 @@ def fetch_collection_queryable_properties(
     :rtype queryable_properties: set
     """
     # Fetch the metadata mapping for collection-specific queryables
-    args = [collection_id, provider] if provider else [collection_id]
-    search_plugin = next(eodag_api._plugins_manager.get_search_plugins(*args))
-    search_plugin.config
-    mapping: Dict[str, Any] = dict(search_plugin.config.metadata_mapping)
+    kwargs = {"product_type": collection_id}
+    if provider is not None:
+        kwargs["provider"] = provider
+    eodag_queryable_properties = eodag_api.get_queryables(**kwargs)
+
     # list of all the STAC standardized collection-specific queryables
     queryable_properties: Set[str] = set()
-    for key, value in mapping.items():
-        if isinstance(value, list) and "TimeFromAscendingNode" not in key:
-            queryable_properties.add(rename_to_stac_standard(key))
+    for prop in eodag_queryable_properties:
+        # remove pure eodag properties
+        if prop not in ["start", "end", "geom", "locations", "id"]:
+            queryable_properties.add(rename_to_stac_standard(prop))
     return queryable_properties
 
 


Test patch:
diff --git a/tests/context.py b/tests/context.py
index 86d21775..3b4c1d73 100644
--- a/tests/context.py
+++ b/tests/context.py
@@ -97,6 +97,7 @@ from eodag.utils.exceptions import (
     PluginImplementationError,
     RequestError,
     UnsupportedDatasetAddressScheme,
+    UnsupportedProductType,
     UnsupportedProvider,
     ValidationError,
     STACOpenerError,
diff --git a/tests/units/test_core.py b/tests/units/test_core.py
index d127ac3c..64fa29bb 100644
--- a/tests/units/test_core.py
+++ b/tests/units/test_core.py
@@ -43,6 +43,7 @@ from tests.context import (
     ProviderConfig,
     RequestError,
     SearchResult,
+    UnsupportedProductType,
     UnsupportedProvider,
     get_geometry_from_various,
     load_default_config,
@@ -980,6 +981,94 @@ class TestCore(TestCoreBase):
         # run a 2nd time: check that it does not raise an error
         self.dag.update_providers_config(new_config)
 
+    def test_get_queryables(self):
+        """get_queryables must return queryables list adapted to provider and product-type"""
+        with self.assertRaises(UnsupportedProvider):
+            self.dag.get_queryables(provider="not_existing_provider")
+
+        with self.assertRaises(UnsupportedProductType):
+            self.dag.get_queryables(product_type="not_existing_product_type")
+
+        expected_result = {"productType", "start", "end", "geom", "locations", "id"}
+        queryables = self.dag.get_queryables()
+        self.assertSetEqual(queryables, expected_result)
+
+        expected_result = {
+            "start",
+            "end",
+            "geom",
+            "locations",
+            "productType",
+            "platformSerialIdentifier",
+            "instrument",
+            "processingLevel",
+            "resolution",
+            "organisationName",
+            "parentIdentifier",
+            "orbitNumber",
+            "orbitDirection",
+            "swathIdentifier",
+            "cloudCover",
+            "snowCover",
+            "sensorMode",
+            "polarizationMode",
+            "id",
+            "tileIdentifier",
+            "geometry",
+        }
+        queryables = self.dag.get_queryables(provider="peps")
+        self.assertSetEqual(queryables, expected_result)
+
+        expected_result = {
+            "start",
+            "end",
+            "geom",
+            "locations",
+            "productType",
+            "platformSerialIdentifier",
+            "instrument",
+            "processingLevel",
+            "resolution",
+            "organisationName",
+            "parentIdentifier",
+            "orbitNumber",
+            "orbitDirection",
+            "swathIdentifier",
+            "snowCover",
+            "sensorMode",
+            "polarizationMode",
+            "id",
+            "tileIdentifier",
+            "geometry",
+        }
+        queryables = self.dag.get_queryables(provider="peps", product_type="S1_SAR_GRD")
+        self.assertSetEqual(queryables, expected_result)
+
+        expected_result = {"productType", "start", "end", "geom", "locations", "id"}
+        queryables = self.dag.get_queryables(product_type="S2_MSI_L1C")
+        self.assertSetEqual(queryables, expected_result)
+
+        expected_result = {
+            "productType",
+            "start",
+            "end",
+            "geom",
+            "locations",
+            "geometry",
+            "platformSerialIdentifier",
+            "title",
+            "cloudCover",
+            "illuminationAzimuthAngle",
+            "illuminationZenithAngle",
+            "awsPath",
+            "productPath",
+            "id",
+        }
+        queryables = self.dag.get_queryables(
+            provider="aws_eos", product_type="S2_MSI_L1C"
+        )
+        self.assertSetEqual(queryables, expected_result)
+
 
 class TestCoreConfWithEnvVar(TestCoreBase):
     @classmethod