Repository: CS-SI/eodag
Pull request #895 (2023-10-20 13:24:15)

fix(server): properly display next page object
### Description
This PR fixes the `next` link display in `search` requests (both `GET` and `POST`).
It provides a URL to the next page and the json body in the case of a `POST` request.


github-actions[bot]: ## Test Results
    1 files   -        3      1 suites   - 3   1m 14s [:stopwatch:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "duration of all tests") - 5m 12s
430 tests ±       0  367 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "passed tests")  -      60  3 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "skipped / disabled tests") ±  0  1 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "failed tests") +1  59 [:fire:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "test errors") +59 
430 runs   - 1 290  367 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "passed tests")  - 1 277  3 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "skipped / disabled tests")  - 73  1 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "failed tests") +1  59 [:fire:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "test errors") +59 

For more details on these failures and errors, see [this check](https://github.com/CS-SI/eodag/runs/17900500149).

Results for commit e5081b61. ± Comparison against base commit 19d7d6b2.

[test-results]:data:application/gzip;base64,H4sIAFGAMmUC/1WMSw6DIBQAr2JYd+ELINDLNPJLSFUahJXp3fvQVupyZpLZiA+TW8m9g1tH1hLyCbakMYe4IAqGjCXXxmj/o8dajEFFB9HUM7yqOoUfw/RdHsKlFBMarlClsrRphevzMG25899x58vQxHkOGYk43kvQA3irjJTCadCcaqYpWAeeK+0pcwokeX8ALtRgOwYBAAA=

github-actions[bot]: <strong>Code Coverage (Ubuntu)</strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `70%` | :white_check_mark: |

_Minimum allowed coverage is `70%`_

<p align="right">Generated by :monkey: cobertura-action against e5081b61fd9c887eb1b53b4b31de1f59bf34e918 </p>
github-actions[bot]: <strong>Code Coverage (Windows)</strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `76%` | :white_check_mark: |

_Minimum allowed coverage is `70%`_

<p align="right">Generated by :monkey: cobertura-action against 1f4f776c0b0f4b2fef79241233a95ac172e0876a </p>

Patch:
diff --git a/.gitignore b/.gitignore
index 0e6aad37..0fcb9b90 100644
--- a/.gitignore
+++ b/.gitignore
@@ -82,7 +82,7 @@ celerybeat-schedule
 .env
 
 # virtualenv
-.venv/
+.venv*/
 venv/
 ENV/
 
diff --git a/docs/api_reference/core.rst b/docs/api_reference/core.rst
index 6a5f8551..3aed3132 100644
--- a/docs/api_reference/core.rst
+++ b/docs/api_reference/core.rst
@@ -80,6 +80,7 @@ Misc
 
    EODataAccessGateway.group_by_extent
    EODataAccessGateway.guess_product_type
+   EODataAccessGateway.get_queryables
 
 .. autoclass:: eodag.api.core.EODataAccessGateway
    :members: set_preferred_provider, get_preferred_provider, update_providers_config, list_product_types,
diff --git a/eodag/api/core.py b/eodag/api/core.py
index 4ebb10de..30dfb600 100644
--- a/eodag/api/core.py
+++ b/eodag/api/core.py
@@ -22,7 +22,7 @@ import os
 import re
 import shutil
 from operator import itemgetter
-from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Set, Tuple, Union
 
 import geojson
 import pkg_resources
@@ -71,6 +71,7 @@ from eodag.utils.exceptions import (
     NoMatchingProductType,
     PluginImplementationError,
     RequestError,
+    UnsupportedProductType,
     UnsupportedProvider,
 )
 from eodag.utils.stac_reader import fetch_stac_items
@@ -1997,3 +1998,58 @@ class EODataAccessGateway:
         plugin_conf = {"name": name}
         plugin_conf.update({key.replace("-", "_"): val for key, val in options.items()})
         return self._plugins_manager.get_crunch_plugin(name, **plugin_conf)
+
+    def get_queryables(
+        self, provider: Optional[str] = None, product_type: Optional[str] = None
+    ) -> Set[str]:
+        """Fetch the queryable properties for a given product type and/or provider.
+
+        :param product_type: (optional) The EODAG product type.
+        :type product_type: str
+        :param provider: (optional) The provider.
+        :type provider: str
+        :returns: A set containing the EODAG queryable properties.
+        :rtype: set
+        """
+        default_queryables = {"productType", "start", "end", "geom", "locations", "id"}
+        if provider is None and product_type is None:
+            return default_queryables
+
+        plugins = self._plugins_manager.get_search_plugins(product_type, provider)
+
+        # dictionary of the queryable properties of the providers supporting the given product type
+        all_queryable_properties = dict()
+        for plugin in plugins:
+            if (
+                product_type
+                and product_type not in plugin.config.products.keys()
+                and provider is None
+            ):
+                raise UnsupportedProductType(product_type)
+            elif product_type and product_type not in plugin.config.products.keys():
+                raise UnsupportedProductType(
+                    f"{product_type} is not available for provider {provider}"
+                )
+
+            provider_queryables = set(default_queryables)
+
+            metadata_mapping = deepcopy(getattr(plugin.config, "metadata_mapping", {}))
+
+            # product_type-specific metadata-mapping
+            metadata_mapping.update(
+                getattr(plugin.config, "products", {})
+                .get(product_type, {})
+                .get("metadata_mapping", {})
+            )
+
+            for key, value in metadata_mapping.items():
+                if isinstance(value, list) and "TimeFromAscendingNode" not in key:
+                    provider_queryables.add(key)
+
+            all_queryable_properties[plugin.provider] = provider_queryables
+
+        if provider is None:
+            # intersection of the queryables among the providers
+            return set.intersection(*all_queryable_properties.values())
+        else:
+            return all_queryable_properties[provider]
diff --git a/eodag/resources/stac.yml b/eodag/resources/stac.yml
index bfea4b15..8c6f2e4d 100644
--- a/eodag/resources/stac.yml
+++ b/eodag/resources/stac.yml
@@ -153,7 +153,8 @@ items:
       href: '$.search_results.next'
       title: Next page
       type: application/geo+json
-      method: GET
+      method: '$.search_results.method'
+      body: '$.search_results.body'
   # time and date when the response was generated
   timeStamp: '$.search_results.timeStamp'
   # count request result
diff --git a/eodag/rest/server.py b/eodag/rest/server.py
index a50b92bf..51b8d32b 100755
--- a/eodag/rest/server.py
+++ b/eodag/rest/server.py
@@ -467,8 +467,8 @@ def list_collection_queryables(
         fetch_collection_queryable_properties(*conf_args)
     )
 
-    for prop in provider_properties:
-        titled_name = re.sub(CAMEL_TO_SPACE_TITLED, " ", prop).title()
+    for prop in sorted(provider_properties):
+        titled_name = re.sub(CAMEL_TO_SPACE_TITLED, " ", prop.split(":")[-1]).title()
         queryables[prop] = QueryableProperty(description=titled_name)
 
     return queryables
@@ -704,7 +704,11 @@ def stac_search(
     provider = arguments.pop("provider", None)
 
     response = search_stac_items(
-        url=url, arguments=arguments, root=url_root, provider=provider
+        url=url,
+        arguments=arguments,
+        root=url_root,
+        provider=provider,
+        method=request.method,
     )
     resp = ORJSONResponse(
         content=response, status_code=200, media_type="application/json"
diff --git a/eodag/rest/stac.py b/eodag/rest/stac.py
index f225abda..9a7835eb 100644
--- a/eodag/rest/stac.py
+++ b/eodag/rest/stac.py
@@ -20,7 +20,6 @@ from __future__ import annotations
 import datetime
 import logging
 import os
-import re
 from collections import defaultdict
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
 from urllib.parse import parse_qs, urlencode, urlparse
@@ -346,22 +345,6 @@ class StacItem(StacCommon):
             for i, _ in enumerate(items_model["links"]):
                 if items_model["links"][i]["rel"] == "self":
                     items_model["links"][i]["href"] = catalog["url"]
-            if "page=" not in self.url:
-                search_results.next = "%s&page=%s" % (
-                    self.url,
-                    search_results.properties["page"] + 1,
-                )
-            else:
-                search_results.next = re.sub(
-                    r"^(.*)(page=[0-9]+)(.*)$",
-                    r"\1page=%s\3" % (search_results.properties["page"] + 1),
-                    self.url,
-                )
-        else:
-            search_results.next = "%s?page=%s" % (
-                self.url,
-                search_results.properties["page"] + 1,
-            )
 
         search_results.timeStamp = (
             datetime.datetime.now(datetime.timezone.utc)
diff --git a/eodag/rest/utils.py b/eodag/rest/utils.py
index 4cee4d85..ccc36ac1 100644
--- a/eodag/rest/utils.py
+++ b/eodag/rest/utils.py
@@ -37,6 +37,7 @@ from typing import (
     Tuple,
     Union,
 )
+from urllib.parse import urlencode
 
 import dateutil.parser
 from dateutil import tz
@@ -818,6 +819,7 @@ def search_stac_items(
     root: str = "/",
     catalogs: List[str] = [],
     provider: Optional[str] = None,
+    method: Optional[str] = "GET",
 ) -> Dict[str, Any]:
     """Get items collection dict for given catalogs list
 
@@ -831,6 +833,8 @@ def search_stac_items(
     :type catalogs: list
     :param provider: (optional) Chosen provider
     :type provider: str
+    :param method: (optional) search request HTTP method ('GET' or 'POST')
+    :type method: str
     :returns: Catalog dictionnary
     :rtype: dict
     """
@@ -838,6 +842,14 @@ def search_stac_items(
 
     catalog_url = url.replace("/items", "")
 
+    next_page_kwargs = {
+        key: value for key, value in arguments.copy().items() if value is not None
+    }
+    next_page_id = (
+        int(next_page_kwargs["page"]) + 1 if "page" in next_page_kwargs else 2
+    )
+    next_page_kwargs["page"] = next_page_id
+
     # use catalogs from path or if it is empty, collections from args
     if catalogs:
         result_catalog = StacCatalog(
@@ -860,7 +872,8 @@ def search_stac_items(
             root=root,
             provider=provider,
             eodag_api=eodag_api,
-            # handle only one collection per request (STAC allows multiple)
+            # handle only one collection
+            # per request (STAC allows multiple)
             catalogs=collections[0:1],
             url=catalog_url.replace("/search", f"/collections/{collections[0]}"),
         )
@@ -956,12 +969,22 @@ def search_stac_items(
                         **{"url": result_catalog.url, "root": result_catalog.root},
                     ),
                 )
+
         search_results = search_products(
             product_type=result_catalog.search_args["product_type"],
             arguments=search_products_arguments,
         )
 
-    return StacItem(
+    search_results.method = method
+    if method == "POST":
+        search_results.next = f"{url}"
+        search_results.body = next_page_kwargs
+
+    elif method == "GET":
+        next_query_string = urlencode(next_page_kwargs)
+        search_results.next = f"{url}?{next_query_string}"
+
+    items = StacItem(
         url=url,
         stac_config=stac_config,
         provider=provider,
@@ -975,6 +998,8 @@ def search_stac_items(
         ),
     )
 
+    return items
+
 
 def get_stac_extension_oseo(url: str) -> Dict[str, str]:
     """Build STAC OGC / OpenSearch Extension for EO
@@ -1068,7 +1093,6 @@ class Queryables(BaseModel):
                 description="Datetime",
                 ref="https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/datetime.json#/properties/datetime",
             ),
-            "ids": QueryableProperty(description="IDs"),
         }
     )
     additional_properties: bool = Field(
@@ -1103,8 +1127,14 @@ def rename_to_stac_standard(key: str) -> str:
     stac_config_properties: Dict[str, Any] = stac_config["item"]["properties"]
 
     for stac_property, value in stac_config_properties.items():
+        if isinstance(value, list):
+            value = value[0]
         if str(value).endswith(key):
             return stac_property
+
+    if key in OSEO_METADATA_MAPPING:
+        return "oseo:" + key
+
     return key
 
 
@@ -1121,15 +1151,17 @@ def fetch_collection_queryable_properties(
     :rtype queryable_properties: set
     """
     # Fetch the metadata mapping for collection-specific queryables
-    args = [collection_id, provider] if provider else [collection_id]
-    search_plugin = next(eodag_api._plugins_manager.get_search_plugins(*args))
-    search_plugin.config
-    mapping: Dict[str, Any] = dict(search_plugin.config.metadata_mapping)
+    kwargs = {"product_type": collection_id}
+    if provider is not None:
+        kwargs["provider"] = provider
+    eodag_queryable_properties = eodag_api.get_queryables(**kwargs)
+
     # list of all the STAC standardized collection-specific queryables
     queryable_properties: Set[str] = set()
-    for key, value in mapping.items():
-        if isinstance(value, list) and "TimeFromAscendingNode" not in key:
-            queryable_properties.add(rename_to_stac_standard(key))
+    for prop in eodag_queryable_properties:
+        # remove pure eodag properties
+        if prop not in ["start", "end", "geom", "locations", "id"]:
+            queryable_properties.add(rename_to_stac_standard(prop))
     return queryable_properties
 
 


Test patch:
diff --git a/tests/context.py b/tests/context.py
index 86d21775..3b4c1d73 100644
--- a/tests/context.py
+++ b/tests/context.py
@@ -97,6 +97,7 @@ from eodag.utils.exceptions import (
     PluginImplementationError,
     RequestError,
     UnsupportedDatasetAddressScheme,
+    UnsupportedProductType,
     UnsupportedProvider,
     ValidationError,
     STACOpenerError,
diff --git a/tests/units/test_core.py b/tests/units/test_core.py
index d127ac3c..64fa29bb 100644
--- a/tests/units/test_core.py
+++ b/tests/units/test_core.py
@@ -43,6 +43,7 @@ from tests.context import (
     ProviderConfig,
     RequestError,
     SearchResult,
+    UnsupportedProductType,
     UnsupportedProvider,
     get_geometry_from_various,
     load_default_config,
@@ -980,6 +981,94 @@ class TestCore(TestCoreBase):
         # run a 2nd time: check that it does not raise an error
         self.dag.update_providers_config(new_config)
 
+    def test_get_queryables(self):
+        """get_queryables must return queryables list adapted to provider and product-type"""
+        with self.assertRaises(UnsupportedProvider):
+            self.dag.get_queryables(provider="not_existing_provider")
+
+        with self.assertRaises(UnsupportedProductType):
+            self.dag.get_queryables(product_type="not_existing_product_type")
+
+        expected_result = {"productType", "start", "end", "geom", "locations", "id"}
+        queryables = self.dag.get_queryables()
+        self.assertSetEqual(queryables, expected_result)
+
+        expected_result = {
+            "start",
+            "end",
+            "geom",
+            "locations",
+            "productType",
+            "platformSerialIdentifier",
+            "instrument",
+            "processingLevel",
+            "resolution",
+            "organisationName",
+            "parentIdentifier",
+            "orbitNumber",
+            "orbitDirection",
+            "swathIdentifier",
+            "cloudCover",
+            "snowCover",
+            "sensorMode",
+            "polarizationMode",
+            "id",
+            "tileIdentifier",
+            "geometry",
+        }
+        queryables = self.dag.get_queryables(provider="peps")
+        self.assertSetEqual(queryables, expected_result)
+
+        expected_result = {
+            "start",
+            "end",
+            "geom",
+            "locations",
+            "productType",
+            "platformSerialIdentifier",
+            "instrument",
+            "processingLevel",
+            "resolution",
+            "organisationName",
+            "parentIdentifier",
+            "orbitNumber",
+            "orbitDirection",
+            "swathIdentifier",
+            "snowCover",
+            "sensorMode",
+            "polarizationMode",
+            "id",
+            "tileIdentifier",
+            "geometry",
+        }
+        queryables = self.dag.get_queryables(provider="peps", product_type="S1_SAR_GRD")
+        self.assertSetEqual(queryables, expected_result)
+
+        expected_result = {"productType", "start", "end", "geom", "locations", "id"}
+        queryables = self.dag.get_queryables(product_type="S2_MSI_L1C")
+        self.assertSetEqual(queryables, expected_result)
+
+        expected_result = {
+            "productType",
+            "start",
+            "end",
+            "geom",
+            "locations",
+            "geometry",
+            "platformSerialIdentifier",
+            "title",
+            "cloudCover",
+            "illuminationAzimuthAngle",
+            "illuminationZenithAngle",
+            "awsPath",
+            "productPath",
+            "id",
+        }
+        queryables = self.dag.get_queryables(
+            provider="aws_eos", product_type="S2_MSI_L1C"
+        )
+        self.assertSetEqual(queryables, expected_result)
+
 
 class TestCoreConfWithEnvVar(TestCoreBase):
     @classmethod
diff --git a/tests/units/test_stac_utils.py b/tests/units/test_stac_utils.py
index 265a8d7c..6b3fd0f1 100644
--- a/tests/units/test_stac_utils.py
+++ b/tests/units/test_stac_utils.py
@@ -486,3 +486,69 @@ class TestStacUtils(unittest.TestCase):
         )
         # check that no other asset have also been added to the response
         self.assertEqual(len(response["features"][0]["assets"]), 2)
+
+    @mock.patch(
+        "eodag.plugins.search.qssearch.QueryStringSearch._request",
+        autospec=True,
+    )
+    def test_search_stac_items_get(self, mock__request):
+        """search_stac_items runs with GET method"""
+        # mock the QueryStringSearch request with the S2_MSI_L1C peps response search dictionary
+        mock__request.return_value = mock.Mock()
+        mock__request.return_value.json.return_value = self.peps_resp_search_json
+
+        response = self.rest_utils.search_stac_items(
+            url="http://foo/search",
+            arguments={"collections": "S2_MSI_L1C"},
+            root="http://foo/",
+            method="GET",
+        )
+
+        mock__request.assert_called()
+
+        next_link = [link for link in response["links"] if link["rel"] == "next"][0]
+
+        self.assertEqual(
+            next_link,
+            {
+                "method": "GET",
+                "body": None,
+                "rel": "next",
+                "href": "http://foo/search?collections=S2_MSI_L1C&page=2",
+                "title": "Next page",
+                "type": "application/geo+json",
+            },
+        )
+
+    @mock.patch(
+        "eodag.plugins.search.qssearch.QueryStringSearch._request",
+        autospec=True,
+    )
+    def test_search_stac_items_post(self, mock__request):
+        """search_stac_items runs with GET method"""
+        # mock the QueryStringSearch request with the S2_MSI_L1C peps response search dictionary
+        mock__request.return_value = mock.Mock()
+        mock__request.return_value.json.return_value = self.peps_resp_search_json
+
+        response = self.rest_utils.search_stac_items(
+            url="http://foo/search",
+            arguments={"collections": ["S2_MSI_L1C"], "page": 2},
+            root="http://foo/",
+            method="POST",
+        )
+
+        mock__request.assert_called()
+
+        next_link = [link for link in response["links"] if link["rel"] == "next"][0]
+
+        self.assertEqual(
+            next_link,
+            {
+                "method": "POST",
+                "rel": "next",
+                "href": "http://foo/search",
+                "title": "Next page",
+                "type": "application/geo+json",
+                "body": {"collections": ["S2_MSI_L1C"], "page": 3},
+            },
+        )