Repository: CS-SI/eodag
Pull request #958 (2023-12-08 15:35:12)

fix(server): CdsApi download using _dc_qs parameter
Fixes server mode `CdsApi` dowloads using a new parameter `_dc_qs` to store query parameters

github-actions[bot]: ## Test Results
    1 files   -        3      1 suites   - 3   50s [:stopwatch:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "duration of all tests") - 2m 37s
439 tests ±       0  434 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "passed tests")  -        2  3 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "skipped / disabled tests") ±  0  2 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "failed tests") +2 
439 runs   - 1 317  434 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "passed tests")  - 1 246  3 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "skipped / disabled tests")  - 73  2 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.11.0/README.md#the-symbols "failed tests") +2 

For more details on these failures, see [this check](https://github.com/CS-SI/eodag/runs/19454407773).

Results for commit cde5f4ce. ± Comparison against base commit 300a2f8c.

[test-results]:data:application/gzip;base64,H4sIAFA4c2UC/1WMSw7CIBQAr9KwdkH5lNbLmPqAhNgWw2dlvLsPjFCXM5PMi1i3mUiuw3gZSMwuNdA5rMn5A1FSZCypNMGXH91iBqhKdPVwT1S8Cbu6DQVrwoTgA5oyDfnozwL/y6/px8qnYeXzD/y+u4RAQBtpBZhptEbf+aS55YtiiipqGcAsQGo6S0PeH6YdBsMEAQAA

github-actions[bot]: <strong>Code Coverage (Ubuntu)</strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `82%` | :white_check_mark: |

_Minimum allowed coverage is `70%`_

<p align="right">Generated by :monkey: cobertura-action against c35393107a920f805503c57a90108f205ab8e891 </p>
github-actions[bot]: <strong>Code Coverage (Windows)</strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `76%` | :white_check_mark: |

_Minimum allowed coverage is `70%`_

<p align="right">Generated by :monkey: cobertura-action against c35393107a920f805503c57a90108f205ab8e891 </p>

Patch:
diff --git a/eodag/api/core.py b/eodag/api/core.py
index db31e04b..31130763 100644
--- a/eodag/api/core.py
+++ b/eodag/api/core.py
@@ -998,6 +998,9 @@ class EODataAccessGateway:
             return self._search_by_id(
                 search_kwargs.pop("id"), provider=provider, **search_kwargs
             )
+        # remove datacube query string from kwargs which was only needed for search-by-id
+        search_kwargs.pop("_dc_qs", None)
+
         search_kwargs.update(
             page=page,
             items_per_page=items_per_page,
@@ -1371,6 +1374,8 @@ class EODataAccessGateway:
         search_plugins = self._plugins_manager.get_search_plugins(
             **get_search_plugins_kwargs
         )
+        # datacube query string
+        _dc_qs = kwargs.pop("_dc_qs", None)
 
         for plugin in search_plugins:
             logger.info(
@@ -1378,7 +1383,10 @@ class EODataAccessGateway:
             )
             logger.debug("Using plugin class for search: %s", plugin.__class__.__name__)
             plugin.clear()
-            results, _ = self._do_search(plugin, id=uid, **kwargs)
+            if isinstance(plugin, BuildPostSearchResult):
+                results, _ = self._do_search(plugin, id=uid, _dc_qs=_dc_qs, **kwargs)
+            else:
+                results, _ = self._do_search(plugin, id=uid, **kwargs)
             if len(results) == 1:
                 if not results[0].product_type:
                     # guess product type from properties
diff --git a/eodag/plugins/apis/cds.py b/eodag/plugins/apis/cds.py
index ae902a02..3a59fe8a 100644
--- a/eodag/plugins/apis/cds.py
+++ b/eodag/plugins/apis/cds.py
@@ -20,6 +20,7 @@ from __future__ import annotations
 import logging
 from datetime import datetime
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
+from urllib.parse import unquote_plus
 
 import cdsapi
 import geojson
@@ -89,6 +90,22 @@ class CdsApi(Download, Api, BuildPostSearchResult):
     ) -> Tuple[List[EOProduct], Optional[int]]:
         """Build ready-to-download SearchResult"""
 
+        _dc_qs = kwargs.get("_dc_qs", None)
+        if _dc_qs is not None:
+            _dc_qp = geojson.loads(unquote_plus(unquote_plus(_dc_qs)))
+            if "/" in _dc_qp.get("date", ""):
+                (
+                    kwargs["startTimeFromAscendingNode"],
+                    kwargs["completionTimeFromAscendingNode"],
+                ) = _dc_qp["date"].split("/")
+            else:
+                kwargs["startTimeFromAscendingNode"] = kwargs[
+                    "completionTimeFromAscendingNode"
+                ] = _dc_qp["date"]
+
+            if "/" in _dc_qp.get("area", ""):
+                kwargs["geometry"] = _dc_qp["area"].split("/")
+
         # check productType, dates, geometry, use defaults if not specified
         # productType
         if not kwargs.get("productType"):
diff --git a/eodag/plugins/search/build_search_result.py b/eodag/plugins/search/build_search_result.py
index 1ee48a87..a043805a 100644
--- a/eodag/plugins/search/build_search_result.py
+++ b/eodag/plugins/search/build_search_result.py
@@ -20,6 +20,7 @@ from __future__ import annotations
 import hashlib
 import logging
 from typing import Any, Dict, List, Optional, Tuple
+from urllib.parse import quote_plus, unquote_plus
 
 import geojson
 import orjson
@@ -109,22 +110,39 @@ class BuildPostSearchResult(PostJsonSearch):
 
         result = results[0]
 
-        # update result with query parameters without pagination (or search-only params)
-        if isinstance(self.config.pagination["next_page_query_obj"], str) and hasattr(
-            self, "query_params_unpaginated"
-        ):
-            unpaginated_query_params = self.query_params_unpaginated
-        elif isinstance(self.config.pagination["next_page_query_obj"], str):
-            next_page_query_obj = orjson.loads(
-                self.config.pagination["next_page_query_obj"].format()
+        # datacube query string got from previous search
+        _dc_qs = kwargs.pop("_dc_qs", None)
+        if _dc_qs is not None:
+            qs = unquote_plus(unquote_plus(_dc_qs))
+            unpaginated_query_params = sorted_unpaginated_query_params = geojson.loads(
+                qs
             )
-            unpaginated_query_params = {
-                k: v
-                for k, v in self.query_params.items()
-                if (k, v) not in next_page_query_obj.items()
-            }
         else:
-            unpaginated_query_params = self.query_params
+            # update result with query parameters without pagination (or search-only params)
+            if isinstance(
+                self.config.pagination["next_page_query_obj"], str
+            ) and hasattr(self, "query_params_unpaginated"):
+                unpaginated_query_params = self.query_params_unpaginated
+            elif isinstance(self.config.pagination["next_page_query_obj"], str):
+                next_page_query_obj = orjson.loads(
+                    self.config.pagination["next_page_query_obj"].format()
+                )
+                unpaginated_query_params = {
+                    k: v[0] if (isinstance(v, list) and len(v) == 1) else v
+                    for k, v in self.query_params.items()
+                    if (k, v) not in next_page_query_obj.items()
+                }
+            else:
+                unpaginated_query_params = self.query_params
+
+            # query hash, will be used to build a product id
+            sorted_unpaginated_query_params = dict_items_recursive_sort(
+                unpaginated_query_params
+            )
+            qs = geojson.dumps(sorted_unpaginated_query_params)
+
+        query_hash = hashlib.sha1(str(qs).encode("UTF-8")).hexdigest()
+
         result = dict(result, **unpaginated_query_params)
 
         # update result with search args if not None (and not auth)
@@ -148,13 +166,6 @@ class BuildPostSearchResult(PostJsonSearch):
             if v not in (NOT_AVAILABLE, NOT_MAPPED)
         }
 
-        # query hash, will be used to build a product id
-        sorted_unpaginated_query_params = dict_items_recursive_sort(
-            unpaginated_query_params
-        )
-        qs = geojson.dumps(sorted_unpaginated_query_params)
-        query_hash = hashlib.sha1(str(qs).encode("UTF-8")).hexdigest()
-
         # build product id
         id_prefix = (product_type or self.provider).upper()
         product_id = "%s_%s_%s" % (
@@ -170,6 +181,7 @@ class BuildPostSearchResult(PostJsonSearch):
 
         # update downloadLink
         product_available_properties["downloadLink"] += f"?{qs}"
+        product_available_properties["_dc_qs"] = quote_plus(qs)
 
         # parse metadata needing downloadLink
         for param, mapping in self.config.metadata_mapping.items():
diff --git a/eodag/resources/stac.yml b/eodag/resources/stac.yml
index 8c6f2e4d..2de3ea7f 100644
--- a/eodag/resources/stac.yml
+++ b/eodag/resources/stac.yml
@@ -222,6 +222,7 @@ item:
         href: "{catalog[url]}/items/{item[id]}/download"
         # https://github.com/radiantearth/stac-spec/blob/master/item-spec/item-spec.md#media-types
         type: 'application/zip'
+        _dc_qs: '$.product.properties._dc_qs'
     thumbnail:
         title: 'Thumbnail'
         href: '$.product.properties.quicklook'
diff --git a/eodag/rest/server.py b/eodag/rest/server.py
index 51b8d32b..7e14d489 100755
--- a/eodag/rest/server.py
+++ b/eodag/rest/server.py
@@ -354,7 +354,7 @@ def stac_collections_item_download(
     provider = arguments.pop("provider", None)
 
     return download_stac_item_by_id_stream(
-        catalogs=[collection_id], item_id=item_id, provider=provider
+        catalogs=[collection_id], item_id=item_id, provider=provider, **arguments
     )
 
 
@@ -373,7 +373,11 @@ def stac_collections_item_download_asset(
     provider = arguments.pop("provider", None)
 
     return download_stac_item_by_id_stream(
-        catalogs=[collection_id], item_id=item_id, provider=provider, asset=asset_filter
+        catalogs=[collection_id],
+        item_id=item_id,
+        provider=provider,
+        asset=asset_filter,
+        **arguments,
     )
 
 
@@ -397,6 +401,7 @@ def stac_collections_item(collection_id: str, item_id: str, request: Request) ->
         root=url_root,
         catalogs=[collection_id],
         provider=provider,
+        **arguments,
     )
 
     if response:
@@ -542,7 +547,7 @@ def stac_catalogs_item_download(
     list_catalog = catalogs.strip("/").split("/")
 
     return download_stac_item_by_id_stream(
-        catalogs=list_catalog, item_id=item_id, provider=provider
+        catalogs=list_catalog, item_id=item_id, provider=provider, **arguments
     )
 
 
@@ -563,7 +568,11 @@ def stac_catalogs_item_download_asset(
     catalogs = catalogs.strip("/").split("/")
 
     return download_stac_item_by_id_stream(
-        catalogs=catalogs, item_id=item_id, provider=provider, asset=asset_filter
+        catalogs=catalogs,
+        item_id=item_id,
+        provider=provider,
+        asset=asset_filter,
+        **arguments,
     )
 
 
@@ -588,6 +597,7 @@ def stac_catalogs_item(catalogs: str, item_id: str, request: Request):
         root=url_root,
         catalogs=list_catalog,
         provider=provider,
+        **arguments,
     )
 
     if response:
diff --git a/eodag/rest/stac.py b/eodag/rest/stac.py
index 6ac05913..2665aaad 100644
--- a/eodag/rest/stac.py
+++ b/eodag/rest/stac.py
@@ -252,16 +252,25 @@ class StacItem(StacCommon):
                 },
             )
 
-            # append provider query-arg to download link
+            # parse download link
+            url_parts = urlparse(str(product_item["assets"]["downloadLink"]["href"]))
+            query_dict = parse_qs(url_parts.query)
+            without_arg_url = (
+                f"{url_parts.scheme}://{url_parts.netloc}{url_parts.path}"
+                if url_parts.scheme
+                else f"{url_parts.netloc}{url_parts.path}"
+            )
+
+            # add provider to query-args
             if self.provider:
-                parts = urlparse(str(product_item["assets"]["downloadLink"]["href"]))
-                query_dict = parse_qs(parts.query)
                 query_dict.update(provider=[self.provider])
-                without_arg_url = (
-                    f"{parts.scheme}://{parts.netloc}{parts.path}"
-                    if parts.scheme
-                    else f"{parts.netloc}{parts.path}"
-                )
+            # add datacube query-string to query-args
+            _dc_qs = product_item["assets"]["downloadLink"].pop("_dc_qs", None)
+            if _dc_qs:
+                query_dict.update(_dc_qs=_dc_qs)
+
+            # update download link with up-to-date query-args
+            if query_dict:
                 product_item["assets"]["downloadLink"][
                     "href"
                 ] = f"{without_arg_url}?{urlencode(query_dict, doseq=True)}"
@@ -271,22 +280,13 @@ class StacItem(StacCommon):
             if getattr(product, "assets", False):
                 # replace origin asset urls with eodag-server ones
                 for asset_key, asset_value in origin_assets.items():
-
-                    parts = urlparse(product_item["assets"]["downloadLink"]["href"])
-                    if parts.query:
-                        query_dict = parse_qs(parts.query)
-                        without_arg_url = (
-                            f"{parts.scheme}://{parts.netloc}{parts.path}"
-                            if parts.scheme
-                            else f"{parts.netloc}{parts.path}"
-                        ) + f"/{asset_key}"
+                    origin_assets[asset_key]["href"] = without_arg_url
+                    if query_dict:
                         origin_assets[asset_key][
                             "href"
-                        ] = f"{without_arg_url}?{urlencode(query_dict, doseq=True)}"
+                        ] += f"/{asset_key}?{urlencode(query_dict, doseq=True)}"
                     else:
-                        origin_assets[asset_key][
-                            "href"
-                        ] = f"{product_item['assets']['downloadLink']['href']}/{asset_key}"
+                        origin_assets[asset_key]["href"] += f"/{asset_key}"
 
                 product_item["assets"] = dict(product_item["assets"], **origin_assets)
 
@@ -318,6 +318,18 @@ class StacItem(StacCommon):
             # remove empty properties
             product_item = self.__filter_item_properties_values(product_item)
 
+            # update item link with datacube query-string
+            if _dc_qs:
+                url_parts = urlparse(str(product_item["links"][0]["href"]))
+                without_arg_url = (
+                    f"{url_parts.scheme}://{url_parts.netloc}{url_parts.path}"
+                    if url_parts.scheme
+                    else f"{url_parts.netloc}{url_parts.path}"
+                )
+                product_item["links"][0][
+                    "href"
+                ] = f"{without_arg_url}?{urlencode(query_dict, doseq=True)}"
+
             item_list.append(product_item)
 
         return item_list
diff --git a/eodag/rest/utils.py b/eodag/rest/utils.py
index d44d6056..3b2c13a4 100644
--- a/eodag/rest/utils.py
+++ b/eodag/rest/utils.py
@@ -535,7 +535,10 @@ def search_products(
 
 
 def search_product_by_id(
-    uid: str, product_type: Optional[str] = None, provider: Optional[str] = None
+    uid: str,
+    product_type: Optional[str] = None,
+    provider: Optional[str] = None,
+    **kwargs: Any,
 ) -> SearchResult:
     """Search a product by its id
 
@@ -545,6 +548,8 @@ def search_product_by_id(
     :type product_type: str
     :param provider: (optional) The provider to be used
     :type provider: str
+    :param kwargs: additional search parameters
+    :type kwargs: Any
     :returns: A search result
     :rtype: :class:`~eodag.api.search_result.SearchResult`
     :raises: :class:`~eodag.utils.exceptions.ValidationError`
@@ -552,7 +557,7 @@ def search_product_by_id(
     """
     try:
         products, _ = eodag_api.search(
-            id=uid, productType=product_type, provider=provider
+            id=uid, productType=product_type, provider=provider, **kwargs
         )
         return products
     except ValidationError:
@@ -638,6 +643,7 @@ def get_stac_item_by_id(
     catalogs: List[str],
     root: str = "/",
     provider: Optional[str] = None,
+    **kwargs: Any,
 ) -> Dict[str, Any]:
     """Build STAC item by id
 
@@ -651,11 +657,17 @@ def get_stac_item_by_id(
     :type root: str
     :param provider: (optional) Chosen provider
     :type provider: str
+    :param kwargs: additional search parameters
+    :type kwargs: Any
     :returns: Collection dictionary
     :rtype: dict
     """
     product_type = catalogs[0]
-    found_products = search_product_by_id(item_id, product_type=product_type)
+    _dc_qs = kwargs.get("_dc_qs", None)
+
+    found_products = search_product_by_id(
+        item_id, product_type=product_type, _dc_qs=_dc_qs
+    )
 
     if len(found_products) > 0:
         found_products[0].product_type = eodag_api.get_alias_from_product_type(
@@ -677,6 +689,7 @@ def download_stac_item_by_id_stream(
     item_id: str,
     provider: Optional[str] = None,
     asset: Optional[str] = None,
+    **kwargs: Any,
 ) -> StreamingResponse:
     """Download item
 
@@ -686,10 +699,14 @@ def download_stac_item_by_id_stream(
     :type item_id: str
     :param provider: (optional) Chosen provider
     :type provider: str
+    :param kwargs: additional download parameters
+    :type kwargs: Any
     :returns: a stream of the downloaded data (zip file)
     :rtype: StreamingResponse
     """
     product_type = catalogs[0]
+    _dc_qs = kwargs.get("_dc_qs", None)
+
     search_plugin = next(
         eodag_api._plugins_manager.get_search_plugins(product_type, provider)
     )
@@ -712,8 +729,9 @@ def download_stac_item_by_id_stream(
         }
         product = EOProduct(provider or product_data["provider"], properties)
     else:
+
         search_results = search_product_by_id(
-            item_id, product_type=product_type, provider=provider
+            item_id, product_type=product_type, provider=provider, _dc_qs=_dc_qs
         )
         if len(search_results) > 0:
             product = search_results[0]


Test patch:
diff --git a/tests/units/test_http_server.py b/tests/units/test_http_server.py
index acc9ad03..a59d4705 100644
--- a/tests/units/test_http_server.py
+++ b/tests/units/test_http_server.py
@@ -679,6 +679,7 @@ class RequestTestCase(unittest.TestCase):
                 "id": "foo",
                 "provider": None,
                 "productType": self.tested_product_type,
+                "_dc_qs": None,
             },
         )
 
@@ -690,6 +691,7 @@ class RequestTestCase(unittest.TestCase):
                 "id": "foo",
                 "provider": None,
                 "productType": self.tested_product_type,
+                "_dc_qs": None,
             },
         )