Repository: CS-SI/eodag
Pull request #1585 (2025-03-31 19:43:19)

fix(core): download asset with special chars
Fixes issue preventing to download an asset if special characters are in its key

github-actions[bot]: ## Test Results
    4 files  ±0      4 suites  ±0   6m 39s ⏱️ +2s
  612 tests ±0    609 ✅ ±0   3 💤 ±0  0 ❌ ±0 
2 448 runs  ±0  2 350 ✅ ±0  98 💤 ±0  0 ❌ ±0 

Results for commit 6b731570. ± Comparison against base commit 28c0bd0b.

[test-results]:data:application/gzip;base64,H4sIAA3x6mcC/02MzQ6DIBAGX8Vw7mH5h75MAygJqUqDcDJ990Kr1OPMtzs78mGeNnQf2G1AWwm5w1iSySGuFanWVdQpt1FgctJjK841BfqvnuHVfrrwJsxVQBdTSjEdJpW1NQlj6qCzSSiHrn5N3W8uzS9fky4uS8gVkLCSYi6BEiBYMqmcEeC5UYAlGb01xgqHJaD3B51EhbsIAQAA

github-actions[bot]: ![badge](https://raw.githubusercontent.com/CS-SI/eodag/_xml_coverage_reports_win/data/asset-values-special-chars/./badge.svg)


## Code Coverage (Windows)

<details>


```
Filename                                     Stmts    Miss  Cover    Missing
-----------------------------------------  -------  ------  -------  ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
__init__.py                                      8       0  100.00%
cli.py                                         323      64  80.19%   62-63, 87, 674-737, 839-890, 894
config.py                                      430      28  93.49%   80-82, 91, 99, 103-105, 176, 187, 692-694, 808-811, 854-855, 864-865, 970, 999, 1029-1034, 1036
crunch.py                                        5       5  0.00%    20-24
api/__init__.py                                  0       0  100.00%
api/core.py                                    783      74  90.55%   316-317, 367, 657, 701-704, 742, 786, 820, 865-870, 896, 987, 1055, 1193, 1278-1290, 1326, 1328, 1356, 1360-1371, 1384-1390, 1473-1476, 1509-1529, 1581, 1598-1602, 1614-1617, 1953, 1977-1983, 2234, 2238-2242, 2251-2253, 2297-2298, 2327-2328
api/search_result.py                            58       4  93.10%   82, 91, 98, 112
api/product/__init__.py                          6       0  100.00%
api/product/_assets.py                          52       7  86.54%   88, 95, 97, 183, 191, 194-198
api/product/_product.py                        196      19  90.31%   70-72, 236-237, 312, 341, 398, 412-415, 428, 452-455, 515-518
api/product/metadata_mapping.py                702      65  90.74%   120-122, 220, 252-253, 299-300, 310-322, 324, 335, 360, 400-401, 438, 459-462, 485, 493-494, 587-588, 612-613, 619-622, 637-638, 787, 838, 991, 1000-1004, 1021-1026, 1153, 1167-1187, 1207, 1212, 1341, 1355, 1380, 1426, 1478, 1501-1502, 1522-1526, 1542, 1550
api/product/drivers/__init__.py                 20       2  90.00%   33-36
api/product/drivers/base.py                     29       1  96.55%   98
api/product/drivers/generic.py                   7       0  100.00%
api/product/drivers/sentinel1.py                15       0  100.00%
api/product/drivers/sentinel2.py                15       0  100.00%
plugins/__init__.py                              0       0  100.00%
plugins/base.py                                 21       3  85.71%   48, 55, 68
plugins/manager.py                             172      14  91.86%   106-111, 161, 199, 221, 225, 249, 389-392, 404-405
plugins/apis/__init__.py                         0       0  100.00%
plugins/apis/base.py                             4       0  100.00%
plugins/apis/ecmwf.py                           97      10  89.69%   167-169, 216-217, 243-245, 298-299
plugins/apis/usgs.py                           180      26  85.56%   156, 258, 292, 327-329, 334, 360-361, 366, 396-403, 414-419, 441-447, 478
plugins/authentication/__init__.py               6       1  83.33%   31
plugins/authentication/aws_auth.py              20       0  100.00%
plugins/authentication/base.py                  17       2  88.24%   45, 58
plugins/authentication/generic.py               14       2  85.71%   51, 61
plugins/authentication/header.py                19       0  100.00%
plugins/authentication/keycloak.py              46       7  84.78%   151-154, 175-180
plugins/authentication/oauth.py                 13       7  46.15%   43-45, 49-52
plugins/authentication/openid_connect.py       208      28  86.54%   80-81, 93-111, 149, 155-183, 191, 323-326, 352, 386
plugins/authentication/qsauth.py                34       1  97.06%   91
plugins/authentication/sas_auth.py              47       1  97.87%   76
plugins/authentication/token.py                100      16  84.00%   141, 170, 172, 207-220, 276-280
plugins/authentication/token_exchange.py        36      14  61.11%   75, 92-120
plugins/crunch/__init__.py                       0       0  100.00%
plugins/crunch/base.py                          10       1  90.00%   43
plugins/crunch/filter_date.py                   59      14  76.27%   52-57, 69, 78, 87, 90, 102-104, 113-115, 122
plugins/crunch/filter_latest_intersect.py       47      33  29.79%   49-54, 67-112
plugins/crunch/filter_latest_tpl_name.py        31       1  96.77%   83
plugins/crunch/filter_overlap.py                66      18  72.73%   28-30, 66-69, 76-79, 85, 93, 104-120
plugins/crunch/filter_property.py               30       5  83.33%   55-60, 63-64
plugins/download/__init__.py                     0       0  100.00%
plugins/download/aws.py                        528     158  70.08%   272, 285, 365-368, 382-386, 476-478, 482, 515-516, 522-526, 556, 589, 593, 600, 630-638, 642, 677-685, 696-698, 729-810, 828-886, 897-902, 955, 970-972, 975, 985-993, 1001-1015, 1025-1048, 1055-1068, 1106, 1132, 1177-1179, 1399
plugins/download/base.py                       253      44  82.61%   127, 155, 222-224, 235-238, 291-292, 340-344, 350-351, 393, 396-410, 422, 426, 490-494, 524-525, 550-558, 560-564, 607, 629, 651, 659
plugins/download/creodias_s3.py                 25       9  64.00%   55-69
plugins/download/http.py                       529      95  82.04%   227, 323-326, 329, 336-341, 359-374, 391, 403, 451, 458-464, 482, 496, 510, 518-520, 536-541, 552, 570, 606-609, 638, 642, 662, 739, 758-769, 777-782, 792-809, 827, 857-866, 902, 927-928, 947-952, 958, 961, 978, 981, 996-997, 1032, 1094, 1109, 1168-1169, 1175, 1185, 1221, 1257, 1277, 1312-1314
plugins/download/s3rest.py                     116      24  79.31%   119, 153, 160, 195, 222-229, 232-234, 238, 249-255, 263-264, 267-271, 294, 315-318
plugins/search/__init__.py                      22       0  100.00%
plugins/search/base.py                         145      11  92.41%   101, 105, 129-135, 276, 296, 429
plugins/search/build_search_result.py          352      38  89.20%   241-242, 278, 281, 412-423, 447, 559, 561, 622, 629, 653, 688, 739, 754, 788-803, 848, 873, 876, 880, 1000-1001, 1007, 1018, 1097
plugins/search/cop_marine.py                   244      47  80.74%   56, 64-66, 76-77, 82, 87-88, 104, 106, 109, 175-176, 228, 234, 238, 242, 255, 266-267, 275, 303, 307, 322, 326, 330, 334, 338-342, 348-351, 354-368, 385, 434-438, 443, 455
plugins/search/creodias_s3.py                   25       1  96.00%   51
plugins/search/csw.py                          105      81  22.86%   98-99, 103-104, 112-160, 166-179, 187-219, 237-278
plugins/search/data_request_search.py          202      69  65.84%   189-192, 208, 219, 223-224, 235, 240, 245, 252, 265-268, 322-323, 327, 337-343, 348, 374-377, 385-396, 413, 415, 422-425, 427-428, 446-450, 483, 493, 504, 517, 523-538, 543
plugins/search/qssearch.py                     701      98  86.02%   448, 506, 520, 524-530, 558-562, 675-687, 732-735, 806-807, 855, 874, 881, 893, 950, 971, 974-975, 984-985, 994-995, 1004-1005, 1032, 1114-1119, 1123-1132, 1166, 1188, 1261-1262, 1268, 1358, 1455-1459, 1519, 1522, 1528-1529, 1550, 1578-1590, 1597, 1629-1631, 1641-1647, 1677, 1700, 1705-1706, 1721, 1737, 1809-1810, 1822-1825, 1830-1833, 1842, 1863-1865, 1874-1878, 1883-1893, 1915
plugins/search/stac_list_assets.py              25      10  60.00%   44-51, 75-85
plugins/search/static_stac_search.py            75      13  82.67%   98-124, 140, 153
rest/__init__.py                                 4       2  50.00%   21-22
rest/cache.py                                   33      22  33.33%   35-37, 44-70
rest/config.py                                  25       1  96.00%   35
rest/constants.py                                6       0  100.00%
rest/core.py                                   260     153  41.15%   157, 159, 161, 164-165, 179-189, 198-199, 205, 208, 249-303, 316-352, 383-420, 435-451, 467-476, 483-522, 539, 581-666, 705, 712-765
rest/errors.py                                  69      49  28.99%   60, 65-100, 105-108, 115-118, 126-147, 155-160, 175-181
rest/server.py                                 192     192  0.00%    18-592
rest/stac.py                                   319      68  78.68%   243, 309, 331, 383-386, 413-440, 471-473, 496, 528-529, 611-651, 673-689, 716, 781-785, 792, 846-847, 853, 908, 946, 979, 998-1000
rest/types/__init__.py                           0       0  100.00%
rest/types/collections_search.py                13      13  0.00%    18-44
rest/types/eodag_search.py                     180      20  88.89%   225-229, 262-264, 282, 285, 291, 295, 353, 371-386
rest/types/queryables.py                        64      20  68.75%   54-55, 60-63, 68-79, 105-110, 119-120, 174
rest/types/stac_search.py                      125      11  91.20%   126-128, 174, 189-191, 199, 203, 251, 254
rest/utils/__init__.py                          94      30  68.09%   72-78, 98, 101-102, 121-123, 136, 143, 168-176, 183-204
rest/utils/cql_evaluate.py                      48       5  89.58%   69, 76, 90, 97, 105
rest/utils/rfc3339.py                           23       5  78.26%   43-44, 48, 60, 62
types/__init__.py                              148      42  71.62%   67, 71, 80-84, 95-107, 135-137, 144-149, 204, 239, 249-265, 270, 272, 294, 299, 307, 317
types/bbox.py                                   43      19  55.81%   46-61, 72-74, 85-87, 99-101, 113-115, 123
types/download_args.py                           9       0  100.00%
types/queryables.py                            108       1  99.07%   63
types/search_args.py                            70      18  74.29%   60-64, 71-88, 103
types/whoosh.py                                 81      16  80.25%   129-132, 136-143, 155-161, 174-176
utils/__init__.py                              538      40  92.57%   81, 198-199, 208-235, 238, 253, 333-337, 410-414, 433-435, 449, 541, 556, 596-597, 626, 1001-1004, 1012-1013, 1051-1052, 1099-1100, 1234
utils/exceptions.py                             46       0  100.00%
utils/import_system.py                          28      19  32.14%   64-78, 89-99
utils/logging.py                                28       1  96.43%   41
utils/notebook.py                               44      23  47.73%   25-29, 36-41, 58-62, 72-78, 83-87
utils/repr.py                                   38       8  78.95%   51, 53, 57, 98, 122-129
utils/requests.py                               55      29  47.27%   51-52, 64, 85-96, 107-124, 128
utils/rest.py                                   36       1  97.22%   55
utils/s3.py                                     69       3  95.65%   162-163, 226
utils/stac_reader.py                           111      45  59.46%   56-57, 63-85, 95-97, 101, 137, 153-156, 203-212, 222-252
TOTAL                                        10180    2027  80.09%
```

</details>


### Diff against develop

```
Filename                  Stmts    Miss  Cover
----------------------  -------  ------  -------
api/product/_assets.py       +4      +2  -3.04%
TOTAL                        +4      +2  -0.01%
```


Results for commit: 6b731570320217478ca60f5a80172dfbaab6c170


_Minimum allowed coverage is `70%`_


:recycle: This comment has been updated with latest results


<!-- Sticky Pull Request Commentartifacts/unit-test-results-python3.13-windows-latest/coverage.xml -->

Patch:
diff --git a/eodag/api/product/_assets.py b/eodag/api/product/_assets.py
index d1ac5c56..1751a94b 100644
--- a/eodag/api/product/_assets.py
+++ b/eodag/api/product/_assets.py
@@ -71,22 +71,29 @@ class AssetsDict(UserDict):
         """
         return {k: v.as_dict() for k, v in self.data.items()}
 
-    def get_values(self, asset_filter: str = "") -> list[Asset]:
+    def get_values(self, asset_filter: str = "", regex=True) -> list[Asset]:
         """
         retrieves the assets matching the given filter
 
         :param asset_filter: regex filter with which the assets should be matched
+        :param regex: Uses regex to match the asset key or simply compare strings
         :return: list of assets
         """
         if asset_filter:
-            filter_regex = re.compile(asset_filter)
-            assets_keys = list(self.keys())
-            assets_keys = list(filter(filter_regex.fullmatch, assets_keys))
+            if regex:
+                filter_regex = re.compile(asset_filter)
+                assets_keys = list(self.keys())
+                assets_keys = list(filter(filter_regex.fullmatch, assets_keys))
+            else:
+                assets_keys = [a for a in self.keys() if a == asset_filter]
             filtered_assets = {}
             if len(assets_keys) > 0:
                 filtered_assets = {a_key: self.get(a_key) for a_key in assets_keys}
             assets_values = [a for a in filtered_assets.values() if a and "href" in a]
-            if not assets_values:
+            if not assets_values and regex:
+                # retry without regex
+                return self.get_values(asset_filter, regex=False)
+            elif not assets_values:
                 raise NotAvailableError(
                     rf"No asset key matching re.fullmatch(r'{asset_filter}') was found in {self.product}"
                 )
diff --git a/eodag/config.py b/eodag/config.py
index 40852a8b..581f217b 100644
--- a/eodag/config.py
+++ b/eodag/config.py
@@ -443,6 +443,9 @@ class PluginConfig(yaml.YAMLObject):
     literal_search_params: dict[str, str]
     #: :class:`~eodag.plugins.search.qssearch.QueryStringSearch` Characters that should not be quoted in the url params
     dont_quote: list[str]
+    #: :class:`~eodag.plugins.search.qssearch.QueryStringSearch` Guess assets keys using their ``href``.
+    #: Use their original key if ``False``
+    asset_key_from_href: bool
     #: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Dict describing free text search request build
     free_text_search_operations: dict[str, Any]
     #: :class:`~eodag.plugins.search.qssearch.ODataV4Search` Set to ``True`` if the metadata is not given in the search
diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py
index 259c9747..fdc31ec2 100644
--- a/eodag/plugins/search/qssearch.py
+++ b/eodag/plugins/search/qssearch.py
@@ -123,6 +123,8 @@ class QueryStringSearch(Search):
           authentication error; only used if ``need_auth=true``
         * :attr:`~eodag.config.PluginConfig.ssl_verify` (``bool``): if the ssl certificates should be verified in
           requests; default: ``True``
+        * :attr:`~eodag.config.PluginConfig.asset_key_from_href` (``bool``): guess assets keys using their ``href``. Use
+          their original key if ``False``; default: ``True``
         * :attr:`~eodag.config.PluginConfig.dont_quote` (``list[str]``): characters that should not be quoted in the
           url params
         * :attr:`~eodag.config.PluginConfig.timeout` (``int``): time to wait until request timeout in seconds;
@@ -1072,6 +1074,7 @@ class QueryStringSearch(Search):
             % normalize_remaining_count
         )
         products: list[EOProduct] = []
+        asset_key_from_href = getattr(self.config, "asset_key_from_href", True)
         for result in results:
             product = EOProduct(
                 self.provider,
@@ -1089,7 +1092,8 @@ class QueryStringSearch(Search):
             # move assets from properties to product's attr, normalize keys & roles
             for key, asset in product.properties.pop("assets", {}).items():
                 norm_key, asset["roles"] = product.driver.guess_asset_key_and_roles(
-                    asset.get("href", ""), product
+                    asset.get("href", "") if asset_key_from_href else key,
+                    product,
                 )
                 if norm_key:
                     product.assets[norm_key] = asset
diff --git a/eodag/resources/providers.yml b/eodag/resources/providers.yml
index 96acd7be..76a677a4 100644
--- a/eodag/resources/providers.yml
+++ b/eodag/resources/providers.yml
@@ -2936,6 +2936,7 @@
     need_auth: true
     auth_error_code: 401
     ssl_verify: true
+    asset_key_from_href: false
     discover_queryables:
       fetch_url: null
       product_type_fetch_url: null
@@ -4527,6 +4528,7 @@
     api_endpoint: https://hda.data.destination-earth.eu/stac/search
     need_auth: true
     timeout: 60
+    asset_key_from_href: false
     metadata_mapping:
       defaultGeometry: 'POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))'
       quicklook: '{thumbnail}'
@@ -5071,6 +5073,7 @@
     dont_quote:
       - '='
       - '&'
+    asset_key_from_href: false
     pagination:
       next_page_url_tpl: '{url}?{search}&c={items_per_page}&pw={page}'
       start_page: 0
@@ -5515,6 +5518,7 @@
     api_endpoint: 'https://stac.marine.copernicus.eu/metadata/{collection}/product.stac.json'
     need_auth: false
     ssl_verify: true
+    asset_key_from_href: false
     timeout: 20
     results_entry: links
     discover_product_types:
@@ -5683,6 +5687,7 @@
     api_endpoint: https://geodes-portal.cnes.fr/api/stac/search
     need_auth: false
     ssl_verify: false
+    asset_key_from_href: false
     discover_queryables:
       fetch_url: null
       product_type_fetch_url: null
@@ -5798,6 +5803,7 @@
     s3_endpoint: https://s3.datalake.cnes.fr
     need_auth: true
     ssl_verify: false
+    asset_key_from_href: false
     discover_queryables:
       fetch_url: null
       product_type_fetch_url: null


Test patch:
diff --git a/tests/context.py b/tests/context.py
index 6bc68157..b5fa90a7 100644
--- a/tests/context.py
+++ b/tests/context.py
@@ -71,6 +71,7 @@ from eodag.plugins.download.http import HTTPDownload
 from eodag.plugins.manager import PluginManager
 from eodag.plugins.search import PreparedSearch
 from eodag.plugins.search.base import Search
+from eodag.plugins.search.qssearch import QueryStringSearch
 from eodag.types import model_fields_to_annotated
 from eodag.types.queryables import CommonQueryables, Queryables
 from eodag.utils import (
diff --git a/tests/units/test_eoproduct.py b/tests/units/test_eoproduct.py
index 2e8f13f3..e2a66e07 100644
--- a/tests/units/test_eoproduct.py
+++ b/tests/units/test_eoproduct.py
@@ -547,3 +547,20 @@ class TestEOProduct(EODagTestCase):
         # asset
         asset_repr = html.fromstring(product.assets._repr_html_())
         self.assertIn("Asset", asset_repr.xpath("//thead/tr/td")[0].text)
+
+    def test_eoproduct_assets_get_values(self):
+        """eoproduct.assets.get_values must return the expected values"""
+        product = self._dummy_product()
+        product.assets.update(
+            {
+                "foo": {"href": "foo.href"},
+                "fooo": {"href": "fooo.href"},
+                "foo?o,o": {"href": "foooo.href"},
+            }
+        )
+        self.assertEqual(len(product.assets.get_values()), 3)
+        self.assertEqual(len(product.assets.get_values("foo.*")), 3)
+        self.assertEqual(len(product.assets.get_values("foo")), 1)
+        self.assertEqual(product.assets.get_values("foo")[0]["href"], "foo.href")
+        self.assertEqual(len(product.assets.get_values("foo?o,o")), 1)
+        self.assertEqual(product.assets.get_values("foo?o,o")[0]["href"], "foooo.href")
diff --git a/tests/units/test_search_plugins.py b/tests/units/test_search_plugins.py
index a4e34a64..07fc6b5e 100644
--- a/tests/units/test_search_plugins.py
+++ b/tests/units/test_search_plugins.py
@@ -54,6 +54,7 @@ from tests.context import (
     NotAvailableError,
     PluginManager,
     PreparedSearch,
+    QueryStringSearch,
     RequestError,
     TimeOutError,
     cached_parse,
@@ -1569,6 +1570,49 @@ class TestSearchPluginStacSearch(BaseSearchPluginTest):
         )
         self.assertEqual(products[1].geometry.bounds, (-180.0, -90.0, 180.0, 90.0))
 
+    @mock.patch(
+        "eodag.api.product.drivers.base.DatasetDriver.guess_asset_key_and_roles",
+        autospec=True,
+    )
+    @mock.patch.dict(QueryStringSearch.extract_properties, {"json": mock.MagicMock()})
+    def test_plugins_search_stacsearch_normalize_asset_key_from_href(
+        self, mock_guess_asset_key_and_roles
+    ):
+        """normalize_results must guess asset key from href if asset_key_from_href is set to True"""
+
+        mock_properties_from_json = QueryStringSearch.extract_properties["json"]
+        mock_properties_from_json.return_value = {
+            "geometry": "POINT (0 0)",
+            "assets": {
+                "foo": {
+                    "href": "https://example.com/foo",
+                    "roles": ["bar"],
+                },
+            },
+        }
+        mock_guess_asset_key_and_roles.return_value = ("normalized_key", ["some_role"])
+
+        # guess asset key from href
+        search_plugin = self.get_search_plugin(self.product_type, "earth_search")
+        self.assertFalse(hasattr(search_plugin.config, "asset_key_from_href"))
+        products = search_plugin.normalize_results([{}])
+        mock_guess_asset_key_and_roles.assert_called_once_with(
+            products[0].driver, "https://example.com/foo", products[0]
+        )
+        self.assertEqual(len(products[0].assets), 1)
+        self.assertEqual(products[0].assets["normalized_key"]["roles"], ["some_role"])
+
+        mock_guess_asset_key_and_roles.reset_mock()
+        # guess asset key from origin key
+        search_plugin = self.get_search_plugin(self.product_type, "geodes")
+        self.assertEqual(search_plugin.config.asset_key_from_href, False)
+        products = search_plugin.normalize_results([{}])
+        mock_guess_asset_key_and_roles.assert_called_once_with(
+            products[0].driver, "foo", products[0]
+        )
+        self.assertEqual(len(products[0].assets), 1)
+        self.assertEqual(products[0].assets["normalized_key"]["roles"], ["some_role"])
+
     @mock.patch("eodag.plugins.search.qssearch.requests.post", autospec=True)
     def test_plugins_search_stacsearch_opened_time_intervals(self, mock_requests_post):
         """Opened time intervals must be handled by StacSearch plugin"""