Repository: CS-SI/eodag
Pull request #660 (2023-02-27 16:11:48)

fix: search errors handling
Fixes [search](https://eodag.readthedocs.io/en/latest/notebooks/api_user_guide/4_search.html#Search-methods) error handling, broken since `v2.9.0` and #632, and adds associated tests

github-actions[bot]: ## Unit Test Results
    2 files  ±  0      2 suites  ±0   8m 2s [:stopwatch:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "duration of all tests") +56s
372 tests +  6  369 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "passed tests") +  6  3 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "skipped / disabled tests") ±0  0 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "failed tests") ±0 
744 runs  +12  738 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "passed tests") +12  6 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "skipped / disabled tests") ±0  0 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "failed tests") ±0 

Results for commit 194077c6. ± Comparison against base commit e925418a.

[test-results]:data:application/gzip;base64,H4sIAFzY/GMC/03MSw7CIBSF4a00jB1QoFDcjLnySG5si+ExMu5d0BY7/L+TnBfxuLhErgO7DCQVzD1siZAxbDXF3KBOuY1c9bqlYkwjqf/0wGejDh5wqUA7uBhD3CWWrX0qIfY4LhWfu/we5dGnw2+f/0xYV8w1yKgFVcpIJjXXd6CjpgBOgnV88h6YBjYJZz15fwAfp1kVBQEAAA==

github-actions[bot]: <strong> </strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `88%` | :white_check_mark: |
| eodag/api/core.py | `92%` | :white_check_mark: |
| eodag/plugins/apis/usgs.py | `81%` | :white_check_mark: |
| eodag/plugins/search/build\_search\_result.py | `93%` | :white_check_mark: |
| eodag/plugins/search/qssearch.py | `86%` | :white_check_mark: |

<strong> </strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `88%` | :white_check_mark: |
| eodag/api/core.py | `92%` | :white_check_mark: |
| eodag/plugins/apis/usgs.py | `81%` | :white_check_mark: |
| eodag/plugins/search/build\_search\_result.py | `93%` | :white_check_mark: |
| eodag/plugins/search/qssearch.py | `86%` | :white_check_mark: |
| eodag/api/core.py | `92%` | :white_check_mark: |
| eodag/plugins/apis/usgs.py | `81%` | :white_check_mark: |
| eodag/plugins/search/build\_search\_result.py | `93%` | :white_check_mark: |
| eodag/plugins/search/qssearch.py | `86%` | :white_check_mark: |

_Minimum allowed coverage is `70%`_

<p align="right">Generated by :monkey: cobertura-action against 194077c626939ba0190aae6ade35ffa29a254edf </p>

Patch:
diff --git a/eodag/api/core.py b/eodag/api/core.py
index c2ae8e32..a08100d5 100644
--- a/eodag/api/core.py
+++ b/eodag/api/core.py
@@ -928,6 +928,8 @@ class EODataAccessGateway(object):
                 products, _ = self._do_search(
                     search_plugin, count=False, raise_errors=True, **search_kwargs
                 )
+            except Exception:
+                products = SearchResult([])
             finally:
                 # we don't want that next(search_iter_page(...)) modifies the plugin
                 # indefinitely. So we reset after each request, but before the generator
diff --git a/eodag/plugins/apis/usgs.py b/eodag/plugins/apis/usgs.py
index 32fef859..3293a073 100644
--- a/eodag/plugins/apis/usgs.py
+++ b/eodag/plugins/apis/usgs.py
@@ -45,7 +45,7 @@ from eodag.utils import (
     format_dict_items,
     path_to_uri,
 )
-from eodag.utils.exceptions import AuthenticationError, NotAvailableError
+from eodag.utils.exceptions import AuthenticationError, NotAvailableError, RequestError
 
 logger = logging.getLogger("eodag.plugins.apis.usgs")
 
@@ -192,9 +192,11 @@ class UsgsApi(Download, Api):
                 )
         except USGSError as e:
             logger.warning(
-                f"Product type {usgs_dataset} does not exist on USGS EE catalog"
+                f"Product type {usgs_dataset} may not exist on USGS EE catalog"
             )
-            logger.warning(f"Skipping error: {e}")
+            api.logout()
+            raise RequestError(e)
+
         api.logout()
 
         if final:
diff --git a/eodag/plugins/search/build_search_result.py b/eodag/plugins/search/build_search_result.py
index 17928f91..acce53cc 100644
--- a/eodag/plugins/search/build_search_result.py
+++ b/eodag/plugins/search/build_search_result.py
@@ -31,7 +31,6 @@ from eodag.api.product.metadata_mapping import (
 )
 from eodag.plugins.search.qssearch import PostJsonSearch
 from eodag.utils import dict_items_recursive_sort
-from eodag.utils.exceptions import RequestError
 
 logger = logging.getLogger("eodag.plugins.search.build_search_result")
 
@@ -76,17 +75,13 @@ class BuildPostSearchResult(PostJsonSearch):
     def do_search(self, *args, **kwargs):
         """Perform the actual search request, and return result in a single element."""
         search_url = self.search_urls[0]
-        try:
-            response = self._request(
-                search_url,
-                info_message=f"Sending search request: {search_url}",
-                exception_message=f"Skipping error while searching for {self.provider} "
-                f"{self.__class__.__name__} instance:",
-            )
-        except RequestError:
-            return []
-        else:
-            return [response.json()]
+        response = self._request(
+            search_url,
+            info_message=f"Sending search request: {search_url}",
+            exception_message=f"Skipping error while searching for {self.provider} "
+            f"{self.__class__.__name__} instance:",
+        )
+        return [response.json()]
 
     def normalize_results(self, results, **kwargs):
         """Build :class:`~eodag.api.product._product.EOProduct` from provider result
diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py
index d87884cc..ae345c81 100644
--- a/eodag/plugins/search/qssearch.py
+++ b/eodag/plugins/search/qssearch.py
@@ -636,121 +636,110 @@ class QueryStringSearch(Search):
 
         results = []
         for search_url in self.search_urls:
-            try:
-                response = self._request(
-                    search_url,
-                    info_message="Sending search request: {}".format(search_url),
-                    exception_message="Skipping error while searching for {} {} "
-                    "instance:".format(self.provider, self.__class__.__name__),
-                )
-            except RequestError:
-                if kwargs.get("raise_errors", False):
-                    raise
-                else:
-                    return []
+            response = self._request(
+                search_url,
+                info_message="Sending search request: {}".format(search_url),
+                exception_message="Skipping error while searching for {} {} "
+                "instance:".format(self.provider, self.__class__.__name__),
+            )
+            next_page_url_key_path = self.config.pagination.get(
+                "next_page_url_key_path", None
+            )
+            next_page_query_obj_key_path = self.config.pagination.get(
+                "next_page_query_obj_key_path", None
+            )
+            next_page_merge_key_path = self.config.pagination.get(
+                "next_page_merge_key_path", None
+            )
+            if self.config.result_type == "xml":
+                root_node = etree.fromstring(response.content)
+                namespaces = {k or "ns": v for k, v in root_node.nsmap.items()}
+                result = [
+                    etree.tostring(entry)
+                    for entry in root_node.xpath(
+                        self.config.results_entry, namespaces=namespaces
+                    )
+                ]
+                if next_page_url_key_path or next_page_query_obj_key_path:
+                    raise NotImplementedError(
+                        "Setting the next page url from an XML response has not "
+                        "been implemented yet"
+                    )
+                if getattr(self, "need_count", False):
+                    # extract total_items_nb from search results
+                    try:
+                        total_nb_results = root_node.xpath(
+                            self.config.pagination["total_items_nb_key_path"],
+                            namespaces={
+                                k or "ns": v for k, v in root_node.nsmap.items()
+                            },
+                        )[0]
+                        _total_items_nb = int(total_nb_results)
+
+                        if getattr(self.config, "merge_responses", False):
+                            total_items_nb = _total_items_nb or 0
+                        else:
+                            total_items_nb += _total_items_nb or 0
+                    except IndexError:
+                        logger.debug(
+                            "Could not extract total_items_nb from search results"
+                        )
             else:
-                next_page_url_key_path = self.config.pagination.get(
-                    "next_page_url_key_path", None
-                )
-                next_page_query_obj_key_path = self.config.pagination.get(
-                    "next_page_query_obj_key_path", None
-                )
-                next_page_merge_key_path = self.config.pagination.get(
-                    "next_page_merge_key_path", None
-                )
-                if self.config.result_type == "xml":
-                    root_node = etree.fromstring(response.content)
-                    namespaces = {k or "ns": v for k, v in root_node.nsmap.items()}
-                    result = [
-                        etree.tostring(entry)
-                        for entry in root_node.xpath(
-                            self.config.results_entry, namespaces=namespaces
+                resp_as_json = response.json()
+                if next_page_url_key_path:
+                    path_parsed = cached_parse(next_page_url_key_path)
+                    try:
+                        self.next_page_url = path_parsed.find(resp_as_json)[0].value
+                        logger.debug(
+                            "Next page URL collected and set for the next search",
                         )
-                    ]
-                    if next_page_url_key_path or next_page_query_obj_key_path:
-                        raise NotImplementedError(
-                            "Setting the next page url from an XML response has not "
-                            "been implemented yet"
+                    except IndexError:
+                        logger.debug("Next page URL could not be collected")
+                if next_page_query_obj_key_path:
+                    path_parsed = cached_parse(next_page_query_obj_key_path)
+                    try:
+                        self.next_page_query_obj = path_parsed.find(resp_as_json)[
+                            0
+                        ].value
+                        logger.debug(
+                            "Next page Query-object collected and set for the next search",
                         )
-                    if getattr(self, "need_count", False):
-                        # extract total_items_nb from search results
-                        try:
-                            total_nb_results = root_node.xpath(
-                                self.config.pagination["total_items_nb_key_path"],
-                                namespaces={
-                                    k or "ns": v for k, v in root_node.nsmap.items()
-                                },
-                            )[0]
-                            _total_items_nb = int(total_nb_results)
+                    except IndexError:
+                        logger.debug("Next page Query-object could not be collected")
+                if next_page_merge_key_path:
+                    path_parsed = cached_parse(next_page_merge_key_path)
+                    try:
+                        self.next_page_merge = path_parsed.find(resp_as_json)[0].value
+                        logger.debug(
+                            "Next page merge collected and set for the next search",
+                        )
+                    except IndexError:
+                        logger.debug("Next page merge could not be collected")
 
-                            if getattr(self.config, "merge_responses", False):
-                                total_items_nb = _total_items_nb or 0
-                            else:
-                                total_items_nb += _total_items_nb or 0
-                        except IndexError:
-                            logger.debug(
-                                "Could not extract total_items_nb from search results"
-                            )
-                else:
-                    resp_as_json = response.json()
-                    if next_page_url_key_path:
-                        path_parsed = cached_parse(next_page_url_key_path)
-                        try:
-                            self.next_page_url = path_parsed.find(resp_as_json)[0].value
-                            logger.debug(
-                                "Next page URL collected and set for the next search",
-                            )
-                        except IndexError:
-                            logger.debug("Next page URL could not be collected")
-                    if next_page_query_obj_key_path:
-                        path_parsed = cached_parse(next_page_query_obj_key_path)
-                        try:
-                            self.next_page_query_obj = path_parsed.find(resp_as_json)[
-                                0
-                            ].value
-                            logger.debug(
-                                "Next page Query-object collected and set for the next search",
-                            )
-                        except IndexError:
-                            logger.debug(
-                                "Next page Query-object could not be collected"
-                            )
-                    if next_page_merge_key_path:
-                        path_parsed = cached_parse(next_page_merge_key_path)
-                        try:
-                            self.next_page_merge = path_parsed.find(resp_as_json)[
-                                0
-                            ].value
-                            logger.debug(
-                                "Next page merge collected and set for the next search",
-                            )
-                        except IndexError:
-                            logger.debug("Next page merge could not be collected")
-
-                    result = resp_as_json.get(self.config.results_entry, [])
-
-                    if getattr(self, "need_count", False):
-                        # extract total_items_nb from search results
-                        try:
-                            _total_items_nb = total_items_nb_key_path_parsed.find(
-                                resp_as_json
-                            )[0].value
-                            if getattr(self.config, "merge_responses", False):
-                                total_items_nb = _total_items_nb or 0
-                            else:
-                                total_items_nb += _total_items_nb or 0
-                        except IndexError:
-                            logger.debug(
-                                "Could not extract total_items_nb from search results"
-                            )
-                if getattr(self.config, "merge_responses", False):
-                    results = (
-                        [dict(r, **result[i]) for i, r in enumerate(results)]
-                        if results
-                        else result
-                    )
-                else:
-                    results.extend(result)
+                result = resp_as_json.get(self.config.results_entry, [])
+
+                if getattr(self, "need_count", False):
+                    # extract total_items_nb from search results
+                    try:
+                        _total_items_nb = total_items_nb_key_path_parsed.find(
+                            resp_as_json
+                        )[0].value
+                        if getattr(self.config, "merge_responses", False):
+                            total_items_nb = _total_items_nb or 0
+                        else:
+                            total_items_nb += _total_items_nb or 0
+                    except IndexError:
+                        logger.debug(
+                            "Could not extract total_items_nb from search results"
+                        )
+            if getattr(self.config, "merge_responses", False):
+                results = (
+                    [dict(r, **result[i]) for i, r in enumerate(results)]
+                    if results
+                    else result
+                )
+            else:
+                results.extend(result)
             if getattr(self, "need_count", False):
                 self.total_items_nb = total_items_nb
                 del self.need_count


Test patch:
diff --git a/tests/integration/test_core_search_errors.py b/tests/integration/test_core_search_errors.py
new file mode 100644
index 00000000..3e08c7e9
--- /dev/null
+++ b/tests/integration/test_core_search_errors.py
@@ -0,0 +1,188 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018, CS GROUP - France, https://www.csgroup.eu/
+#
+# This file is part of EODAG project
+#     https://www.github.com/CS-SI/EODAG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import tempfile
+import unittest
+
+from requests.exceptions import RequestException
+
+from tests import TEST_RESOURCES_PATH
+from tests.context import EODataAccessGateway, RequestError, USGSError
+from tests.utils import mock
+
+
+class TestCoreSearchErrors(unittest.TestCase):
+    def setUp(self):
+        super(TestCoreSearchErrors, self).setUp()
+        # Mock home and eodag conf directory to tmp dir
+        self.tmp_home_dir = tempfile.TemporaryDirectory()
+        self.expanduser_mock = mock.patch(
+            "os.path.expanduser", autospec=True, return_value=self.tmp_home_dir.name
+        )
+        self.expanduser_mock.start()
+        # load fake credentials to prevent providers needing auth for search to be pruned
+        config_path = os.path.join(TEST_RESOURCES_PATH, "wrong_credentials_conf.yml")
+        self.dag = EODataAccessGateway(user_conf_file_path=config_path)
+
+    def tearDown(self):
+        super(TestCoreSearchErrors, self).tearDown()
+        # stop Mock and remove tmp config dir
+        self.expanduser_mock.stop()
+        self.tmp_home_dir.cleanup()
+
+    @mock.patch(
+        "eodag.plugins.search.qssearch.requests.get",
+        autospec=True,
+        side_effect=RequestException,
+    )
+    @mock.patch(
+        "eodag.api.core.EODataAccessGateway.fetch_product_types_list", autospec=True
+    )
+    def test_core_search_errors_qssearch(self, mock_fetch_product_types_list, mock_get):
+
+        # QueryStringSearch (peps)
+        self.dag.set_preferred_provider("peps")
+        products, count = self.dag.search()
+        self.assertEqual(count, 0)
+        self.assertEqual(len(products), 0)
+        self.assertRaises(RequestError, self.dag.search, raise_errors=True)
+        # search iterator
+        self.assertRaises(StopIteration, next, self.dag.search_iter_page())
+        # search_all
+        self.assertEqual(len(self.dag.search_all()), 0)
+
+    @mock.patch(
+        "eodag.plugins.search.qssearch.requests.post",
+        autospec=True,
+        side_effect=RequestException,
+    )
+    @mock.patch(
+        "eodag.api.core.EODataAccessGateway.fetch_product_types_list", autospec=True
+    )
+    def test_core_search_errors_stacsearch(
+        self, mock_fetch_product_types_list, mock_post
+    ):
+        # StacSearch (astraea_eod)
+        self.dag.set_preferred_provider("astraea_eod")
+        products, count = self.dag.search()
+        self.assertEqual(count, 0)
+        self.assertEqual(len(products), 0)
+        self.assertRaises(RequestError, self.dag.search, raise_errors=True)
+        # search iterator
+        self.assertRaises(StopIteration, next, self.dag.search_iter_page())
+        # search_all
+        self.assertEqual(len(self.dag.search_all()), 0)
+
+    @mock.patch(
+        "eodag.plugins.search.qssearch.requests.post",
+        autospec=True,
+        side_effect=RequestException,
+    )
+    @mock.patch(
+        "eodag.api.core.EODataAccessGateway.fetch_product_types_list", autospec=True
+    )
+    def test_core_search_errors_postjson(
+        self, mock_fetch_product_types_list, mock_post
+    ):
+        # PostJsonSearch (aws_eos)
+        self.dag.set_preferred_provider("aws_eos")
+        products, count = self.dag.search()
+        self.assertEqual(count, 0)
+        self.assertEqual(len(products), 0)
+        self.assertRaises(RequestError, self.dag.search, raise_errors=True)
+        # search iterator
+        self.assertRaises(StopIteration, next, self.dag.search_iter_page())
+        # search_all
+        self.assertEqual(len(self.dag.search_all()), 0)
+
+    @mock.patch(
+        "eodag.plugins.search.qssearch.urlopen",
+        autospec=True,
+        side_effect=RequestException,
+    )
+    @mock.patch(
+        "eodag.plugins.search.qssearch.requests.get",
+        autospec=True,
+        side_effect=RequestException,
+    )
+    @mock.patch(
+        "eodag.api.core.EODataAccessGateway.fetch_product_types_list", autospec=True
+    )
+    def test_core_search_errors_odata(
+        self, mock_fetch_product_types_list, mock_get, mock_urlopen
+    ):
+        # ODataV4Search (onda)
+        self.dag.set_preferred_provider("onda")
+        products, count = self.dag.search()
+        self.assertEqual(count, 0)
+        self.assertEqual(len(products), 0)
+        self.assertRaises(RequestError, self.dag.search, raise_errors=True)
+        # search iterator
+        self.assertRaises(StopIteration, next, self.dag.search_iter_page())
+        # search_all
+        self.assertEqual(len(self.dag.search_all()), 0)
+
+    @mock.patch(
+        "eodag.plugins.apis.usgs.api.scene_search", autospec=True, side_effect=USGSError
+    )
+    @mock.patch("eodag.plugins.apis.usgs.api.login", autospec=True)
+    @mock.patch(
+        "eodag.api.core.EODataAccessGateway.fetch_product_types_list", autospec=True
+    )
+    def test_core_search_errors_usgs(
+        self, mock_fetch_product_types_list, mock_login, mock_scene_search
+    ):
+        # UsgsApi (usgs)
+        self.dag.set_preferred_provider("usgs")
+        products, count = self.dag.search(productType="foo")
+        self.assertEqual(count, 0)
+        self.assertEqual(len(products), 0)
+        self.assertRaises(
+            RequestError, self.dag.search, productType="foo", raise_errors=True
+        )
+        # search iterator
+        self.assertRaises(StopIteration, next, self.dag.search_iter_page())
+        # search_all
+        self.assertEqual(len(self.dag.search_all()), 0)
+
+    @mock.patch(
+        "eodag.plugins.search.qssearch.requests.post",
+        autospec=True,
+        side_effect=RequestException,
+    )
+    @mock.patch(
+        "eodag.plugins.authentication.qsauth.HttpQueryStringAuth.authenticate",
+        autospec=True,
+    )
+    @mock.patch(
+        "eodag.api.core.EODataAccessGateway.fetch_product_types_list", autospec=True
+    )
+    def test_core_search_errors_buildpost(
+        self, mock_fetch_product_types_list, mock_authenticate, mock_post
+    ):
+        # BuildPostSearchResult (meteoblue)
+        self.dag.set_preferred_provider("meteoblue")
+        products, count = self.dag.search()
+        self.assertEqual(count, 0)
+        self.assertEqual(len(products), 0)
+        self.assertRaises(RequestError, self.dag.search, raise_errors=True)
+        # search iterator
+        self.assertRaises(StopIteration, next, self.dag.search_iter_page())
+        # search_all
+        self.assertEqual(len(self.dag.search_all()), 0)
diff --git a/tests/resources/wrong_credentials_conf.yml b/tests/resources/wrong_credentials_conf.yml
index c4c72064..b5b514be 100644
--- a/tests/resources/wrong_credentials_conf.yml
+++ b/tests/resources/wrong_credentials_conf.yml
@@ -57,3 +57,8 @@ cop_dataspace:
         credentials:
             username: "wrong_username"
             password: "wrong_password"
+
+meteoblue:
+    auth:
+        credentials:
+            apikey: "wrong_apikey"
diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py
index 79915096..9dde20d6 100644
--- a/tests/test_end_to_end.py
+++ b/tests/test_end_to_end.py
@@ -1002,3 +1002,17 @@ class TestEODagEndToEndWrongCredentials(EndToEndBase):
                     )
                 ),
             )
+
+    def test_end_to_end_wrong_credentials_search_meteoblue(self):
+        # It should already fail while searching for the products.
+        self.eodag.set_preferred_provider(METEOBLUE_SEARCH_ARGS[0])
+        with self.assertRaises(AuthenticationError):
+            results, _ = self.eodag.search(
+                raise_errors=True,
+                **dict(
+                    zip(
+                        ["productType", "start", "end", "geom"],
+                        METEOBLUE_SEARCH_ARGS[1:],
+                    )
+                ),
+            )