Repository: CS-SI/eodag

Issue:
Connection timeout bug in search API
**Describe the bug**

No result from provider 'creodias' due to an error during the search.

`urllib3.exceptions.ConnectTimeoutError: (<urllib3.connection.HTTPSConnection object at 0x7f0614fa07f0>, 'Connection to finder.creodias.eu timed out. (connect timeout=5)')`

Timeout error is not properly handled and propagated back. The error (len() of None) appears already inside the eodag code:

```
normalize_remaining_count = len(results)
TypeError: object of type 'NoneType' has no len()
```

**Code To Reproduce**

`all_products = session.search_all(**search_criteria)` -> with the specific search and creodias provider for example.

**Output**

```
[2022-11-14 23:38:30,795][eodag.core][INFO] - Searching product type 'S2_MSI_L1C' on provider: creodias
[2022-11-14 23:38:30,795][eodag.core][INFO] - Iterate search over multiple pages: page #1
[2022-11-14 23:38:30,795][eodag.plugins.search.qssearch][INFO] - Sending search request: https://finder.creodias.eu/resto/api/collections/Sentinel2/search.json?startDate=2019-05-01T00:00:00&completionDate=2019-10-01T00:00:00&geometry=POLYGON ((9.1010 46.6489, 9.0945 46.6490, 9.0946 46.6534, 9.1011 46.6534, 9.1010 46.6489))&productType=L1C&maxRecords=2000&page=1
[2022-11-14 23:38:35,883][eodag.plugins.search.qssearch][ERROR] - Skipping error while searching for creodias QueryStringSearch instance:
Traceback (most recent call last):
  File ".../lib/python3.9/site-packages/urllib3/connection.py", line 174, in _new_conn
    conn = connection.create_connection(
  File ".../lib/python3.9/site-packages/urllib3/util/connection.py", line 95, in create_connection
    raise err
  File ".../lib/python3.9/site-packages/urllib3/util/connection.py", line 85, in create_connection
    sock.connect(sa)
socket.timeout: timed out

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File ".../lib/python3.9/site-packages/urllib3/connectionpool.py", line 703, in urlopen
    httplib_response = self._make_request(
  File ".../lib/python3.9/site-packages/urllib3/connectionpool.py", line 386, in _make_request
    self._validate_conn(conn)
  File ".../lib/python3.9/site-packages/urllib3/connectionpool.py", line 1042, in _validate_conn
    conn.connect()
  File ".../lib/python3.9/site-packages/urllib3/connection.py", line 358, in connect
    self.sock = conn = self._new_conn()
  File ".../lib/python3.9/site-packages/urllib3/connection.py", line 179, in _new_conn
    raise ConnectTimeoutError(
urllib3.exceptions.ConnectTimeoutError: (<urllib3.connection.HTTPSConnection object at 0x7f0614fa07f0>, 'Connection to finder.creodias.eu timed out. (connect timeout=5)')

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File ".../lib/python3.9/site-packages/requests/adapters.py", line 489, in send
    resp = conn.urlopen(
  File ".../lib/python3.9/site-packages/urllib3/connectionpool.py", line 787, in urlopen
    retries = retries.increment(
  File ".../lib/python3.9/site-packages/urllib3/util/retry.py", line 592, in increment
    raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='finder.creodias.eu', port=443): Max retries exceeded with url: /resto/api/collections/Sentinel2/search.json?startDate=2019-05-01T00:00:00&completionDate=2019-10-01T00:00:00&geometry=POLYGON%20((9.1010%2046.6489,%209.0945%2046.6490,%209.0946%2046.6534,%209.1011%2046.6534,%209.1010%2046.6489))&productType=L1C&maxRecords=2000&page=1 (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x7f0614fa07f0>, 'Connection to finder.creodias.eu timed out. (connect timeout=5)'))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File ".../lib/python3.9/site-packages/eodag/plugins/search/qssearch.py", line 864, in _request
    response = requests.get(url, timeout=HTTP_REQ_TIMEOUT, **kwargs)
  File ".../lib/python3.9/site-packages/requests/api.py", line 73, in get
    return request("get", url, params=params, **kwargs)
  File ".../lib/python3.9/site-packages/requests/api.py", line 59, in request
    return session.request(method=method, url=url, **kwargs)
  File ".../lib/python3.9/site-packages/requests/sessions.py", line 587, in request
    resp = self.send(prep, **send_kwargs)
  File ".../lib/python3.9/site-packages/requests/sessions.py", line 701, in send
    r = adapter.send(request, **kwargs)
  File ".../lib/python3.9/site-packages/requests/adapters.py", line 553, in send
    raise ConnectTimeout(e, request=request)
requests.exceptions.ConnectTimeout: HTTPSConnectionPool(host='finder.creodias.eu', port=443): Max retries exceeded with url: /resto/api/collections/Sentinel2/search.json?startDate=2019-05-01T00:00:00&completionDate=2019-10-01T00:00:00&geometry=POLYGON%20((9.1010%2046.6489,%209.0945%2046.6490,%209.0946%2046.6534,%209.1011%2046.6534,%209.1010%2046.6489))&productType=L1C&maxRecords=2000&page=1 (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x7f0614fa07f0>, 'Connection to finder.creodias.eu timed out. (connect timeout=5)'))
[2022-11-14 23:38:35,888][eodag.core][INFO] - No result from provider 'creodias' due to an error during search. Raise verbosity of log messages for details
```

```
File ".../sentinel_api.py", line 49, in search_products
    all_products = session.search_all(**search_criteria)
  File ".../lib/python3.9/site-packages/eodag/api/core.py", line 1099, in search_all
    for page_results in self.search_iter_page(
  File ".../lib/python3.9/site-packages/eodag/api/core.py", line 953, in search_iter_page
    products, _ = self._do_search(
  File ".../lib/python3.9/site-packages/eodag/api/core.py", line 1339, in _do_search
    res, nb_res = search_plugin.query(count=count, **kwargs)
  File ".../lib/python3.9/site-packages/eodag/plugins/search/qssearch.py", line 341, in query
    eo_products = self.normalize_results(provider_results, **kwargs)
  File ".../lib/python3.9/site-packages/eodag/plugins/search/qssearch.py", line 677, in normalize_results
    normalize_remaining_count = len(results)
TypeError: object of type 'NoneType' has no len()
```

**Environment:**
 - Python version: 3.9
 - EODAG version: 2.6.0

**Additional context**

The above error appeared in a large-scale search for 50k different searches. The error appeared on a specific search, after 10s of thousands of successful searches.


Discussion:
github-actions[bot]: <strong></strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `84%` | :white_check_mark: |
| eodag/plugins/search/qssearch.py | `76%` | :white_check_mark: |
| tests/units/test\_core.py | `98%` | :white_check_mark: |

_Minimum allowed coverage is `70%`_

<p align="right">Generated by :monkey: cobertura-action against 25333fec73fbfc8328fbe55c3d6282eb839dfc31 </p>
github-actions[bot]: ## Unit Test Results
    2 files  ±0      2 suites  ±0   9m 25s [:stopwatch:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "duration of all tests") +22s
319 tests +1  316 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "passed tests") +1  3 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "skipped / disabled tests") ±0  0 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "failed tests") ±0 
638 runs  +2  632 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "passed tests") +2  6 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "skipped / disabled tests") ±0  0 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "failed tests") ±0 

Results for commit 557a6f3f. ± Comparison against base commit 12e3cdc8.

[test-results]:data:application/gzip;base64,H4sIAKK/dGMC/03MSw6DIBSF4a0Yxh3I6wrdTENBElKVhsfIdO8FK9Th/53k7Mi6ZY7oPpDbgGJ2qYfJQSXnt5IceIEypTpSLFs9Ytb6IPjTy70rdbDKLQXGDnMIPpwS8lY/gYoz2iVQ0uX3CK0vh0df/7RfV5dKIM4nBZZaozCbjWTwFIRYIQUfp5ExEKCxxJihzxcUSlynBQEAAA==


Patch:
diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py
index f349b760..a9625b53 100644
--- a/eodag/plugins/search/qssearch.py
+++ b/eodag/plugins/search/qssearch.py
@@ -597,8 +597,7 @@ class QueryStringSearch(Search):
                     "instance:".format(self.provider, self.__class__.__name__),
                 )
             except RequestError:
-                # Signal the end of iteration (see PEP-479)
-                return
+                return []
             else:
                 next_page_url_key_path = self.config.pagination.get(
                     "next_page_url_key_path", None


Test patch:
diff --git a/tests/context.py b/tests/context.py
index f23c149a..47f4b0f7 100644
--- a/tests/context.py
+++ b/tests/context.py
@@ -82,6 +82,7 @@ from eodag.utils.exceptions import (
     NoMatchingProductType,
     NotAvailableError,
     PluginImplementationError,
+    RequestError,
     UnsupportedDatasetAddressScheme,
     UnsupportedProvider,
     ValidationError,
diff --git a/tests/units/test_core.py b/tests/units/test_core.py
index 7b45f284..7bf4ca52 100644
--- a/tests/units/test_core.py
+++ b/tests/units/test_core.py
@@ -40,6 +40,7 @@ from tests.context import (
     EOProduct,
     NoMatchingProductType,
     PluginImplementationError,
+    RequestError,
     SearchResult,
     UnsupportedProvider,
     get_geometry_from_various,
@@ -1704,6 +1705,32 @@ class TestCoreSearch(TestCoreBase):
         dag.search_all(productType="S2_MSI_L1C", items_per_page=7)
         self.assertEqual(mocked_search_iter_page.call_args[1]["items_per_page"], 7)
 
+    @mock.patch(
+        "eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True
+    )
+    def test_search_all_request_error(self, mocked_request):
+        """search_all must stop iteration if a request exception is raised"""
+        dag = EODataAccessGateway()
+        dummy_provider_config = """
+        dummy_provider:
+            search:
+                type: QueryStringSearch
+                api_endpoint: https://api.my_new_provider/search
+                pagination:
+                    next_page_url_tpl: 'dummy_next_page_url_tpl'
+                    next_page_url_key_path: '$.links[?(@.rel="next")].href'
+                metadata_mapping:
+                    dummy: 'dummy'
+            products:
+                S2_MSI_L1C:
+                    productType: '{productType}'
+        """
+        mocked_request.side_effect = RequestError()
+        dag.update_providers_config(dummy_provider_config)
+        dag.set_preferred_provider("dummy_provider")
+        results = dag.search_all(productType="S2_MSI_L1C")
+        self.assertEqual(len(results), 0)
+
 
 class TestCoreDownload(TestCoreBase):
     @classmethod