Repository: CS-SI/eodag
Pull request #495 (2022-08-08 08:44:39)

fix: timeout set for all requests methods
Use default timeouts for all requests methods calls:
- 5s for query/search
- 60s for stream/download (see #394)

github-actions[bot]: ## Unit Test Results
    2 files  ±0      2 suites  ±0   10m 21s [:stopwatch:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "duration of all tests") - 1m 47s
274 tests ±0  267 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "passed tests")  - 4  2 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "skipped / disabled tests")  - 1    5 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "failed tests") +  5 
548 runs  ±0  534 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "passed tests")  - 8  4 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "skipped / disabled tests")  - 2  10 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "failed tests") +10 

For more details on these failures, see [this check](https://github.com/CS-SI/eodag/runs/7721992326).

Results for commit 343a72f5. ± Comparison against base commit 07abe78d.

[test-results]:data:application/gzip;base64,H4sIAKLO8GIC/1WMyw7CIBBFf6Vh7UKGV+PPmOlAE2IfhsLK+O9OLcW4POfmnpcY4xQ2cevg0omtxNzAl4Q5rgujBcmCp/wdnT7pvhWiXVn3U4/4rIlDjBgnFqaJkNKa2FzZpLLsTaP7CmfSKN3MUWxcg/L8/wVpneeYGYTSCh2MBtCRdd4Yi4NHsEDQD2QloVcBCcT7AyLvJaoGAQAA

github-actions[bot]: <strong> </strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `81%` | :white_check_mark: |
| eodag/config.py | `88%` | :white_check_mark: |
| eodag/api/product/\_product.py | `83%` | :white_check_mark: |
| eodag/plugins/authentication/keycloak.py | `52%` | :x: |
| eodag/plugins/authentication/openid\_connect.py | `27%` | :x: |
| eodag/plugins/authentication/token.py | `92%` | :white_check_mark: |
| eodag/plugins/download/aws.py | `46%` | :x: |
| eodag/plugins/download/http.py | `38%` | :x: |
| eodag/plugins/download/s3rest.py | `19%` | :x: |
| eodag/plugins/search/qssearch.py | `74%` | :white_check_mark: |

<strong> </strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `80%` | :white_check_mark: |
| eodag/config.py | `88%` | :white_check_mark: |
| eodag/api/product/\_product.py | `83%` | :white_check_mark: |
| eodag/plugins/authentication/keycloak.py | `52%` | :x: |
| eodag/plugins/authentication/openid\_connect.py | `27%` | :x: |
| eodag/plugins/authentication/token.py | `92%` | :white_check_mark: |
| eodag/plugins/download/aws.py | `46%` | :x: |
| eodag/plugins/download/http.py | `38%` | :x: |
| eodag/plugins/download/s3rest.py | `19%` | :x: |
| eodag/plugins/search/qssearch.py | `74%` | :white_check_mark: |
| eodag/config.py | `88%` | :white_check_mark: |
| eodag/api/product/\_product.py | `83%` | :white_check_mark: |
| eodag/plugins/authentication/keycloak.py | `52%` | :x: |
| eodag/plugins/authentication/openid\_connect.py | `26%` | :x: |
| eodag/plugins/authentication/token.py | `92%` | :white_check_mark: |
| eodag/plugins/download/aws.py | `46%` | :x: |
| eodag/plugins/download/http.py | `38%` | :x: |
| eodag/plugins/download/s3rest.py | `19%` | :x: |
| eodag/plugins/search/qssearch.py | `74%` | :white_check_mark: |

_Minimum allowed coverage is `70%`_

<p align="right">Generated by :monkey: cobertura-action against 343a72f52a7c67d556abda262c28bc61cad3eac2 </p>

Patch:
diff --git a/eodag/api/product/_product.py b/eodag/api/product/_product.py
index 45162a1f..7948f9a4 100644
--- a/eodag/api/product/_product.py
+++ b/eodag/api/product/_product.py
@@ -26,7 +26,11 @@ from shapely import geometry, geos, wkb, wkt
 
 from eodag.api.product.drivers import DRIVERS, NoDriver
 from eodag.api.product.metadata_mapping import NOT_AVAILABLE, NOT_MAPPED
-from eodag.plugins.download.base import DEFAULT_DOWNLOAD_TIMEOUT, DEFAULT_DOWNLOAD_WAIT
+from eodag.plugins.download.base import (
+    DEFAULT_DOWNLOAD_TIMEOUT,
+    DEFAULT_DOWNLOAD_WAIT,
+    DEFAULT_STREAM_REQUESTS_TIMEOUT,
+)
 from eodag.utils import ProgressCallback, get_geometry_from_various
 from eodag.utils.exceptions import DownloadError
 
@@ -399,7 +403,10 @@ class EOProduct(object):
                 else None
             )
             with requests.get(
-                self.properties["quicklook"], stream=True, auth=auth
+                self.properties["quicklook"],
+                stream=True,
+                auth=auth,
+                timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
             ) as stream:
                 try:
                     stream.raise_for_status()
diff --git a/eodag/config.py b/eodag/config.py
index 7a343e65..ee6f8e23 100644
--- a/eodag/config.py
+++ b/eodag/config.py
@@ -36,6 +36,7 @@ from eodag.utils import (
     uri_to_path,
 )
 from eodag.utils.exceptions import ValidationError
+from eodag.utils.stac_reader import HTTP_REQ_TIMEOUT
 
 logger = logging.getLogger("eodag.config")
 
@@ -477,7 +478,7 @@ def get_ext_product_types_conf(conf_uri=EXT_PRODUCT_TYPES_CONF_URI):
     if conf_uri.lower().startswith("http"):
         # read from remote
         try:
-            response = requests.get(conf_uri)
+            response = requests.get(conf_uri, timeout=HTTP_REQ_TIMEOUT)
             response.raise_for_status()
             return response.json()
         except requests.RequestException as e:
diff --git a/eodag/plugins/authentication/keycloak.py b/eodag/plugins/authentication/keycloak.py
index 98dfa18d..6b3e774d 100644
--- a/eodag/plugins/authentication/keycloak.py
+++ b/eodag/plugins/authentication/keycloak.py
@@ -21,6 +21,7 @@ import requests
 from eodag.plugins.authentication import Authentication
 from eodag.plugins.authentication.openid_connect import CodeAuthorizedAuth
 from eodag.utils.exceptions import AuthenticationError
+from eodag.utils.stac_reader import HTTP_REQ_TIMEOUT
 
 
 class KeycloakOIDCPasswordAuth(Authentication):
@@ -50,6 +51,7 @@ class KeycloakOIDCPasswordAuth(Authentication):
                 "password": self.config.credentials["password"],
                 "grant_type": self.GRANT_TYPE,
             },
+            timeout=HTTP_REQ_TIMEOUT,
         )
         try:
             response.raise_for_status()
diff --git a/eodag/plugins/authentication/openid_connect.py b/eodag/plugins/authentication/openid_connect.py
index 41b1e2c2..73f53c8a 100644
--- a/eodag/plugins/authentication/openid_connect.py
+++ b/eodag/plugins/authentication/openid_connect.py
@@ -27,6 +27,7 @@ from requests.auth import AuthBase
 from eodag.plugins.authentication import Authentication
 from eodag.utils import parse_qs, repeatfunc, urlencode, urlparse, urlunparse
 from eodag.utils.exceptions import AuthenticationError, MisconfiguredError
+from eodag.utils.stac_reader import HTTP_REQ_TIMEOUT
 
 
 class OIDCAuthorizationCodeFlowAuth(Authentication):
@@ -172,7 +173,7 @@ class OIDCAuthorizationCodeFlowAuth(Authentication):
             "redirect_uri": self.config.redirect_uri,
         }
         authorization_response = self.session.get(
-            self.config.authorization_uri, params=params
+            self.config.authorization_uri, params=params, timeout=HTTP_REQ_TIMEOUT
         )
 
         login_document = etree.HTML(authorization_response.text)
@@ -194,7 +195,7 @@ class OIDCAuthorizationCodeFlowAuth(Authentication):
             auth_uri = login_form.xpath(
                 self.config.login_form_xpath.rstrip("/") + "/@action"
             )[0]
-        return self.session.post(auth_uri, data=login_data)
+        return self.session.post(auth_uri, data=login_data, timeout=HTTP_REQ_TIMEOUT)
 
     def grant_user_consent(self, authentication_response):
         """Grant user consent"""
@@ -207,7 +208,11 @@ class OIDCAuthorizationCodeFlowAuth(Authentication):
             key: self._constant_or_xpath_extracted(value, user_consent_form)
             for key, value in self.config.user_consent_form_data.items()
         }
-        return self.session.post(self.config.authorization_uri, data=user_consent_data)
+        return self.session.post(
+            self.config.authorization_uri,
+            data=user_consent_data,
+            timeout=HTTP_REQ_TIMEOUT,
+        )
 
     def exchange_code_for_token(self, authorized_url, state):
         """Get exchange code for token"""
@@ -245,7 +250,9 @@ class OIDCAuthorizationCodeFlowAuth(Authentication):
         post_request_kwargs = {
             self.config.token_exchange_post_data_method: token_exchange_data
         }
-        r = self.session.post(self.config.token_uri, **post_request_kwargs)
+        r = self.session.post(
+            self.config.token_uri, timeout=HTTP_REQ_TIMEOUT, **post_request_kwargs
+        )
         return r.json()[self.config.token_key]
 
     def _constant_or_xpath_extracted(self, value, form_element):
diff --git a/eodag/plugins/authentication/token.py b/eodag/plugins/authentication/token.py
index c80c0046..a9e8f1cd 100644
--- a/eodag/plugins/authentication/token.py
+++ b/eodag/plugins/authentication/token.py
@@ -22,6 +22,7 @@ from requests import RequestException
 from eodag.plugins.authentication.base import Authentication
 from eodag.utils import RequestsTokenAuth
 from eodag.utils.exceptions import MisconfiguredError
+from eodag.utils.stac_reader import HTTP_REQ_TIMEOUT
 
 
 class TokenAuth(Authentication):
@@ -51,7 +52,10 @@ class TokenAuth(Authentication):
 
         # First get the token
         response = requests.post(
-            self.config.auth_uri, data=self.config.credentials, **req_kwargs
+            self.config.auth_uri,
+            data=self.config.credentials,
+            timeout=HTTP_REQ_TIMEOUT,
+            **req_kwargs,
         )
         try:
             response.raise_for_status()
diff --git a/eodag/plugins/download/aws.py b/eodag/plugins/download/aws.py
index 35d1ab9c..e27ffe9c 100644
--- a/eodag/plugins/download/aws.py
+++ b/eodag/plugins/download/aws.py
@@ -36,6 +36,7 @@ from eodag.api.product.metadata_mapping import (
 from eodag.plugins.download.base import Download
 from eodag.utils import ProgressCallback, path_to_uri, rename_subfolder, urlparse
 from eodag.utils.exceptions import AuthenticationError, DownloadError
+from eodag.utils.stac_reader import HTTP_REQ_TIMEOUT
 
 logger = logging.getLogger("eodag.plugins.download.aws")
 
@@ -248,7 +249,7 @@ class AwsDownload(Download):
                 **product.properties
             )
             logger.info("Fetching extra metadata from %s" % fetch_url)
-            resp = requests.get(fetch_url)
+            resp = requests.get(fetch_url, timeout=HTTP_REQ_TIMEOUT)
             update_metadata = mtd_cfg_as_jsonpath(update_metadata)
             if fetch_format == "json":
                 json_resp = resp.json()
diff --git a/eodag/plugins/download/http.py b/eodag/plugins/download/http.py
index a599a31d..1c724df1 100644
--- a/eodag/plugins/download/http.py
+++ b/eodag/plugins/download/http.py
@@ -41,6 +41,7 @@ from eodag.utils.exceptions import (
     NotAvailableError,
 )
 from eodag.utils.notebook import NotebookWidgets
+from eodag.utils.stac_reader import HTTP_REQ_TIMEOUT
 
 logger = logging.getLogger("eodag.plugins.download.http")
 
@@ -309,7 +310,9 @@ class HTTPDownload(Download):
         total_size = sum(
             [
                 int(
-                    requests.head(asset_url, auth=auth).headers.get("Content-length", 0)
+                    requests.head(
+                        asset_url, auth=auth, timeout=HTTP_REQ_TIMEOUT
+                    ).headers.get("Content-length", 0)
                 )
                 for asset_url in assets_urls
             ]
@@ -320,9 +323,9 @@ class HTTPDownload(Download):
                 [
                     int(
                         parse_header(
-                            requests.head(asset_url, auth=auth).headers.get(
-                                "content-disposition", ""
-                            )
+                            requests.head(
+                                asset_url, auth=auth, timeout=HTTP_REQ_TIMEOUT
+                            ).headers.get("content-disposition", "")
                         )[-1].get("size", 0)
                     )
                     for asset_url in assets_urls
@@ -334,9 +337,9 @@ class HTTPDownload(Download):
         for asset_url in assets_urls:
 
             # get asset filename from header
-            asset_content_disposition = requests.head(asset_url, auth=auth).headers.get(
-                "content-disposition", None
-            )
+            asset_content_disposition = requests.head(
+                asset_url, auth=auth, timeout=HTTP_REQ_TIMEOUT
+            ).headers.get("content-disposition", None)
             if asset_content_disposition:
                 asset_filename = parse_header(asset_content_disposition)[-1]["filename"]
             else:
diff --git a/eodag/plugins/download/s3rest.py b/eodag/plugins/download/s3rest.py
index 871e753d..db92da7a 100644
--- a/eodag/plugins/download/s3rest.py
+++ b/eodag/plugins/download/s3rest.py
@@ -37,6 +37,7 @@ from eodag.utils.exceptions import (
     NotAvailableError,
     RequestError,
 )
+from eodag.utils.stac_reader import HTTP_REQ_TIMEOUT
 
 logger = logging.getLogger("eodag.plugins.download.s3rest")
 
@@ -100,7 +101,9 @@ class S3RestDownload(AwsDownload):
 
         # get nodes/files list contained in the bucket
         logger.debug("Retrieving product content from %s", nodes_list_url)
-        bucket_contents = requests.get(nodes_list_url, auth=auth)
+        bucket_contents = requests.get(
+            nodes_list_url, auth=auth, timeout=HTTP_REQ_TIMEOUT
+        )
         try:
             bucket_contents.raise_for_status()
         except requests.RequestException as err:
diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py
index c06a7c56..6b098d0d 100644
--- a/eodag/plugins/search/qssearch.py
+++ b/eodag/plugins/search/qssearch.py
@@ -47,6 +47,7 @@ from eodag.utils import (
     urlencode,
 )
 from eodag.utils.exceptions import AuthenticationError, MisconfiguredError, RequestError
+from eodag.utils.stac_reader import HTTP_REQ_TIMEOUT
 
 logger = logging.getLogger("eodag.plugins.search.qssearch")
 
@@ -858,7 +859,7 @@ class QueryStringSearch(Search):
             else:
                 if info_message:
                     logger.info(info_message)
-                response = requests.get(url, **kwargs)
+                response = requests.get(url, timeout=HTTP_REQ_TIMEOUT, **kwargs)
                 response.raise_for_status()
         except (requests.RequestException, urllib_HTTPError) as err:
             err_msg = err.readlines() if hasattr(err, "readlines") else ""
@@ -916,7 +917,7 @@ class ODataV4Search(QueryStringSearch):
             metadata_url = self.get_metadata_search_url(entity)
             try:
                 logger.debug("Sending metadata request: %s", metadata_url)
-                response = requests.get(metadata_url)
+                response = requests.get(metadata_url, timeout=HTTP_REQ_TIMEOUT)
                 response.raise_for_status()
             except requests.RequestException:
                 logger.exception(
@@ -1114,7 +1115,9 @@ class PostJsonSearch(QueryStringSearch):
             if info_message:
                 logger.info(info_message)
             logger.debug("Query parameters: %s" % self.query_params)
-            response = requests.post(url, json=self.query_params, **kwargs)
+            response = requests.post(
+                url, json=self.query_params, timeout=HTTP_REQ_TIMEOUT, **kwargs
+            )
             response.raise_for_status()
         except (requests.RequestException, urllib_HTTPError) as err:
             # check if error is identified as auth_error in provider conf
diff --git a/utils/params_mapping_to_csv.py b/utils/params_mapping_to_csv.py
index 126cfd7d..70b31fa9 100644
--- a/utils/params_mapping_to_csv.py
+++ b/utils/params_mapping_to_csv.py
@@ -25,6 +25,7 @@ from lxml import html
 
 from eodag.api.core import EODataAccessGateway
 from eodag.config import load_stac_provider_config
+from eodag.utils.stac_reader import HTTP_REQ_TIMEOUT
 
 logger = logging.getLogger(__name__)
 logger.addHandler(logging.StreamHandler())
@@ -103,7 +104,7 @@ def params_mapping_to_csv(
 
             # create metadata mapping table rows
             try:
-                page = requests.get(ogc_doc_url)
+                page = requests.get(ogc_doc_url, timeout=HTTP_REQ_TIMEOUT)
                 # page reachable, read infos from remote html
                 tree = html.fromstring(page.content.decode("utf8"))
 


Test patch:
diff --git a/tests/context.py b/tests/context.py
index 52328d1d..ce8ddd59 100644
--- a/tests/context.py
+++ b/tests/context.py
@@ -76,5 +76,5 @@ from eodag.utils.exceptions import (
     UnsupportedProvider,
     ValidationError,
 )
-from eodag.utils.stac_reader import fetch_stac_items
+from eodag.utils.stac_reader import fetch_stac_items, HTTP_REQ_TIMEOUT
 from tests import TESTS_DOWNLOAD_PATH, TEST_RESOURCES_PATH
diff --git a/tests/integration/test_core_config.py b/tests/integration/test_core_config.py
index b8cb592e..4b7bfe1f 100644
--- a/tests/integration/test_core_config.py
+++ b/tests/integration/test_core_config.py
@@ -20,6 +20,7 @@ import tempfile
 from unittest import TestCase, mock
 
 from tests.context import (
+    HTTP_REQ_TIMEOUT,
     AuthenticationError,
     EODataAccessGateway,
     HeaderAuth,
@@ -187,6 +188,7 @@ class TestCoreProductTypesConfig(TestCase):
                 "fetch_url"
             ],
             auth=mock.ANY,
+            timeout=HTTP_REQ_TIMEOUT,
         )
         call_args, call_kwargs = mock_requests_get.call_args
         self.assertIsInstance(call_kwargs["auth"], HeaderAuth)
diff --git a/tests/test_config.py b/tests/test_config.py
index 07400024..40380210 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -28,6 +28,7 @@ from pkg_resources import resource_filename
 
 from tests.context import (
     EXT_PRODUCT_TYPES_CONF_URI,
+    HTTP_REQ_TIMEOUT,
     TEST_RESOURCES_PATH,
     EODataAccessGateway,
     ValidationError,
@@ -470,7 +471,9 @@ class TestConfigFunctions(unittest.TestCase):
         mock_get.return_value.json.return_value = {"some_parameter": "a_value"}
 
         ext_product_types_conf = get_ext_product_types_conf()
-        mock_get.assert_called_once_with(url=EXT_PRODUCT_TYPES_CONF_URI)
+        mock_get.assert_called_once_with(
+            EXT_PRODUCT_TYPES_CONF_URI, timeout=HTTP_REQ_TIMEOUT
+        )
         self.assertEqual(ext_product_types_conf, {"some_parameter": "a_value"})
 
         # local conf file
diff --git a/tests/units/test_eoproduct.py b/tests/units/test_eoproduct.py
index a14ce345..c63e5476 100644
--- a/tests/units/test_eoproduct.py
+++ b/tests/units/test_eoproduct.py
@@ -163,7 +163,10 @@ class TestEOProduct(EODagTestCase):
 
         quicklook_file_path = product.get_quicklook()
         self.requests_http_get.assert_called_with(
-            "https://fake.url.to/quicklook", stream=True, auth=None
+            "https://fake.url.to/quicklook",
+            stream=True,
+            auth=None,
+            timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
         )
         self.assertEqual(quicklook_file_path, "")
 
@@ -183,7 +186,10 @@ class TestEOProduct(EODagTestCase):
 
         quicklook_file_path = product.get_quicklook()
         self.requests_http_get.assert_called_with(
-            "https://fake.url.to/quicklook", stream=True, auth=None
+            "https://fake.url.to/quicklook",
+            stream=True,
+            auth=None,
+            timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
         )
         self.assertEqual(
             os.path.basename(quicklook_file_path), product.properties["id"]
@@ -197,7 +203,10 @@ class TestEOProduct(EODagTestCase):
         # Test the same thing as above but with an explicit name given to the downloaded File
         quicklook_file_path = product.get_quicklook(filename="the_quicklook.png")
         self.requests_http_get.assert_called_with(
-            "https://fake.url.to/quicklook", stream=True, auth=None
+            "https://fake.url.to/quicklook",
+            stream=True,
+            auth=None,
+            timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
         )
         self.assertEqual(self.requests_http_get.call_count, 2)
         self.assertEqual(os.path.basename(quicklook_file_path), "the_quicklook.png")