Repository: CS-SI/eodag
Pull request #506 (2022-09-14 15:11:52)

refactor: download retry decorator and tests
Refactorize [download retry mechanism](https://eodag.readthedocs.io/en/latest/notebooks/api_user_guide/7_download.html?#Download-an-OFFLINE-product) and add tests


github-actions[bot]: ## Unit Test Results
    2 files  ±  0      2 suites  ±0   6m 21s [:stopwatch:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "duration of all tests") -52s
282 tests +  7  279 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "passed tests") +  7  3 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "skipped / disabled tests") ±0  0 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "failed tests") ±0 
564 runs  +14  558 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "passed tests") +14  6 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "skipped / disabled tests") ±0  0 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "failed tests") ±0 

Results for commit 54bfe09f. ± Comparison against base commit e68536c3.

[test-results]:data:application/gzip;base64,H4sIAJrwIWMC/02OSw6DIBBAr2JYdwHKt5dpKJ+EVKVBWJnevYMV6m7em8zL7MiH2W3oPoy3AW0l5A62JJ1DXAEnSUDAKh9LOTZ6bMWYqoT6q1d415suvA4zCNyFSymm06Sy1ibj9ISWZEx28yvyxpfgwdeeicsSMgBi9OkdVt5IqazA1I/wpVJEc0KJkzAILCZi0ecLVfWe0gUBAAA=

github-actions[bot]: <strong> </strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `81%` | :white_check_mark: |
| eodag/api/product/\_product.py | `82%` | :white_check_mark: |
| eodag/plugins/download/base.py | `80%` | :white_check_mark: |
| eodag/plugins/download/http.py | `51%` | :x: |
| tests/units/test\_download\_plugins.py | `94%` | :white_check_mark: |

<strong> </strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `81%` | :white_check_mark: |
| eodag/api/product/\_product.py | `82%` | :white_check_mark: |
| eodag/plugins/download/base.py | `80%` | :white_check_mark: |
| eodag/plugins/download/http.py | `51%` | :x: |
| tests/units/test\_download\_plugins.py | `94%` | :white_check_mark: |
| eodag/api/product/\_product.py | `82%` | :white_check_mark: |
| eodag/plugins/download/base.py | `80%` | :white_check_mark: |
| eodag/plugins/download/http.py | `51%` | :x: |
| tests/units/test\_download\_plugins.py | `94%` | :white_check_mark: |

_Minimum allowed coverage is `70%`_

<p align="right">Generated by :monkey: cobertura-action against 54bfe09fc889d704f2279991a6141e891a70731d </p>

Patch:
diff --git a/eodag/api/product/_product.py b/eodag/api/product/_product.py
index 7948f9a4..cfa50434 100644
--- a/eodag/api/product/_product.py
+++ b/eodag/api/product/_product.py
@@ -32,7 +32,7 @@ from eodag.plugins.download.base import (
     DEFAULT_STREAM_REQUESTS_TIMEOUT,
 )
 from eodag.utils import ProgressCallback, get_geometry_from_various
-from eodag.utils.exceptions import DownloadError
+from eodag.utils.exceptions import DownloadError, MisconfiguredError
 
 try:
     from shapely.errors import TopologicalError
@@ -198,9 +198,14 @@ class EOProduct(object):
     __geo_interface__ = property(as_dict)
 
     def __repr__(self):
-        return "{}(id={}, provider={})".format(
-            self.__class__.__name__, self.properties["id"], self.provider
-        )
+        try:
+            return "{}(id={}, provider={})".format(
+                self.__class__.__name__, self.properties["id"], self.provider
+            )
+        except KeyError as e:
+            raise MisconfiguredError(
+                f"Unable to get {e.args[0]} key from EOProduct.properties"
+            )
 
     def register_downloader(self, downloader, authenticator):
         """Give to the product the information needed to download itself.
@@ -230,7 +235,7 @@ class EOProduct(object):
         progress_callback=None,
         wait=DEFAULT_DOWNLOAD_WAIT,
         timeout=DEFAULT_DOWNLOAD_TIMEOUT,
-        **kwargs
+        **kwargs,
     ):
         """Download the EO product using the provided download plugin and the
         authenticator if necessary.
@@ -295,7 +300,7 @@ class EOProduct(object):
             progress_callback=progress_callback,
             wait=wait,
             timeout=timeout,
-            **kwargs
+            **kwargs,
         )
 
         # close progress bar if needed
diff --git a/eodag/plugins/download/base.py b/eodag/plugins/download/base.py
index e833db21..10130749 100644
--- a/eodag/plugins/download/base.py
+++ b/eodag/plugins/download/base.py
@@ -135,7 +135,7 @@ class Download(PluginTopic):
             # The fs path of a product is either a file (if 'extract' config is False) or a directory
             if os.path.isfile(fs_path) or os.path.isdir(fs_path):
                 logger.info(
-                    "Product already present on this platform. Identifier: %s", fs_path
+                    f"Product already present on this platform. Identifier: {fs_path}",
                 )
                 # Do not download data if we are on site. Instead give back the absolute path to the data
                 return fs_path, None
@@ -143,10 +143,12 @@ class Download(PluginTopic):
         url = product.remote_location
         if not url:
             logger.debug(
-                "Unable to get download url for %s, skipping download", product
+                f"Unable to get download url for {product}, skipping download",
             )
             return None, None
-        logger.info("Download url: %s", url)
+        logger.info(
+            f"Download url: {url}",
+        )
 
         outputs_prefix = (
             kwargs.pop("outputs_prefix", None)
@@ -165,11 +167,7 @@ class Download(PluginTopic):
             collision_avoidance_suffix = "-" + sanitize(product.properties["id"])
         fs_path = os.path.join(
             prefix,
-            "{}{}{}".format(
-                sanitize(product.properties["title"]),
-                collision_avoidance_suffix,
-                outputs_extension,
-            ),
+            f"{sanitize(product.properties['title'])}{collision_avoidance_suffix}{outputs_extension}",
         )
         fs_dir_path = fs_path.replace(outputs_extension, "")
         download_records_dir = os.path.join(prefix, ".downloaded")
@@ -182,18 +180,22 @@ class Download(PluginTopic):
                 import traceback as tb
 
                 logger.warning(
-                    "Unable to create records directory. Got:\n%s", tb.format_exc()
+                    f"Unable to create records directory. Got:\n{tb.format_exc()}",
                 )
         url_hash = hashlib.md5(url.encode("utf-8")).hexdigest()
         record_filename = os.path.join(download_records_dir, url_hash)
         if os.path.isfile(record_filename) and os.path.isfile(fs_path):
-            logger.info("Product already downloaded: %s", fs_path)
+            logger.info(
+                f"Product already downloaded: {fs_path}",
+            )
             return (
                 self._finalize(fs_path, progress_callback=progress_callback, **kwargs),
                 None,
             )
         elif os.path.isfile(record_filename) and os.path.isdir(fs_dir_path):
-            logger.info("Product already downloaded: %s", fs_dir_path)
+            logger.info(
+                f"Product already downloaded: {fs_dir_path}",
+            )
             return (
                 self._finalize(
                     fs_dir_path, progress_callback=progress_callback, **kwargs
@@ -203,9 +205,11 @@ class Download(PluginTopic):
         # Remove the record file if fs_path is absent (e.g. it was deleted while record wasn't)
         elif os.path.isfile(record_filename):
             logger.debug(
-                "Record file found (%s) but not the actual file", record_filename
+                f"Record file found ({record_filename}) but not the actual file",
+            )
+            logger.debug(
+                f"Removing record file : {record_filename}",
             )
-            logger.debug("Removing record file : %s", record_filename)
             os.remove(record_filename)
 
         return fs_path, record_filename
@@ -280,12 +284,8 @@ class Download(PluginTopic):
         product_path_exists = os.path.exists(product_path)
         if product_path_exists and os.path.isfile(product_path):
             logger.info(
-                "Remove existing partially downloaded file: %s (%s/%s)"
-                % (
-                    product_path,
-                    os.stat(product_path).st_size,
-                    os.stat(fs_path).st_size,
-                )
+                f"Remove existing partially downloaded file: {product_path}"
+                f" ({os.stat(product_path).st_size}/{os.stat(fs_path).st_size})"
             )
             os.remove(product_path)
         elif (
@@ -293,9 +293,7 @@ class Download(PluginTopic):
             and os.path.isdir(product_path)
             and len(os.listdir(product_path)) == 0
         ):
-            logger.info(
-                "Remove existing empty destination directory: %s" % product_path
-            )
+            logger.info(f"Remove existing empty destination directory: {product_path}")
             os.rmdir(product_path)
         elif (
             product_path_exists
@@ -303,8 +301,7 @@ class Download(PluginTopic):
             and len(os.listdir(product_path)) > 0
         ):
             logger.info(
-                "Extraction cancelled, destination directory already exists and is not empty: %s"
-                % product_path
+                f"Extraction cancelled, destination directory already exists and is not empty: {product_path}"
             )
             progress_callback(1, total=1)
             product_path = self._resolve_archive_depth(product_path)
@@ -315,8 +312,8 @@ class Download(PluginTopic):
 
         if not os.path.exists(product_path):
             logger.info("Extraction activated")
-            progress_callback.desc = "Extracting files from {}".format(
-                os.path.basename(fs_path)
+            progress_callback.desc = (
+                f"Extracting files from {os.path.basename(fs_path)}"
             )
             progress_callback.refresh()
 
@@ -350,13 +347,11 @@ class Download(PluginTopic):
             tmp_dir.cleanup()
 
             if delete_archive:
-                logger.info("Deleting archive {}".format(os.path.basename(fs_path)))
+                logger.info(f"Deleting archive {os.path.basename(fs_path)}")
                 os.unlink(fs_path)
             else:
                 logger.info(
-                    "Archive deletion is deactivated, keeping {}".format(
-                        os.path.basename(fs_path)
-                    )
+                    f"Archive deletion is deactivated, keeping {os.path.basename(fs_path)}"
                 )
         else:
             progress_callback(1, total=1)
@@ -419,7 +414,7 @@ class Download(PluginTopic):
         paths = []
         # initiate retry loop
         start_time = datetime.now()
-        stop_time = datetime.now() + timedelta(minutes=timeout)
+        stop_time = start_time + timedelta(minutes=timeout)
         nb_products = len(products)
         retry_count = 0
         # another output for notbooks
@@ -477,8 +472,7 @@ class Download(PluginTopic):
 
                         except (AuthenticationError, MisconfiguredError):
                             logger.exception(
-                                "Stopped because of credentials problems with provider %s",
-                                self.provider,
+                                f"Stopped because of credentials problems with provider {self.provider}"
                             )
                             raise
 
@@ -486,22 +480,20 @@ class Download(PluginTopic):
                             import traceback as tb
 
                             logger.error(
-                                "A problem occurred during download of product: %s. "
-                                "Skipping it",
-                                product,
+                                f"A problem occurred during download of product: {product}. "
+                                "Skipping it"
                             )
-                            logger.debug("\n%s", tb.format_exc())
+                            logger.debug(f"\n{tb.format_exc()}")
                             stop_time = datetime.now()
 
                         except Exception:
                             import traceback as tb
 
                             logger.warning(
-                                "A problem occurred during download of product: %s. "
+                                f"A problem occurred during download of product: {product}. "
                                 "Skipping it",
-                                product,
                             )
-                            logger.debug("\n%s", tb.format_exc())
+                            logger.debug(f"\n{tb.format_exc()}")
 
                 if (
                     len(products) > 0
@@ -519,12 +511,114 @@ class Download(PluginTopic):
                     sleep(wait_seconds + 1)
                 elif len(products) > 0 and datetime.now() >= stop_time:
                     logger.warning(
-                        "%s products could not be downloaded: %s",
-                        len(products),
-                        [prod.properties["title"] for prod in products],
+                        f"{len(products)} products could not be downloaded:"
+                        f"{' ' + [prod.properties['title'] for prod in products]}",
                     )
                     break
                 elif len(products) == 0:
                     break
 
         return paths
+
+    def _download_retry(self, product, wait, timeout):
+        """
+        Download retry decorator.
+
+        Retries the wrapped  download method after `wait` minutes if a NotAvailableError
+        exception is thrown until `timeout` minutes.
+
+        :param product: The EO product to download
+        :type product: :class:`~eodag.api.product._product.EOProduct`
+        :param wait: If download fails, wait time in minutes between two download tries
+        :type wait: int
+        :param timeout: If download fails, maximum time in minutes before stop retrying
+                        to download
+        :type timeout: int
+        :returns: decorator
+        :rtype: :class:`typing.Any`
+        """
+
+        def decorator(download):
+            def download_and_retry(*args, **kwargs):
+                # initiate retry loop
+                start_time = datetime.now()
+                stop_time = start_time + timedelta(minutes=timeout)
+                product.next_try = start_time
+                retry_count = 0
+                not_available_info = "The product could not be downloaded"
+                # another output for notebooks
+                nb_info = NotebookWidgets()
+
+                while "Loop until products download succeeds or timeout is reached":
+
+                    datetime_now = datetime.now()
+
+                    if datetime_now >= product.next_try:
+                        product.next_try += timedelta(minutes=wait)
+                        try:
+                            return download(*args, **kwargs)
+
+                        except NotAvailableError as e:
+                            if not getattr(self.config, "order_enabled", False):
+                                raise NotAvailableError(
+                                    f"Product is not available for download and order is not supported for"
+                                    f" {self.provider}, {e}"
+                                )
+                            not_available_info = e
+                            pass
+
+                    if datetime_now >= product.next_try and datetime_now < stop_time:
+                        wait_seconds = (
+                            datetime_now - product.next_try + timedelta(minutes=wait)
+                        ).seconds
+                        retry_count += 1
+                        retry_info = (
+                            f"[Retry #{retry_count}] Waited {wait_seconds}s, trying again to download ordered product"
+                            f" (retry every {wait}' for {timeout}')"
+                        )
+                        logger.debug(not_available_info)
+                        # Retry-After info from Response header
+                        if hasattr(self, "stream"):
+                            retry_server_info = self.stream.headers.get(
+                                "Retry-After", ""
+                            )
+                            if retry_server_info:
+                                logger.debug(
+                                    f"[{self.provider} response] Retry-After: {retry_server_info}"
+                                )
+                        logger.info(retry_info)
+                        nb_info.display_html(retry_info)
+                        product.next_try = datetime_now
+                    elif datetime_now < product.next_try and datetime_now < stop_time:
+                        wait_seconds = (product.next_try - datetime_now).seconds
+                        retry_count += 1
+                        retry_info = (
+                            f"[Retry #{retry_count}] Waiting {wait_seconds}s until next download try"
+                            f" for ordered product (retry every {wait}' for {timeout}')"
+                        )
+                        logger.debug(not_available_info)
+                        # Retry-After info from Response header
+                        retry_server_info = self.stream.headers.get("Retry-After", "")
+                        if retry_server_info:
+                            logger.debug(
+                                f"[{self.provider} response] Retry-After: {retry_server_info}"
+                            )
+                        logger.info(retry_info)
+                        nb_info.display_html(retry_info)
+                        sleep(wait_seconds)
+                    elif datetime_now >= stop_time and timeout > 0:
+                        if "storageStatus" not in product.properties:
+                            product.properties["storageStatus"] = "N/A status"
+                        logger.info(not_available_info)
+                        raise NotAvailableError(
+                            f"{product.properties['title']} is not available ({product.properties['storageStatus']})"
+                            f" and could not be downloaded, timeout reached"
+                        )
+                    elif datetime_now >= stop_time:
+                        raise NotAvailableError(not_available_info)
+
+                return download(*args, **kwargs)
+
+            return download_and_retry
+
+        return decorator
diff --git a/eodag/plugins/download/http.py b/eodag/plugins/download/http.py
index 1c724df1..96caf35a 100644
--- a/eodag/plugins/download/http.py
+++ b/eodag/plugins/download/http.py
@@ -21,8 +21,6 @@ import os
 import shutil
 import zipfile
 from cgi import parse_header
-from datetime import datetime, timedelta
-from time import sleep
 
 import requests
 from requests import HTTPError, RequestException
@@ -40,7 +38,6 @@ from eodag.utils.exceptions import (
     MisconfiguredError,
     NotAvailableError,
 )
-from eodag.utils.notebook import NotebookWidgets
 from eodag.utils.stac_reader import HTTP_REQ_TIMEOUT
 
 logger = logging.getLogger("eodag.plugins.download.http")
@@ -128,151 +125,118 @@ class HTTPDownload(Download):
                         e,
                     )
 
-        # initiate retry loop
-        start_time = datetime.now()
-        stop_time = datetime.now() + timedelta(minutes=timeout)
-        product.next_try = start_time
-        retry_count = 0
-        not_available_info = "The product could not be downloaded"
-        # another output for notebooks
-        nb_info = NotebookWidgets()
-
-        while "Loop until products download succeeds or timeout is reached":
-
-            if datetime.now() >= product.next_try:
-                product.next_try += timedelta(minutes=wait)
+        @self._download_retry(product, wait, timeout)
+        def download_request(
+            product,
+            fs_path,
+            record_filename,
+            auth,
+            progress_callback,
+            ordered_message,
+            **kwargs
+        ):
+            params = kwargs.pop("dl_url_params", None) or getattr(
+                self.config, "dl_url_params", {}
+            )
+            with requests.get(
+                url,
+                stream=True,
+                auth=auth,
+                params=params,
+                timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
+            ) as self.stream:
                 try:
-                    params = kwargs.pop("dl_url_params", None) or getattr(
-                        self.config, "dl_url_params", {}
-                    )
-                    with requests.get(
-                        url,
-                        stream=True,
-                        auth=auth,
-                        params=params,
-                        timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
-                    ) as stream:
-                        try:
-                            stream.raise_for_status()
-                        except RequestException as e:
-                            # check if error is identified as auth_error in provider conf
-                            auth_errors = getattr(
-                                self.config, "auth_error_code", [None]
+                    self.stream.raise_for_status()
+
+                except RequestException as e:
+                    # check if error is identified as auth_error in provider conf
+                    auth_errors = getattr(self.config, "auth_error_code", [None])
+                    if not isinstance(auth_errors, list):
+                        auth_errors = [auth_errors]
+                    if e.response.status_code in auth_errors:
+                        raise AuthenticationError(
+                            "HTTP Error %s returned, %s\nPlease check your credentials for %s"
+                            % (
+                                e.response.status_code,
+                                e.response.text.strip(),
+                                self.provider,
                             )
-                            if not isinstance(auth_errors, list):
-                                auth_errors = [auth_errors]
-                            if e.response.status_code in auth_errors:
-                                raise AuthenticationError(
-                                    "HTTP Error %s returned, %s\nPlease check your credentials for %s"
-                                    % (
-                                        e.response.status_code,
-                                        e.response.text.strip(),
-                                        self.provider,
-                                    )
-                                )
-                            # product not available
-                            elif (
-                                product.properties.get("storageStatus", ONLINE_STATUS)
-                                != ONLINE_STATUS
-                            ):
-                                msg = (
-                                    ordered_message
-                                    if ordered_message and not e.response.text
-                                    else e.response.text
-                                )
-                                raise NotAvailableError(
-                                    "%s(initially %s) requested, returned: %s"
-                                    % (
-                                        product.properties["title"],
-                                        product.properties["storageStatus"],
-                                        msg,
-                                    )
-                                )
-                            else:
-                                import traceback as tb
-
-                                logger.error(
-                                    "Error while getting resource :\n%s",
-                                    tb.format_exc(),
-                                )
-                        else:
-                            stream_size = int(stream.headers.get("content-length", 0))
-                            if (
-                                stream_size == 0
-                                and "storageStatus" in product.properties
-                                and product.properties["storageStatus"] != ONLINE_STATUS
-                            ):
-                                raise NotAvailableError(
-                                    "%s(initially %s) ordered, got: %s"
-                                    % (
-                                        product.properties["title"],
-                                        product.properties["storageStatus"],
-                                        stream.reason,
-                                    )
-                                )
-                            progress_callback.reset(total=stream_size)
-                            with open(fs_path, "wb") as fhandle:
-                                for chunk in stream.iter_content(chunk_size=64 * 1024):
-                                    if chunk:
-                                        fhandle.write(chunk)
-                                        progress_callback(len(chunk))
-
-                            with open(record_filename, "w") as fh:
-                                fh.write(url)
-                            logger.debug("Download recorded in %s", record_filename)
-
-                            # Check that the downloaded file is really a zip file
-                            if not zipfile.is_zipfile(fs_path):
-                                logger.warning(
-                                    "Downloaded product is not a Zip File. Please check its file type before using it"
-                                )
-                                new_fs_path = fs_path[: fs_path.index(".zip")]
-                                shutil.move(fs_path, new_fs_path)
-                                product.location = path_to_uri(new_fs_path)
-                                return new_fs_path
-                            product_path = self._finalize(
-                                fs_path, progress_callback=progress_callback, **kwargs
+                        )
+                    # product not available
+                    elif (
+                        product.properties.get("storageStatus", ONLINE_STATUS)
+                        != ONLINE_STATUS
+                    ):
+                        msg = (
+                            ordered_message
+                            if ordered_message and not e.response.text
+                            else e.response.text
+                        )
+                        raise NotAvailableError(
+                            "%s(initially %s) requested, returned: %s"
+                            % (
+                                product.properties["title"],
+                                product.properties["storageStatus"],
+                                msg,
                             )
-                            product.location = path_to_uri(product_path)
-                            return product_path
+                        )
+                    else:
+                        import traceback as tb
 
-                except NotAvailableError as e:
-                    if not getattr(self.config, "order_enabled", False):
+                        logger.error(
+                            "Error while getting resource :\n%s",
+                            tb.format_exc(),
+                        )
+                else:
+                    stream_size = int(self.stream.headers.get("content-length", 0))
+                    if (
+                        stream_size == 0
+                        and "storageStatus" in product.properties
+                        and product.properties["storageStatus"] != ONLINE_STATUS
+                    ):
                         raise NotAvailableError(
-                            "Product is not available for download and order is not supported for %s, %s"
-                            % (self.provider, e)
+                            "%s(initially %s) ordered, got: %s"
+                            % (
+                                product.properties["title"],
+                                product.properties["storageStatus"],
+                                self.stream.reason,
+                            )
                         )
-                    not_available_info = e
-                    pass
-
-            if datetime.now() < product.next_try and datetime.now() < stop_time:
-                wait_seconds = (product.next_try - datetime.now()).seconds
-                retry_count += 1
-                retry_info = (
-                    "[Retry #%s] Waiting %ss until next download try for ordered product (retry every %s' for %s')"
-                    % (retry_count, wait_seconds, wait, timeout)
-                )
-                logger.debug(not_available_info)
-                # Retry-After info from Response header
-                retry_server_info = stream.headers.get("Retry-After", "")
-                if retry_server_info:
-                    logger.debug(
-                        "[%s response] Retry-After: %s"
-                        % (self.provider, retry_server_info)
+                    progress_callback.reset(total=stream_size)
+                    with open(fs_path, "wb") as fhandle:
+                        for chunk in self.stream.iter_content(chunk_size=64 * 1024):
+                            if chunk:
+                                fhandle.write(chunk)
+                                progress_callback(len(chunk))
+
+                    with open(record_filename, "w") as fh:
+                        fh.write(url)
+                    logger.debug("Download recorded in %s", record_filename)
+
+                    # Check that the downloaded file is really a zip file
+                    if not zipfile.is_zipfile(fs_path):
+                        logger.warning(
+                            "Downloaded product is not a Zip File. Please check its file type before using it"
+                        )
+                        new_fs_path = fs_path[: fs_path.index(".zip")]
+                        shutil.move(fs_path, new_fs_path)
+                        product.location = path_to_uri(new_fs_path)
+                        return new_fs_path
+                    product_path = self._finalize(
+                        fs_path, progress_callback=progress_callback, **kwargs
                     )
-                logger.info(retry_info)
-                nb_info.display_html(retry_info)
-                sleep(wait_seconds + 1)
-            elif datetime.now() >= stop_time and timeout > 0:
-                if "storageStatus" not in product.properties:
-                    product.properties["storageStatus"] = "N/A status"
-                logger.info(not_available_info)
-                raise NotAvailableError(
-                    "%s is not available (%s) and could not be downloaded, timeout reached"
-                    % (product.properties["title"], product.properties["storageStatus"])
-                )
-            elif datetime.now() >= stop_time:
-                raise NotAvailableError(not_available_info)
+                    product.location = path_to_uri(product_path)
+                    return product_path
+
+        return download_request(
+            product,
+            fs_path,
+            record_filename,
+            auth,
+            progress_callback,
+            ordered_message,
+            **kwargs
+        )
 
     def _download_assets(
         self,
diff --git a/setup.cfg b/setup.cfg
index 82532d1c..4e12e59f 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -59,7 +59,7 @@ install_requires =
     ecmwf-api-client
 
 [options.extras_require]
-dev = pytest; pytest-cov; pytest-html; tox; faker; moto; twine; wheel; flake8; pre-commit
+dev = pytest; pytest-cov; pytest-html; tox; faker; moto; twine; wheel; flake8; pre-commit; responses
 notebook = tqdm[notebook]
 tutorials = eodag-cube >= 0.2.0; jupyter; ipyleaflet >= 0.10.0; ipywidgets; matplotlib; folium; imageio; rasterio
 docs = sphinx; sphinx-book-theme >= 0.1.0; sphinx-copybutton; nbsphinx == 0.8.5; markupsafe < 2.1.0


Test patch:
diff --git a/tests/context.py b/tests/context.py
index ce8ddd59..cb294dfd 100644
--- a/tests/context.py
+++ b/tests/context.py
@@ -29,7 +29,11 @@ from eodag.api.core import DEFAULT_ITEMS_PER_PAGE, DEFAULT_MAX_ITEMS_PER_PAGE
 from eodag.api.product import EOProduct
 from eodag.api.product.drivers import DRIVERS
 from eodag.api.product.drivers.base import NoDriver
-from eodag.api.product.metadata_mapping import format_metadata
+from eodag.api.product.metadata_mapping import (
+    format_metadata,
+    OFFLINE_STATUS,
+    ONLINE_STATUS,
+)
 from eodag.api.search_result import SearchResult
 from eodag.cli import download, eodag, list_pt, search_crunch
 from eodag.config import (
@@ -71,6 +75,7 @@ from eodag.utils.exceptions import (
     DownloadError,
     MisconfiguredError,
     NoMatchingProductType,
+    NotAvailableError,
     PluginImplementationError,
     UnsupportedDatasetAddressScheme,
     UnsupportedProvider,
diff --git a/tests/units/test_download_plugins.py b/tests/units/test_download_plugins.py
index 80f70fb8..ba193542 100644
--- a/tests/units/test_download_plugins.py
+++ b/tests/units/test_download_plugins.py
@@ -16,13 +16,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import os
+import shutil
 import stat
 import unittest
 from pathlib import Path
-from tempfile import NamedTemporaryFile, TemporaryDirectory, gettempdir
+from tempfile import NamedTemporaryFile, TemporaryDirectory, gettempdir, mkdtemp
 from unittest import mock
 
-from tests.context import EOProduct, PluginManager, load_default_config, path_to_uri
+import responses
+
+from tests.context import (
+    OFFLINE_STATUS,
+    EOProduct,
+    NotAvailableError,
+    PluginManager,
+    load_default_config,
+    path_to_uri,
+)
 
 
 class BaseDownloadPluginTest(unittest.TestCase):
@@ -50,23 +60,31 @@ class BaseDownloadPluginTest(unittest.TestCase):
         cls.expanduser_mock.stop()
         cls.tmp_home_dir.cleanup()
 
-    def get_download_plugin(self, product):
-        return self.plugins_manager.get_download_plugin(product)
-
-    def get_auth_plugin(self, provider):
-        return self.plugins_manager.get_auth_plugin(provider)
-
-
-class TestDownloadPluginBase(BaseDownloadPluginTest):
     def setUp(self):
+        super(BaseDownloadPluginTest, self).setUp()
         self.product = EOProduct(
             "peps",
             dict(
                 geometry="POINT (0 0)",
                 title="dummy_product",
+                id="dummy",
             ),
         )
+        self.output_dir = mkdtemp()
+
+    def tearDown(self):
+        super(BaseDownloadPluginTest, self).tearDown()
+        if os.path.isdir(self.output_dir):
+            shutil.rmtree(self.output_dir)
+
+    def get_download_plugin(self, product):
+        return self.plugins_manager.get_download_plugin(product)
+
+    def get_auth_plugin(self, provider):
+        return self.plugins_manager.get_auth_plugin(provider)
 
+
+class TestDownloadPluginBase(BaseDownloadPluginTest):
     def test_plugins_download_base_prepare_download_existing(self):
         """Download._prepare_download must detect if product destination already exists"""
 
@@ -124,3 +142,175 @@ class TestDownloadPluginBase(BaseDownloadPluginTest):
                 self.product, outputs_prefix=outdir.name
             )
             self.assertIn("Unable to create records directory", str(cm.output))
+
+
+class TestDownloadPluginHttp(BaseDownloadPluginTest):
+    @mock.patch("eodag.plugins.download.http.requests.get", autospec=True)
+    def test_plugins_download_http_ok(self, mock_requests_get):
+        """HTTPDownload.download() must create an outputfile"""
+
+        plugin = self.get_download_plugin(self.product)
+        self.product.location = self.product.remote_location = "http://somewhere"
+        self.product.properties["id"] = "someproduct"
+
+        path = plugin.download(self.product, outputs_prefix=self.output_dir)
+
+        self.assertEqual(path, os.path.join(self.output_dir, "dummy_product"))
+        self.assertTrue(os.path.isfile(path))
+
+
+class TestDownloadPluginHttpRetry(BaseDownloadPluginTest):
+    def setUp(self):
+        super(TestDownloadPluginHttpRetry, self).setUp()
+
+        self.plugin = self.get_download_plugin(self.product)
+        self.product.location = self.product.remote_location = "http://somewhere"
+        self.product.properties["id"] = "someproduct"
+        self.product.properties["storageStatus"] = OFFLINE_STATUS
+
+    def test_plugins_download_http_retry_error_timeout(self):
+        """HTTPDownload.download() must retry on error until timeout"""
+
+        @responses.activate(registry=responses.registries.FirstMatchRegistry)
+        def run():
+            url = "http://somewhere/?issuerId=peps"
+            responses.add(responses.GET, url, status=500)
+
+            with self.assertRaisesRegex(NotAvailableError, r".*timeout reached"):
+                self.plugin.download(
+                    self.product,
+                    outputs_prefix=self.output_dir,
+                    wait=0.001 / 60,
+                    timeout=0.2 / 60,
+                )
+
+            # there must have been many retries
+            self.assertGreater(len(responses.calls), 2)
+            quick_retry_call_count = len(responses.calls)
+
+            # Same test with longer wait time
+            responses.calls.reset()
+            with self.assertRaisesRegex(NotAvailableError, r".*timeout reached"):
+                self.plugin.download(
+                    self.product,
+                    outputs_prefix=self.output_dir,
+                    wait=0.01 / 60,
+                    timeout=0.2 / 60,
+                )
+
+            # there must have been many retries, but less than before
+            self.assertGreater(len(responses.calls), 2)
+            self.assertLess(len(responses.calls), quick_retry_call_count)
+
+        run()
+
+    def test_plugins_download_http_retry_notready_timeout(self):
+        """HTTPDownload.download() must retry if not ready until timeout"""
+
+        @responses.activate(registry=responses.registries.FirstMatchRegistry)
+        def run():
+            url = "http://somewhere/?issuerId=peps"
+            responses.add(responses.GET, url, status=202)
+
+            with self.assertRaisesRegex(NotAvailableError, r".*timeout reached"):
+                self.plugin.download(
+                    self.product,
+                    outputs_prefix=self.output_dir,
+                    wait=0.001 / 60,
+                    timeout=0.2 / 60,
+                )
+
+            # there must have been many retries
+            self.assertGreater(len(responses.calls), 2)
+
+        run()
+
+    def test_plugins_download_http_retry_ok(self):
+        """HTTPDownload.download() must retry until request succeeds"""
+
+        @responses.activate(registry=responses.registries.OrderedRegistry)
+        def run():
+            # fail, then succeeds
+            url = "http://somewhere/?issuerId=peps"
+            responses.add(responses.GET, url, status=202)
+            responses.add(responses.GET, url, status=202)
+            responses.add(
+                responses.GET,
+                "http://somewhere/?issuerId=peps",
+                status=200,
+                content_type="application/octet-stream",
+                body=b"something",
+                auto_calculate_content_length=True,
+            )
+
+            self.plugin.download(
+                self.product,
+                outputs_prefix=self.output_dir,
+                wait=0.001 / 60,
+                timeout=0.2 / 60,
+            )
+
+            # there must have been 2 retries
+            self.assertEqual(len(responses.calls), 3)
+
+        run()
+
+    def test_plugins_download_http_retry_short_timeout(self):
+        """HTTPDownload.download() must not retry on very short timeout"""
+
+        @responses.activate(registry=responses.registries.FirstMatchRegistry)
+        def run():
+            url = "http://somewhere/?issuerId=peps"
+            responses.add(responses.GET, url, status=202)
+
+            with self.assertRaisesRegex(NotAvailableError, r".*timeout reached"):
+                self.plugin.download(
+                    self.product,
+                    outputs_prefix=self.output_dir,
+                    wait=0.1 / 60,
+                    timeout=0.001 / 60,
+                )
+
+            # there must have been only one try
+            self.assertEqual(len(responses.calls), 1)
+
+        run()
+
+    def test_plugins_download_http_retry_once_timeout(self):
+        """HTTPDownload.download() must retry once if wait time is equal to timeout"""
+
+        @responses.activate(registry=responses.registries.FirstMatchRegistry)
+        def run():
+            url = "http://somewhere/?issuerId=peps"
+            responses.add(responses.GET, url, status=202)
+
+            with self.assertRaisesRegex(NotAvailableError, r".*timeout reached"):
+                self.plugin.download(
+                    self.product,
+                    outputs_prefix=self.output_dir,
+                    wait=0.1 / 60,
+                    timeout=0.1 / 60,
+                )
+
+            # there must have been one retry
+            self.assertEqual(len(responses.calls), 2)
+
+        run()
+
+    def test_plugins_download_http_retry_timeout_disabled(self):
+        """HTTPDownload.download() must not retry on error if timeout is disabled"""
+
+        @responses.activate(registry=responses.registries.FirstMatchRegistry)
+        def run():
+            url = "http://somewhere/?issuerId=peps"
+            responses.add(responses.GET, url, status=202)
+
+            with self.assertRaisesRegex(NotAvailableError, r"^((?!timeout).)*$"):
+                self.plugin.download(
+                    self.product, outputs_prefix=self.output_dir, wait=-1, timeout=-1
+                )
+
+            # there must have been only one try
+            self.assertEqual(len(responses.calls), 1)
+
+        run()