Repository: CS-SI/eodag
Pull request #246 (2021-04-20 16:37:49)

Fix: missed to set limit=None in a whoosh query
Missed that in https://github.com/CS-SI/eodag/pull/201


Patch:
diff --git a/eodag/api/core.py b/eodag/api/core.py
index 6efceb2a..a53b8297 100644
--- a/eodag/api/core.py
+++ b/eodag/api/core.py
@@ -47,6 +47,7 @@ from eodag.utils import (
     _deprecated,
     get_geometry_from_various,
     makedirs,
+    uri_to_path,
 )
 from eodag.utils.exceptions import (
     NoMatchingProductType,
@@ -434,7 +435,7 @@ class EODataAccessGateway(object):
                 if results is None:
                     results = searcher.search(query, limit=None)
                 else:
-                    results.upgrade_and_extend(searcher.search(query))
+                    results.upgrade_and_extend(searcher.search(query, limit=None))
             guesses = [r["ID"] for r in results or []]
         if guesses:
             return guesses
@@ -1281,8 +1282,10 @@ class EODataAccessGateway(object):
         This is an alias to the method of the same name on
         :class:`~eodag.api.product.EOProduct`, but it performs some additional
         checks like verifying that a downloader and authenticator are registered
-        for the product before trying to download it. If the metadata mapping for
-        `downloadLink` is set to something that can be interpreted as a link on a
+        for the product before trying to download it.
+
+        If the metadata mapping for `downloadLink` is set to something that can be
+        interpreted as a link on a
         local filesystem, the download is skipped (by now, only a link starting
         with `file://` is supported). Therefore, any user that knows how to extract
         product location from product metadata on a provider can override the
@@ -1316,10 +1319,15 @@ class EODataAccessGateway(object):
         :rtype: str
         :raises: :class:`~eodag.utils.exceptions.PluginImplementationError`
         :raises: :class:`RuntimeError`
+
+        .. versionchanged:: 2.3.0
+
+           Returns a file system path instead of a file URI ('/tmp' instead of
+           'file:///tmp').
         """
-        if product.location.startswith("file"):
+        if product.location.startswith("file://"):
             logger.info("Local product detected. Download skipped")
-            return product.location
+            return uri_to_path(product.location)
         if product.downloader is None:
             auth = product.downloader_auth
             if auth is None:
diff --git a/eodag/api/product/_product.py b/eodag/api/product/_product.py
index a0d98184..a40e74bd 100644
--- a/eodag/api/product/_product.py
+++ b/eodag/api/product/_product.py
@@ -240,6 +240,11 @@ class EOProduct(object):
         :rtype: str
         :raises: :class:`~eodag.utils.exceptions.PluginImplementationError`
         :raises: :class:`RuntimeError`
+
+        .. versionchanged:: 2.3.0
+
+           Returns a file system path instead of a file URI ('/tmp' instead of
+           'file:///tmp').
         """
         if progress_callback is None:
             progress_callback = ProgressCallback()
@@ -254,7 +259,7 @@ class EOProduct(object):
             if self.downloader_auth is not None
             else self.downloader_auth
         )
-        fs_location = self.downloader.download(
+        fs_path = self.downloader.download(
             self,
             auth=auth,
             progress_callback=progress_callback,
@@ -262,13 +267,8 @@ class EOProduct(object):
             timeout=timeout,
             **kwargs
         )
-        if fs_location is None:
-            raise DownloadError(
-                "Invalid file location returned by download process: '{}'".format(
-                    fs_location
-                )
-            )
-        self.location = "file://{}".format(fs_location)
+        if fs_path is None:
+            raise DownloadError("Missing file location returned by download process")
         logger.debug(
             "Product location updated from '%s' to '%s'",
             self.remote_location,
@@ -279,7 +279,7 @@ class EOProduct(object):
             "'remote_location' property: %s",
             self.remote_location,
         )
-        return self.location
+        return fs_path
 
     def get_quicklook(self, filename=None, base_dir=None, progress_callback=None):
         """Download the quick look image of a given EOProduct from its provider if it
diff --git a/eodag/plugins/apis/base.py b/eodag/plugins/apis/base.py
index 6a775973..b6d385ef 100644
--- a/eodag/plugins/apis/base.py
+++ b/eodag/plugins/apis/base.py
@@ -18,12 +18,40 @@
 import logging
 
 from eodag.plugins.base import PluginTopic
+from eodag.plugins.download.base import DEFAULT_DOWNLOAD_TIMEOUT, DEFAULT_DOWNLOAD_WAIT
 
 logger = logging.getLogger("eodag.plugins.apis.base")
 
 
 class Api(PluginTopic):
-    """Plugins API Base plugin"""
+    """Plugins API Base plugin
+
+    An Api plugin has three download methods that it must implement:
+    - ``query``: search for products
+    - ``download``: download a single ``EOProduct``
+    - ``download_all``: download multiple products from a ``SearchResult``
+
+    The download methods must:
+    - download data in the ``outputs_prefix`` folder defined in the plugin's
+      configuration or passed through kwargs
+    - extract products from their archive (if relevant) if ``extract`` is set to True
+      (True by default)
+    - save a product in an archive/directory (in ``outputs_prefix``) whose name must be
+      the product's ``title`` property
+    - update the product's ``location`` attribute once its data is downloaded (and
+      eventually after it's extracted) to the product's location given as a file URI
+      (e.g. 'file:///tmp/product_folder' on Linux or
+      'file:///C:/Users/username/AppData/LOcal/Temp' on Windows)
+    - save a *record* file in the directory ``outputs_prefix/.downloaded`` whose name
+      is built on the MD5 hash of the product's ``remote_location`` attribute
+      (``hashlib.md5(remote_location.encode("utf-8")).hexdigest()``) and whose content is
+      the product's ``remote_location`` attribute itself.
+    - not try to download a product whose ``location`` attribute already points to an
+      existing file/directory
+    - not try to download a product if its *record* file exists as long as the expected
+      product's file/directory. If the *record* file only is found, it must be deleted
+      (it certainly indicates that the download didn't complete)
+    """
 
     def query(self, *args, count=True, **kwargs):
         """Implementation of how the products must be searched goes here.
@@ -32,14 +60,77 @@ class Api(PluginTopic):
         which will be processed by a Download plugin (2) and the total number of products matching
         the search criteria. If ``count`` is False, the second element returned must be ``None``.
 
-        .. versionchanged::
-            2.1
+        .. versionchanged:: 2.1
 
-                * A new optional boolean parameter ``count`` which defaults to ``True``, it
-                allows to trigger or not a count query.
+           A new optional boolean parameter ``count`` which defaults to ``True``, it
+           allows to trigger or not a count query.
         """
         raise NotImplementedError("A Api plugin must implement a method named query")
 
-    def download(self, *args, **kwargs):
-        """Implementation of how the products must be downloaded."""
-        raise NotImplementedError("A Api plugin must implement a method named download")
+    def download(
+        self,
+        product,
+        auth=None,
+        progress_callback=None,
+        wait=DEFAULT_DOWNLOAD_WAIT,
+        timeout=DEFAULT_DOWNLOAD_TIMEOUT,
+        **kwargs,
+    ):
+        r"""
+        Base download method. Not available, it must be defined for each plugin.
+
+        :param product: EO product to download
+        :type product: :class:`~eodag.api.product.EOProduct`
+        :param progress_callback: A progress callback
+        :type progress_callback: :class:`~eodag.utils.ProgressCallback`, optional
+        :param wait: If download fails, wait time in minutes between two download tries
+        :type wait: int, optional
+        :param timeout: If download fails, maximum time in minutes before stop retrying
+            to download
+        :type timeout: int, optional
+        :param dict kwargs: ``outputs_prefix` (``str``), `extract` (``bool``) and
+            ``dl_url_params`` (``dict``) can be provided as additional kwargs and will
+            override any other values defined in a configuration file or with
+            environment variables.
+        :returns: The absolute path to the downloaded product in the local filesystem
+            (e.g. '/tmp/product.zip' on Linux or
+            'C:\\Users\\username\\AppData\\Local\\Temp\\product.zip' on Windows)
+        :rtype: str
+        """
+        raise NotImplementedError(
+            "An Api plugin must implement a method named download"
+        )
+
+    def download_all(
+        self,
+        products,
+        auth=None,
+        progress_callback=None,
+        wait=DEFAULT_DOWNLOAD_WAIT,
+        timeout=DEFAULT_DOWNLOAD_TIMEOUT,
+        **kwargs,
+    ):
+        """
+        Base download_all method.
+
+        :param products: Products to download
+        :type products: :class:`~eodag.api.search_result.SearchResult`
+        :param progress_callback: A progress callback
+        :type progress_callback: :class:`~eodag.utils.ProgressCallback`, optional
+        :param wait: If download fails, wait time in minutes between two download tries
+        :type wait: int, optional
+        :param timeout: If download fails, maximum time in minutes before stop retrying
+            to download
+        :type timeout: int, optional
+        :param dict kwargs: ``outputs_prefix` (``str``), `extract` (``bool``) and
+            ``dl_url_params`` (``dict``) can be provided as additional kwargs and will
+            override any other values defined in a configuration file or with
+            environment variables.
+        :returns: List of absolute paths to the downloaded products in the local
+            filesystem (e.g. ``['/tmp/product.zip']`` on Linux or
+            ``['C:\\Users\\username\\AppData\\Local\\Temp\\product.zip']`` on Windows)
+        :rtype: list
+        """
+        raise NotImplementedError(
+            "A Api plugin must implement a method named download_all"
+        )
diff --git a/eodag/plugins/apis/usgs.py b/eodag/plugins/apis/usgs.py
index 37ce4443..768778e5 100644
--- a/eodag/plugins/apis/usgs.py
+++ b/eodag/plugins/apis/usgs.py
@@ -37,7 +37,12 @@ from eodag.plugins.download.base import (
     DEFAULT_DOWNLOAD_WAIT,
     Download,
 )
-from eodag.utils import GENERIC_PRODUCT_TYPE, format_dict_items, get_progress_callback
+from eodag.utils import (
+    GENERIC_PRODUCT_TYPE,
+    format_dict_items,
+    get_progress_callback,
+    path_to_uri,
+)
 from eodag.utils.exceptions import AuthenticationError, NotAvailableError
 
 logger = logging.getLogger("eodag.plugins.apis.usgs")
@@ -164,6 +169,8 @@ class UsgsApi(Api, Download):
             product, outputs_extension=".tar.gz", **kwargs
         )
         if not fs_path or not record_filename:
+            if fs_path:
+                product.location = path_to_uri(fs_path)
             return fs_path
 
         # progress bar init
@@ -264,8 +271,11 @@ class UsgsApi(Api, Download):
             )
             new_fs_path = fs_path[: fs_path.index(".tar.gz")]
             shutil.move(fs_path, new_fs_path)
+            product.location = path_to_uri(new_fs_path)
             return new_fs_path
-        return self._finalize(fs_path, outputs_extension=".tar.gz", **kwargs)
+        product_path = self._finalize(fs_path, outputs_extension=".tar.gz", **kwargs)
+        product.location = path_to_uri(product_path)
+        return product_path
 
     def download_all(
         self,
diff --git a/eodag/plugins/download/aws.py b/eodag/plugins/download/aws.py
index 2f0b132d..687985c6 100644
--- a/eodag/plugins/download/aws.py
+++ b/eodag/plugins/download/aws.py
@@ -34,7 +34,7 @@ from eodag.api.product.metadata_mapping import (
     properties_from_xml,
 )
 from eodag.plugins.download.base import Download
-from eodag.utils import get_progress_callback, urlparse
+from eodag.utils import get_progress_callback, path_to_uri, urlparse
 from eodag.utils.exceptions import AuthenticationError, DownloadError
 
 logger = logging.getLogger("eodag.plugins.download.aws")
@@ -185,6 +185,8 @@ class AwsDownload(Download):
         # prepare download & create dirs (before updating metadata)
         product_local_path, record_filename = self._prepare_download(product, **kwargs)
         if not product_local_path or not record_filename:
+            if product_local_path:
+                product.location = path_to_uri(product_local_path)
             return product_local_path
         product_local_path = product_local_path.replace(".zip", "")
         # remove existing incomplete file
@@ -395,6 +397,7 @@ class AwsDownload(Download):
             fh.write(product.remote_location)
         logger.debug("Download recorded in %s", record_filename)
 
+        product.location = path_to_uri(product_local_path)
         return product_local_path
 
     def get_authenticated_objects(self, bucket_name, prefix, auth_dict):
diff --git a/eodag/plugins/download/base.py b/eodag/plugins/download/base.py
index 8d6dbd3e..93001e9a 100644
--- a/eodag/plugins/download/base.py
+++ b/eodag/plugins/download/base.py
@@ -25,7 +25,7 @@ from datetime import datetime, timedelta
 from time import sleep
 
 from eodag.plugins.base import PluginTopic
-from eodag.utils import get_progress_callback, sanitize
+from eodag.utils import get_progress_callback, sanitize, uri_to_path
 from eodag.utils.exceptions import (
     AuthenticationError,
     MisconfiguredError,
@@ -43,6 +43,31 @@ DEFAULT_DOWNLOAD_TIMEOUT = 20
 class Download(PluginTopic):
     """Base Download Plugin.
 
+    A Download plugin has two download methods that it must implement:
+    - ``download``: download a single ``EOProduct``
+    - ``download_all``: download multiple products from a ``SearchResult``
+
+    They must:
+    - download data in the ``outputs_prefix`` folder defined in the plugin's
+      configuration or passed through kwargs
+    - extract products from their archive (if relevant) if ``extract`` is set to True
+      (True by default)
+    - save a product in an archive/directory (in ``outputs_prefix``) whose name must be
+      the product's ``title`` property
+    - update the product's ``location`` attribute once its data is downloaded (and
+      eventually after it's extracted) to the product's location given as a file URI
+      (e.g. 'file:///tmp/product_folder' on Linux or
+      'file:///C:/Users/username/AppData/LOcal/Temp' on Windows)
+    - save a *record* file in the directory ``outputs_prefix/.downloaded`` whose name
+      is built on the MD5 hash of the product's ``remote_location`` attribute
+      (``hashlib.md5(remote_location.encode("utf-8")).hexdigest()``) and whose content is
+      the product's ``remote_location`` attribute itself.
+    - not try to download a product whose ``location`` attribute already points to an
+      existing file/directory
+    - not try to download a product if its *record* file exists as long as the expected
+      product's file/directory. If the *record* file only is found, it must be deleted
+      (it certainly indicates that the download didn't complete)
+
     :param provider: An eodag providers configuration dictionary
     :type provider: dict
     :param config: Path to the user configuration file
@@ -62,8 +87,26 @@ class Download(PluginTopic):
         timeout=DEFAULT_DOWNLOAD_TIMEOUT,
         **kwargs,
     ):
-        """
-        Base download method. Not available, should be defined for each plugin
+        r"""
+        Base download method. Not available, it must be defined for each plugin.
+
+        :param product: EO product to download
+        :type product: :class:`~eodag.api.product.EOProduct`
+        :param progress_callback: A progress callback
+        :type progress_callback: :class:`~eodag.utils.ProgressCallback`, optional
+        :param wait: If download fails, wait time in minutes between two download tries
+        :type wait: int, optional
+        :param timeout: If download fails, maximum time in minutes before stop retrying
+            to download
+        :type timeout: int, optional
+        :param dict kwargs: ``outputs_prefix` (``str``), `extract` (``bool``) and
+            ``dl_url_params`` (``dict``) can be provided as additional kwargs and will
+            override any other values defined in a configuration file or with
+            environment variables.
+        :returns: The absolute path to the downloaded product in the local filesystem
+            (e.g. '/tmp/product.zip' on Linux or
+            'C:\\Users\\username\\AppData\\Local\\Temp\\product.zip' on Windows)
+        :rtype: str
         """
         raise NotImplementedError(
             "A Download plugin must implement a method named download"
@@ -78,8 +121,7 @@ class Download(PluginTopic):
         :rtype: tuple
         """
         if product.location != product.remote_location:
-            scheme_prefix_len = len("file://")
-            fs_path = product.location[scheme_prefix_len:]
+            fs_path = uri_to_path(product.location)
             # The fs path of a product is either a file (if 'extract' config is False) or a directory
             if os.path.isfile(fs_path) or os.path.isdir(fs_path):
                 logger.info(
@@ -249,8 +291,28 @@ class Download(PluginTopic):
         **kwargs,
     ):
         """
-        A sequential download_all implementation
-        using download method for every products
+        Base download_all method.
+
+        This specific implementation uses the ``download`` method implemented by
+        the plugin to **sequentially** attempt to download products.
+
+        :param products: Products to download
+        :type products: :class:`~eodag.api.search_result.SearchResult`
+        :param progress_callback: A progress callback
+        :type progress_callback: :class:`~eodag.utils.ProgressCallback`, optional
+        :param wait: If download fails, wait time in minutes between two download tries
+        :type wait: int, optional
+        :param timeout: If download fails, maximum time in minutes before stop retrying
+            to download
+        :type timeout: int, optional
+        :param dict kwargs: ``outputs_prefix` (``str``), `extract` (``bool``) and
+            ``dl_url_params`` (``dict``) can be provided as additional kwargs and will
+            override any other values defined in a configuration file or with
+            environment variables.
+        :returns: List of absolute paths to the downloaded products in the local
+            filesystem (e.g. ``['/tmp/product.zip']`` on Linux or
+            ``['C:\\Users\\username\\AppData\\Local\\Temp\\product.zip']`` on Windows)
+        :rtype: list
         """
         paths = []
         # initiate retry loop
diff --git a/eodag/plugins/download/http.py b/eodag/plugins/download/http.py
index 11662a18..f54fff86 100644
--- a/eodag/plugins/download/http.py
+++ b/eodag/plugins/download/http.py
@@ -32,7 +32,7 @@ from eodag.plugins.download.base import (
     DEFAULT_DOWNLOAD_WAIT,
     Download,
 )
-from eodag.utils import get_progress_callback
+from eodag.utils import get_progress_callback, path_to_uri
 from eodag.utils.exceptions import (
     AuthenticationError,
     MisconfiguredError,
@@ -70,6 +70,8 @@ class HTTPDownload(Download):
         """
         fs_path, record_filename = self._prepare_download(product, **kwargs)
         if not fs_path or not record_filename:
+            if fs_path:
+                product.location = path_to_uri(fs_path)
             return fs_path
 
         # progress bar init
@@ -80,7 +82,7 @@ class HTTPDownload(Download):
 
         # download assets if exist instead of remote_location
         try:
-            return self._download_assets(
+            fs_path = self._download_assets(
                 product,
                 fs_path.replace(".zip", ""),
                 record_filename,
@@ -88,6 +90,8 @@ class HTTPDownload(Download):
                 progress_callback,
                 **kwargs
             )
+            product.location = path_to_uri(fs_path)
+            return fs_path
         except NotAvailableError:
             pass
 
@@ -218,8 +222,11 @@ class HTTPDownload(Download):
                                 )
                                 new_fs_path = fs_path[: fs_path.index(".zip")]
                                 shutil.move(fs_path, new_fs_path)
+                                product.location = path_to_uri(new_fs_path)
                                 return new_fs_path
-                            return self._finalize(fs_path, **kwargs)
+                            product_path = self._finalize(fs_path, **kwargs)
+                            product.location = path_to_uri(product_path)
+                            return product_path
 
                 except NotAvailableError as e:
                     if not getattr(self.config, "order_enabled", False):
diff --git a/eodag/plugins/download/s3rest.py b/eodag/plugins/download/s3rest.py
index 71876ea4..2f5628b9 100644
--- a/eodag/plugins/download/s3rest.py
+++ b/eodag/plugins/download/s3rest.py
@@ -29,7 +29,7 @@ from requests import HTTPError
 from eodag.api.product.metadata_mapping import OFFLINE_STATUS
 from eodag.plugins.download.aws import AwsDownload
 from eodag.plugins.download.http import HTTPDownload
-from eodag.utils import get_progress_callback, urljoin
+from eodag.utils import get_progress_callback, path_to_uri, urljoin
 from eodag.utils.exceptions import (
     AuthenticationError,
     DownloadError,
@@ -158,6 +158,7 @@ class S3RestDownload(AwsDownload):
         url_hash = hashlib.md5(product.remote_location.encode("utf-8")).hexdigest()
         record_filename = os.path.join(download_records_dir, url_hash)
         if os.path.isfile(record_filename) and os.path.exists(product_local_path):
+            product.location = path_to_uri(product_local_path)
             return product_local_path
         # Remove the record file if product_local_path is absent (e.g. it was deleted while record wasn't)
         elif os.path.isfile(record_filename):
@@ -215,4 +216,5 @@ class S3RestDownload(AwsDownload):
             fh.write(product.remote_location)
         logger.debug("Download recorded in %s", record_filename)
 
+        product.location = path_to_uri(product_local_path)
         return product_local_path
diff --git a/eodag/rest/utils.py b/eodag/rest/utils.py
index f48e3eb0..2c6f7e9d 100644
--- a/eodag/rest/utils.py
+++ b/eodag/rest/utils.py
@@ -463,7 +463,7 @@ def download_stac_item_by_id(catalogs, item_id, provider=None):
 
     product_path = eodag_api.download(product)
 
-    return product_path.replace("file://", "")
+    return product_path
 
 
 def get_stac_catalogs(url, root="/", catalogs=[], provider=None):
diff --git a/eodag/utils/__init__.py b/eodag/utils/__init__.py
index 70798110..444b9e71 100644
--- a/eodag/utils/__init__.py
+++ b/eodag/utils/__init__.py
@@ -35,6 +35,7 @@ import warnings
 from collections import defaultdict
 from datetime import datetime, timezone
 from itertools import repeat, starmap
+from pathlib import Path
 
 # All modules using these should import them from utils package
 from urllib.parse import (  # noqa; noqa
@@ -43,8 +44,10 @@ from urllib.parse import (  # noqa; noqa
     urlencode,
     urljoin,
     urlparse,
+    urlsplit,
     urlunparse,
 )
+from urllib.request import url2pathname
 
 import click
 import shapefile
@@ -203,6 +206,24 @@ def strip_accents(s):
     )
 
 
+def uri_to_path(uri):
+    """
+    Convert a file URI (e.g. 'file:///tmp') to a local path (e.g. '/tmp')
+    """
+    if not uri.startswith("file"):
+        raise ValueError("A file URI must be provided (e.g. 'file:///tmp'")
+    _, _, path, _, _ = urlsplit(uri)
+    # On Windows urlsplit returns the path starting with a slash ('/C:/User)
+    path = url2pathname(path)
+    # url2pathname removes it
+    return path
+
+
+def path_to_uri(path):
+    """Convert a local absolute path to a file URI"""
+    return Path(path).as_uri()
+
+
 def mutate_dict_in_place(func, mapping):
     """Apply func to values of mapping.
 


Test patch:
diff --git a/tests/context.py b/tests/context.py
index 96807bc8..df74276a 100644
--- a/tests/context.py
+++ b/tests/context.py
@@ -44,7 +44,13 @@ from eodag.plugins.manager import PluginManager
 from eodag.plugins.search.base import Search
 from eodag.rest import server as eodag_http_server
 from eodag.rest.utils import eodag_api, get_date
-from eodag.utils import get_geometry_from_various, get_timestamp, makedirs
+from eodag.utils import (
+    get_geometry_from_various,
+    get_timestamp,
+    makedirs,
+    path_to_uri,
+    uri_to_path,
+)
 from eodag.utils.exceptions import (
     AddressNotFound,
     AuthenticationError,
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 94e07cc3..a4171dbb 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -427,7 +427,7 @@ class TestEodagCli(unittest.TestCase):
             TEST_RESOURCES_PATH, "eodag_search_result.geojson"
         )
         config_path = os.path.join(TEST_RESOURCES_PATH, "file_config_override.yml")
-        dag.return_value.download_all.return_value = ["file:///fake_path"]
+        dag.return_value.download_all.return_value = ["/fake_path"]
         result = self.runner.invoke(
             eodag,
             ["download", "--search-results", search_results_path, "-f", config_path],
@@ -435,7 +435,7 @@ class TestEodagCli(unittest.TestCase):
         dag.assert_called_once_with(user_conf_file_path=config_path)
         dag.return_value.deserialize.assert_called_once_with(search_results_path)
         self.assertEqual(dag.return_value.download_all.call_count, 1)
-        self.assertEqual("Downloaded file:///fake_path\n", result.output)
+        self.assertEqual("Downloaded /fake_path\n", result.output)
 
         # Testing the case when no downloaded path is returned
         dag.return_value.download_all.return_value = [None]
diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py
index be2dee90..ca0a145c 100644
--- a/tests/test_end_to_end.py
+++ b/tests/test_end_to_end.py
@@ -18,15 +18,22 @@
 
 import datetime
 import glob
+import hashlib
 import multiprocessing
 import os
 import shutil
+import time
 import unittest
 from pathlib import Path
 
 from eodag.api.product.metadata_mapping import ONLINE_STATUS
 from tests import TEST_RESOURCES_PATH, TESTS_DOWNLOAD_PATH
-from tests.context import AuthenticationError, EODataAccessGateway
+from tests.context import (
+    AuthenticationError,
+    EODataAccessGateway,
+    SearchResult,
+    uri_to_path,
+)
 
 THEIA_SEARCH_ARGS = [
     "theia",
@@ -148,8 +155,8 @@ class EndToEndBase(unittest.TestCase):
         """
         search_criteria = {
             "productType": product_type,
-            "startTimeFromAscendingNode": start,
-            "completionTimeFromAscendingNode": end,
+            "start": start,
+            "end": end,
             "geom": geom,
         }
         if items_per_page:
@@ -190,8 +197,8 @@ class EndToEndBase(unittest.TestCase):
         - Return all the products
         """
         search_criteria = {
-            "startTimeFromAscendingNode": start,
-            "completionTimeFromAscendingNode": end,
+            "start": start,
+            "end": end,
             "geom": geom,
         }
         self.eodag.set_preferred_provider(provider)
@@ -428,6 +435,196 @@ class TestEODagEndToEnd(EndToEndBase):
         self.assertGreater(len(results), 10)
 
 
+class TestEODagEndToEndComplete(unittest.TestCase):
+    """Make real and complete test cases that search for products, download them and
+    extract them. There should be just a tiny number of these tests which can be quite
+    long to run.
+
+    There must be a user conf file in the test resources folder named user_conf.yml
+    """
+
+    @classmethod
+    def setUpClass(cls):
+
+        # use tests/resources/user_conf.yml if exists else default file ~/.config/eodag/eodag.yml
+        tests_user_conf = os.path.join(TEST_RESOURCES_PATH, "user_conf.yml")
+        if not os.path.isfile(tests_user_conf):
+            unittest.SkipTest("Missing user conf file with credentials")
+        cls.eodag = EODataAccessGateway(
+            user_conf_file_path=os.path.join(TEST_RESOURCES_PATH, "user_conf.yml")
+        )
+
+        # create TESTS_DOWNLOAD_PATH is not exists
+        if not os.path.exists(TESTS_DOWNLOAD_PATH):
+            os.makedirs(TESTS_DOWNLOAD_PATH)
+
+        for provider, conf in cls.eodag.providers_config.items():
+            # Change download directory to TESTS_DOWNLOAD_PATH for tests
+            if hasattr(conf, "download") and hasattr(conf.download, "outputs_prefix"):
+                conf.download.outputs_prefix = TESTS_DOWNLOAD_PATH
+            elif hasattr(conf, "api") and hasattr(conf.api, "outputs_prefix"):
+                conf.api.outputs_prefix = TESTS_DOWNLOAD_PATH
+            else:
+                # no outputs_prefix found for provider
+                pass
+            # Force all providers implementing RestoSearch and defining how to retrieve
+            # products by specifying the
+            # location scheme to use https, enabling actual downloading of the product
+            if (
+                getattr(getattr(conf, "search", {}), "product_location_scheme", "https")
+                == "file"
+            ):
+                conf.search.product_location_scheme = "https"
+
+    def tearDown(self):
+        """Clear the test directory"""
+        for p in Path(TESTS_DOWNLOAD_PATH).glob("**/*"):
+            try:
+                os.remove(p)
+            except OSError:
+                shutil.rmtree(p)
+
+    def test_end_to_end_complete_peps(self):
+        """Complete end-to-end test with PEPS for download and download_all"""
+        # Search for products that are ONLINE and as small as possible
+        today = datetime.date.today()
+        month_span = datetime.timedelta(weeks=4)
+        search_results, _ = self.eodag.search(
+            productType="S2_MSI_L1C",
+            start=(today - month_span).isoformat(),
+            end=today.isoformat(),
+            geom={"lonmin": 1, "latmin": 42, "lonmax": 5, "latmax": 46},
+            items_per_page=100,
+        )
+        prods_sorted_by_size = SearchResult(
+            sorted(search_results, key=lambda p: p.properties["resourceSize"])
+        )
+        prods_online = [
+            p for p in prods_sorted_by_size if p.properties["storageStatus"] == "ONLINE"
+        ]
+        if len(prods_online) < 2:
+            unittest.skip(
+                "Not enough ONLINE products found, update the search criteria."
+            )
+
+        # Retrieve one product to work with
+        product = prods_online[0]
+
+        prev_remote_location = product.remote_location
+        prev_location = product.location
+        # The expected product's archive filename is based on the product's title
+        expected_product_name = f"{product.properties['title']}.zip"
+
+        # Download the product, but DON'T extract it
+        archive_file_path = self.eodag.download(product, extract=False)
+
+        # The archive must have been downloaded
+        self.assertTrue(os.path.isfile(archive_file_path))
+        # Its name must be the "{product_title}.zip"
+        self.assertIn(
+            expected_product_name, os.listdir(product.downloader.config.outputs_prefix)
+        )
+        # Its size should be >= 5 KB
+        archive_size = os.stat(archive_file_path).st_size
+        self.assertGreaterEqual(archive_size, 5 * 2 ** 10)
+        # The product remote_location should be the same
+        self.assertEqual(prev_remote_location, product.remote_location)
+        # However its location should have been update
+        self.assertNotEqual(prev_location, product.location)
+        # The location must follow the file URI scheme
+        self.assertTrue(product.location.startswith("file://"))
+        # That points to the downloaded archive
+        self.assertEqual(uri_to_path(product.location), archive_file_path)
+        # A .downloaded folder must have been created
+        record_dir = os.path.join(TESTS_DOWNLOAD_PATH, ".downloaded")
+        self.assertTrue(os.path.isdir(record_dir))
+        # It must contain a file per product downloade, whose name is
+        # the MD5 hash of the product's remote location
+        expected_hash = hashlib.md5(product.remote_location.encode("utf-8")).hexdigest()
+        record_file = os.path.join(record_dir, expected_hash)
+        self.assertTrue(os.path.isfile(record_file))
+        # Its content must be the product's remote location
+        record_content = Path(record_file).read_text()
+        self.assertEqual(record_content, product.remote_location)
+
+        # The product should not be downloaded again if the download method
+        # is executed again
+        previous_archive_file_path = archive_file_path
+        previous_location = product.location
+        start_time = time.time()
+        archive_file_path = self.eodag.download(product, extract=False)
+        end_time = time.time()
+        self.assertLess(end_time - start_time, 2)  # Should be really fast (< 2s)
+        # The paths should be the same as before
+        self.assertEqual(archive_file_path, previous_archive_file_path)
+        self.assertEqual(product.location, previous_location)
+
+        # If we emulate that the product has just been found, it should not
+        # be downloaded again since the record file is still present.
+        product.location = product.remote_location
+        # Pretty much the same checks as with the previous step
+        previous_archive_file_path = archive_file_path
+        start_time = time.time()
+        archive_file_path = self.eodag.download(product, extract=False)
+        end_time = time.time()
+        self.assertLess(end_time - start_time, 2)  # Should be really fast (< 2s)
+        # The returned path should be the same as before
+        self.assertEqual(archive_file_path, previous_archive_file_path)
+        self.assertEqual(uri_to_path(product.location), archive_file_path)
+
+        # Remove the archive
+        os.remove(archive_file_path)
+
+        # Now, the archive is removed but its associated record file
+        # still exists. Downloading the product again should really
+        # download it, if its location points to the remote location.
+        # The product should be automatically extracted.
+        product.location = product.remote_location
+        product_dir_path = self.eodag.download(product)
+
+        # Its size should be >= 5 KB
+        downloaded_size = sum(
+            f.stat().st_size for f in Path(product_dir_path).glob("**/*") if f.is_file()
+        )
+        self.assertGreaterEqual(downloaded_size, 5 * 2 ** 10)
+        # The product remote_location should be the same
+        self.assertEqual(prev_remote_location, product.remote_location)
+        # However its location should have been update
+        self.assertNotEqual(prev_location, product.location)
+        # The location must follow the file URI scheme
+        self.assertTrue(product.location.startswith("file://"))
+        # The location must point to a SAFE directory
+        self.assertTrue(product.location.endswith("SAFE"))
+        # The path must point to a SAFE directory
+        self.assertTrue(os.path.isdir(product_dir_path))
+        self.assertTrue(product_dir_path.endswith("SAFE"))
+
+        # Remove the archive and extracted product and reset the product's location
+        os.remove(archive_file_path)
+        shutil.rmtree(Path(product_dir_path).parent)
+        product.location = product.remote_location
+
+        # Now let's check download_all
+        products = prods_sorted_by_size[:2]
+        # Pass a copy because download_all empties the list
+        archive_paths = self.eodag.download_all(products[:], extract=False)
+
+        # The returned paths must point to the downloaded archives
+        # Each product's location must be a URI path to the archive
+        for product, archive_path in zip(products, archive_paths):
+            self.assertTrue(os.path.isfile(archive_path))
+            self.assertEqual(uri_to_path(product.location), archive_path)
+
+        # Downloading the product again should not download them, since
+        # they are all already there.
+        prev_archive_paths = archive_paths
+        start_time = time.time()
+        archive_paths = self.eodag.download_all(products[:], extract=False)
+        end_time = time.time()
+        self.assertLess(end_time - start_time, 2)  # Should be really fast (< 2s)
+        self.assertEqual(archive_paths, prev_archive_paths)
+
+
 # @unittest.skip("skip auto run")
 class TestEODagEndToEndWrongCredentials(EndToEndBase):
     """Make real case tests with wrong credentials. This assumes the existence of a
@@ -462,7 +659,7 @@ class TestEODagEndToEndWrongCredentials(EndToEndBase):
                 raise_errors=True,
                 **dict(
                     zip(["productType", "start", "end", "geom"], AWSEOS_SEARCH_ARGS[1:])
-                )
+                ),
             )
 
     def test_end_to_end_good_apikey_wrong_credentials_aws_eos(self):
@@ -489,7 +686,7 @@ class TestEODagEndToEndWrongCredentials(EndToEndBase):
                 raise_errors=True,
                 **dict(
                     zip(["productType", "start", "end", "geom"], AWSEOS_SEARCH_ARGS[1:])
-                )
+                ),
             )
             self.assertGreater(len(results), 0)
             one_product = results[0]
@@ -525,5 +722,5 @@ class TestEODagEndToEndWrongCredentials(EndToEndBase):
                 raise_errors=True,
                 **dict(
                     zip(["productType", "start", "end", "geom"], USGS_SEARCH_ARGS[1:])
-                )
+                ),
             )
diff --git a/tests/units/test_eoproduct.py b/tests/units/test_eoproduct.py
index f43aaa84..9222648a 100644
--- a/tests/units/test_eoproduct.py
+++ b/tests/units/test_eoproduct.py
@@ -295,7 +295,6 @@ class TestEOProduct(EODagTestCase):
         self.requests_http_get.assert_called_with(
             self.download_url, stream=True, auth=None, params={}
         )
-        product_file_path = product_file_path[len("file://") :]
         download_records_dir = pathlib.Path(product_file_path).parent / ".downloaded"
         # A .downloaded folder should be created, including a text file that
         # lists the downloaded product by their url
@@ -332,7 +331,7 @@ class TestEOProduct(EODagTestCase):
 
         # Download
         product_dir_path = product.download()
-        product_dir_path = pathlib.Path(product_dir_path[len("file://") :])
+        product_dir_path = pathlib.Path(product_dir_path)
         # The returned path must be a directory.
         self.assertTrue(product_dir_path.is_dir())
         # Check that the extracted dir has at least one file, there are more
@@ -379,7 +378,7 @@ class TestEOProduct(EODagTestCase):
             self.download_url, stream=True, auth=None, params={"fakeparam": "dummy"}
         )
         # Check that "outputs_prefix" is respected.
-        product_dir_path = pathlib.Path(product_dir_path[len("file://") :])
+        product_dir_path = pathlib.Path(product_dir_path)
         self.assertEqual(product_dir_path.parent.name, output_dir_name)
         # We've asked to extract the product so there should be a directory.
         self.assertTrue(product_dir_path.is_dir())
diff --git a/tests/units/test_utils.py b/tests/units/test_utils.py
index 58732712..ef865dff 100644
--- a/tests/units/test_utils.py
+++ b/tests/units/test_utils.py
@@ -16,10 +16,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import sys
 import unittest
 from datetime import datetime
 
-from tests.context import get_timestamp
+from tests.context import get_timestamp, path_to_uri, uri_to_path
 
 
 class TestUtils(unittest.TestCase):
@@ -33,3 +34,21 @@ class TestUtils(unittest.TestCase):
         expected_dt = datetime.strptime(requested_date, date_format)
         actual_utc_dt = datetime.utcfromtimestamp(ts_in_secs)
         self.assertEqual(actual_utc_dt, expected_dt)
+
+    def test_uri_to_path(self):
+        if sys.platform == "win32":
+            expected_path = r"C:\tmp\file.txt"
+            tested_uri = r"file:///C:/tmp/file.txt"
+        else:
+            expected_path = "/tmp/file.txt"
+            tested_uri = "file:///tmp/file.txt"
+        actual_path = uri_to_path(tested_uri)
+        self.assertEqual(actual_path, expected_path)
+        with self.assertRaises(ValueError):
+            uri_to_path("not_a_uri")
+
+    def test_path_to_uri(self):
+        if sys.platform == "win32":
+            self.assertEqual(path_to_uri(r"C:\tmp\file.txt"), "file:///C:/tmp/file.txt")
+        else:
+            self.assertEqual(path_to_uri("/tmp/file.txt"), "file:///tmp/file.txt")