Repository: CS-SI/eodag

Issue:
Add S2_MSI_L1C productType to the USGS provider
When trying to download Sentinel2 images using the USGS provider, the "S2_MSI_L1C" product type is not recognized, even though it is available using the earth explorer website.

It would be great to add this one (and more precisely in my case I'd like that because I only got a bunch of failure when trying to do that using the peps provider...).

Discussion:
github-actions[bot]: ## Unit Test Results
    2 files  ±0      2 suites  ±0   7m 14s [:stopwatch:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "duration of all tests") -42s
285 tests +3  282 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "passed tests") +3  3 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "skipped / disabled tests") ±0  0 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "failed tests") ±0 
570 runs  +6  564 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "passed tests") +6  6 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "skipped / disabled tests") ±0  0 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "failed tests") ±0 

Results for commit 3ecaa30d. ± Comparison against base commit b57fc768.

[test-results]:data:application/gzip;base64,H4sIAEmtJGMC/03MSw7CIBSF4a00jB1A4YK4GcOrCbEthseoce8iCnb4fyc5B1r86hK6TfNlQqn4PMKWqLIPe01GWYU65TZeodc9FWMazX96+GclOmBRfq2AB7gYQ/xJLPvnE0SPfgmcDfk+8t6nw9bnPxO2zecaiDqjFMUWrKaOEOHAMqs5cImVkASYdMA11uj1BtZO9fMFAQAA

github-actions[bot]: <strong> </strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `83%` | :white_check_mark: |
| eodag/plugins/apis/usgs.py | `79%` | :white_check_mark: |
| eodag/plugins/download/base.py | `80%` | :white_check_mark: |
| tests/units/test\_apis\_plugins.py | `98%` | :white_check_mark: |
| tests/units/test\_core.py | `98%` | :white_check_mark: |

<strong> </strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `82%` | :white_check_mark: |
| eodag/plugins/apis/usgs.py | `79%` | :white_check_mark: |
| eodag/plugins/download/base.py | `80%` | :white_check_mark: |
| tests/units/test\_apis\_plugins.py | `98%` | :white_check_mark: |
| tests/units/test\_core.py | `98%` | :white_check_mark: |
| eodag/plugins/apis/usgs.py | `79%` | :white_check_mark: |
| eodag/plugins/download/base.py | `80%` | :white_check_mark: |
| tests/units/test\_apis\_plugins.py | `98%` | :white_check_mark: |
| tests/units/test\_core.py | `98%` | :white_check_mark: |

_Minimum allowed coverage is `70%`_

<p align="right">Generated by :monkey: cobertura-action against 3ecaa30d5db3e117e5d4db65690a791549e56b0b </p>

Patch:
diff --git a/eodag/plugins/apis/usgs.py b/eodag/plugins/apis/usgs.py
index 5a7162bd..ecf1b113 100644
--- a/eodag/plugins/apis/usgs.py
+++ b/eodag/plugins/apis/usgs.py
@@ -19,6 +19,7 @@ import copy
 import logging
 import shutil
 import tarfile
+import zipfile
 
 import requests
 from jsonpath_ng.ext import parse
@@ -35,7 +36,6 @@ from eodag.plugins.apis.base import Api
 from eodag.plugins.download.base import (
     DEFAULT_DOWNLOAD_TIMEOUT,
     DEFAULT_DOWNLOAD_WAIT,
-    DEFAULT_STREAM_REQUESTS_TIMEOUT,
     Download,
 )
 from eodag.utils import (
@@ -52,14 +52,11 @@ logger = logging.getLogger("eodag.plugins.apis.usgs")
 class UsgsApi(Download, Api):
     """A plugin that enables to query and download data on the USGS catalogues"""
 
-    def query(
-        self, product_type=None, items_per_page=None, page=None, count=True, **kwargs
-    ):
-        """Search for data on USGS catalogues"""
-        product_type = kwargs.get("productType")
-        if product_type is None:
-            return [], 0
+    def authenticate(self):
+        """Login to usgs api
 
+        :raises: :class:`~eodag.utils.exceptions.AuthenticationError`
+        """
         for i in range(2):
             try:
                 api.login(
@@ -77,6 +74,16 @@ class UsgsApi(Download, Api):
                     "Please check your USGS credentials."
                 ) from None
 
+    def query(
+        self, product_type=None, items_per_page=None, page=None, count=True, **kwargs
+    ):
+        """Search for data on USGS catalogues"""
+        product_type = kwargs.get("productType")
+        if product_type is None:
+            return [], 0
+
+        self.authenticate()
+
         product_type_def_params = self.config.products.get(
             product_type, self.config.products[GENERIC_PRODUCT_TYPE]
         )
@@ -108,8 +115,7 @@ class UsgsApi(Download, Api):
         else:
             lower_left, upper_right = None, None
         try:
-            results = api.scene_search(
-                usgs_dataset,
+            api_search_kwargs = dict(
                 start_date=start_date,
                 end_date=end_date,
                 ll=lower_left,
@@ -117,6 +123,29 @@ class UsgsApi(Download, Api):
                 max_results=items_per_page,
                 starting_number=(1 + (page - 1) * items_per_page),
             )
+            logger.info(
+                f"Sending search request for {usgs_dataset} with {api_search_kwargs}"
+            )
+
+            results = api.scene_search(usgs_dataset, **api_search_kwargs)
+
+            # update results with storage info from download_options()
+            results_by_entity_id = {
+                res["entityId"]: res for res in results["data"]["results"]
+            }
+            logger.debug(
+                f"Adapting {len(results_by_entity_id)} plugin results to eodag product representation"
+            )
+            download_options = api.download_options(
+                usgs_dataset, list(results_by_entity_id.keys())
+            )
+            if download_options.get("data", None) is not None:
+                for download_option in download_options["data"]:
+                    if "dds" in download_option["downloadSystem"]:
+                        results_by_entity_id[download_option["entityId"]].update(
+                            download_option
+                        )
+            results["data"]["results"] = list(results_by_entity_id.values())
 
             # Same method as in base.py, Search.__init__()
             # Prepare the metadata mapping
@@ -145,10 +174,9 @@ class UsgsApi(Download, Api):
                 )
         except USGSError as e:
             logger.warning(
-                "Product type %s does not exist on USGS EE catalog",
-                usgs_dataset,
+                f"Product type {usgs_dataset} does not exist on USGS EE catalog"
             )
-            logger.warning("Skipping error: %s", e)
+            logger.warning(f"Skipping error: {e}")
         api.logout()
 
         if final:
@@ -160,7 +188,15 @@ class UsgsApi(Download, Api):
 
         return final, total_results
 
-    def download(self, product, auth=None, progress_callback=None, **kwargs):
+    def download(
+        self,
+        product,
+        auth=None,
+        progress_callback=None,
+        wait=DEFAULT_DOWNLOAD_WAIT,
+        timeout=DEFAULT_DOWNLOAD_TIMEOUT,
+        **kwargs,
+    ):
         """Download data from USGS catalogues"""
 
         if progress_callback is None:
@@ -169,130 +205,133 @@ class UsgsApi(Download, Api):
             )
             progress_callback = ProgressCallback(disable=True)
 
+        outputs_extension = self.config.products.get(
+            product.product_type, self.config.products[GENERIC_PRODUCT_TYPE]
+        ).get("outputs_extension", ".tar.gz")
+
         fs_path, record_filename = self._prepare_download(
             product,
             progress_callback=progress_callback,
-            outputs_extension=".tar.gz",
-            **kwargs
+            outputs_extension=outputs_extension,
+            **kwargs,
         )
         if not fs_path or not record_filename:
             if fs_path:
                 product.location = path_to_uri(fs_path)
             return fs_path
 
-        try:
-            api.login(
-                getattr(self.config, "credentials", {}).get("username", ""),
-                getattr(self.config, "credentials", {}).get("password", ""),
-                save=True,
+        self.authenticate()
+
+        if "dds" in product.properties.get("downloadSystem", ""):
+            raise NotAvailableError(
+                f"No USGS products found for {product.properties['id']}"
             )
-        except USGSError:
-            raise AuthenticationError("Please check your USGS credentials.") from None
 
-        download_options = api.download_options(
-            product.properties["productType"], product.properties["id"]
+        download_request = api.download_request(
+            product.properties["productType"],
+            product.properties["entityId"],
+            product.properties["productId"],
         )
 
+        req_urls = []
         try:
-            product_ids = [
-                p["id"]
-                for p in download_options["data"]
-                if p["downloadSystem"] == "dds"
-            ]
+            if len(download_request["data"]["preparingDownloads"]) > 0:
+                req_urls.extend(
+                    [x["url"] for x in download_request["data"]["preparingDownloads"]]
+                )
+            else:
+                req_urls.extend(
+                    [x["url"] for x in download_request["data"]["availableDownloads"]]
+                )
         except KeyError as e:
             raise NotAvailableError(
-                "%s not found in %s's products" % (e, product.properties["id"])
+                f"{e} not found in {product.properties['id']} download_request"
             )
 
-        if not product_ids:
-            raise NotAvailableError(
-                "No USGS products found for %s" % product.properties["id"]
-            )
-
-        req_urls = []
-        for product_id in product_ids:
-            download_request = api.download_request(
-                product.properties["productType"], product.properties["id"], product_id
-            )
-            try:
-                if len(download_request["data"]["preparingDownloads"]) > 0:
-                    req_urls.extend(
-                        [
-                            x["url"]
-                            for x in download_request["data"]["preparingDownloads"]
-                        ]
-                    )
-                else:
-                    req_urls.extend(
-                        [
-                            x["url"]
-                            for x in download_request["data"]["availableDownloads"]
-                        ]
-                    )
-            except KeyError as e:
-                raise NotAvailableError(
-                    "%s not found in %s download_request"
-                    % (e, product.properties["id"])
-                )
-
         if len(req_urls) > 1:
             logger.warning(
-                "%s usgs products found for %s. Only first will be downloaded"
-                % (len(req_urls), product.properties["id"])
+                f"{len(req_urls)} usgs products found for {product.properties['id']}. Only first will be downloaded"
             )
         elif not req_urls:
             raise NotAvailableError(
-                "No usgs request url was found for %s" % product.properties["id"]
+                f"No usgs request url was found for {product.properties['id']}"
             )
 
         req_url = req_urls[0]
         progress_callback.reset()
-        with requests.get(
-            req_url,
-            stream=True,
-            timeout=DEFAULT_STREAM_REQUESTS_TIMEOUT,
-        ) as stream:
+        logger.debug(f"Downloading {req_url}")
+
+        @self._download_retry(product, wait, timeout)
+        def download_request(product, fs_path, progress_callback, **kwargs):
             try:
-                stream.raise_for_status()
-            except RequestException:
-                import traceback as tb
+                with requests.get(
+                    req_url,
+                    stream=True,
+                    timeout=wait * 60,
+                ) as stream:
+                    try:
+                        stream.raise_for_status()
+                    except RequestException:
+                        import traceback as tb
 
-                logger.error(
-                    "Error while getting resource :\n%s",
-                    tb.format_exc(),
-                )
-            else:
-                stream_size = int(stream.headers.get("content-length", 0))
-                progress_callback.reset(total=stream_size)
-                with open(fs_path, "wb") as fhandle:
-                    for chunk in stream.iter_content(chunk_size=64 * 1024):
-                        if chunk:
-                            fhandle.write(chunk)
-                            progress_callback(len(chunk))
+                        logger.error(
+                            f"Error while getting resource :\n{tb.format_exc()}",
+                        )
+                    else:
+                        stream_size = int(stream.headers.get("content-length", 0))
+                        progress_callback.reset(total=stream_size)
+                        with open(fs_path, "wb") as fhandle:
+                            for chunk in stream.iter_content(chunk_size=64 * 1024):
+                                if chunk:
+                                    fhandle.write(chunk)
+                                    progress_callback(len(chunk))
+            except requests.exceptions.Timeout as e:
+                raise NotAvailableError(str(e))
+
+        download_request(product, fs_path, progress_callback, **kwargs)
 
         with open(record_filename, "w") as fh:
             fh.write(product.properties["downloadLink"])
-        logger.debug("Download recorded in %s", record_filename)
+        logger.debug(f"Download recorded in {record_filename}")
 
         api.logout()
 
-        # Check that the downloaded file is really a tar file
-        if not tarfile.is_tarfile(fs_path):
+        # Check downloaded file format
+        if (outputs_extension == ".tar.gz" and tarfile.is_tarfile(fs_path)) or (
+            outputs_extension == ".zip" and zipfile.is_zipfile(fs_path)
+        ):
+            product_path = self._finalize(
+                fs_path,
+                progress_callback=progress_callback,
+                outputs_extension=outputs_extension,
+                **kwargs,
+            )
+            product.location = path_to_uri(product_path)
+            return product_path
+        elif tarfile.is_tarfile(fs_path):
+            logger.info(
+                "Downloaded product detected as a tar File, but was was expected to be a zip file"
+            )
+            new_fs_path = fs_path[: fs_path.index(outputs_extension)] + ".tar.gz"
+            shutil.move(fs_path, new_fs_path)
+            product.location = path_to_uri(new_fs_path)
+            return new_fs_path
+        elif zipfile.is_zipfile(fs_path):
+            logger.info(
+                "Downloaded product detected as a zip File, but was was expected to be a tar file"
+            )
+            new_fs_path = fs_path[: fs_path.index(outputs_extension)] + ".zip"
+            shutil.move(fs_path, new_fs_path)
+            product.location = path_to_uri(new_fs_path)
+            return new_fs_path
+        else:
             logger.warning(
-                "Downloaded product is not a tar File. Please check its file type before using it"
+                "Downloaded product is not a tar or a zip File. Please check its file type before using it"
             )
-            new_fs_path = fs_path[: fs_path.index(".tar.gz")]
+            new_fs_path = fs_path[: fs_path.index(outputs_extension)]
             shutil.move(fs_path, new_fs_path)
             product.location = path_to_uri(new_fs_path)
             return new_fs_path
-        product_path = self._finalize(
-            fs_path,
-            progress_callback=progress_callback,
-            outputs_extension=".tar.gz",
-            **kwargs
-        )
-        product.location = path_to_uri(product_path)
-        return product_path
 
     def download_all(
         self,
@@ -302,7 +341,7 @@ class UsgsApi(Download, Api):
         progress_callback=None,
         wait=DEFAULT_DOWNLOAD_WAIT,
         timeout=DEFAULT_DOWNLOAD_TIMEOUT,
-        **kwargs
+        **kwargs,
     ):
         """
         Download all using parent (base plugin) method
@@ -314,5 +353,5 @@ class UsgsApi(Download, Api):
             progress_callback=progress_callback,
             wait=wait,
             timeout=timeout,
-            **kwargs
+            **kwargs,
         )
diff --git a/eodag/plugins/download/base.py b/eodag/plugins/download/base.py
index 10130749..d3bd7b6e 100644
--- a/eodag/plugins/download/base.py
+++ b/eodag/plugins/download/base.py
@@ -81,7 +81,7 @@ class Download(PluginTopic):
 
     def __init__(self, provider, config):
         super(Download, self).__init__(provider, config)
-        self.authenticate = bool(getattr(self.config, "authenticate", False))
+        self._authenticate = bool(getattr(self.config, "authenticate", False))
 
     def download(
         self,
@@ -598,11 +598,14 @@ class Download(PluginTopic):
                         )
                         logger.debug(not_available_info)
                         # Retry-After info from Response header
-                        retry_server_info = self.stream.headers.get("Retry-After", "")
-                        if retry_server_info:
-                            logger.debug(
-                                f"[{self.provider} response] Retry-After: {retry_server_info}"
+                        if hasattr(self, "stream"):
+                            retry_server_info = self.stream.headers.get(
+                                "Retry-After", ""
                             )
+                            if retry_server_info:
+                                logger.debug(
+                                    f"[{self.provider} response] Retry-After: {retry_server_info}"
+                                )
                         logger.info(retry_info)
                         nb_info.display_html(retry_info)
                         sleep(wait_seconds)
diff --git a/eodag/resources/product_types.yml b/eodag/resources/product_types.yml
index e453de30..96a4eb02 100644
--- a/eodag/resources/product_types.yml
+++ b/eodag/resources/product_types.yml
@@ -187,6 +187,20 @@ LANDSAT_C2L1:
   title: Landsat Collection 2 Level-1 Product
   missionStartDate: "1972-07-25T00:00:00Z"
 
+LANDSAT_C2L2:
+  abstract: |
+    Collection 2 Landsat OLI/TIRS Level-2 Science Products (L2SP) include
+    Surface Reflectance and Surface Temperature scene-based products.
+  instrument: OLI,TIRS
+  platform: LANDSAT
+  platformSerialIdentifier: L8,L9
+  processingLevel: L1
+  keywords: OLI,TIRS,LANDSAT,L8,L9,L2,C2,COLLECTION2
+  sensorType: OPTICAL
+  license: proprietary
+  title: Landsat OLI and TIRS Collection 2 Level-2 Science Products 30-meter multispectral data.
+  missionStartDate: "2013-02-11T00:00:00Z"
+
 LANDSAT_C2L2_SR:
   abstract: |
     The Landsat Surface Reflectance (SR) product measures the fraction of incoming solar radiation that is reflected
@@ -269,6 +283,101 @@ LANDSAT_C2L2ALB_TA:
   title: Landsat Collection 2 Level-2 Albers Top of Atmosphere (TA) Reflectance Product
   missionStartDate: "1982-08-22T00:00:00Z"
 
+LANDSAT_TM_C1:
+  abstract: |
+    Landsat 4-5 TM image data files consist of seven spectral bands (See band designations).
+    The resolution is 30 meters for bands 1 to 7.
+    (Thermal infrared band 6 was collected at 120 meters, but was resampled to 30 meters.)
+    The approximate scene size is 170 km north-south by 183 km east-west (106 mi by 114 mi).
+  instrument: TM
+  platform: LANDSAT
+  platformSerialIdentifier: L4,L5
+  processingLevel: L1
+  keywords: TM,LANDSAT,L4,L5,L1,C1,COLLECTION1
+  sensorType: OPTICAL
+  license: proprietary
+  title: Landsat 4-5 Thematic Mapper (TM) Collection-1 Level-1 Data Products
+  missionStartDate: "1982-08-22T00:00:00Z"
+  missionEndDate: "2013-06-06T00:00:00Z"
+
+LANDSAT_TM_C2L1:
+  abstract: |
+    Landsat 4-5 TM image data files consist of seven spectral bands (See band designations).
+    The resolution is 30 meters for bands 1 to 7.
+    (Thermal infrared band 6 was collected at 120 meters, but was resampled to 30 meters.)
+    The approximate scene size is 170 km north-south by 183 km east-west (106 mi by 114 mi).
+  instrument: TM
+  platform: LANDSAT
+  platformSerialIdentifier: L4,L5
+  processingLevel: L1
+  keywords: TM,LANDSAT,L4,L5,L1,C2,COLLECTION2
+  sensorType: OPTICAL
+  license: proprietary
+  title: Landsat 4-5 Thematic Mapper (TM) Collection-2 Level-1 Data Products
+  missionStartDate: "1982-08-22T00:00:00Z"
+  missionEndDate: "2013-06-06T00:00:00Z"
+
+LANDSAT_TM_C2L2:
+  abstract: |
+    Collection 2 Landsat 4-5 Thematic Mapper (TM) Level-2 Science Products (L2SP) include
+    Surface Reflectance and Surface Temperature scene-based products.
+  instrument: TM
+  platform: LANDSAT
+  platformSerialIdentifier: L4,L5
+  processingLevel: L2
+  keywords: TM,LANDSAT,L4,L5,L2,C2,COLLECTION2
+  sensorType: OPTICAL
+  license: proprietary
+  title: Landsat 4-5 Thematic Mapper (TM) Collection-2 Level-2 Data Products
+  missionStartDate: "1982-08-22T00:00:00Z"
+  missionEndDate: "2013-06-06T00:00:00Z"
+
+LANDSAT_ETM_C1:
+  abstract: |
+    Landsat 7 ETM+ images consist of eight spectral bands with a spatial resolution of 30 meters for bands 1 to 7.
+    The panchromatic band 8 has a resolution of 15 meters. All bands can collect one of two gain settings (high or low)
+    for increased radiometric sensitivity and dynamic range, while Band 6 collects both high and low gain for all
+    scenes. Approximate scene size is 170 km north-south by 183 km east-west (106 mi by 114 mi).
+  instrument: ETM+
+  platform: LANDSAT
+  platformSerialIdentifier: L7
+  processingLevel: L1
+  keywords: ETM,ETM+,LANDSAT,L7,L1,C1,COLLECTION1
+  sensorType: OPTICAL
+  license: proprietary
+  title: Enhanced Thematic Mapper Plus (ETM+) 15- to 30-meter multispectral Collection-1 Level-1 data from Landsat 7
+  missionStartDate: "1999-04-15T00:00:00Z"
+
+LANDSAT_ETM_C2L1:
+  abstract: |
+    Landsat 7 ETM+ images consist of eight spectral bands with a spatial resolution of 30 meters for bands 1 to 7.
+    The panchromatic band 8 has a resolution of 15 meters. All bands can collect one of two gain settings (high or low)
+    for increased radiometric sensitivity and dynamic range, while Band 6 collects both high and low gain for all
+    scenes. Approximate scene size is 170 km north-south by 183 km east-west (106 mi by 114 mi).
+  instrument: ETM+
+  platform: LANDSAT
+  platformSerialIdentifier: L7
+  processingLevel: L1
+  keywords: ETM,ETM+,LANDSAT,L7,L1,C2,COLLECTION2
+  sensorType: OPTICAL
+  license: proprietary
+  title: Enhanced Thematic Mapper Plus (ETM+) 15- to 30-meter multispectral Collection-2 Level-1 data from Landsat 7
+  missionStartDate: "1999-04-15T00:00:00Z"
+
+LANDSAT_ETM_C2L2:
+  abstract: |
+    Collection 2 Landsat 7 ETM+ Level-2 Science Products (L2SP) include Surface Reflectance and Surface Temperature
+    scene-based products.
+  instrument: ETM+
+  platform: LANDSAT
+  platformSerialIdentifier: L7
+  processingLevel: L2
+  keywords: ETM,ETM+,LANDSAT,L7,L2,C2,COLLECTION2
+  sensorType: OPTICAL
+  license: proprietary
+  title: Enhanced Thematic Mapper Plus (ETM+) 15- to 30-meter multispectral Collection-2 Level-2 data from Landsat 7
+  missionStartDate: "1999-04-15T00:00:00Z"
+
 # MODIS -----------------------------------------------------------------------
 MODIS_MCD43A4:
   abstract: |
diff --git a/eodag/resources/providers.yml b/eodag/resources/providers.yml
index 7094127f..1f60a8a3 100644
--- a/eodag/resources/providers.yml
+++ b/eodag/resources/providers.yml
@@ -31,7 +31,7 @@
       max_items_per_page: 5000
       total_items_nb_key_path: '$.totalHits'
     metadata_mapping:
-      id: '$.entityId'
+      id: '$.displayId'
       geometry: '$.spatialBounds'
       productType: '$.productType'
       title: '$.displayId'
@@ -39,15 +39,49 @@
       cloudCover: '$.cloudCover'
       startTimeFromAscendingNode: '$.temporalCoverage.startDate'
       completionTimeFromAscendingNode: '$.temporalCoverage.endDate'
-      quicklook: '$.browse[0].thumbnailPath'
-      # storageStatus set to ONLINE for consistency between providers
-      storageStatus: '{$.null#replace_str("Not Available","ONLINE")}'
-      downloadLink: 'https://dds.cr.usgs.gov/{id}'
+      publicationDate: '$.publishDate'
+      thumbnail: '$.browse[0].thumbnailPath'
+      quicklook: '$.browse[0].browsePath'
+      storageStatus: '{$.available#get_group_name((?P<ONLINE>True)|(?P<OFFLINE>False))}'
+      downloadLink: 'https://earthexplorer.usgs.gov/download/external/options/{productType}/{entityId}/M2M/'
+      # metadata needed for download
+      entityId: '$.entityId'
+      productId: '$.id'
     extract: True
+    order_enabled: true
   products:
-    # datasets list http://kapadia.github.io/usgs/_sources/reference/catalog/ee.txt
+    # datasets list http://kapadia.github.io/usgs/_sources/reference/catalog/ee.txt may be outdated
+    # see also https://dds.cr.usgs.gov/ee-data/coveragemaps/shp/ee/
     L8_OLI_TIRS_C1L1:
       dataset: LANDSAT_8_C1
+      outputs_extension: .tar.gz
+    LANDSAT_C2L1:
+      dataset: landsat_ot_c2_l1
+      outputs_extension: .tar.gz
+    LANDSAT_C2L2:
+      dataset: landsat_ot_c2_l2
+      outputs_extension: .tar.gz
+    LANDSAT_TM_C1:
+      dataset: landsat_tm_c1
+      outputs_extension: .tar.gz
+    LANDSAT_TM_C2L1:
+      dataset: landsat_tm_c2_l1
+      outputs_extension: .tar.gz
+    LANDSAT_TM_C2L2:
+      dataset: landsat_tm_c2_l2
+      outputs_extension: .tar.gz
+    LANDSAT_ETM_C1:
+      dataset: landsat_etm_c1
+      outputs_extension: .tar.gz
+    LANDSAT_ETM_C2L1:
+      dataset: landsat_etm_c2_l1
+      outputs_extension: .tar.gz
+    LANDSAT_ETM_C2L2:
+      dataset: landsat_etm_c2_l2
+      outputs_extension: .tar.gz
+    S2_MSI_L1C:
+      dataset: SENTINEL_2A
+      outputs_extension: .zip
     GENERIC_PRODUCT_TYPE:
       dataset: '{productType}'
 


Test patch:
diff --git a/tests/context.py b/tests/context.py
index cb294dfd..1428ddf0 100644
--- a/tests/context.py
+++ b/tests/context.py
@@ -52,7 +52,11 @@ from eodag.plugins.crunch.filter_latest_tpl_name import FilterLatestByName
 from eodag.plugins.crunch.filter_property import FilterProperty
 from eodag.plugins.crunch.filter_overlap import FilterOverlap
 from eodag.plugins.download.aws import AwsDownload
-from eodag.plugins.download.base import Download, DEFAULT_STREAM_REQUESTS_TIMEOUT
+from eodag.plugins.download.base import (
+    Download,
+    DEFAULT_STREAM_REQUESTS_TIMEOUT,
+    DEFAULT_DOWNLOAD_WAIT,
+)
 from eodag.plugins.download.http import HTTPDownload
 from eodag.plugins.manager import PluginManager
 from eodag.plugins.search.base import Search
@@ -83,3 +87,4 @@ from eodag.utils.exceptions import (
 )
 from eodag.utils.stac_reader import fetch_stac_items, HTTP_REQ_TIMEOUT
 from tests import TESTS_DOWNLOAD_PATH, TEST_RESOURCES_PATH
+from usgs.api import USGSAuthExpiredError, USGSError
diff --git a/tests/units/test_apis_plugins.py b/tests/units/test_apis_plugins.py
index d9b4304c..2e2b2980 100644
--- a/tests/units/test_apis_plugins.py
+++ b/tests/units/test_apis_plugins.py
@@ -21,13 +21,23 @@ import unittest
 from tempfile import TemporaryDirectory
 from unittest import mock
 
+import responses
 from ecmwfapi.api import ANONYMOUS_APIKEY_VALUES
+from shapely.geometry import shape
 
 from tests.context import (
+    DEFAULT_DOWNLOAD_WAIT,
+    ONLINE_STATUS,
+    AuthenticationError,
     EODataAccessGateway,
+    EOProduct,
+    NotAvailableError,
     PluginManager,
     SearchResult,
+    USGSAuthExpiredError,
+    USGSError,
     ValidationError,
+    get_geometry_from_various,
     load_default_config,
     parse_qsl,
     path_to_uri,
@@ -317,3 +327,251 @@ class TestApisPluginEcmwfApi(BaseApisPluginTest):
         )
         assert mock_ecmwfdataserver_retrieve.call_count == 2
         assert len(paths) == 2
+
+
+class TestApisPluginUsgsApi(BaseApisPluginTest):
+    def setUp(self):
+        self.provider = "usgs"
+        self.api_plugin = self.get_search_plugin(provider=self.provider)
+
+    @mock.patch("usgs.api.login", autospec=True)
+    @mock.patch("usgs.api.logout", autospec=True)
+    def test_plugins_apis_usgs_authenticate(self, mock_api_logout, mock_api_login):
+        """UsgsApi.authenticate must return try to login"""
+        # no credential
+        self.api_plugin.authenticate()
+        mock_api_login.assert_called_once_with("", "", save=True)
+        mock_api_logout.assert_not_called()
+        mock_api_login.reset_mock()
+
+        # with credentials
+        self.api_plugin.config.credentials = {
+            "username": "foo",
+            "password": "bar",
+        }
+        self.api_plugin.authenticate()
+        mock_api_login.assert_called_once_with("foo", "bar", save=True)
+        mock_api_logout.assert_not_called()
+        mock_api_login.reset_mock()
+
+        # with USGSAuthExpiredError
+        mock_api_login.side_effect = [USGSAuthExpiredError(), None]
+        self.api_plugin.authenticate()
+        self.assertEqual(mock_api_login.call_count, 2)
+        self.assertEqual(mock_api_logout.call_count, 1)
+        mock_api_login.reset_mock()
+        mock_api_logout.reset_mock()
+
+        # with invalid credentials / USGSError
+        mock_api_login.side_effect = USGSError()
+        with self.assertRaises(AuthenticationError):
+            self.api_plugin.authenticate()
+            self.assertEqual(mock_api_login.call_count, 1)
+            mock_api_logout.assert_not_called()
+
+    @mock.patch("usgs.api.login", autospec=True)
+    @mock.patch("usgs.api.logout", autospec=True)
+    @mock.patch(
+        "usgs.api.scene_search",
+        autospec=True,
+        return_value={
+            "data": {
+                "results": [
+                    {
+                        "browse": [
+                            {
+                                "browsePath": "https://path/to/quicklook.jpg",
+                                "thumbnailPath": "https://path/to/thumbnail.jpg",
+                            },
+                            {},
+                            {},
+                        ],
+                        "cloudCover": "77.46",
+                        "entityId": "LC81780382020041LGN00",
+                        "displayId": "LC08_L1GT_178038_20200210_20200224_01_T2",
+                        "spatialBounds": {
+                            "type": "Polygon",
+                            "coordinates": [
+                                [
+                                    [28.03905, 30.68073],
+                                    [28.03905, 32.79057],
+                                    [30.46294, 32.79057],
+                                    [30.46294, 30.68073],
+                                    [28.03905, 30.68073],
+                                ]
+                            ],
+                        },
+                        "temporalCoverage": {
+                            "endDate": "2020-02-10 00:00:00",
+                            "startDate": "2020-02-10 00:00:00",
+                        },
+                        "publishDate": "2020-02-10 08:24:46",
+                    },
+                ],
+                "recordsReturned": 5,
+                "totalHits": 139,
+                "startingNumber": 6,
+                "nextRecord": 11,
+            }
+        },
+    )
+    @mock.patch(
+        "usgs.api.download_options",
+        autospec=True,
+        return_value={
+            "data": [
+                {
+                    "id": "5e83d0b8e7f6734c",
+                    "entityId": "LC81780382020041LGN00",
+                    "available": True,
+                    "filesize": 9186067,
+                    "productName": "LandsatLook Natural Color Image",
+                    "downloadSystem": "dds",
+                },
+                {
+                    "entityId": "LC81780382020041LGN00",
+                    "available": False,
+                    "downloadSystem": "wms",
+                },
+            ]
+        },
+    )
+    def test_plugins_apis_usgs_query(
+        self,
+        mock_api_download_options,
+        mock_api_scene_search,
+        mock_api_logout,
+        mock_api_login,
+    ):
+        """UsgsApi.query must search using usgs api"""
+
+        search_kwargs = {
+            "productType": "L8_OLI_TIRS_C1L1",
+            "startTimeFromAscendingNode": "2020-02-01",
+            "completionTimeFromAscendingNode": "2020-02-10",
+            "geometry": get_geometry_from_various(geometry=[10, 20, 30, 40]),
+            "items_per_page": 5,
+            "page": 2,
+        }
+        search_results, total_count = self.api_plugin.query(**search_kwargs)
+        mock_api_scene_search.assert_called_once_with(
+            "LANDSAT_8_C1",
+            start_date="2020-02-01",
+            end_date="2020-02-10",
+            ll={"longitude": 10.0, "latitude": 20.0},
+            ur={"longitude": 30.0, "latitude": 40.0},
+            max_results=5,
+            starting_number=6,
+        )
+        self.assertEqual(search_results[0].provider, "usgs")
+        self.assertEqual(search_results[0].product_type, "L8_OLI_TIRS_C1L1")
+        self.assertEqual(
+            search_results[0].geometry,
+            shape(
+                mock_api_scene_search.return_value["data"]["results"][0][
+                    "spatialBounds"
+                ]
+            ),
+        )
+        self.assertEqual(
+            search_results[0].properties["id"],
+            mock_api_scene_search.return_value["data"]["results"][0]["displayId"],
+        )
+        self.assertEqual(
+            search_results[0].properties["cloudCover"],
+            float(
+                mock_api_scene_search.return_value["data"]["results"][0]["cloudCover"]
+            ),
+        )
+        self.assertEqual(search_results[0].properties["storageStatus"], ONLINE_STATUS)
+        self.assertEqual(
+            total_count, mock_api_scene_search.return_value["data"]["totalHits"]
+        )
+
+    @mock.patch("usgs.api.login", autospec=True)
+    @mock.patch("usgs.api.logout", autospec=True)
+    @mock.patch("usgs.api.download_request", autospec=True)
+    def test_plugins_apis_usgs_download(
+        self,
+        mock_api_download_request,
+        mock_api_logout,
+        mock_api_login,
+    ):
+        """UsgsApi.download must donwload using usgs api"""
+
+        @responses.activate
+        def run():
+
+            product = EOProduct(
+                "peps",
+                dict(
+                    geometry="POINT (0 0)",
+                    title="dummy_product",
+                    id="dummy",
+                    entityId="dummyEntityId",
+                    productId="dummyProductId",
+                    productType="L8_OLI_TIRS_C1L1",
+                ),
+            )
+            product.location = product.remote_location = product.properties[
+                "downloadLink"
+            ] = "http://somewhere"
+            product.properties["id"] = "someproduct"
+
+            responses.add(
+                responses.GET,
+                "http://path/to/product",
+                status=200,
+                content_type="application/octet-stream",
+                body=b"something",
+                auto_calculate_content_length=True,
+                match=[
+                    responses.matchers.request_kwargs_matcher(
+                        dict(stream=True, timeout=DEFAULT_DOWNLOAD_WAIT * 60)
+                    )
+                ],
+            )
+
+            # missing download_request return value
+            with self.assertRaises(NotAvailableError):
+                self.api_plugin.download(product, outputs_prefix=self.tmp_home_dir.name)
+
+            # download 1 available product
+            mock_api_download_request.return_value = {
+                "data": {"preparingDownloads": [{"url": "http://path/to/product"}]}
+            }
+
+            path = self.api_plugin.download(
+                product, outputs_prefix=self.tmp_home_dir.name
+            )
+
+            self.assertEqual(len(responses.calls), 1)
+            responses.calls.reset()
+
+            self.assertEqual(
+                path, os.path.join(self.tmp_home_dir.name, "dummy_product")
+            )
+            self.assertTrue(os.path.isfile(path))
+
+            # check file extension
+            # tar
+            os.remove(path)
+            with mock.patch("tarfile.is_tarfile", return_value=True, autospec=True):
+                path = self.api_plugin.download(
+                    product, outputs_prefix=self.tmp_home_dir.name, extract=False
+                )
+                self.assertEqual(
+                    path, os.path.join(self.tmp_home_dir.name, "dummy_product.tar.gz")
+                )
+            # zip
+            os.remove(path)
+            product.product_type = "S2_MSI_L1C"
+            with mock.patch("zipfile.is_zipfile", return_value=True, autospec=True):
+                path = self.api_plugin.download(
+                    product, outputs_prefix=self.tmp_home_dir.name, extract=False
+                )
+                self.assertEqual(
+                    path, os.path.join(self.tmp_home_dir.name, "dummy_product.zip")
+                )
+
+        run()
diff --git a/tests/units/test_core.py b/tests/units/test_core.py
index edaac541..9df038b6 100644
--- a/tests/units/test_core.py
+++ b/tests/units/test_core.py
@@ -86,13 +86,20 @@ class TestCore(TestCoreBase):
             "earth_search",
             "earth_search_gcs",
         ],
-        "LANDSAT_C2L1": ["usgs_satapi_aws", "astraea_eod"],
+        "LANDSAT_C2L1": ["usgs_satapi_aws", "astraea_eod", "usgs"],
+        "LANDSAT_C2L2": ["usgs"],
         "LANDSAT_C2L2_SR": ["usgs_satapi_aws"],
         "LANDSAT_C2L2_ST": ["usgs_satapi_aws"],
         "LANDSAT_C2L2ALB_BT": ["usgs_satapi_aws"],
         "LANDSAT_C2L2ALB_SR": ["usgs_satapi_aws"],
         "LANDSAT_C2L2ALB_ST": ["usgs_satapi_aws"],
         "LANDSAT_C2L2ALB_TA": ["usgs_satapi_aws"],
+        "LANDSAT_TM_C1": ["usgs"],
+        "LANDSAT_TM_C2L1": ["usgs"],
+        "LANDSAT_TM_C2L2": ["usgs"],
+        "LANDSAT_ETM_C1": ["usgs"],
+        "LANDSAT_ETM_C2L1": ["usgs"],
+        "LANDSAT_ETM_C2L2": ["usgs"],
         "S1_SAR_GRD": [
             "peps",
             "sobloo",
@@ -130,6 +137,7 @@ class TestCore(TestCoreBase):
             "astraea_eod",
             "earth_search",
             "earth_search_gcs",
+            "usgs",
         ],
         "S3_ERR": ["onda", "creodias"],
         "S3_EFR": ["onda", "creodias"],