Repository: CS-SI/eodag (python)

Issue:
Sentinel SAFE / non-SAFE product types distinction
All providers do not provide Sentinel products in SAFE format, which means for example that searching `S2_MSI_L1C` products on `peps` or `astraea_eod` can return products packaged differently, and differently formatted ids.

We should add more product types to be able to distinguish:
- Sentinel products in SAFE / jp2 format (S2_MSI_L1C peps)
- Sentinel products in non-SAFE / jp2 format (S2_MSI_L1C astraea_eod)
- Sentinel products in cog format (sentinel-s2-l2a-cogs earth_search)

See https://github.com/CS-SI/eodag/issues/136#issuecomment-808082569, #171 

Pull request description:
feat: sentinel non-safe product types
Add new Sentinel non-SAFE product types, fixes #216

And updates AWS download SAFE formatting, please note that :
- to keep the ability to search by id on AWS STAC providers (_astraea_eod_ and _earth_search_), product id will not be SAFE formatted but will use the formatting used by the provider (see also #218). 
I.e. `id="S2B_31TCH_20210318_0_L1C"` for _earth_search_  and `id="S2B_OPER_MSI_L1C_TL_EPAE_20201201T115506_A019525_T30TYP_N02.09"` for _astraea_eod_
- If the product title (used as download destination path) can be SAFE formatted from search results, it will be. 
For example with _earth_search_ we can have `product.properties["id"]="S2B_31TCH_20210318_0_L1C"` and `product.properties["title"]="S2B_MSIL1C_20210318T103649_N0209_R008_T31TCH_20210318T143809"`. Destination path will be `"{title}/{title}.SAFE"`
- If the product title needs and extra request to be SAFE formatted, as for _astraea_eod_, this will not be done during search, but during download. 
The product destination will be in this case: `"{old_title}/{safe_title}.SAFE"` with `old_title=product.properties["title"]="S2B_OPER_MSI_L1C_TL_EPAE_20201201T115506_A019525_T30TYP_N02.09"` and `safe_title="S2B_MSIL1C_20201201T105419_N0209_R051_T30TYP_20201201T115506"`

Patch:
diff --git a/eodag/plugins/download/aws.py b/eodag/plugins/download/aws.py
index 84fc9ea9..2f0b132d 100644
--- a/eodag/plugins/download/aws.py
+++ b/eodag/plugins/download/aws.py
@@ -163,6 +163,12 @@ class AwsDownload(Download):
     def download(self, product, auth=None, progress_callback=None, **kwargs):
         """Download method for AWS S3 API.
 
+        The product can be downloaded as it is, or as SAFE-formatted product.
+        SAFE-build is configured for a given provider and product type.
+        If the product title is configured to be updated during download and
+        SAFE-formatted, its destination path will be:
+        `{outputs_prefix}/{title}/{updated_title}.SAFE`
+
         :param product: The EO product to download
         :type product: :class:`~eodag.api.product.EOProduct`
         :param auth: (optional) The configuration of a plugin of type Authentication
@@ -176,6 +182,18 @@ class AwsDownload(Download):
         :return: The absolute path to the downloaded product in the local filesystem
         :rtype: str
         """
+        # prepare download & create dirs (before updating metadata)
+        product_local_path, record_filename = self._prepare_download(product, **kwargs)
+        if not product_local_path or not record_filename:
+            return product_local_path
+        product_local_path = product_local_path.replace(".zip", "")
+        # remove existing incomplete file
+        if os.path.isfile(product_local_path):
+            os.remove(product_local_path)
+        # create product dest dir
+        if not os.path.isdir(product_local_path):
+            os.makedirs(product_local_path)
+
         product_conf = getattr(self.config, "products", {}).get(
             product.product_type, {}
         )
@@ -228,18 +246,6 @@ class AwsDownload(Download):
                 )
             )
 
-        # prepare download & create dirs
-        product_local_path, record_filename = self._prepare_download(product, **kwargs)
-        if not product_local_path or not record_filename:
-            return product_local_path
-        product_local_path = product_local_path.replace(".zip", "")
-        # remove existing incomplete file
-        if os.path.isfile(product_local_path):
-            os.remove(product_local_path)
-        # create product dest dir
-        if not os.path.isdir(product_local_path):
-            os.makedirs(product_local_path)
-
         # progress bar init
         if progress_callback is None:
             progress_callback = get_progress_callback()
diff --git a/eodag/resources/product_types.yml b/eodag/resources/product_types.yml
index 004a1d3a..6a9a313b 100644
--- a/eodag/resources/product_types.yml
+++ b/eodag/resources/product_types.yml
@@ -360,6 +360,8 @@ S1_SAR_OCN:
     derived from internally generated Level-1 GRD images of SM, IW or EW modes.
     The RVL is a ground range gridded difference between the measured Level-2 Doppler grid and the Level-1 calculated
     geometrical Doppler.
+    SAFE formatted product, see
+    https://sentinel.esa.int/web/sentinel/user-guides/sentinel-1-sar/data-formats/safe-specification
   instrument: SAR
   platform: SENTINEL1
   platformSerialIdentifier: S1A,S1B
@@ -381,6 +383,31 @@ S1_SAR_GRD:
       Medium Resolution (MR).
     The resolution is dependent upon the amount of multi-looking performed. Level-1 GRD products are available in MR
     and HR for IW and EW modes, MR for WV mode and MR, HR and FR for SM mode.
+    SAFE formatted product, see
+    https://sentinel.esa.int/web/sentinel/user-guides/sentinel-1-sar/data-formats/safe-specification
+  instrument: SAR
+  platform: SENTINEL1
+  platformSerialIdentifier: S1A,S1B
+  processingLevel: L1
+  sensorType: RADAR
+  license: proprietary
+  title: SENTINEL1 Level-1 Ground Range Detected
+  missionStartDate: "2014-04-03T00:00:00Z"
+
+S1_SAR_GRD_JP2:
+  abstract: |
+    Level-1 Ground Range Detected (GRD) products consist of focused SAR data that has been detected, multi-looked and
+    projected to ground range using an Earth ellipsoid model. Phase information is lost. The resulting product has
+    approximately square spatial resolution pixels and square pixel spacing with reduced speckle at the cost of worse
+    spatial resolution.
+    GRD products can be in one of three resolutions:
+      Full Resolution (FR),
+      High Resolution (HR),
+      Medium Resolution (MR).
+    The resolution is dependent upon the amount of multi-looking performed. Level-1 GRD products are available in MR
+    and HR for IW and EW modes, MR for WV mode and MR, HR and FR for SM mode.
+    Product without SAFE formatting, see
+    https://sentinel.esa.int/web/sentinel/user-guides/sentinel-1-sar/data-formats/safe-specification
   instrument: SAR
   platform: SENTINEL1
   platformSerialIdentifier: S1A,S1B
@@ -396,6 +423,8 @@ S1_SAR_SLC:
     data from the satellite and provided in zero-Doppler slant-range geometry. The products include a single look in
     each dimension using the full transmit signal bandwidth and consist of complex samples preserving the phase
     information.
+    SAFE formatted product, see
+    https://sentinel.esa.int/web/sentinel/user-guides/sentinel-1-sar/data-formats/safe-specification
   instrument: SAR
   platform: SENTINEL1
   platformSerialIdentifier: S1A,S1B
@@ -410,6 +439,8 @@ S1_SAR_RAW:
     The SAR Level-0 products consist of the sequence of Flexible Dynamic Block Adaptive Quantization (FDBAQ) compressed
     unfocused SAR raw data. For the data to be usable, it will need to be decompressed and processed using a SAR
     processor.
+    SAFE formatted product, see
+    https://sentinel.esa.int/web/sentinel/user-guides/sentinel-1-sar/data-formats/safe-specification
   instrument: SAR
   platform: SENTINEL1
   platformSerialIdentifier: S1A,S1B
@@ -429,6 +460,7 @@ S2_MSI_L1C:
     meters depending on the native resolution of the different spectral bands. In Level-1C  products, pixel
     coordinates refer to the upper left corner of the pixel. Level-1C products will additionally include Cloud Masks
     and ECMWF data (total column of ozone, total column of water vapour and mean sea level pressure).
+    SAFE formatted product, see https://sentinel.esa.int/web/sentinel/user-guides/sentinel-2-msi/data-formats
   instrument: MSI
   platform: SENTINEL2
   platformSerialIdentifier: S2A,S2B
@@ -499,10 +531,59 @@ S2_MSI_L1C:
       full_width_half_max: 0.242
       gsd: 20
 
+S2_MSI_L1C_JP2:
+  abstract: |
+    The Level-1C product is composed of 100x100 km2 tiles (ortho-images in UTM/WGS84 projection). It results from
+    using a Digital Elevation Model (DEM) to project the image in cartographic geometry. Per-pixel radiometric
+    measurements are provided in Top Of Atmosphere (TOA) reflectances along with the parameters to transform them
+    into radiances. Level-1C products are resampled with a constant Ground Sampling Distance (GSD) of 10, 20 and 60
+    meters depending on the native resolution of the different spectral bands. In Level-1C  products, pixel
+    coordinates refer to the upper left corner of the pixel. Level-1C products will additionally include Cloud Masks
+    and ECMWF data (total column of ozone, total column of water vapour and mean sea level pressure).
+    Product without SAFE formatting, see https://sentinel.esa.int/web/sentinel/user-guides/sentinel-2-msi/data-formats
+  instrument: MSI
+  platform: SENTINEL2
+  platformSerialIdentifier: S2A,S2B
+  processingLevel: L1
+  sensorType: OPTICAL
+  license: proprietary
+  missionStartDate: "2015-06-23T00:00:00Z"
+  title: SENTINEL2 Level-1C
+
+
 S2_MSI_L2A:
   abstract: |
     The Level-2A product provides Bottom Of Atmosphere (BOA) reflectance images derived from the associated Level-1C
     products. Each Level-2A product is composed of 100x100 km2 tiles in cartographic geometry (UTM/WGS84 projection).
+    SAFE formatted product, see https://sentinel.esa.int/web/sentinel/user-guides/sentinel-2-msi/data-formats
+  instrument: MSI
+  platform: SENTINEL2
+  platformSerialIdentifier: S2A,S2B
+  processingLevel: L2
+  sensorType: OPTICAL
+  license: proprietary
+  title: SENTINEL2 Level-2A
+  missionStartDate: "2015-06-23T00:00:00Z"
+
+S2_MSI_L2A_JP2:
+  abstract: |
+    The Level-2A product provides Bottom Of Atmosphere (BOA) reflectance images derived from the associated Level-1C
+    products. Each Level-2A product is composed of 100x100 km2 tiles in cartographic geometry (UTM/WGS84 projection).
+    Product without SAFE formatting, see https://sentinel.esa.int/web/sentinel/user-guides/sentinel-2-msi/data-formats
+  instrument: MSI
+  platform: SENTINEL2
+  platformSerialIdentifier: S2A,S2B
+  processingLevel: L2
+  sensorType: OPTICAL
+  license: proprietary
+  title: SENTINEL2 Level-2A
+  missionStartDate: "2015-06-23T00:00:00Z"
+
+S2_MSI_L2A_COG:
+  abstract: |
+    The Level-2A product provides Bottom Of Atmosphere (BOA) reflectance images derived from the associated Level-1C
+    products. Each Level-2A product is composed of 100x100 km2 tiles in cartographic geometry (UTM/WGS84 projection).
+    Product containing Cloud Optimized GeoTIFF images, without SAFE formatting.
   instrument: MSI
   platform: SENTINEL2
   platformSerialIdentifier: S2A,S2B
diff --git a/eodag/resources/providers.yml b/eodag/resources/providers.yml
index 83f702d7..6b3bd5f6 100644
--- a/eodag/resources/providers.yml
+++ b/eodag/resources/providers.yml
@@ -1672,3 +1672,35 @@
           - tilePath
   auth: !plugin
     type: AwsAuth
+
+---
+!provider
+  name: earth_search_cog
+  priority: 0
+  roles:
+    - host
+  description: Earth Search
+  url: https://www.element84.com/earth-search/
+  search: !plugin
+    type: StacSearch
+    api_endpoint: https://earth-search.aws.element84.com/v0/search
+    metadata_mapping:
+      id:
+        - 'ids=["{id}"]'
+        - '$.id'
+      productType:
+        - 'collections=["{productType}"]'
+        - '$.collection'
+      geometry:
+        - 'bbox=[{geometry.bounds#csv_list}]'
+        - '$.geometry'
+      platformSerialIdentifier: '$.id.`split(_, 0, -1)`'
+      polarizationMode: '$.id.`sub(/.{14}([A-Z]{2}).*/, \\1)`'
+  products:
+    S2_MSI_L2A_COG:
+      productType: sentinel-s2-l2a-cogs
+    GENERIC_PRODUCT_TYPE:
+      productType: '{productType}'
+  download: !plugin
+    type: HTTPDownload
+    base_uri: 'https://sentinel-cogs.s3.us-west-2.amazonaws.com'
diff --git a/eodag/resources/user_conf_template.yml b/eodag/resources/user_conf_template.yml
index 2edfc6b3..a3fdf94e 100644
--- a/eodag/resources/user_conf_template.yml
+++ b/eodag/resources/user_conf_template.yml
@@ -127,3 +127,8 @@ earth_search:
             aws_profile:
     download:
         outputs_prefix:
+earth_search_cog:
+    priority: # Lower value means lower priority (Default: 0)
+    search:   # Search parameters configuration
+    download:
+        outputs_prefix:


Test patch:
diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py
index 294d0d5e..be2dee90 100644
--- a/tests/test_end_to_end.py
+++ b/tests/test_end_to_end.py
@@ -78,6 +78,13 @@ EARTH_SEARCH_SEARCH_ARGS = [
     "2020-01-15",
     [0.2563590566012408, 43.19555008715042, 2.379835675499976, 43.907759172380565],
 ]
+EARTH_SEARCH_COG_SEARCH_ARGS = [
+    "earth_search_cog",
+    "S2_MSI_L2A_COG",
+    "2020-01-01",
+    "2020-01-15",
+    [0.2563590566012408, 43.19555008715042, 2.379835675499976, 43.907759172380565],
+]
 USGS_SATAPI_AWS_SEARCH_ARGS = [
     "usgs_satapi_aws",
     "LANDSAT_C2L1",
@@ -360,13 +367,18 @@ class TestEODagEndToEnd(EndToEndBase):
     def test_end_to_end_search_download_astraea_eod(self):
         product = self.execute_search(*ASTRAE_EOD_SEARCH_ARGS)
         expected_filename = "{}".format(product.properties["title"])
-        self.execute_download(product, expected_filename)
+        self.execute_download(product, expected_filename, wait_sec=15)
 
     def test_end_to_end_search_download_earth_search(self):
         product = self.execute_search(*EARTH_SEARCH_SEARCH_ARGS)
         expected_filename = "{}".format(product.properties["title"])
         self.execute_download(product, expected_filename)
 
+    def test_end_to_end_search_download_earth_search_cog(self):
+        product = self.execute_search(*EARTH_SEARCH_COG_SEARCH_ARGS)
+        expected_filename = "{}".format(product.properties["title"])
+        self.execute_download(product, expected_filename, wait_sec=20)
+
     def test_end_to_end_search_download_usgs_satapi_aws(self):
         product = self.execute_search(*USGS_SATAPI_AWS_SEARCH_ARGS)
         expected_filename = "{}".format(product.properties["title"])
diff --git a/tests/units/test_core.py b/tests/units/test_core.py
index 2cc06b56..cb6022e4 100644
--- a/tests/units/test_core.py
+++ b/tests/units/test_core.py
@@ -77,6 +77,7 @@ class TestCore(unittest.TestCase):
             "astraea_eod",
             "earth_search",
         ],
+        "S2_MSI_L2A_COG": ["earth_search_cog"],
         "S2_MSI_L2A_MAJA": ["theia"],
         "S2_MSI_L2B_MAJA_SNOW": ["theia"],
         "S2_MSI_L2B_MAJA_WATER": ["theia"],
@@ -135,6 +136,7 @@ class TestCore(unittest.TestCase):
             "astraea_eod",
             "usgs_satapi_aws",
             "earth_search",
+            "earth_search_cog",
         ],
     }
     SUPPORTED_PROVIDERS = [
@@ -150,6 +152,7 @@ class TestCore(unittest.TestCase):
         "astraea_eod",
         "usgs_satapi_aws",
         "earth_search",
+        "earth_search_cog",
     ]
 
     @classmethod
@@ -510,6 +513,7 @@ class TestCoreSearch(unittest.TestCase):
         expected = [
             "S2_MSI_L1C",
             "S2_MSI_L2A",
+            "S2_MSI_L2A_COG",
             "S2_MSI_L2A_MAJA",
             "S2_MSI_L2B_MAJA_SNOW",
             "S2_MSI_L2B_MAJA_WATER",