Repository: CS-SI/eodag (python)

Issue:
finalize_s2_safe_product: Fails when missing manifest.safe
When downloading some products on `earth_search`, I have a problem when converting the product format to `.SAFE`:
![2022-02-22_14h47_27](https://user-images.githubusercontent.com/67311115/155145249-38c42542-b9bd-4888-9573-2f7d1cc24c6d.png)
![2022-02-22_14h48_44](https://user-images.githubusercontent.com/67311115/155145285-a0e12781-4c2d-442b-8910-48ddae12dd7a.png)

It seems some L2A products are arriving without the `manifest.safe` file 😓 


Environment:

    Python version: Python 3.7.12
    EODAG version: eodag (Earth Observation Data Access Gateway): version 2.3.4


Pull request:
fix: productPath regex for earth_search
fixes #402, answers [#404 ](https://github.com/CS-SI/eodag/discussions/404)

- handle missing files error during SAFE build 
- fixes issue in `productPath` property extraction regex for `earth_search` provider

Patch:
diff --git a/eodag/plugins/download/aws.py b/eodag/plugins/download/aws.py
index 3202c76a..fe7d3573 100644
--- a/eodag/plugins/download/aws.py
+++ b/eodag/plugins/download/aws.py
@@ -253,11 +253,17 @@ class AwsDownload(Download):
             bucket_names_and_prefixes = [self.get_bucket_name_and_prefix(product)]
 
         # add complementary urls
-        for complementary_url_key in product_conf.get("complementary_url_key", []):
-            bucket_names_and_prefixes.append(
-                self.get_bucket_name_and_prefix(
-                    product, product.properties[complementary_url_key]
+        try:
+            for complementary_url_key in product_conf.get("complementary_url_key", []):
+                bucket_names_and_prefixes.append(
+                    self.get_bucket_name_and_prefix(
+                        product, product.properties[complementary_url_key]
+                    )
                 )
+        except KeyError:
+            logger.error(
+                "complementary_url_key %s is missing in %s properties"
+                % (complementary_url_key, product.properties["id"])
             )
 
         # authenticate
@@ -566,13 +572,18 @@ class AwsDownload(Download):
 
     def check_manifest_file_list(self, product_path):
         """Checks if products listed in manifest.safe exist"""
-        manifest_path = [
+        manifest_path_list = [
             os.path.join(d, x)
             for d, _, f in os.walk(product_path)
             for x in f
             if x == "manifest.safe"
-        ][0]
-        safe_path = os.path.dirname(manifest_path)
+        ]
+        if len(manifest_path_list) == 0:
+            raise FileNotFoundError(
+                f"No manifest.safe could be found in {product_path}"
+            )
+        else:
+            safe_path = os.path.dirname(manifest_path_list[0])
 
         root = etree.parse(os.path.join(safe_path, "manifest.safe")).getroot()
         for safe_file in root.xpath("//fileLocation"):
@@ -587,13 +598,18 @@ class AwsDownload(Download):
         """Add missing dirs to downloaded product"""
         try:
             logger.debug("Finalize SAFE product")
-            manifest_path = [
+            manifest_path_list = [
                 os.path.join(d, x)
                 for d, _, f in os.walk(product_path)
                 for x in f
                 if x == "manifest.safe"
-            ][0]
-            safe_path = os.path.dirname(manifest_path)
+            ]
+            if len(manifest_path_list) == 0:
+                raise FileNotFoundError(
+                    f"No manifest.safe could be found in {product_path}"
+                )
+            else:
+                safe_path = os.path.dirname(manifest_path_list[0])
 
             # create empty missing dirs
             auxdata_path = os.path.join(safe_path, "AUX_DATA")
@@ -868,7 +884,7 @@ class AwsDownload(Download):
         auth=None,
         downloaded_callback=None,
         progress_callback=None,
-        **kwargs
+        **kwargs,
     ):
         """
         download_all using parent (base plugin) method
@@ -878,5 +894,5 @@ class AwsDownload(Download):
             auth=auth,
             downloaded_callback=downloaded_callback,
             progress_callback=progress_callback,
-            **kwargs
+            **kwargs,
         )
diff --git a/eodag/resources/providers.yml b/eodag/resources/providers.yml
index 0dc859d4..2b792b94 100644
--- a/eodag/resources/providers.yml
+++ b/eodag/resources/providers.yml
@@ -1718,7 +1718,7 @@
         platformSerialIdentifier: '$.id.`split(_, 0, -1)`'
         polarizationMode: '$.id.`sub(/.{14}([A-Z]{2}).*/, \\1)`'
         productPath: |
-          $.properties."sentinel:product_id".`sub(/([S2AB]{3})_MSIL1C_([0-9]{4})(0?)([1-9]+)([0-9]{2})(T.*)/, products/\\2/\\4/\\5/\\1_MSIL1C_\\2\\3\\4\\5\\6)`
+          $.properties."sentinel:product_id".`sub(/([S2AB]{3})_MSIL1C_([0-9]{4})([0-9]{2})([0-9]{2})(T.*)/, products/\\2/\\3/\\4/\\1_MSIL1C_\\2\\3\\4\\5)`.`sub(/\/0+/, \/)`
         tilePath: |
           $.assets.info.href.`sub(/.*/sentinel-s2-l1c\/(tiles\/.*)\/tileInfo\.json/, \\1)`
     S2_MSI_L2A:
@@ -1728,7 +1728,7 @@
         platformSerialIdentifier: '$.id.`split(_, 0, -1)`'
         polarizationMode: '$.id.`sub(/.{14}([A-Z]{2}).*/, \\1)`'
         productPath: |
-          $.properties."sentinel:product_id".`sub(/([S2AB]{3})_MSIL2A_([0-9]{4})(0?)([1-9]+)([0-9]{2})(T.*)/, products/\\2/\\4/\\5/\\1_MSIL2A_\\2\\3\\4\\5\\6)`
+          $.properties."sentinel:product_id".`sub(/([S2AB]{3})_MSIL2A_([0-9]{4})([0-9]{2})([0-9]{2})(T.*)/, products/\\2/\\3/\\4/\\1_MSIL2A_\\2\\3\\4\\5)`.`sub(/\/0+/, \/)`
         tilePath: |
           $.assets.info.href.`sub(/.*/sentinel-s2-l2a\/(tiles\/.*)\/tileInfo\.json/, \\1)`
     L8_OLI_TIRS_C1L1:


Test patch:
diff --git a/tests/units/test_search_plugins.py b/tests/units/test_search_plugins.py
index 405451a8..47ced2e6 100644
--- a/tests/units/test_search_plugins.py
+++ b/tests/units/test_search_plugins.py
@@ -297,3 +297,50 @@ class TestSearchPluginODataV4Search(BaseSearchPluginTest):
         self.assertEqual(estimate, self.onda_products_count)
         self.assertEqual(len(products), number_of_products)
         self.assertIsInstance(products[0], EOProduct)
+
+
+class TestSearchPluginStacSearch(BaseSearchPluginTest):
+    @mock.patch("eodag.plugins.search.qssearch.StacSearch._request", autospec=True)
+    def test_plugins_search_stacsearch_mapping_earthsearch(self, mock__request):
+        """The metadata mapping for earth_search should return well formatted results"""  # noqa
+
+        geojson_geometry = self.search_criteria_s2_msi_l1c["geometry"].__geo_interface__
+
+        mock__request.return_value = mock.Mock()
+        mock__request.return_value.json.side_effect = [
+            {
+                "context": {"page": 1, "limit": 2, "matched": 1, "returned": 2},
+            },
+            {
+                "features": [
+                    {
+                        "id": "foo",
+                        "geometry": geojson_geometry,
+                        "properties": {
+                            "sentinel:product_id": "S2B_MSIL1C_20201009T012345_N0209_R008_T31TCJ_20201009T123456",
+                        },
+                    },
+                    {
+                        "id": "bar",
+                        "geometry": geojson_geometry,
+                        "properties": {
+                            "sentinel:product_id": "S2B_MSIL1C_20200910T012345_N0209_R008_T31TCJ_20200910T123456",
+                        },
+                    },
+                ]
+            },
+        ]
+
+        search_plugin = self.get_search_plugin(self.product_type, "earth_search")
+
+        products, estimate = search_plugin.query(
+            page=1, items_per_page=2, auth=None, **self.search_criteria_s2_msi_l1c
+        )
+        self.assertEqual(
+            products[0].properties["productPath"],
+            "products/2020/10/9/S2B_MSIL1C_20201009T012345_N0209_R008_T31TCJ_20201009T123456",
+        )
+        self.assertEqual(
+            products[1].properties["productPath"],
+            "products/2020/9/10/S2B_MSIL1C_20200910T012345_N0209_R008_T31TCJ_20200910T123456",
+        )