Repository: CS-SI/eodag
Pull request #622 (2023-01-27 15:37:38)

feat: simplify odata metadata mapping using pre-mapping
`metadata_pre_mapping` as new entry in `ODataV4Search` configuration.

Enables a 2 step  `normalize_results()` with a pre-mapping that simplifies metadata parsing performed in the 2nd step.

For example, goes from `$.Metadata[?(@.id="foo")].value` to `$.Metadata.foo.value`

github-actions[bot]: ## Unit Test Results
    2 files  ±0      2 suites  ±0   11m 55s [:stopwatch:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "duration of all tests") -36s
355 tests +1  352 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "passed tests") +1  3 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "skipped / disabled tests") ±0  0 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "failed tests") ±0 
710 runs  +2  704 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "passed tests") +2  6 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "skipped / disabled tests") ±0  0 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "failed tests") ±0 

Results for commit abbb3804. ± Comparison against base commit 37eb4e53.

[test-results]:data:application/gzip;base64,H4sIAJHy02MC/03NSw7CIBSF4a00jB1QnuJmDNxCQmyL4TEy7l2Kgh3+303OfSHnV5vQbSKXCaXi84ilRJ192GvKmVeop3wcKR91TwWgEfnTwz8PGuC0XyvgATbGEH8Sy57ahx59UmI25Lsoep8GW5/3IGybzzWQNsbQK2ZCcaEoCClgdpw4aTFT3OHFSQLMGPT+ACAbYwEFAQAA

github-actions[bot]: <strong> </strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `87%` | :white_check_mark: |
| eodag/api/product/metadata\_mapping.py | `81%` | :white_check_mark: |
| eodag/plugins/search/qssearch.py | `82%` | :white_check_mark: |
| tests/units/test\_search\_plugins.py | `99%` | :white_check_mark: |

<strong> </strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `86%` | :white_check_mark: |
| eodag/api/product/metadata\_mapping.py | `81%` | :white_check_mark: |
| eodag/plugins/search/qssearch.py | `82%` | :white_check_mark: |
| tests/units/test\_search\_plugins.py | `99%` | :white_check_mark: |
| eodag/api/product/metadata\_mapping.py | `80%` | :white_check_mark: |
| eodag/plugins/search/qssearch.py | `82%` | :white_check_mark: |
| tests/units/test\_search\_plugins.py | `99%` | :white_check_mark: |

_Minimum allowed coverage is `70%`_

<p align="right">Generated by :monkey: cobertura-action against abbb3804695693c676c1f52f7e0495f0df72c4bb </p>

Patch:
diff --git a/eodag/api/product/metadata_mapping.py b/eodag/api/product/metadata_mapping.py
index f3247002..73d77dfc 100644
--- a/eodag/api/product/metadata_mapping.py
+++ b/eodag/api/product/metadata_mapping.py
@@ -558,6 +558,12 @@ def properties_from_json(json, mapping, discovery_config=None):
                     # default value got from metadata_path
                     properties[found_key] = found_jsonpath.value
 
+                # properties as python objects when possible (format_metadata returns only strings)
+                try:
+                    properties[found_key] = ast.literal_eval(properties[found_key])
+                except Exception:
+                    pass
+
     return properties
 
 
diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py
index 9a48a2ac..16cba15c 100644
--- a/eodag/plugins/search/qssearch.py
+++ b/eodag/plugins/search/qssearch.py
@@ -983,6 +983,34 @@ class ODataV4Search(QueryStringSearch):
             self.config.api_endpoint.rstrip("/"), entity["id"]
         )
 
+    def normalize_results(self, results, **kwargs):
+        """Build EOProducts from provider results
+
+        If configured, a metadata pre-mapping can be applied to simplify further metadata extraction.
+        For example, going from '$.Metadata[?(@.id="foo")].value' to '$.Metadata.foo.value'
+        """
+        metadata_pre_mapping = getattr(self.config, "metadata_pre_mapping", {})
+        metadata_path = metadata_pre_mapping.get("metadata_path", None)
+        metadata_path_id = metadata_pre_mapping.get("metadata_path_id", None)
+        metadata_path_value = metadata_pre_mapping.get("metadata_path_value", None)
+
+        if metadata_path and metadata_path_id and metadata_path_value:
+            parsed_metadata_path = cached_parse(metadata_path)
+            for result in results:
+                found_metadata = parsed_metadata_path.find(result)
+                if found_metadata:
+                    metadata_update = {}
+                    for metadata_dict in found_metadata[0].value:
+                        metada_id = metadata_dict[metadata_path_id]
+                        metada_value = metadata_dict[metadata_path_value]
+                        metadata_update[metada_id] = metada_value
+                    parsed_metadata_path.update(result, metadata_update)
+
+        # once metadata pre-mapping applied execute QueryStringSearch.normalize_results()
+        products = super(ODataV4Search, self).normalize_results(results, **kwargs)
+
+        return products
+
 
 class PostJsonSearch(QueryStringSearch):
     """A specialisation of a QueryStringSearch that uses POST method"""
diff --git a/eodag/resources/providers.yml b/eodag/resources/providers.yml
index 5f0d4436..d2111f49 100644
--- a/eodag/resources/providers.yml
+++ b/eodag/resources/providers.yml
@@ -1354,10 +1354,13 @@
             operations:
               AND:
                 - '{metadata}:{{{metadata}}}'
-      metadata_path: '$.Metadata[*]'
+      metadata_path: '$.Metadata.*'
+    per_product_metadata_query: false
+    metadata_pre_mapping:
+      metadata_path: '$.Metadata'
       metadata_path_id: 'id'
       metadata_path_value: 'value'
-    per_product_metadata_query: false
+
     metadata_mapping:
       # Opensearch resource identifier within the search engine context (in our case
       # within the context of the data provider)
@@ -1367,61 +1370,61 @@
       # OpenSearch Parameters for Collection Search (Table 3)
       productType:
         - null
-        - '$.Metadata[?(@.id="productType")].value'
+        - '$.Metadata.productType'
       platform:
         - null
-        - '$.Metadata[?(@.id="platformName")].value'
+        - '$.Metadata.platformName'
       platformSerialIdentifier:
         - null
-        - '$.Metadata[?(@.id="platformSerialIdentifier")].value'
+        - '$.Metadata.platformSerialIdentifier'
       instrument:
         - null
-        - '$.Metadata[?(@.id="instrumentShortName")].value'
+        - '$.Metadata.instrumentShortName'
       processingLevel:
         - null
-        - '$.Metadata[?(@.id="processingLevel")].value'
+        - '$.Metadata.processingLevel'
       sensorType:
         - null
-        - '$.Metadata[?(@.id="sensorType")].value'
+        - '$.Metadata.sensorType'
 
       # INSPIRE obligated OpenSearch Parameters for Collection Search (Table 4)
-      title: '$.Metadata[?(@.id="filename")].value'
+      title: '$.Metadata.filename'
       topicCategory:
         - null
-        - '$.Metadata[?(@.id="topicCategory")].value'
+        - '$.Metadata.topicCategory'
       lineage:
         - null
-        - '$.Metadata[?(@.id="lineage")].value'
+        - '$.Metadata.lineage'
 
       # OpenSearch Parameters for Product Search (Table 5)
       orbitNumber:
         - null
-        - '$.Metadata[?(@.id="orbitNumber")].value'
+        - '$.Metadata.orbitNumber'
       orbitDirection:
         - null
-        - '$.Metadata[?(@.id="orbitDirection")].value'
+        - '$.Metadata.orbitDirection'
       cloudCover:
         - null
-        - '$.Metadata[?(@.id="cloudCoverPercentage")].value'
+        - '$.Metadata.cloudCoverPercentage'
       productVersion:
         - null
-        - '$.Metadata[?(@.id="processingBaseline")].value'
+        - '$.Metadata.processingBaseline'
       productQualityStatus:
         - null
-        - '$.Metadata[?(@.id="generalQualityFlag")].value'
-      creationDate: '$.Metadata[?(@.id="creationDate")].value'
-      processingDate: '$.Metadata[?(@.id="processingDate")].value'
+        - '$.Metadata.generalQualityFlag'
+      creationDate: '$.Metadata.creationDate'
+      processingDate: '$.Metadata.processingDate'
       sensorMode:
         - null
-        - '$.Metadata[?(@.id="sensorOperationalMode")].value'
+        - '$.Metadata.sensorOperationalMode'
 
       # OpenSearch Parameters for Acquistion Parameters Search (Table 6)
       startTimeFromAscendingNode: '$.beginPosition'
       completionTimeFromAscendingNode: '$.endPosition'
-      polarizationChannels: '{$.Metadata[?(@.id="polarisationChannels")].value#replace_str(","," ")}'
+      polarizationChannels: '{$.Metadata.polarisationChannels#replace_str(","," ")}'
 
       # Custom parameters (not defined in the base document referenced above)
-      id: '{$.Metadata[?(@.id="filename")].value#remove_extension}'
+      id: '{$.Metadata.filename#remove_extension}'
       # The geographic extent of the product
       geometry: '$.footprint'
       # The url of the quicklook


Test patch:
diff --git a/tests/units/test_search_plugins.py b/tests/units/test_search_plugins.py
index f02accbf..75fb411b 100644
--- a/tests/units/test_search_plugins.py
+++ b/tests/units/test_search_plugins.py
@@ -394,6 +394,30 @@ class TestSearchPluginODataV4Search(BaseSearchPluginTest):
 
         self.onda_products_count = 47
 
+    def test_plugins_search_odatav4search_normalize_results_onda(self):
+        """ODataV4Search.normalize_results must use metada pre-mapping if configured"""
+
+        self.assertTrue(hasattr(self.onda_search_plugin.config, "metadata_pre_mapping"))
+        self.assertDictEqual(
+            self.onda_search_plugin.config.metadata_pre_mapping,
+            {
+                "metadata_path": "$.Metadata",
+                "metadata_path_id": "id",
+                "metadata_path_value": "value",
+            },
+        )
+        self.assertEqual(
+            self.onda_search_plugin.config.discover_metadata["metadata_path"],
+            "$.Metadata.*",
+        )
+
+        raw_results = [
+            {"Metadata": [{"id": "foo", "value": "bar"}], "footprint": "POINT (0 0)"}
+        ]
+        products = self.onda_search_plugin.normalize_results(raw_results)
+
+        self.assertEqual(products[0].properties["foo"], "bar")
+
     @mock.patch("eodag.plugins.search.qssearch.requests.get", autospec=True)
     @mock.patch(
         "eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True