Repository: CS-SI/eodag
Pull request #483 (2022-07-20 09:50:54)

fix: metadata-mapping with providers update
Fixes metadata-mapping issues when updating a provider's configuration after having already searched on it:
```py
>>> from eodag import EODataAccessGateway
>>> dag = EODataAccessGateway()
>>> dag.set_preferred_provider("earth_search")
>>> dag.search()
([EOProduct(id=S2B_46XDG_20220720_0_L2A, provider=earth_search), EOProduct(id=S2B_46XDG_20220720_0_L2A, provider=earth_search), EOProduct(id=S2B_46XFF_20220720_0_L2A, provider=earth_search), EOProduct(id=S2B_47XMA_20220720_0_L2A, provider=earth_search), EOProduct(id=S2B_46XFF_20220720_0_L2A, provider=earth_search), EOProduct(id=S2B_47XMA_20220720_0_L2A, provider=earth_search), EOProduct(id=S2B_46XEG_20220720_0_L2A, provider=earth_search), EOProduct(id=S2B_46XEG_20220720_0_L2A, provider=earth_search), EOProduct(id=S2B_47XNA_20220720_0_L2A, provider=earth_search), EOProduct(id=S2B_46XDH_20220720_0_L2A, provider=earth_search), EOProduct(id=S2B_47XNA_20220720_0_L2A, provider=earth_search), EOProduct(id=S2B_46XDH_20220720_0_L2A, provider=earth_search), EOProduct(id=S2B_47XMB_20220720_0_L2A, provider=earth_search), EOProduct(id=S2B_47XMB_20220720_0_L2A, provider=earth_search), EOProduct(id=S2B_46XEH_20220720_0_L2A, provider=earth_search), EOProduct(id=S2B_46XEH_20220720_0_L2A, provider=earth_search), EOProduct(id=S2B_46XDJ_20220720_0_L2A, provider=earth_search), EOProduct(id=S2B_46XDJ_20220720_0_L2A, provider=earth_search), EOProduct(id=S2B_47XNB_20220720_0_L2A, provider=earth_search), EOProduct(id=S2B_47XNB_20220720_0_L2A, provider=earth_search)], 52742652)
>>> dag.update_providers_config(
...     """
...     earth_search:
...         search:
...             foo: bar
...     """
... )
>>> dag.search()
Could not match regex on metadata path '(None, 'Not Mapped')'
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/sylvain/workspace/eodag/eodag/api/core.py", line 805, in search
    search_kwargs = self._prepare_search(
  File "/home/sylvain/workspace/eodag/eodag/api/core.py", line 1192, in _prepare_search
    search_plugin = next(
  File "/home/sylvain/workspace/eodag/eodag/plugins/manager.py", line 167, in get_search_plugins
    yield get_plugin()
  File "/home/sylvain/workspace/eodag/eodag/plugins/manager.py", line 147, in get_plugin
    plugin = self._build_plugin(config.name, config.search, Search)
  File "/home/sylvain/workspace/eodag/eodag/plugins/manager.py", line 290, in _build_plugin
    plugin = plugin_class(provider, plugin_conf)
  File "/home/sylvain/workspace/eodag/eodag/plugins/search/qssearch.py", line 1152, in __init__
    super(StacSearch, self).__init__(provider, config)
  File "/home/sylvain/workspace/eodag/eodag/plugins/search/qssearch.py", line 945, in __init__
    super(PostJsonSearch, self).__init__(provider, config)
  File "/home/sylvain/workspace/eodag/eodag/plugins/search/qssearch.py", line 156, in __init__
    super(QueryStringSearch, self).__init__(provider, config)
  File "/home/sylvain/workspace/eodag/eodag/plugins/search/base.py", line 43, in __init__
    self.config.metadata_mapping = mtd_cfg_as_jsonpath(
  File "/home/sylvain/workspace/eodag/eodag/api/product/metadata_mapping.py", line 638, in mtd_cfg_as_jsonpath
    conversion, path = get_metadata_path(dest_dict[metadata])
  File "/home/sylvain/workspace/eodag/eodag/api/product/metadata_mapping.py", line 93, in get_metadata_path
    raise e
  File "/home/sylvain/workspace/eodag/eodag/api/product/metadata_mapping.py", line 90, in get_metadata_path
    match = INGEST_CONVERSION_REGEX.match(path)
TypeError: expected string or bytes-like object
```

github-actions[bot]: ## Unit Test Results
    2 files  ±0      2 suites  ±0   11m 32s [:stopwatch:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "duration of all tests") - 3m 39s
265 tests +1  263 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "passed tests") +1  2 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "skipped / disabled tests") ±0  0 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "failed tests") ±0 
530 runs  +2  526 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "passed tests") +2  4 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "skipped / disabled tests") ±0  0 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.20/README.md#the-symbols "failed tests") ±0 

Results for commit 839b41e5. ± Comparison against base commit 4cacc1cf.

[test-results]:data:application/gzip;base64,H4sIABrS12IC/02MSQ7CIAAAv9Jw9gBl9zMGWRJiWwzLyfh3KS3Y48wk8wHOLzaB+zTfJpCKzwNMiSr7sFVkchc15RYZ7fRIReum8F+9/PtcHMIpv1QBh7AxhniaWLb9SXGHvqQzG+Y4ks6XYePrT4d19bkCEFg+CbIUS0SQhNooKqCVliKBOWdYcgeZNRx8f7oUE/4FAQAA

github-actions[bot]: <strong> </strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `80%` | :white_check_mark: |
| eodag/api/core.py | `87%` | :white_check_mark: |
| eodag/api/product/metadata\_mapping.py | `80%` | :white_check_mark: |

<strong> </strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `80%` | :white_check_mark: |
| eodag/api/core.py | `87%` | :white_check_mark: |
| eodag/api/product/metadata\_mapping.py | `80%` | :white_check_mark: |
| eodag/api/core.py | `87%` | :white_check_mark: |
| eodag/api/product/metadata\_mapping.py | `78%` | :white_check_mark: |

_Minimum allowed coverage is `70%`_

<p align="right">Generated by :monkey: cobertura-action against 839b41e53914190cda580e9e5183776397f06ed7 </p>

Patch:
diff --git a/eodag/api/core.py b/eodag/api/core.py
index f29e238a..498aea3e 100644
--- a/eodag/api/core.py
+++ b/eodag/api/core.py
@@ -30,6 +30,7 @@ from whoosh.fields import Schema
 from whoosh.index import create_in, exists_in, open_dir
 from whoosh.qparser import QueryParser
 
+from eodag.api.product.metadata_mapping import mtd_cfg_as_jsonpath
 from eodag.api.search_result import SearchResult
 from eodag.config import (
     SimpleYamlProxyConfig,
@@ -338,17 +339,58 @@ class EODataAccessGateway(object):
         >>> dag.providers_config["my_new_provider"].priority
         0
         >>> # run 2nd time (update provider)
+        >>> update_config = '''
+        ...     my_new_provider:
+        ...         search:
+        ...             type: StacSearch
+        ...             api_endpoint: https://api.my_new_provider/search
+        ...         products:
+        ...             GENERIC_PRODUCT_TYPE:
+        ...                 productType: '{productType}'
+        ... '''
         >>> dag.update_providers_config(new_config)
 
         :param yaml_conf: YAML formated provider configuration
         :type yaml_conf: str
         """
         conf_update = yaml.safe_load(yaml_conf)
+        # check if metada-mapping as already been built as jsonpath in providers_config
+        for provider, provider_conf in conf_update.items():
+            if (
+                provider in self.providers_config
+                and "metadata_mapping" in provider_conf.get("search", {})
+            ):
+                search_plugin_key = "search"
+            elif (
+                provider in self.providers_config
+                and "metadata_mapping" in provider_conf.get("api", {})
+            ):
+                search_plugin_key = "api"
+            else:
+                continue
+            # get some already configured value
+            configured_metadata_mapping = getattr(
+                self.providers_config[provider], search_plugin_key
+            ).metadata_mapping
+            some_configured_value = next(iter(configured_metadata_mapping.values()))
+            # check if the configured value has already been built as jsonpath
+            if (
+                isinstance(some_configured_value, list)
+                and isinstance(some_configured_value[1], tuple)
+                or isinstance(some_configured_value, tuple)
+            ):
+                # also build as jsonpath the incoming conf
+                mtd_cfg_as_jsonpath(
+                    conf_update[provider][search_plugin_key]["metadata_mapping"],
+                    conf_update[provider][search_plugin_key]["metadata_mapping"],
+                )
+
         override_config_from_mapping(self.providers_config, conf_update)
         stac_provider_config = load_stac_provider_config()
         for provider in conf_update.keys():
             provider_config_init(self.providers_config[provider], stac_provider_config)
-        self._plugins_manager = PluginManager(self.providers_config)
+        # re-create _plugins_manager using up-to-date providers_config
+        self._plugins_manager.build_product_type_to_provider_config_map()
 
     def _prune_providers_list(self):
         """Removes from config providers needing auth that have no credentials set."""
@@ -675,7 +717,6 @@ class EODataAccessGateway(object):
             self.providers_config[provider].product_types_fetched = True
 
         # re-create _plugins_manager using up-to-date providers_config
-        # self._plugins_manager = PluginManager(self.providers_config)
         self._plugins_manager.build_product_type_to_provider_config_map()
 
         # rebuild index after product types list update
diff --git a/eodag/api/product/metadata_mapping.py b/eodag/api/product/metadata_mapping.py
index 62167d3a..f399bc0a 100644
--- a/eodag/api/product/metadata_mapping.py
+++ b/eodag/api/product/metadata_mapping.py
@@ -18,6 +18,7 @@
 import ast
 import logging
 import re
+from copy import deepcopy
 from datetime import datetime, timedelta
 from string import Formatter
 
@@ -630,7 +631,7 @@ def mtd_cfg_as_jsonpath(src_dict, dest_dict={}):
     :rtype: dict
     """
     if not dest_dict:
-        dest_dict = src_dict
+        dest_dict = deepcopy(src_dict)
     for metadata in src_dict:
         if metadata not in dest_dict:
             dest_dict[metadata] = (None, NOT_MAPPED)


Test patch:
diff --git a/tests/integration/test_core_config.py b/tests/integration/test_core_config.py
new file mode 100644
index 00000000..9109420f
--- /dev/null
+++ b/tests/integration/test_core_config.py
@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+# Copyright 2022, CS GROUP - France, https://www.csgroup.eu/
+#
+# This file is part of EODAG project
+#     https://www.github.com/CS-SI/EODAG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import tempfile
+from unittest import TestCase, mock
+
+from tests.context import EODataAccessGateway
+
+
+class TestCoreProvidersConfig(TestCase):
+    def setUp(self):
+        super(TestCoreProvidersConfig, self).setUp()
+        # Mock home and eodag conf directory to tmp dir
+        self.tmp_home_dir = tempfile.TemporaryDirectory()
+        self.expanduser_mock = mock.patch(
+            "os.path.expanduser", autospec=True, return_value=self.tmp_home_dir.name
+        )
+        self.expanduser_mock.start()
+        self.dag = EODataAccessGateway()
+
+    def tearDown(self):
+        super(TestCoreProvidersConfig, self).tearDown()
+        # stop Mock and remove tmp config dir
+        self.expanduser_mock.stop()
+        self.tmp_home_dir.cleanup()
+
+    @mock.patch("eodag.plugins.search.qssearch.PostJsonSearch._request", autospec=True)
+    def test_core_providers_config_update(self, mock__request):
+        """Providers config must be updatable"""
+        mock__request.return_value = mock.Mock()
+        mock__request_side_effect = [
+            {
+                "context": {
+                    "matched": 1,
+                }
+            },
+            {
+                "features": [
+                    {
+                        "id": "foo",
+                        "bar": "baz",
+                        "geometry": "POLYGON((180 -90, 180 90, -180 90, -180 -90, 180 -90))",
+                    }
+                ]
+            },
+        ]
+        mock__request.return_value.json.side_effect = mock__request_side_effect
+
+        # add new provider and search on it
+        self.dag.update_providers_config(
+            """
+            foo_provider:
+                search:
+                    type: StacSearch
+                    api_endpoint: https://foo.bar/search
+                products:
+                    GENERIC_PRODUCT_TYPE:
+                        productType: '{productType}'
+                download:
+                    type: HTTPDownload
+                    base_uri: https://foo.bar
+            """
+        )
+        self.dag.set_preferred_provider("foo_provider")
+
+        prods, _ = self.dag.search(raise_errors=True)
+        self.assertEqual(prods[0].properties["title"], "foo")
+
+        # update provider which have metadata_mapping already built as jsonpath by search()
+        self.dag.update_providers_config(
+            """
+            foo_provider:
+                search:
+                    metadata_mapping:
+                        title: '$.bar'
+            """
+        )
+        mock__request.return_value.json.side_effect = mock__request_side_effect
+        prods, _ = self.dag.search(raise_errors=True)
+        self.assertEqual(prods[0].properties["title"], "baz")