Repository: CS-SI/eodag
Pull request #189 (2021-03-19 17:06:32)

fix: conf update with external plugin, fixes GH-184


Patch:
diff --git a/docs/_static/custom.css b/docs/_static/custom.css
new file mode 100644
index 00000000..6a54a42e
--- /dev/null
+++ b/docs/_static/custom.css
@@ -0,0 +1,4 @@
+div.sphinxsidebar {
+    max-height: 100%;
+    overflow-y: auto;
+}
diff --git a/docs/conf.py b/docs/conf.py
index 5357ea66..3c6fddcb 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -111,7 +111,7 @@ html_theme_options = {
     "logo": "eodag_logo.png",
     "logo_name": False,
     "show_related": True,
-    "fixed_sidebar": False,
+    "fixed_sidebar": True,
     "show_powered_by": False,
     "github_user": "CS-SI",
     "github_repo": "eodag",
diff --git a/eodag/api/core.py b/eodag/api/core.py
index 15ada6d6..3bb9b089 100644
--- a/eodag/api/core.py
+++ b/eodag/api/core.py
@@ -100,6 +100,8 @@ class EODataAccessGateway(object):
         override_config_from_env(self.providers_config)
 
         self._plugins_manager = PluginManager(self.providers_config)
+        # use updated providers_config
+        self.providers_config = self._plugins_manager.providers_config
 
         # Build a search index for product types
         self._product_types_index = None
diff --git a/eodag/config.py b/eodag/config.py
index 2a9e45d7..2334ce75 100644
--- a/eodag/config.py
+++ b/eodag/config.py
@@ -335,6 +335,38 @@ def override_config_from_mapping(config, mapping):
                 logger.warning("%s skipped: %s", provider, e)
 
 
+def merge_configs(config, other_config):
+    """Override a configuration with the values of another configuration
+
+    :param dict config: An eodag providers configuration dictionary
+    :param dict other_config: An eodag providers configuration dictionary
+    """
+    # configs union with other_config values as default
+    other_config = dict(config, **other_config)
+
+    for provider, new_conf in other_config.items():
+        old_conf = config.get(provider, None)
+
+        if old_conf:
+            # update non-objects values
+            new_conf = dict(old_conf.__dict__, **new_conf.__dict__)
+
+            for conf_k, conf_v in new_conf.items():
+                old_conf_v = getattr(old_conf, conf_k, None)
+
+                if isinstance(conf_v, PluginConfig) and isinstance(
+                    old_conf_v, PluginConfig
+                ):
+                    old_conf_v.update(conf_v.__dict__)
+                    new_conf[conf_k] = old_conf_v
+                elif isinstance(old_conf_v, PluginConfig):
+                    new_conf[conf_k] = old_conf_v
+
+                setattr(config[provider], conf_k, new_conf[conf_k])
+        else:
+            config[provider] = new_conf
+
+
 def load_yml_config(yml_path):
     """Build a conf dictionnary from given yml absolute path
 
diff --git a/eodag/plugins/manager.py b/eodag/plugins/manager.py
index 8abf723b..42b359e7 100644
--- a/eodag/plugins/manager.py
+++ b/eodag/plugins/manager.py
@@ -21,7 +21,7 @@ from pathlib import Path
 
 import pkg_resources
 
-from eodag.config import load_config
+from eodag.config import load_config, merge_configs
 from eodag.plugins.apis.base import Api
 from eodag.plugins.authentication.base import Authentication
 from eodag.plugins.base import EODAGPluginMount
@@ -79,17 +79,21 @@ class PluginManager(object):
                         "Check that the plugin module (%s) is importable",
                         entry_point.module_name,
                     )
-                if entry_point.dist.project_name != "eodag":
+                if entry_point.dist.key != "eodag":
                     # use plugin providers if any
                     plugin_providers_config_path = [
                         x
-                        for x in Path(entry_point.dist.location).rglob("providers.yml")
-                        if all(i not in str(x) for i in ["build", ".tox"])
+                        for x in Path(
+                            entry_point.dist.location,
+                            pkg_resources.to_filename(entry_point.dist.key),
+                        ).rglob("providers.yml")
                     ]
                     if plugin_providers_config_path:
-                        self.providers_config.update(
-                            load_config(plugin_providers_config_path[0])
+                        plugin_providers_config = load_config(
+                            plugin_providers_config_path[0]
                         )
+                        merge_configs(plugin_providers_config, self.providers_config)
+                        self.providers_config = plugin_providers_config
 
         self.product_type_to_provider_config_map = {}
         for provider_config in self.providers_config.values():


Test patch:
diff --git a/tests/context.py b/tests/context.py
index c0f3a8b0..bd969304 100644
--- a/tests/context.py
+++ b/tests/context.py
@@ -31,7 +31,7 @@ from eodag.api.product.drivers import DRIVERS
 from eodag.api.product.drivers.base import NoDriver
 from eodag.api.search_result import SearchResult
 from eodag.cli import download, eodag, list_pt, search_crunch
-from eodag.config import load_default_config
+from eodag.config import load_default_config, merge_configs
 from eodag.plugins.authentication.base import Authentication
 from eodag.plugins.crunch.filter_date import FilterDate
 from eodag.plugins.crunch.filter_latest_tpl_name import FilterLatestByName
diff --git a/tests/resources/fake_ext_plugin/eodag_fakeplugin/__init__.py b/tests/resources/fake_ext_plugin/eodag_fakeplugin/__init__.py
new file mode 100644
index 00000000..ec2fd1e3
--- /dev/null
+++ b/tests/resources/fake_ext_plugin/eodag_fakeplugin/__init__.py
@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# Copyright 2021, CS GROUP - France, http://www.c-s.fr
+#
+# This file is part of EODAG project
+#     https://www.github.com/CS-SI/EODAG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Filter Cython warnings
+# Apparently (see
+# https://stackoverflow.com/questions/40845304/runtimewarning-numpy-dtype-size-changed-may-indicate-binary-incompatibility     # noqa
+# and https://github.com/numpy/numpy/issues/11788) this is caused by Cython and affect pre-built Python packages that
+# depends on numpy and ship with a pre-built version of numpy that is older than 1.15.1 (where the warning is silenced
+# exactly as below)
+"""EODAG fake_ext_plugin."""
+
+
+class FakePluginAPI:
+    pass
diff --git a/tests/resources/fake_ext_plugin/providers.yml b/tests/resources/fake_ext_plugin/eodag_fakeplugin/providers.yml
similarity index 100%
rename from tests/resources/fake_ext_plugin/providers.yml
rename to tests/resources/fake_ext_plugin/eodag_fakeplugin/providers.yml
diff --git a/tests/test_config.py b/tests/test_config.py
index 2d01ad25..dac5a227 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -23,7 +23,7 @@ from io import StringIO
 
 import yaml.parser
 
-from tests.context import ValidationError, config
+from tests.context import ValidationError, config, merge_configs
 
 
 class TestProviderConfig(unittest.TestCase):
@@ -136,6 +136,48 @@ class TestProviderConfig(unittest.TestCase):
         self.assertTrue(hasattr(provider_config.api, "newParam"))
         self.assertEqual(provider_config.api.newParam, "val")
 
+    def test_provider_config_merge(self):
+        """Merge 2 providers configs"""
+        config_stream1 = StringIO(
+            """!provider
+                name: provider1
+                provider_param: val
+                provider_param2: val2
+                api: !plugin
+                    type: MyPluginClass
+                    plugin_param1: value1
+                    pluginParam2: value2
+        """
+        )
+        config_stream2 = StringIO(
+            """!provider
+                name: provider1
+                provider_param: val1
+                provider_param3: val3
+                api: !plugin
+                    type: MyPluginClass
+                    pluginParam2: value3
+        """
+        )
+        provider_config1 = yaml.load(config_stream1, Loader=yaml.Loader)
+        provider_config2 = yaml.load(config_stream2, Loader=yaml.Loader)
+
+        providers_config = {
+            "provider1": provider_config1,
+            "provider2": provider_config1,
+        }
+
+        merge_configs(
+            providers_config,
+            {"provider1": provider_config2, "provider3": provider_config1},
+        )
+        self.assertEqual(len(providers_config), 3)
+        self.assertEqual(providers_config["provider1"].provider_param, "val1")
+        self.assertEqual(providers_config["provider1"].provider_param2, "val2")
+        self.assertEqual(providers_config["provider1"].provider_param3, "val3")
+        self.assertEqual(providers_config["provider1"].api.plugin_param1, "value1")
+        self.assertEqual(providers_config["provider1"].api.pluginParam2, "value3")
+
 
 class TestPluginConfig(unittest.TestCase):
     def test_plugin_config_valid(self):