Repository: CS-SI/eodag
Pull request #844 (2023-09-26 08:49:42)

feat: restore pruned providers when configuration is updated
The pruned providers are saved in `EODataAccessGateway._pruned_providers_config` and restored when the configuration is updated.

github-actions[bot]: ## Test Results
    2 files   -        2      2 suites   - 2   2m 26s [:stopwatch:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.10.0/README.md#the-symbols "duration of all tests") - 4m 28s
431 tests ±       0  394 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.10.0/README.md#the-symbols "passed tests")  -      34  35 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.10.0/README.md#the-symbols "skipped / disabled tests") +32  2 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.10.0/README.md#the-symbols "failed tests") +2 
576 runs   - 1 148  537 [:heavy_check_mark:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.10.0/README.md#the-symbols "passed tests")  - 1 111  35 [:zzz:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.10.0/README.md#the-symbols "skipped / disabled tests")  - 41  4 [:x:](https://github.com/EnricoMi/publish-unit-test-result-action/blob/v2.10.0/README.md#the-symbols "failed tests") +4 

For more details on these failures, see [this check](https://github.com/CS-SI/eodag/runs/17132157050).

Results for commit 18cafe41. ± Comparison against base commit 5e389e4e.

<details>
  <summary>This pull request <b>skips</b> 32 tests.</summary>

```
tests.units.test_download_plugins.TestDownloadPluginBase ‑ test_plugins_download_base_prepare_download_dir_permission
tests.units.test_http_server.RequestTestCase ‑ test_auth_error
tests.units.test_http_server.RequestTestCase ‑ test_catalog_browse
tests.units.test_http_server.RequestTestCase ‑ test_catalog_browse_date_search
tests.units.test_http_server.RequestTestCase ‑ test_cloud_cover_post_search
tests.units.test_http_server.RequestTestCase ‑ test_collection
tests.units.test_http_server.RequestTestCase ‑ test_conformance
tests.units.test_http_server.RequestTestCase ‑ test_date_post_search
tests.units.test_http_server.RequestTestCase ‑ test_date_search
tests.units.test_http_server.RequestTestCase ‑ test_date_search_from_catalog_items
…
```
</details>

[test-results]:data:application/gzip;base64,H4sIAEycEmUC/03MTQ7CIBCG4as0rF1AGYp4GQNTSIj9MRRWxrs71YLdTPI9k7wvFuLkN3br+kvHthJzG2NJNsd1oSlgIKBX3p8gRV33rSASSQN/esTnTqpJsHE6oj/wKa2JhJOksuxRpYdj1KaSuskp+YWjCHWfg7jOc8w0mLiiDR4EcmmC8xxQaackclDOGTtqOtCbgb0/hFk38AcBAAA=

github-actions[bot]: <strong>Code Coverage (Ubuntu)</strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `82%` | :white_check_mark: |

_Minimum allowed coverage is `70%`_

<p align="right">Generated by :monkey: cobertura-action against 82e23db2e152e94d1cd59ba226956af81e8415d2 </p>
github-actions[bot]: <strong>Code Coverage (Windows)</strong>

| File | Coverage |   |
| - | :-: | :-: |
| **All files** | `53%` | :x: |

_Minimum allowed coverage is `70%`_

<p align="right">Generated by :monkey: cobertura-action against 82e23db2e152e94d1cd59ba226956af81e8415d2 </p>

Patch:
diff --git a/eodag/api/core.py b/eodag/api/core.py
index 45c31d6e..10b414b7 100644
--- a/eodag/api/core.py
+++ b/eodag/api/core.py
@@ -134,6 +134,8 @@ class EODataAccessGateway(object):
         # use updated and checked providers_config
         self.providers_config = self._plugins_manager.providers_config
 
+        # store pruned providers configs
+        self._pruned_providers_config = {}
         # filter out providers needing auth that have no credentials set
         self._prune_providers_list()
 
@@ -301,6 +303,18 @@ class EODataAccessGateway(object):
         :type yaml_conf: str
         """
         conf_update = yaml.safe_load(yaml_conf)
+
+        # restore the pruned configuration
+        for provider in list(self._pruned_providers_config.keys()):
+            if provider in conf_update:
+                logger.info(
+                    "%s: provider restored from the pruned configurations",
+                    provider,
+                )
+                self.providers_config[provider] = self._pruned_providers_config.pop(
+                    provider
+                )
+
         # check if metada-mapping as already been built as jsonpath in providers_config
         for provider, provider_conf in conf_update.items():
             if (
@@ -335,6 +349,7 @@ class EODataAccessGateway(object):
                 )
 
         override_config_from_mapping(self.providers_config, conf_update)
+
         stac_provider_config = load_stac_provider_config()
         for provider in conf_update.keys():
             provider_config_init(self.providers_config[provider], stac_provider_config)
@@ -356,7 +371,9 @@ class EODataAccessGateway(object):
                 )
                 if not credentials_exist:
                     # credentials needed but not found
-                    del self.providers_config[provider]
+                    self._pruned_providers_config[provider] = self.providers_config.pop(
+                        provider
+                    )
                     update_needed = True
                     logger.info(
                         "%s: provider needing auth for search has been pruned because no crendentials could be found",
@@ -365,7 +382,9 @@ class EODataAccessGateway(object):
             elif hasattr(conf, "search") and getattr(conf.search, "need_auth", False):
                 if not hasattr(conf, "auth"):
                     # credentials needed but no auth plugin was found
-                    del self.providers_config[provider]
+                    self._pruned_providers_config[provider] = self.providers_config.pop(
+                        provider
+                    )
                     update_needed = True
                     logger.info(
                         "%s: provider needing auth for search has been pruned because no auth plugin could be found",
@@ -380,7 +399,9 @@ class EODataAccessGateway(object):
                 )
                 if not credentials_exist:
                     # credentials needed but not found
-                    del self.providers_config[provider]
+                    self._pruned_providers_config[provider] = self.providers_config.pop(
+                        provider
+                    )
                     update_needed = True
                     logger.info(
                         "%s: provider needing auth for search has been pruned because no crendentials could be found",
@@ -388,7 +409,9 @@ class EODataAccessGateway(object):
                     )
             elif not hasattr(conf, "api") and not hasattr(conf, "search"):
                 # provider should have at least an api or search plugin
-                del self.providers_config[provider]
+                self._pruned_providers_config[provider] = self.providers_config.pop(
+                    provider
+                )
                 logger.info(
                     "%s: provider has been pruned because no api or search plugin could be found",
                     provider,
diff --git a/eodag/api/product/metadata_mapping.py b/eodag/api/product/metadata_mapping.py
index ae0b74db..62bbc05c 100644
--- a/eodag/api/product/metadata_mapping.py
+++ b/eodag/api/product/metadata_mapping.py
@@ -1013,6 +1013,17 @@ def mtd_cfg_as_conversion_and_querypath(src_dict, dest_dict={}, result_type="jso
     :returns: dest_dict
     :rtype: dict
     """
+    # check if the configuration has already been converted
+    some_configured_value = (
+        next(iter(dest_dict.values())) if dest_dict else next(iter(src_dict.values()))
+    )
+    if (
+        isinstance(some_configured_value, list)
+        and isinstance(some_configured_value[1], tuple)
+        or isinstance(some_configured_value, tuple)
+    ):
+        return dest_dict or src_dict
+
     if not dest_dict:
         dest_dict = deepcopy(src_dict)
     for metadata in src_dict:


Test patch:
diff --git a/tests/integration/test_core_config.py b/tests/integration/test_core_config.py
index cbd6cc75..525a5391 100644
--- a/tests/integration/test_core_config.py
+++ b/tests/integration/test_core_config.py
@@ -119,6 +119,55 @@ class TestCoreProvidersConfig(TestCase):
             self.dag.providers_config["foo_provider"].auth.type, "GenericAuth"
         )
 
+        # update pruned provider with credentials
+        self.assertNotIn("usgs", self.dag.providers_config)
+        self.dag.update_providers_config(
+            """
+            usgs:
+                api:
+                    credentials:
+                        username: foo
+                        password: bar
+            """
+        )
+        self.assertIsInstance(self.dag.providers_config["usgs"].api, PluginConfig)
+        self.assertEqual(
+            self.dag.providers_config["usgs"].api.credentials["username"], "foo"
+        )
+
+        # add new provider that requires auth but without credentials
+        self.dag.update_providers_config(
+            """
+            bar_provider:
+                search:
+                    type: StacSearch
+                    api_endpoint: https://foo.bar/search
+                    need_auth: True
+                products:
+                    GENERIC_PRODUCT_TYPE:
+                        productType: '{productType}'
+            """
+        )
+        self.assertIn("bar_provider", self.dag.providers_config)
+
+        # update provider with credentials
+        self.dag.update_providers_config(
+            """
+            bar_provider:
+                auth:
+                    credentials:
+                        username: bar
+                        password: foo
+            """
+        )
+        self.assertIsInstance(
+            self.dag.providers_config["bar_provider"].auth, PluginConfig
+        )
+        self.assertEqual(
+            self.dag.providers_config["bar_provider"].auth.credentials["username"],
+            "bar",
+        )
+
 
 class TestCoreProductTypesConfig(TestCase):
     def setUp(self):