Repository: kedro-org/kedro

Issue:
[BUG] Passing runtime nested params to `kedro run --params` overwrites params from config files in latest release
Hello team - nice to be back in these discussions :)

## Description
There was a regression in the latest release (0.18.5), which introduced `OmegaConf` for parsing parameters provided to the `kedro run --params`.

More specifically, in previous versions

```kedro run --params foo.bar=a```

was properly merged with an existing parameter ```{"foo": {"baz": 1}}```.

Now, `kedro.framework.cli.hooks.utils._split_params` returns an `omegaconf.DictConfig` object (rather than a plain dictionary), and the second part of the if clause [here](https://github.com/kedro-org/kedro/blob/45c345d5a8109938e2735c1056a62182eb85d2f3/kedro/framework/context/context.py#L157) returns false:

```python
if isinstance(old_dict[key], dict) and isinstance(value, dict):
    _update_nested_dict(old_dict[key], value)
```

Which means that after merging we get 
```
{"foo": {"bar": a}}
```

instead of the expected

```
{"foo": {"bar": a, "baz": 1}}
```

I could open a PR to fix but it ll take way longer that letting you fix it :)
In the meantime I 've got a local workaround by subclassing `KedroContext` and casting params to a dict!



Discussion:
This is also a problem at the line [here](https://github.com/kedro-org/kedro/blob/94e87ef6b5a7eee1021a877d90691e070b6d3220/kedro/framework/context/context.py#L326) 
The run time params provided through
`kedro run —params`
is/are an instance of `omegaconf.dictconfig.DictConfig` even when the configloader is not set to OmegaConf.

Thus, the run time params are not added/merged with the catalog which subsequently fails to override the same from a parameters file and also causes failure of the pipeline because it is not able to get those parameters in the catalog.

This is my first ever comment on an open source project. Please feel free to correct me.
Hello @tsanikgr nice to see you on here! Even if it's to report a bug 😂  Thanks for reporting this and also thank you @tejaassolanki to report your findings. We will have a look at this asap!

Resolving patch:
diff --git a/kedro/framework/cli/utils.py b/kedro/framework/cli/utils.py
--- a/kedro/framework/cli/utils.py
+++ b/kedro/framework/cli/utils.py
@@ -469,7 +469,7 @@ def _split_params(ctx, param, value):
             )
         dot_list.append(item)
     conf = OmegaConf.from_dotlist(dot_list)
-    return conf
+    return OmegaConf.to_container(conf)
 
 
 def _split_load_versions(ctx, param, value):
diff --git a/kedro/framework/context/context.py b/kedro/framework/context/context.py
--- a/kedro/framework/context/context.py
+++ b/kedro/framework/context/context.py
@@ -7,6 +7,7 @@
 from urllib.parse import urlparse
 from warnings import warn
 
+from omegaconf import DictConfig
 from pluggy import PluginManager
 
 from kedro.config import ConfigLoader, MissingConfigException
@@ -154,7 +155,9 @@ def _update_nested_dict(old_dict: Dict[Any, Any], new_dict: Dict[Any, Any]) -> N
         if key not in old_dict:
             old_dict[key] = value
         else:
-            if isinstance(old_dict[key], dict) and isinstance(value, dict):
+            if isinstance(old_dict[key], (dict, DictConfig)) and isinstance(
+                value, (dict, DictConfig)
+            ):
                 _update_nested_dict(old_dict[key], value)
             else:
                 old_dict[key] = value
@@ -322,8 +325,7 @@ def _add_param_to_feed_dict(param_name, param_value):
             """
             key = f"params:{param_name}"
             feed_dict[key] = param_value
-
-            if isinstance(param_value, dict):
+            if isinstance(param_value, (dict, DictConfig)):
                 for key, val in param_value.items():
                     _add_param_to_feed_dict(f"{param_name}.{key}", val)
 


Test patch:
diff --git a/tests/framework/context/test_context.py b/tests/framework/context/test_context.py
--- a/tests/framework/context/test_context.py
+++ b/tests/framework/context/test_context.py
@@ -10,6 +10,7 @@
 import pytest
 import toml
 import yaml
+from omegaconf import OmegaConf
 from pandas.util.testing import assert_frame_equal
 
 from kedro import __version__ as kedro_version
@@ -163,7 +164,6 @@ def dummy_dataframe():
     '  --from-nodes "nodes3"'
 )
 
-
 expected_message_head = (
     "There are 4 nodes that have not run.\n"
     "You can resume the pipeline run by adding the following "
@@ -484,6 +484,11 @@ def test_validate_layers_error(layers, conflicting_datasets, mocker):
             {"a": {"a.c": {"a.c.b": 4}}},
             {"a": {"a.a": 1, "a.b": 2, "a.c": {"a.c.a": 3, "a.c.b": 4}}},
         ),
+        (
+            {"a": OmegaConf.create({"b": 1}), "x": 3},
+            {"a": {"c": 2}},
+            {"a": {"b": 1, "c": 2}, "x": 3},
+        ),
     ],
 )
 def test_update_nested_dict(old_dict: Dict, new_dict: Dict, expected: Dict):
diff --git a/tests/framework/session/test_session.py b/tests/framework/session/test_session.py
--- a/tests/framework/session/test_session.py
+++ b/tests/framework/session/test_session.py
@@ -2,14 +2,17 @@
 import re
 import subprocess
 import textwrap
+from collections.abc import Mapping
 from pathlib import Path
 
 import pytest
 import toml
 import yaml
+from omegaconf import OmegaConf
 
 from kedro import __version__ as kedro_version
 from kedro.config import AbstractConfigLoader, ConfigLoader, OmegaConfigLoader
+from kedro.framework.cli.utils import _split_params
 from kedro.framework.context import KedroContext
 from kedro.framework.project import (
     ValidationError,
@@ -920,3 +923,26 @@ def test_setup_logging_using_omega_config_loader_class(
     ).as_posix()
     actual_log_filepath = call_args["handlers"]["info_file_handler"]["filename"]
     assert actual_log_filepath == expected_log_filepath
+
+
+def get_all_values(mapping: Mapping):
+    for value in mapping.values():
+        yield value
+        if isinstance(value, Mapping):
+            yield from get_all_values(value)
+
+
+@pytest.mark.parametrize("params", ["a=1,b.c=2", "a=1,b=2,c=3", ""])
+def test_no_DictConfig_in_store(
+    params,
+    mock_package_name,
+    fake_project,
+):
+    extra_params = _split_params(None, None, params)
+    session = KedroSession.create(
+        mock_package_name, fake_project, extra_params=extra_params
+    )
+
+    assert not any(
+        OmegaConf.is_config(value) for value in get_all_values(session._store)
+    )