Repository: dynaconf/dynaconf (python)

Issue:
BoxKeyError when using setdefault
The following code raises `BoxKeyError`.
```python
my_flask_dynaconf.init_app(app)
asdf = app.config.setdefault("ASDF", {})
asdf.setdefault("qwer", "zxcv")
qwer = app.config["ASDF"]["qwer"]
```

The reason for bringing this up is because the Flask extension Flask-Alembic uses this code pattern.

Please see [the Flask-Alembic source code](https://github.com/pallets-eco/flask-alembic/blob/3.1.0/src/flask_alembic/extension.py).
On line 150 and 151 `script_location` is defined.
On line 215 `script_location` is accessed, and this is where Dynaconf raises `BoxKeyError`.

It would be good if these libraries could be used at the same time.
Is there anything that could be changed in Dynaconf to accommodate for this pattern? Or is this pattern incompatible with Dynaconf?

It is possible to workaround the issue, by moving the call to Flask-Alembic's `init_app` so that it occurs before the call to Dynaconf's Flask `init_app`.
But that probably does not count as a proper solution. Ideally, Dynaconf's Flask `init_app` should be the first one to be called, so that config is loaded before everything else.

Pull request:
fix(flask):Allow disabling transformation of dict to Dynabox
fix #1113

## Compatibility mode

Dynaconf transform nested data structures to a new DynaBox instance, this is an
object that allows dot notation access such as `app.config.key.value.other`.
However this feature is imcompatible with some Flask extensions, for example:
`Flask-Alembic`.

So in case you see a `BoxKeyError` when running your app with you can run it on
compatibility mode.

```python
FlaskDynaconf(app, compatibility_mode=True)
```

The downside is that now you cannot access nested elements using dot notation
but the usual access interface via dict subscription will work normally:
`app.config["KEY"]["value"]["other"]`

!!! info
    On Dynaconf 4.0 compatibility_mode will default to True when running
    on Flask.

Patch:
diff --git a/docs/flask.md b/docs/flask.md
index d8f54a8..49886df 100644
--- a/docs/flask.md
+++ b/docs/flask.md
@@ -18,6 +18,29 @@ FlaskDynaconf(app)
 
 > You can optionally use `init_app` as well.
 
+
+## Compatibility mode
+
+Dynaconf transform nested data structures to a new DynaBox instance, this is an
+object that allows dot notation access such as `app.config.key.value.other`.
+However this feature is imcompatible with some Flask extensions, for example:
+`Flask-Alembic`.
+
+So in case you see a `BoxKeyError` when running your app with you can run it on
+compatibility mode.
+
+```python
+FlaskDynaconf(app, compatibility_mode=True)
+```
+
+The downside is that now you cannot access nested elements using dot notation
+but the usual access interface via dict subscription will work normally:
+`app.config["KEY"]["value"]["other"]`
+
+!!! info
+    On Dynaconf 4.0 compatibility_mode will default to True when running
+    on Flask.
+
 ## Use `FLASK_` environment variables
 
 Then the `app.config` will work as a `dynaconf.settings` instance and `FLASK_` will be the global prefix for exporting environment variables.
diff --git a/dynaconf/base.py b/dynaconf/base.py
index 3edc9a1..e22fef9 100644
--- a/dynaconf/base.py
+++ b/dynaconf/base.py
@@ -239,7 +239,12 @@ class Settings:
         self._loaded_py_modules = []
         self._loaded_files = []
         self._deleted = set()
-        self._store = DynaBox(box_settings=self)
+
+        if kwargs.get("DYNABOXIFY", True):
+            self._store = DynaBox(box_settings=self)
+        else:
+            self._store = {}  # Disabled dot access
+
         self._env_cache = {}
         self._loaded_by_loaders: dict[SourceMetadata | str, Any] = {}
         self._loaders = []
@@ -526,7 +531,7 @@ class Settings:
                 default = self.get_environ(key, cast=True)
 
         # default values should behave exactly Dynaconf parsed values
-        if default is not None:
+        if default is not None and self._store.get("DYNABOXIFY", True):
             if isinstance(default, list):
                 default = BoxList(default)
             elif isinstance(default, dict):
@@ -907,7 +912,10 @@ class Settings:
         # Add a "." before "[" to help splitting
         split_keys = dotted_key.replace("[", ".[").split(".")
         existing_data = self.get(split_keys[0], {})
-        new_data = tree = DynaBox(box_settings=self)
+        if self.get("DYNABOXIFY", True):
+            new_data = tree = DynaBox(box_settings=self)
+        else:
+            new_data = tree = {}
         value = parse_conf_data(value, tomlfy=tomlfy, box_settings=self)
 
         for n, k in enumerate(split_keys):
@@ -934,9 +942,13 @@ class Settings:
                 raise (ValueError("Invalid field:", k))
 
         if existing_data:
-            old_data = DynaBox(
-                {split_keys[0]: existing_data}, box_settings=self
-            )
+            if self.get("DYNABOXIFY", True):
+                old_data = DynaBox(
+                    {split_keys[0]: existing_data}, box_settings=self
+                )
+            else:
+                old_data = {split_keys[0]: existing_data}
+
             new_data = object_merge(
                 old=old_data,
                 new=new_data,
@@ -1060,7 +1072,11 @@ class Settings:
                     else "merge",  # fix 905
                 )
 
-        if isinstance(parsed, dict) and not isinstance(parsed, DynaBox):
+        if (
+            self.get("DYNABOXIFY", True)
+            and isinstance(parsed, dict)
+            and not isinstance(parsed, DynaBox)
+        ):
             parsed = DynaBox(parsed, box_settings=self)
 
         # Set the parsed value
diff --git a/dynaconf/contrib/flask_dynaconf.py b/dynaconf/contrib/flask_dynaconf.py
index 5d9ba7a..5a4d624 100644
--- a/dynaconf/contrib/flask_dynaconf.py
+++ b/dynaconf/contrib/flask_dynaconf.py
@@ -111,6 +111,9 @@ class FlaskDynaconf:
 
     def init_app(self, app, **kwargs):
         """kwargs holds initial dynaconf configuration"""
+        if kwargs.pop("compatibility_mode", None) is True:
+            kwargs["DYNABOXIFY"] = False
+
         self.kwargs.update(kwargs)
         self.settings = self.dynaconf_instance or dynaconf.LazySettings(
             **self.kwargs
diff --git a/dynaconf/utils/parse_conf.py b/dynaconf/utils/parse_conf.py
index 524e857..dae8f74 100644
--- a/dynaconf/utils/parse_conf.py
+++ b/dynaconf/utils/parse_conf.py
@@ -412,7 +412,7 @@ def _parse_conf_data(data, tomlfy=False, box_settings=None):
     else:
         value = parse_with_toml(data) if tomlfy else data
 
-    if isinstance(value, dict):
+    if isinstance(value, dict) and box_settings.get("DYNABOXIFY", True):
         value = DynaBox(value, box_settings=box_settings)
 
     return value
@@ -441,21 +441,23 @@ def parse_conf_data(data, tomlfy=False, box_settings=None):
 
     if isinstance(data, DynaBox):
         # recursively parse inner dict items
-        _parsed = DynaBox({}, box_settings=box_settings)
+        # It is important to keep the same object id because
+        # of mutability
         for k, v in data._safe_items():
-            _parsed[k] = parse_conf_data(
+            data[k] = parse_conf_data(
                 v, tomlfy=tomlfy, box_settings=box_settings
             )
-        return _parsed
+        return data
 
     if isinstance(data, dict):
         # recursively parse inner dict items
-        _parsed = {}
+        # It is important to keep the same object id because
+        # of mutability
         for k, v in data.items():
-            _parsed[k] = parse_conf_data(
+            data[k] = parse_conf_data(
                 v, tomlfy=tomlfy, box_settings=box_settings
             )
-        return _parsed
+        return data
 
     # return parsed string value
     return _parse_conf_data(data, tomlfy=tomlfy, box_settings=box_settings)


Test patch:
diff --git a/tests/test_flask.py b/tests/test_flask.py
index cbd2189..be33292 100644
--- a/tests/test_flask.py
+++ b/tests/test_flask.py
@@ -163,3 +163,31 @@ def test_setting_instance_options_works_case_insensitive():
     # oddly, using '_for_dynaconf' won't work, although
     # the option functionality seems to work as expected
     assert app.config.load_dotenv is False
+
+
+def test_compatibility_mode():
+    app = Flask(__name__)
+    FlaskDynaconf(app, compatibility_mode=True)
+
+    data = {}
+    new_data = app.config.setdefault("asdf", data)
+    assert new_data is data
+    assert new_data is app.config._store["ASDF"]
+
+    asdf = app.config.setdefault("ASDF", {})
+    asdf.setdefault("qwer", "zxcv")
+    qwer = app.config["ASDF"]["qwer"]
+    assert qwer == "zxcv"
+
+    # Attribute access works in the first level
+    assert app.config.ASDF == {"qwer": "zxcv"}
+
+    # But not on nested levels
+    with pytest.raises(AttributeError):
+        app.config.ASDF.qwer
+
+    # Unless explicitly casted to a DynaBox
+    from dynaconf.utils.boxing import DynaBox
+
+    asdf = DynaBox(app.config.ASDF)
+    assert asdf.qwer == "zxcv"