Repository: janehmueller/python-json-config

Issue:
Handle environment variables that overwrite config values with underscores in their names.
Currently if a field has an underscore in its name (e.g., `log_file`), the field can't be overwritten via environment variables. This can be enabled by escaping these underscores with another underscore. (In this case setting the variable `PREFIX_LOG__FILE`)

Discussion:

Patch:
diff --git a/python_json_config/config_node.py b/python_json_config/config_node.py
index 07a8d04..ce524d2 100644
--- a/python_json_config/config_node.py
+++ b/python_json_config/config_node.py
@@ -5,7 +5,7 @@ from typing import List, Union, Tuple
 
 import msgpack
 
-from .utils import normalize_path
+from .utils import normalize_path, parse_env_variable_name
 
 
 class ConfigNode(object):
@@ -20,7 +20,7 @@ class ConfigNode(object):
         :param config_dict: Source dictionary containing the part of the config that will be in this node and its
                             children. Each dictionary value in this dictionary will become another ConfigNode that is
                             a child of this node.
-        :param path: The access path to this node, i.e. the config keys that are used to access thos node.
+        :param path: The access path to this node, i.e. the config keys that are used to access this node.
         :param strict_access: If True, an error will be thrown if a non-existing field is accessed. If False,
                                     None will be returned instead.
         :param required_fields: A list of field names, for which an error will be thrown if they are accessed but don't
@@ -146,8 +146,7 @@ class ConfigNode(object):
             for prefix in prefixes:
                 if key.startswith(f"{prefix}_"):
                     value = os.environ[key]
-                    cleaned_key = key[len(prefix) + 1:]
-                    cleaned_key = cleaned_key.lower().split("_")
+                    cleaned_key = parse_env_variable_name(key[len(prefix) + 1:])
                     self.update(path=cleaned_key, value=value, upsert=True)
 
     """
diff --git a/python_json_config/utils.py b/python_json_config/utils.py
index 555be79..b6c885a 100644
--- a/python_json_config/utils.py
+++ b/python_json_config/utils.py
@@ -1,3 +1,4 @@
+import re
 from typing import Union, List
 
 
@@ -5,3 +6,32 @@ def normalize_path(path: Union[str, List[str]]) -> List[str]:
     if isinstance(path, str):
         path = path.split(".")
     return path
+
+
+def parse_env_variable_name(variable: str) -> List[str]:
+    """
+    Parse the name of an environment variable into a config path. Underscores in the names of variables are escaped with
+    an extra underscore. (E.g., the field test_value will be set by the variable TEST__VALUE).
+    :param variable: the name of the environment variable
+    :return: the path extracted from the name of the environment variable
+    """
+    # Split only on single underscores
+    split_variable = re.split("([^_]+)_([^_]+)", variable.lower())
+    path = [element for element in split_variable if element]
+
+    # If the path contains single elements only consisting of underscores, join the previous and next element together.
+    # (e.g., ["test", "__", "value"] becomes ["test__value"])
+    underscore_indices = [i for i, item in enumerate(path) if re.search("^_+$", item)]
+    for index in underscore_indices:
+        path[index - 1:index + 2] = ["".join(path[index - 1:index + 2])]
+
+    # Remove the extra underscore from names containing underscores (that was used to escape the underscore).
+    # (e.g., ["test__value"] becomes ["test_value"])
+    for index, element in enumerate(path):
+        if "_" in element:
+            underscore_match = re.search("[_]+", element)
+            underscores = underscore_match.group(0)
+            if len(underscores) > 1:
+                path[index] = element.replace(underscores, underscores[:-1])
+
+    return path


Test patch:
diff --git a/tests/config_node/test_crud.py b/tests/config_node/test_crud.py
index 16737a7..764e339 100644
--- a/tests/config_node/test_crud.py
+++ b/tests/config_node/test_crud.py
@@ -155,3 +155,28 @@ def test_merge_env_variable():
 
     for key, value in variables.items():
         del os.environ[key]
+
+
+def test_merge_env_variable_underscore_name():
+    prefix = "PYTHONJSONCONFIG"
+    variables = {
+        f"{prefix}_TEST__VALUE1": "bla",
+        f"{prefix}_TEST_VALUE2": "1",
+        f"{prefix}_TEST_TEST2__VALUES_VALUE2": "3"
+    }
+    for key, value in variables.items():
+        os.environ[key] = value
+
+    config = ConfigNode({"test_value1": "blub",
+                         "test": {
+                             "value2": "5",
+                             "test2_values": {"value2": 1}
+                         }})
+    config.merge_with_env_variables(prefix)
+
+    assert config.test_value1 == "bla"
+    assert config.test.value2 == "1"
+    assert config.test.test2_values.value2 == "3"
+
+    for key, value in variables.items():
+        del os.environ[key]