Repository: janehmueller/python-json-config

Issue:
Merge with ENV variables
It would be nice if one could specify the prefix or list of prefixes for environment variables that should be merged into the configuration. For example to merge all environment variables starting with `"FOO_"` into the configuration. If a key already exists, the corresponding value will be overwritten, it doesn't exist, it will be added.

A possible implementation might look like this:

```python
    def _merge_with_env_variables(self, prefix: Union[str, List[str]]):
        """
        This method takes all environment variables that start with the specified prefixes,
        and merges them into the config. If the key is not already present in the config,
        it will be added, if the key is present in the config it will be overwritten

        :param prefix: a prefix or list of prefixes for the env variables to merge
        """
        prefixes = []
        if isinstance(prefix, str):
            prefixes = [prefix]

        for key in os.environ.keys():
            for prefix in prefixes:
                if key.startswith(prefix):
                    value = os.environ[key]
                    key = key.replace(prefix, '').lower().replace('_', '.')
                    # TODO: Merge key and value into the configurtation
                    self.config.update(key, value)
```

Comments:

Patch:
diff --git a/README.md b/README.md
index 41490a2..22f8ea1 100644
--- a/README.md
+++ b/README.md
@@ -112,3 +112,22 @@ assert config.server.port == 1025
 config.update("server.user", "user", upsert=True)
 assert config.server.user == "user"
 ```
+
+## Overwrite fields with environment variables
+First, set environment variables (e.g., via bash):
+```
+$ MYPROJECT_SERVER_HOST="localhost"
+$ MYPROJECT_CACHE="redis"
+$ MYPYTHONPROJECTS_USER="user"
+```
+Then just tell the builder, which prefixes should be merged:
+```
+builder = ConfigBuilder()
+# you can also just pass a single prefix (builder.merge_with_env_variables("MYPROJECT")
+builder.merge_with_env_variables(["MYPROJECT", "MYPYTHONPROJECTS"])
+config = builder.parse_config({"server.host": "0.0.0.0"})
+
+assert config.server.host == "localhost"
+assert config.cache == "redis"
+assert config.user == "user"
+```
diff --git a/python_json_config/__version__.py b/python_json_config/__version__.py
index 7b344ec..7bb021e 100644
--- a/python_json_config/__version__.py
+++ b/python_json_config/__version__.py
@@ -1,1 +1,1 @@
-__version__ = '1.1.2'
+__version__ = '1.1.3'
diff --git a/python_json_config/config_builder.py b/python_json_config/config_builder.py
index 5eb57f0..9a9d622 100644
--- a/python_json_config/config_builder.py
+++ b/python_json_config/config_builder.py
@@ -1,4 +1,5 @@
 import json
+import os
 from typing import Dict, Union, List
 
 import jsonschema
@@ -22,7 +23,16 @@ class ConfigBuilder(object):
         self.__strict_access: bool = None
         self.__field_access_settings: Dict[str, bool] = {}
 
+        # environment variables that will be merged into the config (pointing from path joined with dots to the value)
+        self.__environment_variables: Dict[str, str] = {}
+
     def validate_field_type(self, field_name: str, field_type: type):
+        """
+        Validate that the given field is of the given type when the final config is built.
+        :param field_name: The field that is validated.
+        :param field_type: The type that the field value should have.
+        :return: The builder object for chaining of calls.
+        """
         self.__validation_types[field_name] = field_type
         return self
 
@@ -32,9 +42,18 @@ class ConfigBuilder(object):
             # skip optional fields that do not exist
             if value is None:
                 continue
-            assert isinstance(value, field_type), f'Config field "{field_name}" with value "{value}" is not of type {field_type}'
+            assert isinstance(value, field_type), f'Config field "{field_name}" with value "{value}" is not of ' \
+                f'type {field_type}'
 
     def validate_field_value(self, field_name: str, validation_function):
+        """
+        Validate that the validation function returns true with the value of the given field when the final config is
+        built.
+        :param field_name: The field that is validated.
+        :param validation_function: Function that takes the field value as input and validates it (returns True if the
+                                    value is valid and False if it is invalid).
+        :return: The builder object for chaining of calls.
+        """
         if field_name not in self.__validation_functions:
             self.__validation_functions[field_name] = []
 
@@ -62,6 +81,14 @@ class ConfigBuilder(object):
                     assert validation_result, error_message
 
     def transform_field_value(self, field_name: str, transformation_function):
+        """
+        Transform the given field value with the transformation function when the final config is built.
+        The transformation function takes the field value as input and returns a new value.
+        :param field_name: The field that is validated.
+        :param transformation_function: Function that takes the field value as input and transforms it into another
+                                        value.
+        :return: The builder object for chaining of calls.
+        """
         self.__transformation_functions[field_name] = transformation_function
         return self
 
@@ -87,38 +114,113 @@ class ConfigBuilder(object):
                 self.__json_schema = json.load(json_file)
 
     def set_field_access_optional(self):
+        """
+        Set the access mode of all fields to optional (if the field doesn't exist, None is returned).
+        :return: The builder object for chaining of calls.
+        """
         self.__strict_access = False
+        return self
 
     def set_field_access_required(self):
+        """
+        Set the access mode of all fields to required (if the field doesn't exist, an exception is raised).
+        :return: The builder object for chaining of calls.
+        """
         self.__strict_access = True
+        return self
 
     def add_required_field(self, field_name: str):
+        """
+        Set the access mode of the given field to required (if the field doesn't exist, an exception is raised).
+        :param field_name: The field whose access mode is set to required.
+        :return: The builder object for chaining of calls.
+        """
         self.__field_access_settings[field_name] = True
+        return self
 
     def add_required_fields(self, field_names: List[str]):
+        """
+        Set the access mode of the given fields to required (if the field doesn't exist, an exception is raised).
+        :param field_names: List of fields whose access mode is set to required.
+        :return: The builder object for chaining of calls.
+        """
         for field in field_names:
             self.add_required_field(field)
+        return self
 
     def add_optional_field(self, field_name: str):
+        """
+        Set the access mode of the given field to optional (if the field doesn't exist, None is returned).
+        :param field_name: The field whose access mode is set to required.
+        :return: The builder object for chaining of calls.
+        """
         self.__field_access_settings[field_name] = False
+        return self
 
     def add_optional_fields(self, field_names: List[str]):
+        """
+        Set the access mode of the given fields to optional (if the field doesn't exist, None is returned).
+        :param field_names: List of fields whose access mode is set to required.
+        :return: The builder object for chaining of calls.
+        """
         for field in field_names:
             self.add_optional_field(field)
+        return self
+
+    def merge_with_env_variables(self, prefix: Union[str, List[str]]):
+        """
+        Take all environment variables that start with the specified prefix or one of the specific prefixes and merge
+        them into the config. These values will be added before the validations and transformations happen.
+        The environment variable names will be split on underscores (_) and changed to lowercase to determine the
+        different keys (e.g., "FOO_BAR_TEST_ME" will result in the keys ["bar", "test", "me"] (with the prefix "FOO").
+        If the keys already exist in the config, the existing values will be overwritten by the values of the
+        environment variables.
+        :param prefix: Either a single or a list of prefixes of the environment variables (e.g., "FOO_").
+        :return: The builder object for chaining of calls.
+        """
+        prefixes = [prefix] if isinstance(prefix, str) else prefix
+        for key in os.environ:
+            for prefix in prefixes:
+                if key.startswith(prefix):
+                    value = os.environ[key]
+                    cleaned_key = key[len(prefix):]
+                    if cleaned_key[0] == "_":
+                        cleaned_key = cleaned_key[1:]
+                    cleaned_key = cleaned_key.lower().replace("_", ".")
+                    self.__environment_variables[cleaned_key] = value
+        return self
 
     def parse_config(self, config: Union[str, dict]) -> Config:
+        """
+        Build the config. This method should be called last and uses the settings set via the other methods of this
+        class (e.g., validate field type).
+        :param config: Path to the config json file or a dictionary that contains the config values
+        :return: The built config (that is validated and transformed according to the passed functions).
+        """
+        # Either parse the JSON file or use the passed dictionary directly
         if isinstance(config, dict):
             config_dict = config
         else:
             with open(config, "r") as json_file:
                 config_dict = json.load(json_file)
 
+        # Validate with JSON schema if it exists
         if self.__json_schema is not None:
             jsonschema.validate(config_dict, self.__json_schema)
+
+        # Build the config object. Dictionaries in the input data are resolved recursively in the object creation.
         self.__config = Config(config_dict,
                                strict_access=self.__strict_access,
-                               required_fields=[field for field, status in self.__field_access_settings.items() if status],
-                               optional_fields=[field for field, status in self.__field_access_settings.items() if not status])
+                               required_fields=[field for field, status in self.__field_access_settings.items()
+                                                if status],
+                               optional_fields=[field for field, status in self.__field_access_settings.items()
+                                                if not status])
+
+        # Add/Overwrite values set via environment variables
+        for path, value in self.__environment_variables.items():
+            self.__config.add(path, value)
+
+        # Apply the custom validation and transformation function
         self.__validate_types()
         self.__validate_field_values()
         self.__transform_field_values()
diff --git a/python_json_config/config_node.py b/python_json_config/config_node.py
index 8684037..1195954 100644
--- a/python_json_config/config_node.py
+++ b/python_json_config/config_node.py
@@ -195,11 +195,12 @@ class ConfigNode(object):
 
 
 class Config(ConfigNode):
-    def __init__(self, config_dict: dict,
+    def __init__(self,
+                 config_dict: dict,
                  strict_access: bool = True,
                  required_fields: List[Union[str, List[str]]] = None,
                  optional_fields: List[Union[str, List[str]]] = None):
-        super(Config, self).__init__(config_dict,
+        super(Config, self).__init__(config_dict=config_dict,
                                      path=[],
                                      strict_access=strict_access,
                                      required_fields=required_fields,


Test patch:
diff --git a/tests/config_builder_test.py b/tests/config_builder_test.py
index 6e5c8dc..0997bd8 100644
--- a/tests/config_builder_test.py
+++ b/tests/config_builder_test.py
@@ -1,4 +1,5 @@
 import json
+import os
 from unittest import TestCase
 
 from jsonschema import ValidationError
@@ -141,7 +142,6 @@ class ConfigBuilderTest(TestCase):
         with self.assertRaises(AttributeError):
             config.cache.name2
 
-
     def test_optional_validation(self):
         builder = ConfigBuilder()
         builder.set_field_access_optional()
@@ -149,3 +149,18 @@ class ConfigBuilderTest(TestCase):
         builder.validate_field_value('cache.host', 'localhost')
         builder.transform_field_value('cache.host', lambda name: f"https://{name}")
         builder.parse_config(self.path)
+
+    def test_merge_env_variable(self):
+        builder = ConfigBuilder()
+        prefix = "PYTHONJSONCONFIG"
+        variables = {f"{prefix}_TESTVALUE1": "bla", f"{prefix}_TESTVALUE2": "1"}
+        for key, value in variables.items():
+            os.environ[key] = value
+        builder.merge_with_env_variables(prefix)
+        for key, value in variables.items():
+            del os.environ[key]
+
+        config = builder.parse_config({"testvalue1": "blub", "testvalue3": 5})
+        self.assertEqual(config.testvalue1, "bla")
+        self.assertEqual(config.testvalue2, "1")
+        self.assertEqual(config.testvalue3, 5)