Repository: markusressel/container-app-conf

Issue:
Automatically load data source values
**Is your feature request related to a problem? Please describe.**
When using a custom YAML data source the dev has to call `load()` manually which is cumbersome and easy to forget.

**Describe the solution you'd like**
Automatically load values of data sources when loading config.

Discussion:

Patch:
diff --git a/container_app_conf/__init__.py b/container_app_conf/__init__.py
index edc7853..8100383 100644
--- a/container_app_conf/__init__.py
+++ b/container_app_conf/__init__.py
@@ -98,6 +98,7 @@ class ConfigBase:
         Loads the configuration from all available sources
         """
         for source in reversed(self.data_sources):
+            source.load()
             for entry in self._config_entries.values():
                 if source.has(entry):
                     entry.value = source.get(entry)
diff --git a/container_app_conf/source/__init__.py b/container_app_conf/source/__init__.py
index 775e152..f34e59d 100644
--- a/container_app_conf/source/__init__.py
+++ b/container_app_conf/source/__init__.py
@@ -28,13 +28,38 @@ LOGGER = logging.getLogger(__name__)
 
 class DataSource:
 
+    def __init__(self):
+        self.root = {}
+
+    def load(self):
+        """
+        Loads all values of this data source into memory
+        """
+        self.root = self._load()
+
+    def _load(self) -> dict:
+        """
+        Loads all values of this data source and returns them as a dictionary tree
+        :return: value tree
+        """
+        raise NotImplementedError()
+
     def has(self, entry: ConfigEntry) -> bool:
         """
         Checks whether the data source has a value for the given config entry
         :param entry: the config entry to check
         :return: True if the source contains a value for the given entry, False otherwise
         """
-        raise NotImplementedError()
+        value = self.root
+        if value is None:
+            return False
+
+        for key in entry.key_path:
+            value = value.get(key)
+            if value is None:
+                return False
+
+        return True
 
     def get(self, entry: ConfigEntry) -> any:
         """
@@ -42,7 +67,16 @@ class DataSource:
         :param entry: config entry
         :return: value
         """
-        raise NotImplementedError()
+        value = self.root
+        if value is None:
+            return None
+
+        for key in entry.key_path:
+            value = value.get(key)
+            if value is None:
+                return entry.value
+
+        return value
 
     def write_reference(self, reference: dict):
         """
@@ -63,7 +97,7 @@ class FilesystemSource(DataSource):
         :param file_name: allowed config file name(s)
         :param file_extension: allowed config file extension(s)
         """
-        self.root = {}
+        super().__init__()
         if path is None:
             from container_app_conf import DEFAULT_CONFIG_FILE_PATHS
             self.paths = DEFAULT_CONFIG_FILE_PATHS
@@ -75,18 +109,15 @@ class FilesystemSource(DataSource):
         else:
             self.file_extensions = file_extension if isinstance(file_extension, list) else [file_extension]
 
-    def load(self):
-        """
-        Loads the config file content into memory
-        """
+    def _load(self) -> dict:
         file_path = self._find_config_file()
         if file_path is None:
             LOGGER.debug("No config file found in paths: {}".format(self.paths))
-            return
+            return {}
 
-        self.root = self._load(file_path)
+        return self._load_file(file_path)
 
-    def _load(self, file_path: str) -> dict:
+    def _load_file(self, file_path: str) -> dict:
         """
         Parses the file content into memory
         :param file_path: the path of the file to parse
@@ -94,30 +125,6 @@ class FilesystemSource(DataSource):
         """
         raise NotImplementedError()
 
-    def has(self, entry: ConfigEntry) -> bool:
-        value = self.root
-        if value is None:
-            return False
-
-        for key in entry.key_path:
-            value = value.get(key)
-            if value is None:
-                return False
-
-        return True
-
-    def get(self, entry: ConfigEntry) -> any:
-        value = self.root
-        if value is None:
-            return None
-
-        for key in entry.key_path:
-            value = value.get(key)
-            if value is None:
-                return entry.value
-
-        return value
-
     def write_reference(self, reference: dict):
         """
         Writes a reference config file
diff --git a/container_app_conf/source/env_source.py b/container_app_conf/source/env_source.py
index db87c44..1ec9d95 100644
--- a/container_app_conf/source/env_source.py
+++ b/container_app_conf/source/env_source.py
@@ -28,6 +28,7 @@ class EnvSource(DataSource):
     """
     Data source utilizing environment variables
     """
+    KEY_SPLIT_CHAR = "_"
 
     def has(self, entry: ConfigEntry) -> bool:
         return self.env_key(entry) in os.environ.keys()
@@ -36,6 +37,10 @@ class EnvSource(DataSource):
         key = self.env_key(entry)
         return os.environ.get(key, None)
 
-    @staticmethod
-    def env_key(entry: ConfigEntry) -> str:
-        return "_".join(entry.key_path).upper()
+    def env_key(self, entry: ConfigEntry) -> str:
+        return self.KEY_SPLIT_CHAR.join(entry.key_path).upper()
+
+    def _load(self) -> dict:
+        # loading env is pointless since it is already in memory
+        # instead we override has() and get() methods to directly access the environment
+        pass
diff --git a/container_app_conf/source/json_source.py b/container_app_conf/source/json_source.py
index 50bb032..9769355 100644
--- a/container_app_conf/source/json_source.py
+++ b/container_app_conf/source/json_source.py
@@ -31,7 +31,7 @@ class JsonSource(FilesystemSource):
     """
     DEFAULT_FILE_EXTENSIONS = ['json']
 
-    def _load(self, file_path: str) -> dict:
+    def _load_file(self, file_path: str) -> dict:
         with open(file_path, 'r') as file:
             return json.load(file)
 
diff --git a/container_app_conf/source/toml_source.py b/container_app_conf/source/toml_source.py
index 357008b..9fc8e57 100644
--- a/container_app_conf/source/toml_source.py
+++ b/container_app_conf/source/toml_source.py
@@ -32,7 +32,7 @@ class TomlSource(FilesystemSource):
     """
     DEFAULT_FILE_EXTENSIONS = ['toml', 'tml']
 
-    def _load(self, file_path: str) -> dict:
+    def _load_file(self, file_path: str) -> dict:
         with open(file_path, 'r') as file:
             return toml.load(file)
 
diff --git a/container_app_conf/source/yaml_source.py b/container_app_conf/source/yaml_source.py
index 61b294e..eb9b9b5 100644
--- a/container_app_conf/source/yaml_source.py
+++ b/container_app_conf/source/yaml_source.py
@@ -32,7 +32,7 @@ class YamlSource(FilesystemSource):
     """
     DEFAULT_FILE_EXTENSIONS = ['yaml', 'yml']
 
-    def _load(self, file_path: str) -> dict:
+    def _load_file(self, file_path: str) -> dict:
         with open(file_path, 'r') as ymlfile:
             return yaml.load(ymlfile, Loader=yaml.FullLoader)
 


Test patch:
diff --git a/tests/data_source/__init__.py b/tests/data_source/__init__.py
new file mode 100644
index 0000000..3ccbd83
--- /dev/null
+++ b/tests/data_source/__init__.py
@@ -0,0 +1,43 @@
+#  Copyright (c) 2019 Markus Ressel
+#  .
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#  .
+#  The above copyright notice and this permission notice shall be included in all
+#  copies or substantial portions of the Software.
+#  .
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+#  SOFTWARE.
+from typing import Dict
+
+from container_app_conf import DataSource, ConfigEntry
+
+
+class MemoryDataSource(DataSource):
+
+    def items(self) -> Dict[ConfigEntry, any]:
+        """
+        :return: dictionary of "config entry" -> "value" items
+        """
+        raise NotImplementedError()
+
+    def _load(self) -> dict:
+        data = {}
+        for entry, value in self.items().items():
+            key_path = entry.key_path
+            current_level = data
+            for key in key_path[:-1]:
+                current_level[key] = {}
+                current_level = current_level[key]
+            current_level[key_path[-1]] = value
+
+        return data
diff --git a/tests/source_test.py b/tests/data_source/source_test.py
similarity index 87%
rename from tests/source_test.py
rename to tests/data_source/source_test.py
index 34875aa..9a186b7 100644
--- a/tests/source_test.py
+++ b/tests/data_source/source_test.py
@@ -17,45 +17,40 @@
 #  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 #  SOFTWARE.
-from container_app_conf import DataSource, ConfigEntry
+from typing import Dict
+
+from container_app_conf import ConfigEntry
 from container_app_conf.entry.int import IntConfigEntry
 from container_app_conf.entry.string import StringConfigEntry
 from container_app_conf.source.json_source import JsonSource
 from container_app_conf.source.toml_source import TomlSource
 from container_app_conf.source.yaml_source import YamlSource
 from tests import TestBase
+from tests.data_source import MemoryDataSource
 from tests.singleton_test import TestConfigBase2
 
 
-def _key(entry: ConfigEntry) -> str:
-    return "_".join(entry.key_path)
-
-
-class MemoryDataSource(DataSource):
-    data = {
-        _key(TestConfigBase2.BOOL): True
-    }
-
-    def has(self, entry: ConfigEntry) -> bool:
-        key = _key(entry)
-        return key in self.data.keys()
+class MemoryDataSource1(MemoryDataSource):
 
-    def get(self, entry: ConfigEntry) -> any:
-        key = _key(entry)
-        return self.data[key]
+    def items(self) -> Dict[ConfigEntry, any]:
+        return {
+            TestConfigBase2.BOOL: True
+        }
 
 
 class MemoryDataSource2(MemoryDataSource):
-    data = {
-        _key(TestConfigBase2.BOOL): False
-    }
+
+    def items(self) -> Dict[ConfigEntry, any]:
+        return {
+            TestConfigBase2.BOOL: False
+        }
 
 
 class TestDataSource(TestBase):
 
     def test_priority(self):
         conf = TestConfigBase2(data_sources=[
-            MemoryDataSource(),
+            MemoryDataSource1(),
             MemoryDataSource2()
         ], singleton=False)
 
@@ -63,7 +58,7 @@ class TestDataSource(TestBase):
 
         conf2 = TestConfigBase2(data_sources=[
             MemoryDataSource2(),
-            MemoryDataSource()
+            MemoryDataSource1()
         ], singleton=False)
 
         self.assertFalse(conf2.BOOL.value)