Repository: DARMA-tasking/LB-analysis-framework

Issue:
Create baseline/global YAML configuration file

Discussion:

Patch:
diff --git a/config/global.yaml b/config/global.yaml
new file mode 100644
index 0000000..11d700e
--- /dev/null
+++ b/config/global.yaml
@@ -0,0 +1,3 @@
+# Global configuration
+
+output_dir: ../output
\ No newline at end of file
diff --git a/src/lbaf/Applications/LBAF_app.py b/src/lbaf/Applications/LBAF_app.py
index e6ab649..abac235 100644
--- a/src/lbaf/Applications/LBAF_app.py
+++ b/src/lbaf/Applications/LBAF_app.py
@@ -67,13 +67,11 @@ class InternalParameters:
 
     def validate_configuration(self, config: dict):
         """Configuration file validation."""
-
         ConfigurationValidator(
             config_to_validate=config, logger=self.__logger).main()
 
     def init_parameters(self, config: dict, base_dir: str):
         """Execute when YAML configuration file was found and checked"""
-
         # Get top-level allowed configuration keys
         self.__allowed_config_keys = cast(list, ConfigurationValidator.allowed_keys())
 
@@ -185,8 +183,7 @@ class LBAFApplication:
                             )
         self.__args = parser.parse_args()
 
-    def __configure(self, path: str):
-        """Configure the application using the configuration file at the given path."""
+    def __read_configuration_file(self, path: str):
         if os.path.splitext(path)[-1] in [".yml", ".yaml"]:
             # Try to open configuration file in read+text mode
             try:
@@ -203,24 +200,59 @@ class LBAFApplication:
                 raise SystemExit(1) from err
         else:
             raise SystemExit(1)
+        return data
+
+    def __merge(self, src: dict, dest: dict) -> dict:
+        """Merges dictionaries. Internally used to merge configuration data"""
+        data = dest.copy()
+        for k in src:
+            if not k in data:
+                # if new key
+                data[k] = src[k]
+            else:
+                # if key exists in both src and dest
+                if isinstance(src[k], dict) and isinstance(data[k], dict):
+                    data[k] = self.__merge(src[k], dest[k])
+                else:
+                    data[k] = src[k]
+        return data
+
+    def __merge_configurations(self, *config_path):
+        """Generates a unique configuration dict from multiple configurations from a path list"""
+        config = {}
+        for path in config_path:
+            next_config = self.__read_configuration_file(path)
+            config = self.__merge(next_config, config)
+        return config
 
-        # Change logger (with parameters from the configuration data)
-        lvl = cast(str, data.get("logging_level", "info"))
-        config_dir = os.path.dirname(path)
-        log_to_file = data.get("log_to_file", None)
+    def __configure(self, *config_path):
+        """Configure the application using the configuration file(s) at the given path(s).
+
+        :param config_path: The configuration file path.
+            If multiple then provide it from the most generic to the most specialized.
+        :returns: The configuration as a dictionary
+        """
+
+        # merge configurations
+        config = self.__merge_configurations(*config_path)
+
+        # Change logger (with parameters from the configuration)
+        lvl = cast(str, config.get("logging_level", "info"))
+        config_dir = os.path.dirname(config_path[-1]) # Directory of the most specialized configuration
+        log_to_file = config.get("log_to_file", None)
         self.__logger = get_logger(
             name="lbaf",
             level=lvl,
-            log_to_console=data.get("log_to_console", None) is None,
-            log_to_file=None if log_to_file is None else abspath(data.get("log_to_file"), relative_to=config_dir)
+            log_to_console=config.get("log_to_console", None) is None,
+            log_to_file=None if log_to_file is None else abspath(config.get("log_to_file"), relative_to=config_dir)
         )
         self.__logger.info(f"Logging level: {lvl.lower()}")
         if log_to_file is not None:
-            log_to_file_path = abspath(data.get("log_to_file"), relative_to=config_dir)
+            log_to_file_path = abspath(config.get("log_to_file"), relative_to=config_dir)
             self.__logger.info(f"Logging to file: {log_to_file_path}")
 
         # Instantiate the application internal parameters
-        self.__parameters = InternalParameters(config=data, base_dir=os.path.dirname(path), logger=self.__logger)
+        self.__parameters = InternalParameters(config=config, base_dir=config_dir, logger=self.__logger)
 
         # Create VT writer except when explicitly turned off
         self.__json_writer = VTDataWriter(
@@ -229,39 +261,33 @@ class LBAFApplication:
             self.__parameters.output_file_stem,
             self.__parameters.json_params) if self.__parameters.json_params else None
 
-        return data
+        return config
 
-    def __get_config_path(self) -> str:
+    def __resolve_config_path(self, config_path) -> str:
         """Find the config file from the '-configuration' command line argument and returns its absolute path
         (if configuration file path is relative it is searched in the current working directory and at the end in the
         {PROJECT_PATH}/config directory)
 
         :raises FileNotFoundError: if configuration file cannot be found
         """
-        path = None
-        path_list = []
-
         # search config file in the current working directory if relative
-        path = abspath(self.__args.configuration)
+        path = config_path
+        path_list = []
         path_list.append(path)
         if (
             path is not None and
             not os.path.isfile(path) and
-            not os.path.isabs(self.__args.configuration) and PROJECT_PATH is not None
+            not os.path.isabs(config_path) and PROJECT_PATH is not None
         ):
             # then search config file relative to the config folder
             search_dir = abspath("config", relative_to=PROJECT_PATH)
-            path = search_dir + '/' + self.__args.configuration
+            path = search_dir + '/' + config_path
             path_list.append(path)
 
         if not os.path.isfile(path):
-            error_message = "The configuration file cannot be found at\n"
-            for invalid_path in path_list:
-                error_message += " " + invalid_path + " -> not found\n"
-            error_message += (
-                "If you provide a relative path, please verify that the file exists relative to the "
-                "current working directory or to the `config` directory"
-            )
+            error_message = "The configuration file cannot be found." \
+                            " If you provide a relative path, please verify that the file exists in the " \
+                            "current working directory or in the `<project_path>/config` directory"
             raise FileNotFoundError(error_message)
         else:
             self.__logger.info(f"Found configuration file at path {path}")
@@ -360,11 +386,22 @@ class LBAFApplication:
                                   "working directory or in the project config directory !")
             self.__args.configuration = "conf.yaml"
 
-        # Find configuration file absolute path
-        config_file = self.__get_config_path()
+        # Find configuration files
+        config_file_list = []
+        # Global configuration (optional)
+        try:
+            config_file_list.append(self.__resolve_config_path("global.yaml"))
+        except FileNotFoundError:
+            pass
+        # Local/Specialized configuration (required)
+        try:
+            config_file_list.append(self.__resolve_config_path(self.__args.configuration))
+        except(FileNotFoundError) as err:
+            self.__logger.error(err)
+            raise SystemExit(-1) from err
 
         # Apply configuration
-        cfg = self.__configure(config_file)
+        cfg = self.__configure(*config_file_list)
 
         # Download JSON data files validator (JSON data files validator is required to continue)
         loader = JSONDataFilesValidatorLoader()


Test patch:
diff --git a/tests/unit/test_lbaf_app.py b/tests/unit/test_lbaf_app.py
new file mode 100644
index 0000000..de97ade
--- /dev/null
+++ b/tests/unit/test_lbaf_app.py
@@ -0,0 +1,77 @@
+import unittest
+
+from lbaf.Applications.LBAF_app import LBAFApplication
+
+class TestLBAFApplication(unittest.TestCase):
+    """Tests for LBAFApplication"""
+
+    def setUp(self):
+        pass
+
+    def _global_conf(self) -> dict:
+        """Sample global configuration subset"""
+        return {
+            "algorithm": {
+                "name": "BruteForce",
+                "parameters": {
+                    "transfer_strategy": "Recursive",
+                }
+            },
+            "output_dir": "../output",
+            "write_JSON": {
+                "compressed": False,
+                "suffix": "json",
+                "communications": True,
+                "offline_LB_compatible": False
+            }
+        }
+
+    def _local_conf(self) -> dict:
+        """Sample local configuration subset"""
+        return {
+            "from_data": {
+                "data_stem": "../data/synthetic_lb_data/data",
+                "phase_ids": [0]
+            },
+            "check_schema": False,
+            "algorithm": {
+                "name": "InformAndTransfer",
+                "parameters": {
+                    "fanout": 3
+                }
+            }
+        }
+
+    def test_configuration_merge(self):
+        """Test that 2 dictionaries generates a single dictionay as expected.
+
+        The following are tested:
+        - Keys defined only in global configuration
+        - Keys defined only in local configuration
+        - Keys that must be overriden by the local configuration
+        """
+
+        app = LBAFApplication()
+        data = {}
+        # Inject some global configuration
+        data = app._LBAFApplication__merge(self._global_conf(), data)
+        global_conf = self._global_conf()
+        local_conf = self._local_conf()
+        # Test merge results in a copy of the global configuration
+        self.assertDictEqual(data, global_conf)
+
+        # Inject some local configuration
+        data = app._LBAFApplication__merge(local_conf, data)
+        # Keys defined only in global config must still be there
+        self.assertIsNotNone(data.get("write_JSON", None))
+        self.assertDictEqual(data.get("write_JSON"), global_conf.get("write_JSON")) # dict should also be the same
+        self.assertEqual(data.get("algorithm", {}).get("parameters", {}).get("transfer_strategy"),"Recursive")
+        self.assertEqual(data.get("output_dir", {}),"../output")
+        # Keys defined only in local config must also be there
+        self.assertIsNotNone(data.get("from_data", None))
+        # Keys that must be overriden by the local configuration
+        self.assertEqual(data.get("algorithm", {}).get("name"), "InformAndTransfer")
+        self.assertEqual(data.get("algorithm", {}).get("parameters", {}).get("fanout"), 3)
+
+if __name__ == "__main__":
+    unittest.main()