Repository: XKNX/xknx
Pull request #517 (2020-11-29 12:44:22)

Allow advanced YAML commands
<!--
  You are awesome! Thanks for contributing to our project!
  Please, DO NOT DELETE ANY TEXT from this template!.
-->
## Description
<!--
Please include a summary of the change and which issue is fixed.
Please also include relevant motivation and context.
List any dependencies that are required for this change.
-->
Allows custom YAML commands. See docs.

Fixes # (issue)

## Type of change
<!--
Please tick the applicable options.
NOTE: Ticking multiple options most likely indicates
that your change is to big and it is suggested to split it into several smaller PRs.
-->

- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
- [ ] This change requires a documentation update

## Checklist:

- [ ] I have performed a self-review of my own code
- [ ] The documentation has been adjusted accordingly
- [ ] The changes generate no new warnings
- [ ] Tests have been added that prove the fix is effective or that the feature works
- [ ] The changes are documented in the changelog
- [ ] The Homeassistant plugin has been adjusted in case of new config options



Patch:
diff --git a/docs/configuration.md b/docs/configuration.md
index 839a15e4..5755f961 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1,7 +1,8 @@
 ---
 layout: default
 title: Configuration
-nav_order: 12
+nav_order: 5
+has_children: true
 ---
 
 # Configuration
diff --git a/docs/splitting_up.md b/docs/splitting_up.md
new file mode 100644
index 00000000..b4d9068a
--- /dev/null
+++ b/docs/splitting_up.md
@@ -0,0 +1,25 @@
+---
+layout: default
+title: Splitting up your configuration
+nav_order: 1
+parent: Configuration
+---
+
+# Splitting up your configuration
+
+## Basic usage
+
+XKNX is controlled via a configuration file. Per default the configuration file is named `xknx.yaml`. You can change this by providing the `config` option like so: `XKNX(config='xknx.yaml')`.
+
+Throughout the time this configuration has grown a lot meaning that it can be really hard to maintain for bigger installations.
+
+There are several ways to improve the readability of your configuration:
+
+* `lights: !include lights.yaml` will load all lights from a dedicated `lights.yaml`.
+* `host: !env_var XKNX_HOST 192.168.0.200` will load the host variable from an environment variable `XKNX_HOST` and if `XKNX_HOST` does not exist falls back to `192.168.0.200`.
+
+## Advanced usage
+
+There is one advanced method to load a whole directory of YAML files at once. Your files must have the `.yaml` file extension.
+
+* `!include_dir_list` will return the complete content of a directory as a list with each file content being and entry in the list. The list entries are unordered.
\ No newline at end of file
diff --git a/xknx.yaml b/xknx.yaml
index ed119385..18dd1142 100644
--- a/xknx.yaml
+++ b/xknx.yaml
@@ -1,5 +1,3 @@
-version: 1
-
 general:
     own_address: "15.15.249"
     rate_limit: 18
diff --git a/xknx/config/config.py b/xknx/config/config.py
index 5a9fbff7..60e724c4 100644
--- a/xknx/config/config.py
+++ b/xknx/config/config.py
@@ -7,9 +7,8 @@ Module for reading config files (xknx.yaml).
 from enum import Enum
 import logging
 
-import yaml
-
 from .config_v1 import ConfigV1
+from .yaml_loader import load_yaml
 
 logger = logging.getLogger("xknx.log")
 
@@ -31,12 +30,8 @@ class Config:
     def read(self, file="xknx.yaml"):
         """Read config."""
         logger.debug("Reading %s", file)
-        try:
-            with open(file) as filehandle:
-                doc = yaml.safe_load(filehandle)
-                self.parse(doc)
-        except FileNotFoundError as ex:
-            logger.error("Error while reading %s: %s", file, ex)
+        doc = load_yaml(file)
+        self.parse(doc)
 
     @staticmethod
     def parse_version(doc):
diff --git a/xknx/config/objects.py b/xknx/config/objects.py
new file mode 100644
index 00000000..cae95774
--- /dev/null
+++ b/xknx/config/objects.py
@@ -0,0 +1,9 @@
+"""Custom yaml object types."""
+
+
+class NodeListClass(list):
+    """Wrapper class to be able to add attributes on a list."""
+
+
+class NodeStrClass(str):
+    """Wrapper class to be able to add attributes on a string."""
diff --git a/xknx/config/yaml_loader.py b/xknx/config/yaml_loader.py
new file mode 100644
index 00000000..27581566
--- /dev/null
+++ b/xknx/config/yaml_loader.py
@@ -0,0 +1,116 @@
+"""Custom YAML loader."""
+from collections import OrderedDict
+import fnmatch
+import logging
+import os
+from typing import Dict, Iterator, List, TypeVar, Union
+
+from xknx.config.objects import NodeListClass, NodeStrClass
+from xknx.exceptions import XKNXException
+import yaml
+
+logger = logging.getLogger("xknx.log")
+
+JSON_TYPE = Union[List, Dict, str]  # pylint: disable=invalid-name
+DICT_T = TypeVar("DICT_T", bound=Dict)  # pylint: disable=invalid-name
+
+
+class SafeLineLoader(yaml.SafeLoader):
+    """Loader class that keeps track of line numbers."""
+
+    def compose_node(self, parent: yaml.nodes.Node, index: int) -> yaml.nodes.Node:
+        """Annotate a node with the first line it was seen."""
+        last_line: int = self.line
+        node: yaml.nodes.Node = super().compose_node(parent, index)
+        node.__line__ = last_line + 1  # type: ignore
+        return node
+
+
+def _add_reference(obj, loader: SafeLineLoader, node: yaml.nodes.Node):  # type: ignore
+    """Add file reference information to an object."""
+    if isinstance(obj, list):
+        obj = NodeListClass(obj)
+    if isinstance(obj, str):
+        obj = NodeStrClass(obj)
+    setattr(obj, "__config_file__", loader.name)
+    setattr(obj, "__line__", node.start_mark.line)
+    return obj
+
+
+def load_yaml(fname: str) -> JSON_TYPE:
+    """Load a YAML file."""
+    try:
+        with open(fname, encoding="utf-8") as conf_file:
+            # If configuration file is empty YAML returns None
+            # We convert that to an empty dict
+            return yaml.load(conf_file, Loader=SafeLineLoader) or OrderedDict()
+    except yaml.YAMLError as exc:
+        logger.error(str(exc))
+        raise XKNXException(exc) from exc
+    except UnicodeDecodeError as exc:
+        logger.error("Unable to read file %s: %s", fname, exc)
+        raise XKNXException(exc) from exc
+
+
+def _include_yaml(loader: SafeLineLoader, node: yaml.nodes.Node) -> JSON_TYPE:
+    """Load another YAML file and embeds it using the !include tag.
+
+    Example:
+        device_tracker: !include device_tracker.yaml
+
+    """
+    fname = os.path.join(os.path.dirname(loader.name), node.value)
+    try:
+        return _add_reference(load_yaml(fname), loader, node)
+    except FileNotFoundError as exc:
+        raise XKNXException(f"{node.start_mark}: Unable to read file {fname}.") from exc
+
+
+def _is_file_valid(name: str) -> bool:
+    """Decide if a file is valid."""
+    return not name.startswith(".")
+
+
+def _find_files(directory: str, pattern: str) -> Iterator[str]:
+    """Recursively load files in a directory."""
+    for root, dirs, files in os.walk(directory, topdown=True):
+        dirs[:] = [d for d in dirs if _is_file_valid(d)]
+        for basename in sorted(files):
+            if _is_file_valid(basename) and fnmatch.fnmatch(basename, pattern):
+                filename = os.path.join(root, basename)
+                yield filename
+
+
+def _construct_seq(loader: SafeLineLoader, node: yaml.nodes.Node) -> JSON_TYPE:
+    """Add line number and file name to Load YAML sequence."""
+    (obj,) = loader.construct_yaml_seq(node)
+    return _add_reference(obj, loader, node)
+
+
+def _include_dir_list_yaml(
+    loader: SafeLineLoader, node: yaml.nodes.Node
+) -> List[JSON_TYPE]:
+    """Load multiple files from directory as a list."""
+    loc = os.path.join(os.path.dirname(loader.name), node.value)
+    return [load_yaml(f) for f in _find_files(loc, "*.yaml")]
+
+
+def _env_var_yaml(loader: SafeLineLoader, node: yaml.nodes.Node) -> str:
+    """Load environment variables and embed it into the configuration YAML."""
+    args = node.value.split()
+
+    # Check for a default value
+    if len(args) > 1:
+        return os.getenv(args[0], " ".join(args[1:]))
+    if args[0] in os.environ:
+        return os.environ[args[0]]
+    logger.error("Environment variable %s not defined", node.value)
+    raise XKNXException(node.value)
+
+
+yaml.SafeLoader.add_constructor("!include", _include_yaml)
+yaml.SafeLoader.add_constructor(
+    yaml.resolver.BaseResolver.DEFAULT_SEQUENCE_TAG, _construct_seq
+)
+yaml.SafeLoader.add_constructor("!env_var", _env_var_yaml)
+yaml.SafeLoader.add_constructor("!include_dir_list", _include_dir_list_yaml)


Test patch:
diff --git a/test/__init__.py b/test/__init__.py
new file mode 100644
index 00000000..4c7db209
--- /dev/null
+++ b/test/__init__.py
@@ -0,0 +1,1 @@
+"""Tests for XKNX."""
diff --git a/test/config_tests/config_v1_test.py b/test/config_tests/config_v1_test.py
index 25b1b563..5479f439 100644
--- a/test/config_tests/config_v1_test.py
+++ b/test/config_tests/config_v1_test.py
@@ -565,12 +565,6 @@ class TestConfig(unittest.TestCase):
         )
         self.assertFalse(TestConfig.xknx.devices.__contains__("Remote_wind_alarm"))
 
-    def test_config_file_not_found(self):
-        """Test error message when reading a non exisiting config file."""
-        with patch("logging.Logger.error") as mock_err:
-            XKNX(config="xknx_does_not_exist.yaml")
-            self.assertEqual(mock_err.call_count, 1)
-
     def test_config_file_error(self):
         """Test error message when reading an errornous config file."""
         with patch("logging.Logger.error") as mock_err, patch(
diff --git a/test/config_tests/yaml_loader_test.py b/test/config_tests/yaml_loader_test.py
new file mode 100644
index 00000000..e1475218
--- /dev/null
+++ b/test/config_tests/yaml_loader_test.py
@@ -0,0 +1,105 @@
+import io
+import os
+from unittest.mock import patch
+
+import pytest
+from xknx.config import yaml_loader
+from xknx.exceptions import XKNXException
+import yaml
+
+from test.util import patch_yaml_files
+
+
+def test_simple_list():
+    """Test simple list."""
+    conf = "config:\n  - simple\n  - list"
+    with io.StringIO(conf) as file:
+        doc = yaml_loader.yaml.safe_load(file)
+    assert doc["config"] == ["simple", "list"]
+
+
+def test_simple_dict():
+    """Test simple dict."""
+    conf = "key: value"
+    with io.StringIO(conf) as file:
+        doc = yaml_loader.yaml.safe_load(file)
+    assert doc["key"] == "value"
+
+
+@patch("xknx.config.yaml_loader.open", create=True)
+def test_load_yaml_encoding_error(mock_open):
+    """Test raising a UnicodeDecodeError."""
+    mock_open.side_effect = UnicodeDecodeError("", b"", 1, 0, "")
+    with pytest.raises(XKNXException):
+        yaml_loader.load_yaml("test")
+
+
+@patch("xknx.config.yaml_loader.open", create=True)
+def test_load_yaml_loading_error(mock_open):
+    """Test raising a YAMLError."""
+    mock_open.side_effect = yaml.error.YAMLError
+    with pytest.raises(XKNXException):
+        yaml_loader.load_yaml("test")
+
+
+@patch("xknx.config.yaml_loader.os.walk")
+def test_include_dir_list(mock_walk):
+    """Test include dir list yaml."""
+    mock_walk.return_value = [["/test", [], ["two.yaml", "one.yaml"]]]
+
+    with patch_yaml_files({"/test/one.yaml": "one", "/test/two.yaml": "two"}):
+        conf = "key: !include_dir_list /test"
+        with io.StringIO(conf) as file:
+            doc = yaml_loader.yaml.safe_load(file)
+            assert doc["key"] == sorted(["one", "two"])
+
+
+def test_environment_variable():
+    """Test config file with environment variable."""
+    os.environ["XKNX_HOST"] = "192.168.30.32"
+    conf = "host: !env_var XKNX_HOST"
+    with io.StringIO(conf) as file:
+        doc = yaml_loader.yaml.safe_load(file)
+    assert doc["host"] == "192.168.30.32"
+    del os.environ["XKNX_HOST"]
+
+
+def test_environment_variable_default():
+    """Test config file with default value for environment variable."""
+    conf = "host: !env_var XKNX_HOST 127.0.0.1"
+    with io.StringIO(conf) as file:
+        doc = yaml_loader.yaml.safe_load(file)
+    assert doc["host"] == "127.0.0.1"
+
+
+def test_invalid_environment_variable():
+    """Test config file with no environment variable sat."""
+    conf = "host: !env_var XKNX_HOST"
+    with pytest.raises(XKNXException):
+        with io.StringIO(conf) as file:
+            yaml_loader.yaml.safe_load(file)
+
+
+def test_include_yaml():
+    """Test include yaml."""
+    with patch_yaml_files({"test.yaml": "value"}):
+        conf = "key: !include test.yaml"
+        with io.StringIO(conf) as file:
+            doc = yaml_loader.yaml.safe_load(file)
+            assert doc["key"] == "value"
+
+    with patch_yaml_files({"test.yaml": None}):
+        conf = "key: !include test.yaml"
+        with io.StringIO(conf) as file:
+            doc = yaml_loader.yaml.safe_load(file)
+            assert doc["key"] == {}
+
+
+def test_include_yaml_error():
+    """Test include yaml error."""
+    with patch_yaml_files({"test.yaml": "value"}):
+        conf = "key: !include test2.yaml"
+        with io.StringIO(conf) as file:
+            with pytest.raises(XKNXException):
+                doc = yaml_loader.yaml.safe_load(file)
+                assert doc["key"] == "value"
diff --git a/test/util.py b/test/util.py
new file mode 100644
index 00000000..538a19cb
--- /dev/null
+++ b/test/util.py
@@ -0,0 +1,36 @@
+"""Utility functions for testing."""
+from io import StringIO
+import logging
+from unittest.mock import patch
+
+from xknx.config import yaml_loader
+
+logger = logging.getLogger("xknx.log")
+
+
+def patch_yaml_files(files_dict, endswith=True):
+    """Patch load_yaml with a dictionary of yaml files."""
+    # match using endswith, start search with longest string
+    matchlist = sorted(list(files_dict.keys()), key=len) if endswith else []
+
+    def mock_open_f(fname, **_):
+        """Mock open() in the yaml module, used by load_yaml."""
+        # Return the mocked file on full match
+        if fname in files_dict:
+            logger.debug("patch_yaml_files match %s", fname)
+            res = StringIO(files_dict[fname])
+            setattr(res, "name", fname)
+            return res
+
+        # Match using endswith
+        for ends in matchlist:
+            if fname.endswith(ends):
+                logger.debug("patch_yaml_files end match %s: %s", ends, fname)
+                res = StringIO(files_dict[ends])
+                setattr(res, "name", fname)
+                return res
+
+        # Not found
+        raise FileNotFoundError(f"File not found: {fname}")
+
+    return patch.object(yaml_loader, "open", mock_open_f, create=True)