Repository: opsdroid/opsdroid

Issue:
Add filters to logging
Currently, opsdroid has a lot of things logging into debug mode, which makes it hard to try and find things while doing the actual debugging.

It would be good to implement filters that could be added to the logging through the `config.yaml` file. 

After sharing this in the channel Jacob suggested to implement filters by creating a class that goes through all those logs and then use the config to add a list of things to filter and that class would check every log against that list and either show that log or drop it.


Discussion:
Sounds good!

We should add some kind of config option like:

```yaml
logging:
  level: debug
  console: true
  filters:
    - aiosqlite
    - websockets.protocol
```

We would need to implement a log filter class which would compare the log domain with the filter list and return `False` for anything which starts with one of the filters.

Thank you that helps as it gives a bit more guidance on how to tackle this issue 😄 👍 

Resolving patch:
diff --git a/opsdroid/__main__.py b/opsdroid/__main__.py
--- a/opsdroid/__main__.py
+++ b/opsdroid/__main__.py
@@ -6,13 +6,13 @@
 import logging
 import gettext
 import time
-import contextlib
 
 import click
 
 from opsdroid import __version__
 from opsdroid.core import OpsDroid
 from opsdroid.loader import Loader
+from opsdroid.logging import configure_logging
 from opsdroid.const import (
     DEFAULT_LOG_FILENAME,
     LOCALE_DIR,
@@ -41,81 +41,6 @@ def configure_lang(config):
         lang.install()
 
 
-def configure_logging(config):
-    """Configure the root logger based on user config."""
-    rootlogger = logging.getLogger()
-    while rootlogger.handlers:
-        rootlogger.handlers.pop()
-
-    try:
-        if config["logging"]["path"]:
-            logfile_path = os.path.expanduser(config["logging"]["path"])
-        else:
-            logfile_path = config["logging"]["path"]
-    except KeyError:
-        logfile_path = DEFAULT_LOG_FILENAME
-
-    try:
-        log_level = get_logging_level(config["logging"]["level"])
-    except KeyError:
-        log_level = logging.INFO
-
-    rootlogger.setLevel(log_level)
-    formatter = logging.Formatter("%(levelname)s %(name)s: %(message)s")
-
-    console_handler = logging.StreamHandler()
-    console_handler.setLevel(log_level)
-    console_handler.setFormatter(formatter)
-    rootlogger.addHandler(console_handler)
-
-    with contextlib.suppress(KeyError):
-        if not config["logging"]["console"]:
-            console_handler.setLevel(logging.CRITICAL)
-
-    if logfile_path:
-        logdir = os.path.dirname(os.path.realpath(logfile_path))
-        if not os.path.isdir(logdir):
-            os.makedirs(logdir)
-        file_handler = logging.FileHandler(logfile_path)
-        file_handler.setLevel(log_level)
-        file_handler.setFormatter(formatter)
-        rootlogger.addHandler(file_handler)
-    _LOGGER.info("=" * 40)
-    _LOGGER.info(_("Started opsdroid %s"), __version__)
-
-
-def get_logging_level(logging_level):
-    """Get the logger level based on the user configuration.
-
-    Args:
-        logging_level: logging level from config file
-
-    Returns:
-        logging LEVEL ->
-            CRITICAL = 50
-            FATAL = CRITICAL
-            ERROR = 40
-            WARNING = 30
-            WARN = WARNING
-            INFO = 20
-            DEBUG = 10
-            NOTSET = 0
-
-    """
-    if logging_level == "critical":
-        return logging.CRITICAL
-
-    if logging_level == "error":
-        return logging.ERROR
-    if logging_level == "warning":
-        return logging.WARNING
-
-    if logging_level == "debug":
-        return logging.DEBUG
-
-    return logging.INFO
-
-
 def check_dependencies():
     """Check for system dependencies required by opsdroid."""
     if sys.version_info.major < 3 or sys.version_info.minor < 6:
diff --git a/opsdroid/logging.py b/opsdroid/logging.py
new file mode 100644
--- /dev/null
+++ b/opsdroid/logging.py
@@ -0,0 +1,149 @@
+"""Class for Filter logs and logging logic."""
+
+import os
+import logging
+import contextlib
+
+from opsdroid.const import DEFAULT_LOG_FILENAME, __version__
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class ParsingFilter(logging.Filter):
+    """Class that filters logs."""
+
+    def __init__(self, config, *parse_list):
+        """Create object to implement filtering."""
+        super(ParsingFilter, self).__init__()
+        self.config = config["logging"]
+        try:
+            if (
+                self.config["filter"]["whitelist"]
+                and self.config["filter"]["blacklist"]
+            ):
+                _LOGGER.warning(
+                    "Both whitelist and blacklist "
+                    "filters found in configuration. "
+                    "Only one can be used at a time - "
+                    "only the whitelist filter will be used."
+                )
+                self.parse_list = [
+                    logging.Filter(name) for name in parse_list[0]["whitelist"]
+                ]
+        except KeyError:
+            self.parse_list = parse_list[0].get("whitelist") or parse_list[0].get(
+                "blacklist"
+            )
+
+            self.parse_list = [logging.Filter(name) for name in self.parse_list]
+
+    def filter(self, record):
+        """Apply filter to the log message.
+
+        This is a subset of Logger.filter, this method applies the logger
+        filters and returns a bool. If the value is true the record will
+        be passed to the handlers and the log shown. If the value is
+        false it will be ignored.
+
+        Args:
+            record: a log record containing the log message and the
+                name of the log - example: opsdroid.core.
+
+        Returns:
+            Boolean: If True - pass the log to handler.
+
+        """
+
+        if self.config["filter"].get("whitelist"):
+            return any(name.filter(record) for name in self.parse_list)
+        return not any(name.filter(record) for name in self.parse_list)
+
+
+def configure_logging(config):
+    """Configure the root logger based on user config."""
+    rootlogger = logging.getLogger()
+    while rootlogger.handlers:
+        rootlogger.handlers.pop()
+
+    try:
+        if config["logging"]["path"]:
+            logfile_path = os.path.expanduser(config["logging"]["path"])
+        else:
+            logfile_path = config["logging"]["path"]
+    except KeyError:
+        logfile_path = DEFAULT_LOG_FILENAME
+
+    try:
+        log_level = get_logging_level(config["logging"]["level"])
+    except KeyError:
+        log_level = logging.INFO
+
+    rootlogger.setLevel(log_level)
+
+    try:
+        if config["logging"]["extended"]:
+            formatter = logging.Formatter(
+                "%(levelname)s %(name)s.%(funcName)s(): %(message)s"
+            )
+    except KeyError:
+        formatter = logging.Formatter("%(levelname)s %(name)s: %(message)s")
+
+    console_handler = logging.StreamHandler()
+    console_handler.setLevel(log_level)
+    console_handler.setFormatter(formatter)
+
+    with contextlib.suppress(KeyError):
+        console_handler.addFilter(ParsingFilter(config, config["logging"]["filter"]))
+
+    rootlogger.addHandler(console_handler)
+
+    with contextlib.suppress(KeyError):
+        if not config["logging"]["console"]:
+            console_handler.setLevel(logging.CRITICAL)
+
+    if logfile_path:
+        logdir = os.path.dirname(os.path.realpath(logfile_path))
+        if not os.path.isdir(logdir):
+            os.makedirs(logdir)
+        file_handler = logging.FileHandler(logfile_path)
+        file_handler.setLevel(log_level)
+        file_handler.setFormatter(formatter)
+
+        with contextlib.suppress(KeyError):
+            file_handler.addFilter(ParsingFilter(config, config["logging"]["filter"]))
+
+        rootlogger.addHandler(file_handler)
+    _LOGGER.info("=" * 40)
+    _LOGGER.info(_("Started opsdroid %s"), __version__)
+
+
+def get_logging_level(logging_level):
+    """Get the logger level based on the user configuration.
+
+    Args:
+        logging_level: logging level from config file
+
+    Returns:
+        logging LEVEL ->
+            CRITICAL = 50
+            FATAL = CRITICAL
+            ERROR = 40
+            WARNING = 30
+            WARN = WARNING
+            INFO = 20
+            DEBUG = 10
+            NOTSET = 0
+
+    """
+    if logging_level == "critical":
+        return logging.CRITICAL
+
+    if logging_level == "error":
+        return logging.ERROR
+    if logging_level == "warning":
+        return logging.WARNING
+
+    if logging_level == "debug":
+        return logging.DEBUG
+
+    return logging.INFO


Test patch:
diff --git a/tests/test_logging.py b/tests/test_logging.py
new file mode 100644
--- /dev/null
+++ b/tests/test_logging.py
@@ -0,0 +1,167 @@
+import unittest
+import unittest.mock as mock
+import logging
+import os
+import tempfile
+import contextlib
+
+import opsdroid.logging as opsdroid
+from opsdroid.__main__ import configure_lang
+
+
+class TestLogging(unittest.TestCase):
+    """Test the logging module."""
+
+    def setUp(self):
+        self._tmp_dir = os.path.join(tempfile.gettempdir(), "opsdroid_tests")
+        with contextlib.suppress(FileExistsError):
+            os.makedirs(self._tmp_dir, mode=0o777)
+        configure_lang({})
+
+    def test_set_logging_level(self):
+        self.assertEqual(logging.DEBUG, opsdroid.get_logging_level("debug"))
+        self.assertEqual(logging.INFO, opsdroid.get_logging_level("info"))
+        self.assertEqual(logging.WARNING, opsdroid.get_logging_level("warning"))
+        self.assertEqual(logging.ERROR, opsdroid.get_logging_level("error"))
+        self.assertEqual(logging.CRITICAL, opsdroid.get_logging_level("critical"))
+        self.assertEqual(logging.INFO, opsdroid.get_logging_level(""))
+
+    def test_configure_no_logging(self):
+        config = {"logging": {"path": False, "console": False}}
+        opsdroid.configure_logging(config)
+        rootlogger = logging.getLogger()
+        self.assertEqual(len(rootlogger.handlers), 1)
+        self.assertEqual(logging.StreamHandler, type(rootlogger.handlers[0]))
+        self.assertEqual(rootlogger.handlers[0].level, logging.CRITICAL)
+
+    def test_configure_file_logging(self):
+        config = {
+            "logging": {
+                "path": os.path.join(self._tmp_dir, "output.log"),
+                "console": False,
+            }
+        }
+        opsdroid.configure_logging(config)
+        rootlogger = logging.getLogger()
+        self.assertEqual(len(rootlogger.handlers), 2)
+        self.assertEqual(logging.StreamHandler, type(rootlogger.handlers[0]))
+        self.assertEqual(rootlogger.handlers[0].level, logging.CRITICAL)
+        self.assertEqual(logging.FileHandler, type(rootlogger.handlers[1]))
+        self.assertEqual(rootlogger.handlers[1].level, logging.INFO)
+
+    def test_configure_file_blacklist(self):
+        config = {
+            "logging": {
+                "path": os.path.join(self._tmp_dir, "output.log"),
+                "console": False,
+                "filter": {"blacklist": "opsdroid.logging"},
+            }
+        }
+        opsdroid.configure_logging(config)
+        rootlogger = logging.getLogger()
+        self.assertEqual(len(rootlogger.handlers), 2)
+        self.assertEqual(logging.StreamHandler, type(rootlogger.handlers[0]))
+        self.assertEqual(rootlogger.handlers[0].level, logging.CRITICAL)
+        self.assertEqual(logging.FileHandler, type(rootlogger.handlers[1]))
+        self.assertEqual(rootlogger.handlers[1].level, logging.INFO)
+        self.assertLogs("_LOGGER", None)
+
+    def test_configure_file_logging_directory_not_exists(self):
+        with mock.patch("logging.getLogger") as logmock:
+            mocklogger = mock.MagicMock()
+            mocklogger.handlers = [True]
+            logmock.return_value = mocklogger
+            config = {
+                "logging": {
+                    "path": os.path.join(
+                        self._tmp_dir, "mynonexistingdirectory", "output.log"
+                    ),
+                    "console": False,
+                }
+            }
+            opsdroid.configure_logging(config)
+
+    def test_configure_console_logging(self):
+        config = {"logging": {"path": False, "level": "error", "console": True}}
+        opsdroid.configure_logging(config)
+        rootlogger = logging.getLogger()
+        self.assertEqual(len(rootlogger.handlers), 1)
+        self.assertEqual(logging.StreamHandler, type(rootlogger.handlers[0]))
+        self.assertEqual(rootlogger.handlers[0].level, logging.ERROR)
+
+    def test_configure_console_blacklist(self):
+        config = {
+            "logging": {
+                "path": False,
+                "level": "error",
+                "console": True,
+                "filter": {"blacklist": "opsdroid"},
+            }
+        }
+        opsdroid.configure_logging(config)
+        rootlogger = logging.getLogger()
+        self.assertEqual(len(rootlogger.handlers), 1)
+        self.assertEqual(logging.StreamHandler, type(rootlogger.handlers[0]))
+        self.assertLogs("_LOGGER", None)
+
+    def test_configure_console_whitelist(self):
+        config = {
+            "logging": {
+                "path": False,
+                "level": "info",
+                "console": True,
+                "filter": {"whitelist": "opsdroid.logging"},
+            }
+        }
+        opsdroid.configure_logging(config)
+        rootlogger = logging.getLogger()
+        self.assertEqual(len(rootlogger.handlers), 1)
+        self.assertEqual(logging.StreamHandler, type(rootlogger.handlers[0]))
+        self.assertLogs("_LOGGER", "info")
+
+    def test_configure_extended_logging(self):
+        config = {
+            "logging": {
+                "path": False,
+                "level": "error",
+                "console": True,
+                "extended": True,
+            }
+        }
+        opsdroid.configure_logging(config)
+        rootlogger = logging.getLogger()
+        self.assertEqual(len(rootlogger.handlers), 1)
+        self.assertEqual(logging.StreamHandler, type(rootlogger.handlers[0]))
+
+    def test_configure_default_logging(self):
+        config = {}
+        opsdroid.configure_logging(config)
+        rootlogger = logging.getLogger()
+        print(rootlogger)
+        print(config)
+        self.assertEqual(len(rootlogger.handlers), 2)
+        self.assertEqual(logging.StreamHandler, type(rootlogger.handlers[0]))
+        self.assertEqual(rootlogger.handlers[0].level, logging.INFO)
+        self.assertEqual(logging.FileHandler, type(rootlogger.handlers[1]))
+        self.assertEqual(rootlogger.handlers[1].level, logging.INFO)
+        self.assertLogs("_LOGGER", "info")
+
+
+class TestWhiteAndBlackFilter(unittest.TestCase):
+    def setUp(self):
+        self.config = {
+            "logging": {
+                "path": False,
+                "level": "info",
+                "console": True,
+                "filter": {"whitelist": ["opsdroid"], "blacklist": ["opsdroid.core"]},
+            }
+        }
+
+    def tearDown(self):
+        self.config = {}
+        opsdroid.configure_logging(self.config)
+
+    def test_configure_whitelist_and_blacklist(self):
+        opsdroid.configure_logging(self.config)
+        self.assertLogs("_LOGGER", "warning")
diff --git a/tests/test_main.py b/tests/test_main.py
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -1,6 +1,5 @@
 import unittest
 import unittest.mock as mock
-import logging
 import os
 import sys
 import shutil
@@ -54,71 +53,6 @@ def test_configure_lang(self):
             opsdroid.configure_lang({"lang": "es"})
             self.assertTrue(translation.return_value.install.called)
 
-    def test_set_logging_level(self):
-        self.assertEqual(logging.DEBUG, opsdroid.get_logging_level("debug"))
-        self.assertEqual(logging.INFO, opsdroid.get_logging_level("info"))
-        self.assertEqual(logging.WARNING, opsdroid.get_logging_level("warning"))
-        self.assertEqual(logging.ERROR, opsdroid.get_logging_level("error"))
-        self.assertEqual(logging.CRITICAL, opsdroid.get_logging_level("critical"))
-        self.assertEqual(logging.INFO, opsdroid.get_logging_level(""))
-
-    def test_configure_no_logging(self):
-        config = {"logging": {"path": False, "console": False}}
-        opsdroid.configure_logging(config)
-        rootlogger = logging.getLogger()
-        self.assertEqual(len(rootlogger.handlers), 1)
-        self.assertEqual(logging.StreamHandler, type(rootlogger.handlers[0]))
-        self.assertEqual(rootlogger.handlers[0].level, logging.CRITICAL)
-
-    def test_configure_file_logging(self):
-        config = {
-            "logging": {
-                "path": os.path.join(self._tmp_dir, "output.log"),
-                "console": False,
-            }
-        }
-        opsdroid.configure_logging(config)
-        rootlogger = logging.getLogger()
-        self.assertEqual(len(rootlogger.handlers), 2)
-        self.assertEqual(logging.StreamHandler, type(rootlogger.handlers[0]))
-        self.assertEqual(rootlogger.handlers[0].level, logging.CRITICAL)
-        self.assertEqual(logging.FileHandler, type(rootlogger.handlers[1]))
-        self.assertEqual(rootlogger.handlers[1].level, logging.INFO)
-
-    def test_configure_file_logging_directory_not_exists(self):
-        with mock.patch("logging.getLogger") as logmock:
-            mocklogger = mock.MagicMock()
-            mocklogger.handlers = [True]
-            logmock.return_value = mocklogger
-            config = {
-                "logging": {
-                    "path": os.path.join(
-                        self._tmp_dir, "mynonexistingdirectory", "output.log"
-                    ),
-                    "console": False,
-                }
-            }
-            opsdroid.configure_logging(config)
-            # self.assertEqual(os.path.isfile(config['logging']['path']), True)
-
-    def test_configure_console_logging(self):
-        config = {"logging": {"path": False, "level": "error", "console": True}}
-        opsdroid.configure_logging(config)
-        rootlogger = logging.getLogger()
-        self.assertEqual(len(rootlogger.handlers), 1)
-        self.assertEqual(logging.StreamHandler, type(rootlogger.handlers[0]))
-        self.assertEqual(rootlogger.handlers[0].level, logging.ERROR)
-
-    def test_configure_default_logging(self):
-        config = {}
-        opsdroid.configure_logging(config)
-        rootlogger = logging.getLogger()
-        self.assertEqual(len(rootlogger.handlers), 2)
-        self.assertEqual(logging.StreamHandler, type(rootlogger.handlers[0]))
-        self.assertEqual(rootlogger.handlers[0].level, logging.INFO)
-        self.assertEqual(logging.FileHandler, type(rootlogger.handlers[1]))
-        self.assertEqual(rootlogger.handlers[1].level, logging.INFO)
-
     def test_welcome_message(self):
         config = {"welcome-message": True}
         opsdroid.welcome_message(config)