Repository: dask/distributed

Issue:
Improve configuration options regarding logging
Python's standard library offers two useful ways to initialize loggers from a [file](https://docs.python.org/2/library/logging.config.html#logging.config.fileConfig)
or a [dictionary](https://docs.python.org/2/library/logging.config.html#logging.config.dictConfig).

Dask's `config.yaml` currently contains a `logging` field. I propose to change the field so that it can be fed to the `logging.config.dictConfig` function. The relevant part of the default `config.yaml` would then for example look like this:

```yaml
logging:
  formatters:
    default:
      format: '%(name)s - %(levelname)s - %(message)s'
  handlers:
    console:
      class: logging.StreamHandler
      stream: ext://sys.stderr
  loggers:
    distributed:
      level: info
      handler: console
      formatter: default
    distributed.client:
      level: warning
      handler: console
      formatter: default
    bokeh:
      level: warning
      handler: console
      formatter: default
    tornado:
      level: critical
      handler: console
      formatter: default
    tornado.application:
      level: error
      handler: console
      formatter: default
``` 

To also gain the possibility to provide a file config, it could either be fed to `logging.config.fileConfig` in case it is a string. Or another field like `logging-file-config` could be introduced for that purpose.

The downside is increasing complexity of the config file. But gaining more flexibility in setting up logging would be a huge gain.

What are your thoughts on this?


Discussion:
If we want to keep compatibility with existing config files, we need to detect both schemes. For example by special-casing keys `formatters`, `handlers` and `loggers` (that's assuming no-one has top-level loggers named like that).

As for `logging-file-config`, why not...
We may also want all three configuration schemes to be exclusive, to avoid confusion?
Yes, compatibility would be an issue. For the dict config, we could introduce a field `logging-dict-config`. Then there'd be three different mutual exclusive ways to set the logging options.

Resolving patch:
diff --git a/distributed/config.py b/distributed/config.py
--- a/distributed/config.py
+++ b/distributed/config.py
@@ -1,6 +1,7 @@
 from __future__ import print_function, division, absolute_import
 
 import logging
+import logging.config
 import os
 import sys
 import warnings
@@ -68,7 +69,17 @@ def load_env_vars(config):
             config[varname] = value
 
 
-def initialize_logging(config):
+def _initialize_logging_old_style(config):
+    """
+    Initialize logging using the "old-style" configuration scheme, e.g.:
+        {
+        'logging': {
+            'distributed': 'info',
+            'tornado': 'critical',
+            'tornado.application': 'error',
+            }
+        }
+    """
     loggers = config.get('logging', {})
     loggers.setdefault('distributed', 'info')
     # We could remove those lines and let the default config.yaml handle it
@@ -84,10 +95,29 @@ def initialize_logging(config):
             level = logging_names[level.upper()]
         logger = logging.getLogger(name)
         logger.setLevel(level)
+        logger.handlers[:] = []
         logger.addHandler(handler)
         logger.propagate = False
 
 
+def _initialize_logging_new_style(config):
+    """
+    Initialize logging using logging's "Configuration dictionary schema".
+    (ref.: https://docs.python.org/2/library/logging.config.html#logging-config-dictschema)
+    """
+    logging.config.dictConfig(config['logging'])
+
+
+def initialize_logging(config):
+    log_config = config.get('logging', {})
+    if 'version' in log_config:
+        # logging module mandates version to be an int
+        log_config['version'] = int(log_config['version'])
+        _initialize_logging_new_style(config)
+    else:
+        _initialize_logging_old_style(config)
+
+
 try:
     import yaml
 except ImportError:


Test patch:
diff --git a/distributed/tests/test_config.py b/distributed/tests/test_config.py
new file mode 100644
--- /dev/null
+++ b/distributed/tests/test_config.py
@@ -0,0 +1,206 @@
+from __future__ import print_function, division, absolute_import
+
+import logging
+import subprocess
+import sys
+
+import pytest
+
+from distributed.utils_test import (captured_handler, captured_logger,
+                                    new_config, new_config_file)
+
+
+def dump_logger_list():
+    root = logging.getLogger()
+    loggers = root.manager.loggerDict
+    print()
+    print("== Loggers (name, level, effective level, propagate) ==")
+
+    def logger_info(name, logger):
+        return (name, logging.getLevelName(logger.level),
+                logging.getLevelName(logger.getEffectiveLevel()),
+                logger.propagate)
+
+    infos = []
+    infos.append(logger_info('<root>', root))
+
+    for name, logger in sorted(loggers.items()):
+        if not isinstance(logger, logging.Logger):
+            # Skip 'PlaceHolder' objects
+            continue
+        assert logger.name == name
+        infos.append(logger_info(name, logger))
+
+    for info in infos:
+        print("%-40s %-8s %-8s %-5s" % info)
+
+    print()
+
+
+def test_logging_default():
+    """
+    Test default logging configuration.
+    """
+    d = logging.getLogger('distributed')
+    assert len(d.handlers) == 1
+    assert isinstance(d.handlers[0], logging.StreamHandler)
+
+    # Work around Bokeh messing with the root logger level
+    # https://github.com/bokeh/bokeh/issues/5793
+    root = logging.getLogger('')
+    old_root_level = root.level
+    root.setLevel('WARN')
+
+    try:
+        dfb = logging.getLogger('distributed.foo.bar')
+        f = logging.getLogger('foo')
+        fb = logging.getLogger('foo.bar')
+
+        with captured_handler(d.handlers[0]) as distributed_log:
+            with captured_logger(root) as foreign_log:
+                h = logging.StreamHandler(foreign_log)
+                fmt = '[%(levelname)s in %(name)s] - %(message)s'
+                h.setFormatter(logging.Formatter(fmt))
+                fb.addHandler(h)
+                fb.propagate = False
+
+                # For debugging
+                dump_logger_list()
+
+                d.debug("1: debug")
+                d.info("2: info")
+                dfb.info("3: info")
+                fb.info("4: info")
+                fb.error("5: error")
+                f.info("6: info")
+                f.error("7: error")
+
+        distributed_log = distributed_log.getvalue().splitlines()
+        foreign_log = foreign_log.getvalue().splitlines()
+
+        # distributed log is configured at INFO level by default
+        assert distributed_log == [
+            "distributed - INFO - 2: info",
+            "distributed.foo.bar - INFO - 3: info",
+            ]
+
+        # foreign logs should be unaffected by distributed's logging
+        # configuration.  They get the default ERROR level from logging.
+        assert foreign_log == [
+            "[ERROR in foo.bar] - 5: error",
+            "7: error",
+            ]
+
+    finally:
+        root.setLevel(old_root_level)
+
+
+def test_logging_empty_simple():
+    with new_config({}):
+        test_logging_default()
+
+
+def test_logging_simple():
+    """
+    Test simple ("old-style") logging configuration.
+    """
+    c = {
+        'logging': {
+            'distributed.foo': 'info',
+            'distributed.foo.bar': 'error',
+            }
+        }
+    # Must test using a subprocess to avoid wrecking pre-existing configuration
+    with new_config_file(c):
+        code = """if 1:
+            import logging
+
+            from distributed.utils_test import captured_handler
+
+            d = logging.getLogger('distributed')
+            assert len(d.handlers) == 1
+            assert isinstance(d.handlers[0], logging.StreamHandler)
+            df = logging.getLogger('distributed.foo')
+            dfb = logging.getLogger('distributed.foo.bar')
+
+            with captured_handler(d.handlers[0]) as distributed_log:
+                df.info("1: info")
+                dfb.warning("2: warning")
+                dfb.error("3: error")
+
+            distributed_log = distributed_log.getvalue().splitlines()
+
+            assert distributed_log == [
+                "distributed.foo - INFO - 1: info",
+                "distributed.foo.bar - ERROR - 3: error",
+                ]
+            """
+
+        subprocess.check_call([sys.executable, "-c", code])
+
+
+def test_logging_extended():
+    """
+    Test extended ("new-style") logging configuration.
+    """
+    c= {
+        'logging': {
+            'version': '1',
+            'formatters': {
+                'simple': {
+                    'format': '%(levelname)s: %(name)s: %(message)s',
+                    },
+                },
+            'handlers': {
+                'console': {
+                    'class': 'logging.StreamHandler',
+                    'stream': 'ext://sys.stderr',
+                    'formatter': 'simple',
+                    },
+                },
+            'loggers': {
+                'distributed.foo': {
+                    'level': 'INFO',
+                    #'handlers': ['console'],
+                    },
+                'distributed.foo.bar': {
+                    'level': 'ERROR',
+                    #'handlers': ['console'],
+                    },
+                },
+            'root': {
+                'level': 'WARNING',
+                'handlers': ['console'],
+                },
+            },
+        }
+    # Must test using a subprocess to avoid wrecking pre-existing configuration
+    with new_config_file(c):
+        code = """if 1:
+            import logging
+
+            from distributed.utils_test import captured_handler
+
+            root = logging.getLogger()
+            d = logging.getLogger('distributed')
+            df = logging.getLogger('distributed.foo')
+            dfb = logging.getLogger('distributed.foo.bar')
+
+            with captured_handler(root.handlers[0]) as root_log:
+                df.info("1: info")
+                dfb.warning("2: warning")
+                dfb.error("3: error")
+                d.info("4: info")
+                d.warning("5: warning")
+
+            root_log = root_log.getvalue().splitlines()
+            print(root_log)
+
+            assert root_log == [
+                "INFO: distributed.foo: 1: info",
+                "ERROR: distributed.foo.bar: 3: error",
+                "WARNING: distributed: 5: warning",
+                ]
+            """
+
+        subprocess.check_call([sys.executable, "-c", code])
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -24,8 +24,7 @@
         truncate_exception, get_traceback, queue_to_iterator,
         iterator_to_queue, _maybe_complex, read_block, seek_delimiter,
         funcname, ensure_bytes, open_port, get_ip_interface, nbytes)
-from distributed.utils_test import (loop, inc, throws, div, captured_handler,
-                                    captured_logger, has_ipv6)
+from distributed.utils_test import loop, inc, throws, div, has_ipv6
 
 
 def test_All(loop):
@@ -332,90 +331,6 @@ def test_nbytes():
     assert nbytes(memoryview(scalar)) == scalar.nbytes
     assert nbytes(memoryview(multi_dim)) == multi_dim.nbytes
 
-def dump_logger_list():
-    root = logging.getLogger()
-    loggers = root.manager.loggerDict
-    print()
-    print("== Loggers (name, level, effective level, propagate) ==")
-
-    def logger_info(name, logger):
-        return (name, logging.getLevelName(logger.level),
-                logging.getLevelName(logger.getEffectiveLevel()),
-                logger.propagate)
-
-    infos = []
-    infos.append(logger_info('<root>', root))
-
-    for name, logger in sorted(loggers.items()):
-        if not isinstance(logger, logging.Logger):
-            # Skip 'PlaceHolder' objects
-            continue
-        assert logger.name == name
-        infos.append(logger_info(name, logger))
-
-    for info in infos:
-        print("%-40s %-8s %-8s %-5s" % info)
-
-    print()
-
-
-def test_logging():
-    """
-    Test default logging configuration.
-    """
-    d = logging.getLogger('distributed')
-    assert len(d.handlers) == 1
-    assert isinstance(d.handlers[0], logging.StreamHandler)
-
-    # Work around Bokeh messing with the root logger level
-    # https://github.com/bokeh/bokeh/issues/5793
-    root = logging.getLogger('')
-    old_root_level = root.level
-    root.setLevel('WARN')
-
-    try:
-        dfb = logging.getLogger('distributed.foo.bar')
-        f = logging.getLogger('foo')
-        fb = logging.getLogger('foo.bar')
-
-        with captured_handler(d.handlers[0]) as distributed_log:
-            with captured_logger(root) as foreign_log:
-                h = logging.StreamHandler(foreign_log)
-                fmt = '[%(levelname)s in %(name)s] - %(message)s'
-                h.setFormatter(logging.Formatter(fmt))
-                fb.addHandler(h)
-                fb.propagate = False
-
-                # For debugging
-                dump_logger_list()
-
-                d.debug("1: debug")
-                d.info("2: info")
-                dfb.info("3: info")
-                fb.info("4: info")
-                fb.error("5: error")
-                f.info("6: info")
-                f.error("7: error")
-
-        distributed_log = distributed_log.getvalue().splitlines()
-        foreign_log = foreign_log.getvalue().splitlines()
-
-        # distributed log is configured at INFO level by default
-        assert distributed_log == [
-            "distributed - INFO - 2: info",
-            "distributed.foo.bar - INFO - 3: info",
-            ]
-
-        # foreign logs should be unaffected by distributed's logging
-        # configuration.  They get the default ERROR level from logging.
-        assert foreign_log == [
-            "[ERROR in foo.bar] - 5: error",
-            "7: error",
-            ]
-
-    finally:
-        root.setLevel(old_root_level)
-
 
 def test_open_port():
     port = open_port()
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -6,6 +6,7 @@
 from glob import glob
 import inspect
 import logging
+import logging.config
 import os
 import shutil
 import signal
@@ -24,7 +25,7 @@
 from tornado.gen import TimeoutError
 from tornado.ioloop import IOLoop
 
-from .config import config
+from .config import config, initialize_logging
 from .core import connect, rpc, CommClosedError
 from .metrics import time
 from .nanny import Nanny
@@ -830,10 +831,12 @@ def new_config(new_config):
     try:
         config.clear()
         config.update(new_config)
+        initialize_logging(config)
         yield
     finally:
         config.clear()
         config.update(orig_config)
+        initialize_logging(config)
 
 
 @contextmanager