Repository: dask/distributed

Issue:
distributed hijacks the logger
Simply importing the Client will muck with my logging setup - is it using the root logger?

Will try to add a repro in the morning...


Discussion:
Hrm, we do do this:

``` python
logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s',
                    level=logging.INFO)
```

Presumably we should be doing this on `logging.getLogger('distributed')` instead?

Yes, I think so. If you point out the line I can monkey-patch my local version and report back (tomorrow)

distributed/utils.py around 362

On Wed, Nov 2, 2016 at 8:51 AM, Dave Hirschfeld notifications@github.com
wrote:

> Yes, I think so. If you point out the line I can monkey-patch my local
> version and report back (tomorrow)
> 
> —
> You are receiving this because you commented.
> Reply to this email directly, view it on GitHub
> https://github.com/dask/distributed/issues/618#issuecomment-257855266,
> or mute the thread
> https://github.com/notifications/unsubscribe-auth/AASszJS8vsS5Ej2St9_bRCHNDQoS6Gakks5q6IdYgaJpZM4KnMgE
> .

Cheers, will let you know if that fixes it for me...

![image](https://cloud.githubusercontent.com/assets/881019/19950041/e55ae50a-a1a0-11e6-8954-ed10bfc54f0c.png)
...so it looks like it mucks with the root logger. I Am Not A Logging Expert but I'm not sure that a good thing for a library to do. Quite likely I should be doing something to protect my logging from another application calling `basicConfig`. 

In any case PR #623 seems to fix the issue for me...


Resolving patch:
diff --git a/distributed/config.py b/distributed/config.py
--- a/distributed/config.py
+++ b/distributed/config.py
@@ -1,7 +1,11 @@
 from __future__ import print_function, division, absolute_import
 
+import logging
 import os
-from .compatibility import FileExistsError
+import sys
+
+from .compatibility import FileExistsError, logging_names
+
 try:
     import yaml
 except ImportError:
@@ -33,3 +37,26 @@ def load_config_file(path=dask_config_path):
 
     ensure_config_file()
     config = load_config_file()
+
+
+def initialize_logging(config):
+    loggers = config.get('logging', {})
+    loggers.setdefault('distributed', 'info')
+
+    fmt = '%(name)s - %(levelname)s - %(message)s'
+
+    handler = logging.StreamHandler(sys.stderr)
+    handler.setFormatter(logging.Formatter(fmt))
+    for name, level in loggers.items():
+        LEVEL = logging_names[level.upper()]
+        logger = logging.getLogger(name)
+        logger.setLevel(LEVEL)
+        logger.addHandler(handler)
+        logger.propagate = False
+
+    # http://stackoverflow.com/questions/21234772/python-tornado-disable-logging-to-stderr
+    # XXX this silences legitimate error messages
+    logging.getLogger('tornado').setLevel(logging.CRITICAL)
+
+
+initialize_logging(config)
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1129,8 +1129,8 @@ def worker_stream(self, worker):
                     self.transitions(recommendations)
 
                     if self.validate:
-                        logger.info("Messages: %s\nRecommendations: %s",
-                                    msgs, recommendations)
+                        logger.debug("Messages: %s\nRecommendations: %s",
+                                     msgs, recommendations)
                 self.ensure_occupied()
 
         except (StreamClosedError, IOError, OSError):
@@ -2454,8 +2454,8 @@ def transition(self, key, finish, *args, **kwargs):
             finish2 = self.task_state.get(key, 'forgotten')
             self.transition_log.append((key, start, finish2, recommendations))
             if self.validate:
-                logger.info("Transition %s->%s: %s New: %s",
-                            start, finish2, key, recommendations)
+                logger.debug("Transition %s->%s: %s New: %s",
+                             start, finish2, key, recommendations)
             for plugin in self.plugins:
                 try:
                     plugin.transition(key, start, finish2, *args, **kwargs)
diff --git a/distributed/utils.py b/distributed/utils.py
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -5,6 +5,7 @@
 import logging
 import os
 import re
+import shutil
 import socket
 import six
 import sys
@@ -357,26 +358,6 @@ def convert(task):
     return {tokey(k): convert(v) for k, v in dsk.items()}
 
 
-import logging
-from .compatibility import logging_names
-logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s',
-                    level=logging.INFO)
-
-for name, level in config.get('logging', {}).items():
-    LEVEL = logging_names[level.upper()]
-    logging.getLogger(name).setLevel(LEVEL)
-
-# http://stackoverflow.com/questions/21234772/python-tornado-disable-logging-to-stderr
-stream = logging.StreamHandler(sys.stderr)
-stream.setLevel(logging.CRITICAL)
-logging.getLogger('tornado').addHandler(stream)
-logging.getLogger('tornado').propagate = False
-
-
-from contextlib import contextmanager
-import shutil
-
-
 def seek_delimiter(file, delimiter, blocksize):
     """ Seek current file to next byte after a delimiter bytestring
 


Test patch:
diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -53,11 +53,6 @@ def test_bokeh(loop):
         with Client('127.0.0.1:%d' % Scheduler.default_port, loop=loop) as c:
             pass
 
-        while True:
-            line = proc.stderr.readline()
-            if b'Web UI' in line:
-                break
-
         names = ['localhost', '127.0.0.1', get_ip()]
         if 'linux' in sys.platform:
             names.append(socket.gethostname())
@@ -87,11 +82,6 @@ def test_bokeh_non_standard_ports(loop):
         with Client('127.0.0.1:3448', loop=loop) as c:
             pass
 
-        while True:
-            line = proc.stderr.readline()
-            if b'Web UI' in line:
-                break
-
         start = time()
         while True:
             try:
@@ -102,7 +92,7 @@ def test_bokeh_non_standard_ports(loop):
                 sleep(0.1)
                 assert time() < start + 20
     with pytest.raises(Exception):
-        requests.get('http://127.0.0.1:4832/status/')
+        requests.get('http://localhost:4832/status/')
 
 
 @pytest.mark.skipif(not sys.platform.startswith('linux'),
@@ -117,11 +107,6 @@ def test_bokeh_whitelist(loop):
         with Client('127.0.0.1:%d' % Scheduler.default_port, loop=loop) as c:
             pass
 
-        while True:
-            line = proc.stderr.readline()
-            if b'Web UI' in line:
-                break
-
         start = time()
         while True:
             try:
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -3,6 +3,7 @@
 from collections import Iterator
 from functools import partial
 import io
+import logging
 from time import time, sleep
 from threading import Thread
 import threading
@@ -18,7 +19,8 @@
         truncate_exception, get_traceback, queue_to_iterator,
         iterator_to_queue, _maybe_complex, read_block, seek_delimiter,
         funcname, ensure_bytes)
-from distributed.utils_test import loop, inc, throws, div
+from distributed.utils_test import (loop, inc, throws, div, captured_handler,
+                                    captured_logger)
 from distributed.compatibility import Queue, isqueue, PY2
 
 
@@ -282,3 +284,49 @@ def test_ensure_bytes():
         result = ensure_bytes(d)
         assert isinstance(result, bytes)
         assert result == b'1'
+
+
+def test_logging():
+    """
+    Test default logging configuration.
+    """
+    d = logging.getLogger('distributed')
+    assert len(d.handlers) == 1
+    assert isinstance(d.handlers[0], logging.StreamHandler)
+
+    root = logging.getLogger('')
+    dfb = logging.getLogger('distributed.foo.bar')
+    f = logging.getLogger('foo')
+    fb = logging.getLogger('foo.bar')
+
+    with captured_handler(d.handlers[0]) as distributed_log:
+        with captured_logger(root) as foreign_log:
+            h = logging.StreamHandler(foreign_log)
+            fmt = '[%(levelname)s in %(name)s] - %(message)s'
+            h.setFormatter(logging.Formatter(fmt))
+            fb.addHandler(h)
+            fb.propagate = False
+
+            d.debug("1: debug")
+            d.info("2: info")
+            dfb.info("3: info")
+            fb.info("4: info")
+            fb.error("5: error")
+            f.info("6: info")
+            f.error("7: error")
+
+    distributed_log = distributed_log.getvalue().splitlines()
+    foreign_log = foreign_log.getvalue().splitlines()
+
+    # distributed log is configured at INFO level by default
+    assert distributed_log == [
+        "distributed - INFO - 2: info",
+        "distributed.foo.bar - INFO - 3: info",
+        ]
+
+    # foreign logs should be unaffected by distributed's logging
+    # configuration.  They get the default ERROR level from logging.
+    assert foreign_log == [
+        "[ERROR in foo.bar] - 5: error",
+        "7: error",
+        ]
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -13,6 +13,8 @@
 from time import time, sleep
 import uuid
 
+import six
+
 from toolz import merge
 from tornado import gen
 from tornado.ioloop import IOLoop, TimeoutError
@@ -148,11 +150,9 @@ def slowadd(x, y, delay=0.02):
 def run_scheduler(q, scheduler_port=0, **kwargs):
     from distributed import Scheduler
     from tornado.ioloop import IOLoop, PeriodicCallback
-    import logging
     IOLoop.clear_instance()
     loop = IOLoop(); loop.make_current()
     PeriodicCallback(lambda: None, 500).start()
-    logging.getLogger("tornado").setLevel(logging.CRITICAL)
 
     scheduler = Scheduler(loop=loop, validate=True, **kwargs)
     done = scheduler.start(scheduler_port)
@@ -167,12 +167,10 @@ def run_scheduler(q, scheduler_port=0, **kwargs):
 def run_worker(q, scheduler_port, **kwargs):
     from distributed import Worker
     from tornado.ioloop import IOLoop, PeriodicCallback
-    import logging
     with log_errors():
         IOLoop.clear_instance()
         loop = IOLoop(); loop.make_current()
         PeriodicCallback(lambda: None, 500).start()
-        logging.getLogger("tornado").setLevel(logging.CRITICAL)
         worker = Worker('127.0.0.1', scheduler_port, ip='127.0.0.1',
                         loop=loop, **kwargs)
         loop.run_sync(lambda: worker._start(0))
@@ -186,12 +184,10 @@ def run_worker(q, scheduler_port, **kwargs):
 def run_nanny(q, scheduler_port, **kwargs):
     from distributed import Nanny
     from tornado.ioloop import IOLoop, PeriodicCallback
-    import logging
     with log_errors():
         IOLoop.clear_instance()
         loop = IOLoop(); loop.make_current()
         PeriodicCallback(lambda: None, 500).start()
-        logging.getLogger("tornado").setLevel(logging.CRITICAL)
         worker = Nanny('127.0.0.1', scheduler_port, ip='127.0.0.1',
                        loop=loop, **kwargs)
         loop.run_sync(lambda: worker._start(0))
@@ -456,3 +452,29 @@ def popen(*args, **kwargs):
                 while line:
                     print(line)
                     line = proc.stdout.readline()
+
+
+@contextmanager
+def captured_logger(logger):
+    """Capture output from the given Logger.
+    """
+    orig_handlers = logger.handlers[:]
+    sio = six.StringIO()
+    logger.handlers[:] = [logging.StreamHandler(sio)]
+    try:
+        yield sio
+    finally:
+        logger.handlers[:] = orig_handlers
+
+
+@contextmanager
+def captured_handler(handler):
+    """Capture output from the given logging.StreamHandler.
+    """
+    assert isinstance(handler, logging.StreamHandler)
+    orig_stream = handler.stream
+    handler.stream = six.StringIO()
+    try:
+        yield handler.stream
+    finally:
+        handler.stream = orig_stream