Repository: getpelican/pelican

Issue:
fix log formatting of iterable objects
If logged object is a dictionary (or any other iterable object), 1 argument is extected to a number of items in the object by `BaseFormatter`  in attempt to prettify a message. This would result in a invalid message format with unexpected numbers of arguments.

```
import logging
logger = logging.getLogger(__name__)
logger.debug('my dict: %s', {'here': 'is', 'my': 'dict'})
```

Which results in the following error:

```
Traceback (most recent call last):
  File "/usr/lib/python2.7/logging/__init__.py", line 868, in emit
    msg = self.format(record)
  File "/usr/lib/python2.7/logging/__init__.py", line 741, in format
    return fmt.format(record)
  File "/usr/local/lib/python2.7/dist-packages/pelican/log.py", line 34, in format
    return super(BaseFormatter, self).format(record)
  File "/usr/lib/python2.7/logging/__init__.py", line 465, in format
    record.message = record.getMessage()
  File "/usr/lib/python2.7/logging/__init__.py", line 329, in getMessage
    msg = msg % self.args
TypeError: not all arguments converted during string formatting
```

introduce by https://github.com/getpelican/pelican/commit/dd76c7158f7e05b0d203818d3fe18bea26e48c3f in #2438

"Solution": try to prettify arguments only if it's a tuple: `logger.debug('my message: %s and %s', 'foo', 'bar')`

This requires review and thourugh testing.


Discussion:
Fair enough, but this won't be an issue for "any iterable", [at best any `Mapping` as the only argument](https://github.com/python/cpython/blob/3.7/Lib/logging/__init__.py#L291-L310). `logging` singles out `Mapping`s because string formatting handles it specially.

Resolving patch:
diff --git a/pelican/log.py b/pelican/log.py
--- a/pelican/log.py
+++ b/pelican/log.py
@@ -2,6 +2,7 @@
 import os
 import sys
 from collections import defaultdict
+from collections.abc import Mapping
 
 __all__ = [
     'init'
@@ -18,9 +19,10 @@ def format(self, record):
         record.__dict__['customlevelname'] = customlevel
         # format multiline messages 'nicely' to make it clear they are together
         record.msg = record.msg.replace('\n', '\n  | ')
-        record.args = tuple(arg.replace('\n', '\n  | ') if
-                            isinstance(arg, str) else
-                            arg for arg in record.args)
+        if not isinstance(record.args, Mapping):
+            record.args = tuple(arg.replace('\n', '\n  | ') if
+                                isinstance(arg, str) else
+                                arg for arg in record.args)
         return super().format(record)
 
     def formatException(self, ei):


Test patch:
diff --git a/pelican/tests/support.py b/pelican/tests/support.py
--- a/pelican/tests/support.py
+++ b/pelican/tests/support.py
@@ -195,6 +195,15 @@ def count_logs(self, msg=None, level=None):
                (level is None or l.levelno == level)
         ])
 
+    def count_formatted_logs(self, msg=None, level=None):
+        return len([
+            l
+            for l
+            in self.buffer
+            if (msg is None or re.search(msg, self.format(l))) and
+               (level is None or l.levelno == level)
+        ])
+
 
 class LoggedTestCase(unittest.TestCase):
     """A test case that captures log messages."""
diff --git a/pelican/tests/test_log.py b/pelican/tests/test_log.py
--- a/pelican/tests/test_log.py
+++ b/pelican/tests/test_log.py
@@ -1,6 +1,7 @@
 import logging
 import unittest
 from collections import defaultdict
+from contextlib import contextmanager
 
 from pelican import log
 from pelican.tests.support import LogCountHandler
@@ -11,6 +12,7 @@ def setUp(self):
         super().setUp()
         self.logger = logging.getLogger(__name__)
         self.handler = LogCountHandler()
+        self.handler.setFormatter(log.get_formatter())
         self.logger.addHandler(self.handler)
 
     def tearDown(self):
@@ -24,55 +26,107 @@ def _reset_limit_filter(self):
         log.LimitFilter._threshold = 5
         log.LimitFilter._group_count = defaultdict(int)
 
+    @contextmanager
+    def reset_logger(self):
+        try:
+            yield None
+        finally:
+            self._reset_limit_filter()
+            self.handler.flush()
+
+    def test_log_formatter(self):
+        counter = self.handler.count_formatted_logs
+        with self.reset_logger():
+            # log simple case
+            self.logger.warning('Log %s', 'test')
+            self.assertEqual(
+                counter('Log test', logging.WARNING),
+                1)
+
+        with self.reset_logger():
+            # log multiline message
+            self.logger.warning('Log\n%s', 'test')
+            # Log
+            # | test
+            self.assertEqual(
+                counter('Log', logging.WARNING),
+                1)
+            self.assertEqual(
+                counter(' | test', logging.WARNING),
+                1)
+
+        with self.reset_logger():
+            # log multiline argument
+            self.logger.warning('Log %s', 'test1\ntest2')
+            # Log test1
+            # | test2
+            self.assertEqual(
+                counter('Log test1', logging.WARNING),
+                1)
+            self.assertEqual(
+                counter(' | test2', logging.WARNING),
+                1)
+
+        with self.reset_logger():
+            # log single list
+            self.logger.warning('Log %s', ['foo', 'bar'])
+            self.assertEqual(
+                counter(r"Log \['foo', 'bar'\]", logging.WARNING),
+                1)
+
+        with self.reset_logger():
+            # log single dict
+            self.logger.warning('Log %s', {'foo': 1, 'bar': 2})
+            self.assertEqual(
+                # dict order is not guaranteed
+                counter(r"Log {'.*': \d, '.*': \d}", logging.WARNING),
+                1)
+
     def test_log_filter(self):
         def do_logging():
             for i in range(5):
                 self.logger.warning('Log %s', i)
                 self.logger.warning('Another log %s', i)
         # no filter
-        do_logging()
-        self.assertEqual(
-            self.handler.count_logs('Log \\d', logging.WARNING),
-            5)
-        self.assertEqual(
-            self.handler.count_logs('Another log \\d', logging.WARNING),
-            5)
-        self.handler.flush()
-        self._reset_limit_filter()
+        with self.reset_logger():
+            do_logging()
+            self.assertEqual(
+                self.handler.count_logs('Log \\d', logging.WARNING),
+                5)
+            self.assertEqual(
+                self.handler.count_logs('Another log \\d', logging.WARNING),
+                5)
 
         # filter by template
-        log.LimitFilter._ignore.add((logging.WARNING, 'Log %s'))
-        do_logging()
-        self.assertEqual(
-            self.handler.count_logs('Log \\d', logging.WARNING),
-            0)
-        self.assertEqual(
-            self.handler.count_logs('Another log \\d', logging.WARNING),
-            5)
-        self.handler.flush()
-        self._reset_limit_filter()
+        with self.reset_logger():
+            log.LimitFilter._ignore.add((logging.WARNING, 'Log %s'))
+            do_logging()
+            self.assertEqual(
+                self.handler.count_logs('Log \\d', logging.WARNING),
+                0)
+            self.assertEqual(
+                self.handler.count_logs('Another log \\d', logging.WARNING),
+                5)
 
         # filter by exact message
-        log.LimitFilter._ignore.add((logging.WARNING, 'Log 3'))
-        do_logging()
-        self.assertEqual(
-            self.handler.count_logs('Log \\d', logging.WARNING),
-            4)
-        self.assertEqual(
-            self.handler.count_logs('Another log \\d', logging.WARNING),
-            5)
-        self.handler.flush()
-        self._reset_limit_filter()
+        with self.reset_logger():
+            log.LimitFilter._ignore.add((logging.WARNING, 'Log 3'))
+            do_logging()
+            self.assertEqual(
+                self.handler.count_logs('Log \\d', logging.WARNING),
+                4)
+            self.assertEqual(
+                self.handler.count_logs('Another log \\d', logging.WARNING),
+                5)
 
         # filter by both
-        log.LimitFilter._ignore.add((logging.WARNING, 'Log 3'))
-        log.LimitFilter._ignore.add((logging.WARNING, 'Another log %s'))
-        do_logging()
-        self.assertEqual(
-            self.handler.count_logs('Log \\d', logging.WARNING),
-            4)
-        self.assertEqual(
-            self.handler.count_logs('Another log \\d', logging.WARNING),
-            0)
-        self.handler.flush()
-        self._reset_limit_filter()
+        with self.reset_logger():
+            log.LimitFilter._ignore.add((logging.WARNING, 'Log 3'))
+            log.LimitFilter._ignore.add((logging.WARNING, 'Another log %s'))
+            do_logging()
+            self.assertEqual(
+                self.handler.count_logs('Log \\d', logging.WARNING),
+                4)
+            self.assertEqual(
+                self.handler.count_logs('Another log \\d', logging.WARNING),
+                0)