Repository: itamarst/eliot

Issue:
Point to the call site where a log mesage that fails validation was emitted
The traceback associated with a validation failure is pretty much useless.  It points to test runner innards and Eliot innards.  It doesn't point to the call site where the log message was emitted.  It doesn't even say what the action or message type was.  It only reports the field that failed validation.

Debugging validation failures would be easier with an action or message type.  It would be _much_ easier with a filename and line number.

I imagine there are many good reasons not to collect this information during normal operation but I think it may be feasible to collect when running with the intent of validating.

Discussion:

Patch:
diff --git a/docs/source/news.rst b/docs/source/news.rst
index d1f2871..886bb3d 100644
--- a/docs/source/news.rst
+++ b/docs/source/news.rst
@@ -13,9 +13,17 @@ Features:
 * Generating messages is much faster.
 * ``eliot.ValidationError``, as raised by e.g. ``capture_logging``, is now part of the public API. Fixed issue #146.
 * ``eliot.twisted.DeferredContext.addCallbacks`` now supports omitting the errback, for compatibility with Twisted's ``Deferred``. Thanks to Jean-Paul Calderone for the fix. Fixed issue #366.
+* The testing infrastructure now has slightly more informative error messages. Thanks to Jean-Paul Calderone for the bug report. Fixes issue #373.
+* ``@validate_logging`` and ``@capture_logging`` now make it clearer what caused validation errors by printing the original traceback. Thanks to Jean-Paul Calderone for the bug report. Fixes issue #365.
 * Added lower-level testing infrastructure—``eliot.testing.swap_logger`` and ``eliot.testing.check_for_errors``—which is useful for cases when the ``@capture_logging`` decorator is insufficient. For example, test methods that are async, or return Twisted ``Deferred``. See the :doc:`testing documentation<generating/testing>` for details. Thanks to Jean-Paul Calderone for the feature request. Fixes #364.
 * The testing API now has slightly more informative error messages. Thanks to Jean-Paul Calderone for the bug report. Fixes issue #373.
-* ILogger.write is now explicitly thread-safe.  The MemoryLogger implementation of this method which was previously not thread-safe is now thread-safe.
+
+Bug fixes:
+
+* ``ILogger.write`` is now explicitly thread-safe. The ``MemoryLogger`` (as used
+  by tests) implementation of this method which was previously not thread-safe
+  is now thread-safe. Thanks to Jean-Paul Calderone for the patch. Fixes issue
+  #382.
 
 1.6.0
 ^^^^^
diff --git a/eliot/_output.py b/eliot/_output.py
index ab29884..df03421 100644
--- a/eliot/_output.py
+++ b/eliot/_output.py
@@ -5,6 +5,8 @@ Implementation of hooks and APIs for outputting log messages.
 from __future__ import unicode_literals, absolute_import
 
 import sys
+import traceback
+import inspect
 import json as pyjson
 from threading import Lock
 from functools import wraps
@@ -25,6 +27,7 @@ from ._message import (
 )
 from ._util import saferepr, safeunicode
 from .json import EliotJSONEncoder
+from ._validation import ValidationError
 
 
 class _DestinationsSendError(Exception):
@@ -297,11 +300,64 @@ class MemoryLogger(object):
         """
         Add the dictionary to list of messages.
         """
+        # Validate copy of the dictionary, to ensure what we store isn't
+        # mutated.
+        try:
+            self._validate_message(dictionary.copy(), serializer)
+        except Exception as e:
+            # Skip irrelevant frames that don't help pinpoint the problem:
+            from . import _output, _message, _action
+            skip_filenames = [
+                _output.__file__, _message.__file__, _action.__file__
+            ]
+            for frame in inspect.stack():
+                if frame[1] not in skip_filenames:
+                    break
+            self._failed_validations.append(
+                "{}: {}".format(
+                    e, "".join(traceback.format_stack(frame[0])))
+            )
         self.messages.append(dictionary)
         self.serializers.append(serializer)
         if serializer is TRACEBACK_MESSAGE._serializer:
             self.tracebackMessages.append(dictionary)
 
+    def _validate_message(self, dictionary, serializer):
+        """Validate an individual message.
+
+        As a side-effect, the message is replaced with its serialized contents.
+
+        @param dictionary: A message C{dict} to be validated.  Might be mutated
+            by the serializer!
+
+        @param serializer: C{None} or a serializer.
+
+        @raises TypeError: If a field name is not unicode, or the dictionary
+            fails to serialize to JSON.
+
+        @raises eliot.ValidationError: If serializer was given and validation
+            failed.
+        """
+        if serializer is not None:
+            serializer.validate(dictionary)
+        for key in dictionary:
+            if not isinstance(key, unicode):
+                if isinstance(key, bytes):
+                    key.decode("utf-8")
+                else:
+                    raise TypeError(
+                        dictionary, "%r is not unicode" % (key, )
+                    )
+        if serializer is not None:
+            serializer.serialize(dictionary)
+
+        try:
+            bytesjson.dumps(dictionary)
+            pyjson.dumps(dictionary)
+        except Exception as e:
+            raise TypeError("Message %s doesn't encode to JSON: %s" % (
+                dictionary, e))
+
     @exclusively
     def validate(self):
         """
@@ -310,6 +366,9 @@ class MemoryLogger(object):
         Does minimal validation of types, and for messages with corresponding
         serializers use those to do additional validation.
 
+        As a side-effect, the messages are replaced with their serialized
+        contents.
+
         @raises TypeError: If a field name is not unicode, or the dictionary
             fails to serialize to JSON.
 
@@ -317,25 +376,14 @@ class MemoryLogger(object):
             failed.
         """
         for dictionary, serializer in zip(self.messages, self.serializers):
-            if serializer is not None:
-                serializer.validate(dictionary)
-            for key in dictionary:
-                if not isinstance(key, unicode):
-                    if isinstance(key, bytes):
-                        key.decode("utf-8")
-                    else:
-                        raise TypeError(
-                            dictionary, "%r is not unicode" % (key, )
-                        )
-            if serializer is not None:
-                serializer.serialize(dictionary)
-
             try:
-                bytesjson.dumps(dictionary)
-                pyjson.dumps(dictionary)
-            except Exception as e:
-                raise TypeError("Message %s doesn't encode to JSON: %s" % (
-                    dictionary, e))
+                self._validate_message(dictionary, serializer)
+            except (TypeError, ValidationError) as e:
+                # We already figured out which messages failed validation
+                # earlier. This just lets us figure out which exception type to
+                # raise.
+                raise e.__class__("\n\n".join(self._failed_validations))
+
 
     @exclusively
     def serialize(self):
@@ -367,7 +415,7 @@ class MemoryLogger(object):
         self.messages = []
         self.serializers = []
         self.tracebackMessages = []
-
+        self._failed_validations = []
 
 class FileDestination(PClass):
     """


Test patch:
diff --git a/eliot/tests/test_output.py b/eliot/tests/test_output.py
index 55ea76e..ff91938 100644
--- a/eliot/tests/test_output.py
+++ b/eliot/tests/test_output.py
@@ -80,7 +80,7 @@ class MemoryLoggerTests(TestCase):
     def test_serializer(self):
         """
         L{MemoryLogger.validate} calls the given serializer's C{validate()}
-        method with the message.
+        method with the message, as does L{MemoryLogger.write}.
         """
 
         class FakeValidator(list):
@@ -94,9 +94,9 @@ class MemoryLoggerTests(TestCase):
         logger = MemoryLogger()
         message = {"message_type": "mymessage", "X": 1}
         logger.write(message, validator)
-        self.assertEqual(validator, [])
-        logger.validate()
         self.assertEqual(validator, [message])
+        logger.validate()
+        self.assertEqual(validator, [message, message])
 
     def test_failedValidation(self):
         """
diff --git a/eliot/tests/test_testing.py b/eliot/tests/test_testing.py
index 3d60450..6619864 100644
--- a/eliot/tests/test_testing.py
+++ b/eliot/tests/test_testing.py
@@ -491,9 +491,12 @@ class ValidateLoggingTestsMixin(object):
                     "message_type": "wrongmessage"}, MESSAGE._serializer)
 
         test = MyTest()
-        self.assertRaises(ValidationError, test.debug)
-        self.assertEqual(
-            list(test.logger.messages[0].keys()), ["message_type"])
+        with self.assertRaises(ValidationError) as context:
+            test.debug()
+        # Some reference to the reason:
+        self.assertIn("wrongmessage", str(context.exception))
+        # Some reference to which file caused the problem:
+        self.assertIn("test_testing.py", str(context.exception))
 
     def test_addCleanupTracebacks(self):
         """