Repository: kiwicom/structlog-sentry

Issue:
Log messages are sent twice when using processors.JSONRenderer
First of all I would like to thank you guys for this package. It's really nice.

In production environment I use `processors.JSONRenderer` to format logs properly for log storages like [elk-stack](https://www.elastic.co/elk-stack). Now I am trying to configure sentry in compound with structlog and your package. 

The problem is that the log messages are sent twice.

Here's the code:
```python

## config = dict(...)

if config.sentry.enabled:
    sentry_sdk.init(
        dsn=config.sentry.dsn,
        send_default_pii=True,
        attach_stacktrace="On",
    )

logging.basicConfig(format='%(message)s', stream=sys.stdout, level=config.level)

chain = [
    structlog.stdlib.add_log_level,
    SentryProcessor(level=logging.WARNING, active=config.sentry.enabled),
    structlog.processors.JSONRenderer(),
]

structlog.configure_once(
    processors=chain,
    context_class=dict,
    logger_factory=structlog.stdlib.LoggerFactory(),
    wrapper_class=structlog.stdlib.BoundLogger,
    cache_logger_on_first_use=True,
)
```

Result screenshot:

![image](https://user-images.githubusercontent.com/6891299/58564118-0c22de00-8235-11e9-8243-d5f832ba36f5.png)

Any suggestions on how to workaround this?



Comments:

Patch:
diff --git a/README.md b/README.md
index 436ea1e..37f9b09 100644
--- a/README.md
+++ b/README.md
@@ -77,6 +77,68 @@ optional argument to logger methods, like this:
 log.error(sentry_skip=True)
 ```
 
+### Sentry Tags
+
+You can set some or all of key/value pairs of structlog `event_dict` as sentry `tags`:
+
+```python
+structlog.configure(
+    processors=[
+        structlog.stdlib.add_log_level,
+        SentryProcessor(level=logging.ERROR, tag_keys=["city", "timezone"]),
+    ],...
+)
+
+log.error("error message", city="Tehran", timezone="UTC+3:30", movie_title="Some title")
+```
+
+this will report the error and the sentry event will have **city** and **timezone** tags.
+If you want to have all event data as tags, create the `SentryProcessor` with `tag_keys="__all__"`.  
+
+```python
+structlog.configure(
+    processors=[
+        structlog.stdlib.add_log_level,
+        SentryProcessor(level=logging.ERROR, tag_keys="__all__"),
+    ],...
+)
+```
+
+### Skip Extra
+
+By default `SentryProcessor` will send `event_dict` key/value pairs as extra info to the sentry. 
+Sometimes you may want to skip this, specially when sending the `event_dict` as sentry tags:
+
+```python
+structlog.configure(
+    processors=[
+        structlog.stdlib.add_log_level,
+        SentryProcessor(level=logging.ERROR, as_extra=False, tag_keys="__all__"),
+    ],...
+)
+```
+
+### Logging as JSON
+
+If you want to configure `structlog` to format the output as **JSON** 
+(maybe for [elk-stack](https://www.elastic.co/elk-stack)) you have to use `SentryJsonProcessor` to prevent
+duplication of an event reported to sentry.
+
+```python
+from structlog_sentry import SentryJsonProcessor
+
+structlog.configure(
+    processors=[
+        structlog.stdlib.add_logger_name,  # required before SentryJsonProcessor()
+        structlog.stdlib.add_log_level,
+        SentryJsonProcessor(level=logging.ERROR, tag_keys="__all__"),
+        structlog.processors.JSONRenderer()
+    ],...
+)
+```
+
+This processor tells sentry to *ignore* the logger and captures the events manually.
+
 ## Testing
 
 To run all tests:
diff --git a/setup.py b/setup.py
index cddb9fc..dc9ed96 100644
--- a/setup.py
+++ b/setup.py
@@ -11,7 +11,7 @@ with open("README.md") as f:
 
 setup(
     name="structlog-sentry",
-    version="1.0.0",
+    version="1.1.0",
     url="https://github.com/kiwicom/structlog-sentry",
     long_description=readme,
     long_description_content_type="text/markdown",
diff --git a/structlog_sentry/__init__.py b/structlog_sentry/__init__.py
index 0c43731..9d9d1b7 100644
--- a/structlog_sentry/__init__.py
+++ b/structlog_sentry/__init__.py
@@ -1,19 +1,40 @@
 import logging
-import os
 import sys
+from typing import List, Optional, Tuple, Union
 
 from sentry_sdk import capture_event
+from sentry_sdk.integrations.logging import ignore_logger
 from sentry_sdk.utils import event_from_exception
 
 
 class SentryProcessor:
-    """Sentry processor for structlog. Uses Sentry SDK to capture events in Sentry."""
 
-    def __init__(self, level=logging.WARNING, active=True):
+    def __init__(
+            self,
+            level: int = logging.WARNING,
+            active: bool = True,
+            as_extra: bool = True,
+            tag_keys: Union[List[str], str] = None) -> None:
+        """Sentry processor for structlog. Uses Sentry SDK to capture events in Sentry.
+
+        :param level: events of this or higher levels will be reported to Sentry.
+        :param active: a flag to make this processor enabled/disabled.
+        :param as_extra: send `event_dict` as extra info to Sentry.
+        :param tag_keys: a list of keys. If any if these keys appear in `event_dict`,
+        the key and its corresponding value in `event_dict` will be used as Sentry event tags. use `"__all__"` to report
+        all key/value pairs of event as tags.
+        """
         self.level = level
         self.active = active
+        self.tag_keys = tag_keys
+        self._as_extra = as_extra
+        self._original_event_dict = None
+
+    def _get_event_and_hint(self, event_dict: dict) -> Tuple[dict, Optional[str]]:
+        """Create a sentry event and hint from structlog `event_dict` and sys.exc_info.
 
-    def _log(self, event_dict, level):
+        :param event_dict: structlog event_dict
+        """
         exc_info = event_dict.pop("exc_info", sys.exc_info())
         has_exc_info = exc_info and exc_info != (None, None, None)
 
@@ -23,24 +44,71 @@ class SentryProcessor:
             event, hint = {}, None
 
         event["message"] = event_dict.get("event")
-        event["level"] = level
-        event["extra"] = event_dict
+        event["level"] = event_dict.get("level")
+
+        if self._as_extra:
+            event["extra"] = self._original_event_dict
+        if self.tag_keys == "__all__":
+            event["tags"] = self._original_event_dict
+        elif isinstance(self.tag_keys, list):
+            event["tags"] = {key: event_dict[key] for key in self.tag_keys if key in event_dict}
+
+        return event, hint
+
+    def _log(self, event_dict: dict) -> str:
+        """Send an event to Sentry and return sentry event id.
 
+        :param event_dict: structlog event_dict
+        """
+        event, hint = self._get_event_and_hint(event_dict)
         return capture_event(event, hint=hint)
 
-    def __call__(self, logger, method, event_dict):
-        event_dict["sentry"] = "skipped"
+    def __call__(self, logger, method, event_dict) -> dict:
+        """A middleware to process structlog `event_dict` and send it to Sentry."""
+        self._original_event_dict = event_dict.copy()
         sentry_skip = event_dict.pop("sentry_skip", False)
-
-        level = event_dict["level"]
-        do_log = getattr(logging, level.upper()) >= self.level
+        do_log = getattr(logging, event_dict["level"].upper()) >= self.level
 
         if sentry_skip or not self.active or not do_log:
+            event_dict["sentry"] = "skipped"
             return event_dict
 
-        sid = self._log(event_dict, level=level)
-
-        event_dict["sentry_id"] = sid
+        sid = self._log(event_dict)
         event_dict["sentry"] = "sent"
+        event_dict["sentry_id"] = sid
 
         return event_dict
+
+
+class SentryJsonProcessor(SentryProcessor):
+    """Sentry processor for structlog which uses JSONRenderer. Uses Sentry SDK to capture events in Sentry."""
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self._is_logger_ignored = False
+
+    def __call__(self, logger, method, event_dict) -> dict:
+        if not self._is_logger_ignored:
+            self._ignore_logger(logger, event_dict)
+        return super().__call__(logger, method, event_dict)
+
+    def _ignore_logger(self, logger, event_dict: dict) -> None:
+        """Tell Sentry to ignore logger. This is temporary workaround to prevent duplication of a JSON event in Sentry.
+
+        :param logger: logger instance
+        :param event_dict: structlog event_dict
+        """
+        record = event_dict.get("_record")
+        l_name = event_dict.get("logger")
+        if l_name:
+            logger_name = l_name
+        elif record is None:
+            logger_name = logger.name
+        else:
+            logger_name = record.name
+
+        if not logger_name:
+            raise Exception("Cannot ignore logger without a name.")
+
+        ignore_logger(logger_name)
+        self._is_logger_ignored = True


Test patch:
diff --git a/test/test_sentry_processor.py b/test/test_sentry_processor.py
index ab0f618..8f15de2 100644
--- a/test/test_sentry_processor.py
+++ b/test/test_sentry_processor.py
@@ -2,7 +2,12 @@ import logging
 
 import pytest
 
-from structlog_sentry import SentryProcessor
+from structlog_sentry import SentryJsonProcessor, SentryProcessor
+
+
+class MockLogger:
+    def __init__(self, name):
+        self.name = name
 
 
 def test_sentry_disabled():
@@ -28,6 +33,7 @@ def test_sentry_log(mocker, level):
     m_capture_event = mocker.patch("structlog_sentry.capture_event")
 
     event_data = {"level": level, "event": level + " message"}
+    sentry_event_data = event_data.copy()
     processor = SentryProcessor(level=getattr(logging, level.upper()))
     processor(None, None, event_data)
 
@@ -35,7 +41,7 @@ def test_sentry_log(mocker, level):
         {
             "level": level,
             "message": event_data["event"],
-            "extra": event_data,
+            "extra": sentry_event_data,
         },
         hint=None,
     )
@@ -57,6 +63,7 @@ def test_sentry_log_failure(mocker, level):
     )
 
     event_data = {"level": level, "event": level + " message"}
+    sentry_event_data = event_data.copy()
     processor = SentryProcessor(level=getattr(logging, level.upper()))
     try:
         1 / 0
@@ -68,7 +75,133 @@ def test_sentry_log_failure(mocker, level):
             "level": level,
             "message": event_data["event"],
             "exception": mocker.sentinel.exception,
-            "extra": event_data,
+            "extra": sentry_event_data,
         },
         hint=mocker.sentinel.hint,
     )
+
+
+@pytest.mark.parametrize("level", ["debug", "info", "warning"])
+def test_sentry_log_no_extra(mocker, level):
+    m_capture_event = mocker.patch("structlog_sentry.capture_event")
+
+    event_data = {"level": level, "event": level + " message"}
+    processor = SentryProcessor(level=getattr(logging, level.upper()), as_extra=False)
+    processor(None, None, event_data)
+
+    m_capture_event.assert_called_once_with(
+        {
+            "level": level,
+            "message": event_data["event"],
+        },
+        hint=None,
+    )
+
+    processor_only_errors = SentryProcessor(level=logging.ERROR)
+    event_dict = processor_only_errors(
+        None, None, {"level": level, "event": level + " message"}
+    )
+
+    assert event_dict.get("sentry") != "sent"
+
+
+@pytest.mark.parametrize("level", ["debug", "info", "warning"])
+def test_sentry_log_all_as_tags(mocker, level):
+    m_capture_event = mocker.patch("structlog_sentry.capture_event")
+
+    event_data = {"level": level, "event": level + " message"}
+    sentry_event_data = event_data.copy()
+    processor = SentryProcessor(level=getattr(logging, level.upper()), tag_keys="__all__")
+    processor(None, None, event_data)
+
+    m_capture_event.assert_called_once_with(
+        {
+            "level": level,
+            "message": event_data["event"],
+            "extra": sentry_event_data,
+            "tags": sentry_event_data,
+        },
+        hint=None,
+    )
+
+    processor_only_errors = SentryProcessor(level=logging.ERROR)
+    event_dict = processor_only_errors(
+        None, None, {"level": level, "event": level + " message"}
+    )
+
+    assert event_dict.get("sentry") != "sent"
+
+
+@pytest.mark.parametrize("level", ["debug", "info", "warning"])
+def test_sentry_log_specific_keys_as_tags(mocker, level):
+    m_capture_event = mocker.patch("structlog_sentry.capture_event")
+
+    event_data = {"level": level, "event": level + " message", "info1": "info1", "required": True}
+    tag_keys = ["info1", "required", "some non existing key"]
+    sentry_event_data = event_data.copy()
+    processor = SentryProcessor(level=getattr(logging, level.upper()), tag_keys=tag_keys)
+    processor(None, None, event_data)
+
+    m_capture_event.assert_called_once_with(
+        {
+            "level": level,
+            "message": event_data["event"],
+            "extra": sentry_event_data,
+            "tags": {k: sentry_event_data[k] for k in tag_keys if k in sentry_event_data},
+        },
+        hint=None,
+    )
+
+    processor_only_errors = SentryProcessor(level=logging.ERROR)
+    event_dict = processor_only_errors(
+        None, None, {"level": level, "event": level + " message"}
+    )
+
+    assert event_dict.get("sentry") != "sent"
+
+
+def test_sentry_json_ignore_logger_using_event_dict_logger_name(mocker):
+    m_ignore_logger = mocker.patch("structlog_sentry.ignore_logger")
+    m_logger = MockLogger("MockLogger")
+    event_data = {"level": "info", "event": "message", "logger": "EventLogger", "_record": MockLogger("RecordLogger")}
+    processor = SentryJsonProcessor()
+
+    assert not processor._is_logger_ignored
+    processor._ignore_logger(logger=m_logger, event_dict=event_data)
+    m_ignore_logger.assert_called_once_with(event_data["logger"])
+    assert processor._is_logger_ignored
+
+
+def test_sentry_json_ignore_logger_using_event_dict_record(mocker):
+    m_ignore_logger = mocker.patch("structlog_sentry.ignore_logger")
+    m_logger = MockLogger("MockLogger")
+    event_data = {"level": "info", "event": "message", "_record": MockLogger("RecordLogger")}
+    processor = SentryJsonProcessor()
+
+    assert not processor._is_logger_ignored
+    processor._ignore_logger(logger=m_logger, event_dict=event_data)
+    m_ignore_logger.assert_called_once_with(event_data["_record"].name)
+    assert processor._is_logger_ignored
+
+
+def test_sentry_json_ignore_logger_using_logger_instance_name(mocker):
+    m_ignore_logger = mocker.patch("structlog_sentry.ignore_logger")
+    m_logger = MockLogger("MockLogger")
+    event_data = {"level": "info", "event": "message"}
+    processor = SentryJsonProcessor()
+
+    assert not processor._is_logger_ignored
+    processor._ignore_logger(logger=m_logger, event_dict=event_data)
+    m_ignore_logger.assert_called_once_with(m_logger.name)
+    assert processor._is_logger_ignored
+
+
+def test_sentry_json_call_ignores_logger_once(mocker):
+    processor = SentryJsonProcessor()
+    m_ignore_logger = mocker.patch("structlog_sentry.ignore_logger")
+    event_data = {"level": "warning", "event": "message", "sentry_skip": True}
+    logger = MockLogger("MockLogger")
+    processor(logger, None, event_data)
+    processor(logger, None, event_data)
+    processor(logger, None, event_data)
+    m_ignore_logger.assert_called_once_with(logger.name)