Repository: getsentry/sentry-python

Issue:
0.9.1 + pytest + DjangoIntegration + Django-REST-Framework: ImproperlyConfigured, missing SECRET_KEY
Using Python 3.7. I have the following libraries installed, among others:
```
django==2.2.2
djangorestframework==3.9.4
pytest-django==3.5.0
pytest==4.6.3
sentry-sdk==0.9.1
```
In my `settings.py`:
```python
# ...
SECRET_KEY = "some value"
# ...
sentry_sdk.init(
    dsn=SENTRY_DSN,
    environment=ENVIRONMENT,
    release=f"project@{VERSION}",
    integrations=[DjangoIntegration()],
)
```
Starting the application via the builtin test server works fine. When invoking `pytest`, I'm getting the following error:
```
Traceback (most recent call last):
  File "/usr/local/bin/pytest", line 10, in <module>
    sys.exit(main())
  File "/usr/local/lib/python3.7/site-packages/_pytest/config/__init__.py", line 63, in main
    config = _prepareconfig(args, plugins)
  File "/usr/local/lib/python3.7/site-packages/_pytest/config/__init__.py", line 207, in _prepareconfig
    pluginmanager=pluginmanager, args=args
  File "/usr/local/lib/python3.7/site-packages/pluggy/hooks.py", line 289, in __call__
    return self._hookexec(self, self.get_hookimpls(), kwargs)
  File "/usr/local/lib/python3.7/site-packages/pluggy/manager.py", line 87, in _hookexec
    return self._inner_hookexec(hook, methods, kwargs)
  File "/usr/local/lib/python3.7/site-packages/pluggy/manager.py", line 81, in <lambda>
    firstresult=hook.spec.opts.get("firstresult") if hook.spec else False,
  File "/usr/local/lib/python3.7/site-packages/pluggy/callers.py", line 203, in _multicall
    gen.send(outcome)
  File "/usr/local/lib/python3.7/site-packages/_pytest/helpconfig.py", line 94, in pytest_cmdline_parse
    config = outcome.get_result()
  File "/usr/local/lib/python3.7/site-packages/pluggy/callers.py", line 80, in get_result
    raise ex[1].with_traceback(ex[2])
  File "/usr/local/lib/python3.7/site-packages/pluggy/callers.py", line 187, in _multicall
    res = hook_impl.function(*args)
  File "/usr/local/lib/python3.7/site-packages/_pytest/config/__init__.py", line 687, in pytest_cmdline_parse
    self.parse(args)
  File "/usr/local/lib/python3.7/site-packages/_pytest/config/__init__.py", line 895, in parse
    self._preparse(args, addopts=addopts)
  File "/usr/local/lib/python3.7/site-packages/_pytest/config/__init__.py", line 851, in _preparse
    early_config=self, args=args, parser=self._parser
  File "/usr/local/lib/python3.7/site-packages/pluggy/hooks.py", line 289, in __call__
    return self._hookexec(self, self.get_hookimpls(), kwargs)
  File "/usr/local/lib/python3.7/site-packages/pluggy/manager.py", line 87, in _hookexec
    return self._inner_hookexec(hook, methods, kwargs)
  File "/usr/local/lib/python3.7/site-packages/pluggy/manager.py", line 81, in <lambda>
    firstresult=hook.spec.opts.get("firstresult") if hook.spec else False,
  File "/usr/local/lib/python3.7/site-packages/pluggy/callers.py", line 208, in _multicall
    return outcome.get_result()
  File "/usr/local/lib/python3.7/site-packages/pluggy/callers.py", line 80, in get_result
    raise ex[1].with_traceback(ex[2])
  File "/usr/local/lib/python3.7/site-packages/pluggy/callers.py", line 187, in _multicall
    res = hook_impl.function(*args)
  File "/usr/local/lib/python3.7/site-packages/pytest_django/plugin.py", line 333, in pytest_load_initial_conftests
    dj_settings.DATABASES
  File "/usr/local/lib/python3.7/site-packages/django/conf/__init__.py", line 79, in __getattr__
    self._setup(name)
  File "/usr/local/lib/python3.7/site-packages/django/conf/__init__.py", line 66, in _setup
    self._wrapped = Settings(settings_module)
  File "/usr/local/lib/python3.7/site-packages/django/conf/__init__.py", line 157, in __init__
    mod = importlib.import_module(self.SETTINGS_MODULE)
  File "/usr/local/lib/python3.7/importlib/__init__.py", line 127, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "<frozen importlib._bootstrap>", line 1006, in _gcd_import
  File "<frozen importlib._bootstrap>", line 983, in _find_and_load
  File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
  File "<frozen importlib._bootstrap_external>", line 728, in exec_module
  File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
  File "/app/backend/testing_settings.py", line 1, in <module>
    from .settings import *  # noqa
  File "/app/backend/settings.py", line 155, in <module>
    integrations=[DjangoIntegration()]
  File "/usr/local/lib/python3.7/site-packages/sentry_sdk/hub.py", line 74, in init
    client = Client(*args, **kwargs)
  File "/usr/local/lib/python3.7/site-packages/sentry_sdk/client.py", line 88, in __init__
    options["integrations"], with_defaults=options["default_integrations"]
  File "/usr/local/lib/python3.7/site-packages/sentry_sdk/integrations/__init__.py", line 80, in setup_integrations
    type(integration).setup_once()
  File "/usr/local/lib/python3.7/site-packages/sentry_sdk/integrations/django/__init__.py", line 128, in setup_once
    from rest_framework.views import APIView  # type: ignore
  File "/usr/local/lib/python3.7/site-packages/rest_framework/views.py", line 19, in <module>
    from rest_framework.schemas import DefaultSchema
  File "/usr/local/lib/python3.7/site-packages/rest_framework/schemas/__init__.py", line 32, in <module>
    authentication_classes=api_settings.DEFAULT_AUTHENTICATION_CLASSES,
  File "/usr/local/lib/python3.7/site-packages/rest_framework/settings.py", line 220, in __getattr__
    val = self.user_settings[attr]
  File "/usr/local/lib/python3.7/site-packages/rest_framework/settings.py", line 211, in user_settings
    self._user_settings = getattr(settings, 'REST_FRAMEWORK', {})
  File "/usr/local/lib/python3.7/site-packages/django/conf/__init__.py", line 79, in __getattr__
    self._setup(name)
  File "/usr/local/lib/python3.7/site-packages/django/conf/__init__.py", line 66, in _setup
    self._wrapped = Settings(settings_module)
  File "/usr/local/lib/python3.7/site-packages/django/conf/__init__.py", line 176, in __init__
    raise ImproperlyConfigured("The SECRET_KEY setting must not be empty.")
django.core.exceptions.ImproperlyConfigured: The SECRET_KEY setting must not be empty.
```

It's basically a kind of a circular import error.

It seems that this is due to the REST framework importing settings while settings are still being imported (but due to the proxy nature of `django.conf.settings`, this is slightly hidden).

The issue is triggered from the `from rest_framework.views import APIView` line, which was introduced in 0.9.1 (I've tried 0.9.0 and it's fine).

I'm not entirely sure yet, why it only happens with `pytest`; it's a complex interaction between specific (latest) versions of several packages, but none of them are obscure.


Discussion:
Thanks for the report, this is a bug in the SDK. I imagine that moving your
init code into `views.py` fixes the issue, but I'd recommend downgrading to
0.9.0.

I will fix this bug on Monday.

On Sat, Jun 22, 2019 at 3:04 PM Kamil <notifications@github.com> wrote:

> Using Python 3.7. I have the following libraries installed, among others:
>
> django==2.2.2
> djangorestframework==3.9.4
> pytest-django==3.5.0
> pytest==4.6.3
> sentry-sdk==0.9.1
>
> In my settings.py:
>
> # ...SECRET_KEY = "some value"# ...
> sentry_sdk.init(
>     dsn=SENTRY_DSN,
>     environment=ENVIRONMENT,
>     release=f"project@{VERSION}",
>     integrations=[DjangoIntegration()],
> )
>
> Starting the application via the builtin test server works fine. When
> invoking pytest, I'm getting the following error:
>
> Traceback (most recent call last):
>   File "/usr/local/bin/pytest", line 10, in <module>
>     sys.exit(main())
>   File "/usr/local/lib/python3.7/site-packages/_pytest/config/__init__.py", line 63, in main
>     config = _prepareconfig(args, plugins)
>   File "/usr/local/lib/python3.7/site-packages/_pytest/config/__init__.py", line 207, in _prepareconfig
>     pluginmanager=pluginmanager, args=args
>   File "/usr/local/lib/python3.7/site-packages/pluggy/hooks.py", line 289, in __call__
>     return self._hookexec(self, self.get_hookimpls(), kwargs)
>   File "/usr/local/lib/python3.7/site-packages/pluggy/manager.py", line 87, in _hookexec
>     return self._inner_hookexec(hook, methods, kwargs)
>   File "/usr/local/lib/python3.7/site-packages/pluggy/manager.py", line 81, in <lambda>
>     firstresult=hook.spec.opts.get("firstresult") if hook.spec else False,
>   File "/usr/local/lib/python3.7/site-packages/pluggy/callers.py", line 203, in _multicall
>     gen.send(outcome)
>   File "/usr/local/lib/python3.7/site-packages/_pytest/helpconfig.py", line 94, in pytest_cmdline_parse
>     config = outcome.get_result()
>   File "/usr/local/lib/python3.7/site-packages/pluggy/callers.py", line 80, in get_result
>     raise ex[1].with_traceback(ex[2])
>   File "/usr/local/lib/python3.7/site-packages/pluggy/callers.py", line 187, in _multicall
>     res = hook_impl.function(*args)
>   File "/usr/local/lib/python3.7/site-packages/_pytest/config/__init__.py", line 687, in pytest_cmdline_parse
>     self.parse(args)
>   File "/usr/local/lib/python3.7/site-packages/_pytest/config/__init__.py", line 895, in parse
>     self._preparse(args, addopts=addopts)
>   File "/usr/local/lib/python3.7/site-packages/_pytest/config/__init__.py", line 851, in _preparse
>     early_config=self, args=args, parser=self._parser
>   File "/usr/local/lib/python3.7/site-packages/pluggy/hooks.py", line 289, in __call__
>     return self._hookexec(self, self.get_hookimpls(), kwargs)
>   File "/usr/local/lib/python3.7/site-packages/pluggy/manager.py", line 87, in _hookexec
>     return self._inner_hookexec(hook, methods, kwargs)
>   File "/usr/local/lib/python3.7/site-packages/pluggy/manager.py", line 81, in <lambda>
>     firstresult=hook.spec.opts.get("firstresult") if hook.spec else False,
>   File "/usr/local/lib/python3.7/site-packages/pluggy/callers.py", line 208, in _multicall
>     return outcome.get_result()
>   File "/usr/local/lib/python3.7/site-packages/pluggy/callers.py", line 80, in get_result
>     raise ex[1].with_traceback(ex[2])
>   File "/usr/local/lib/python3.7/site-packages/pluggy/callers.py", line 187, in _multicall
>     res = hook_impl.function(*args)
>   File "/usr/local/lib/python3.7/site-packages/pytest_django/plugin.py", line 333, in pytest_load_initial_conftests
>     dj_settings.DATABASES
>   File "/usr/local/lib/python3.7/site-packages/django/conf/__init__.py", line 79, in __getattr__
>     self._setup(name)
>   File "/usr/local/lib/python3.7/site-packages/django/conf/__init__.py", line 66, in _setup
>     self._wrapped = Settings(settings_module)
>   File "/usr/local/lib/python3.7/site-packages/django/conf/__init__.py", line 157, in __init__
>     mod = importlib.import_module(self.SETTINGS_MODULE)
>   File "/usr/local/lib/python3.7/importlib/__init__.py", line 127, in import_module
>     return _bootstrap._gcd_import(name[level:], package, level)
>   File "<frozen importlib._bootstrap>", line 1006, in _gcd_import
>   File "<frozen importlib._bootstrap>", line 983, in _find_and_load
>   File "<frozen importlib._bootstrap>", line 967, in _find_and_load_unlocked
>   File "<frozen importlib._bootstrap>", line 677, in _load_unlocked
>   File "<frozen importlib._bootstrap_external>", line 728, in exec_module
>   File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
>   File "/app/backend/testing_settings.py", line 1, in <module>
>     from .settings import *  # noqa
>   File "/app/backend/settings.py", line 155, in <module>
>     integrations=[DjangoIntegration()]
>   File "/usr/local/lib/python3.7/site-packages/sentry_sdk/hub.py", line 74, in init
>     client = Client(*args, **kwargs)
>   File "/usr/local/lib/python3.7/site-packages/sentry_sdk/client.py", line 88, in __init__
>     options["integrations"], with_defaults=options["default_integrations"]
>   File "/usr/local/lib/python3.7/site-packages/sentry_sdk/integrations/__init__.py", line 80, in setup_integrations
>     type(integration).setup_once()
>   File "/usr/local/lib/python3.7/site-packages/sentry_sdk/integrations/django/__init__.py", line 128, in setup_once
>     from rest_framework.views import APIView  # type: ignore
>   File "/usr/local/lib/python3.7/site-packages/rest_framework/views.py", line 19, in <module>
>     from rest_framework.schemas import DefaultSchema
>   File "/usr/local/lib/python3.7/site-packages/rest_framework/schemas/__init__.py", line 32, in <module>
>     authentication_classes=api_settings.DEFAULT_AUTHENTICATION_CLASSES,
>   File "/usr/local/lib/python3.7/site-packages/rest_framework/settings.py", line 220, in __getattr__
>     val = self.user_settings[attr]
>   File "/usr/local/lib/python3.7/site-packages/rest_framework/settings.py", line 211, in user_settings
>     self._user_settings = getattr(settings, 'REST_FRAMEWORK', {})
>   File "/usr/local/lib/python3.7/site-packages/django/conf/__init__.py", line 79, in __getattr__
>     self._setup(name)
>   File "/usr/local/lib/python3.7/site-packages/django/conf/__init__.py", line 66, in _setup
>     self._wrapped = Settings(settings_module)
>   File "/usr/local/lib/python3.7/site-packages/django/conf/__init__.py", line 176, in __init__
>     raise ImproperlyConfigured("The SECRET_KEY setting must not be empty.")
> django.core.exceptions.ImproperlyConfigured: The SECRET_KEY setting must not be empty.
>
> It's basically a kind of a circular import error.
>
> It seems that this is due to the REST framework importing settings while
> settings are still being imported (but due to the proxy nature of
> django.conf.settings, this is slightly hidden).
>
> The issue is triggered from the from rest_framework.views import APIView
> line, which was introduced in 0.9.1 (I've tried 0.9.0 and it's fine).
>
> I'm not entirely sure yet, why it only happens with pytest; it's a
> complex interaction between specific (latest) versions of several packages,
> but none of them are obscure.
>
> —
> You are receiving this because you are subscribed to this thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/getsentry/sentry-python/issues/397?email_source=notifications&email_token=AAGMPRKGHSHOAXN7R3NZTWTP3YPMZA5CNFSM4H2WYHW2YY3PNVWWK3TUL52HS4DFUVEXG43VMWVGG33NNVSW45C7NFSM4G3C7WXA>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AAGMPRIDFXRIFRH5DDUGZRDP3YPMZANCNFSM4H2WYHWQ>
> .
>


Resolving patch:
diff --git a/sentry_sdk/integrations/django/__init__.py b/sentry_sdk/integrations/django/__init__.py
--- a/sentry_sdk/integrations/django/__init__.py
+++ b/sentry_sdk/integrations/django/__init__.py
@@ -2,10 +2,10 @@
 from __future__ import absolute_import
 
 import sys
+import threading
 import weakref
 
 from django import VERSION as DJANGO_VERSION  # type: ignore
-from django.db.models.query import QuerySet  # type: ignore
 from django.core import signals  # type: ignore
 
 if False:
@@ -116,6 +116,8 @@ def sentry_patched_get_response(self, request):
             hub = Hub.current
             integration = hub.get_integration(DjangoIntegration)
             if integration is not None:
+                _patch_drf()
+
                 with hub.configure_scope() as scope:
                     scope.add_event_processor(
                         _make_event_processor(weakref.ref(request), integration)
@@ -124,25 +126,6 @@ def sentry_patched_get_response(self, request):
 
         BaseHandler.get_response = sentry_patched_get_response
 
-        try:
-            from rest_framework.views import APIView  # type: ignore
-        except ImportError:
-            pass
-        else:
-            # DRF's request type (which wraps the Django request and proxies
-            # all attrs) has some attributes such as `data` which buffer
-            # request data.  We want to use those in the RequestExtractor to
-            # get body data more reliably.
-            old_drf_initial = APIView.initial
-
-            def sentry_patched_drf_initial(self, request, *args, **kwargs):
-                with capture_internal_exceptions():
-                    request._request._sentry_drf_request_backref = weakref.ref(request)
-                    pass
-                return old_drf_initial(self, request, *args, **kwargs)
-
-            APIView.initial = sentry_patched_drf_initial
-
         signals.got_request_exception.connect(_got_request_exception)
 
         @add_global_event_processor
@@ -190,6 +173,17 @@ def process_django_templates(event, hint):
 
         @add_global_repr_processor
         def _django_queryset_repr(value, hint):
+            try:
+                # Django 1.6 can fail to import `QuerySet` when Django settings
+                # have not yet been initialized.
+                #
+                # If we fail to import, return `NotImplemented`. It's at least
+                # unlikely that we have a query set in `value` when importing
+                # `QuerySet` fails.
+                from django.db.models.query import QuerySet  # type: ignore
+            except Exception:
+                return NotImplemented
+
             if not isinstance(value, QuerySet) or value._result_cache:
                 return NotImplemented
 
@@ -205,6 +199,63 @@ def _django_queryset_repr(value, hint):
             )
 
 
+_DRF_PATCHED = False
+_DRF_PATCH_LOCK = threading.Lock()
+
+
+def _patch_drf():
+    """
+    Patch Django Rest Framework for more/better request data. DRF's request
+    type is a wrapper around Django's request type. The attribute we're
+    interested in is `request.data`, which is a cached property containing a
+    parsed request body. Reading a request body from that property is more
+    reliable than reading from any of Django's own properties, as those don't
+    hold payloads in memory and therefore can only be accessed once.
+
+    We patch the Django request object to include a weak backreference to the
+    DRF request object, such that we can later use either in
+    `DjangoRequestExtractor`.
+
+    This function is not called directly on SDK setup, because importing almost
+    any part of Django Rest Framework will try to access Django settings (where
+    `sentry_sdk.init()` might be called from in the first place). Instead we
+    run this function on every request and do the patching on the first
+    request.
+    """
+
+    global _DRF_PATCHED
+
+    if _DRF_PATCHED:
+        # Double-checked locking
+        return
+
+    with _DRF_PATCH_LOCK:
+        if _DRF_PATCHED:
+            return
+
+        # We set this regardless of whether the code below succeeds or fails.
+        # There is no point in trying to patch again on the next request.
+        _DRF_PATCHED = True
+
+        with capture_internal_exceptions():
+            try:
+                from rest_framework.views import APIView  # type: ignore
+            except ImportError:
+                pass
+            else:
+                old_drf_initial = APIView.initial
+
+                def sentry_patched_drf_initial(self, request, *args, **kwargs):
+                    with capture_internal_exceptions():
+                        request._request._sentry_drf_request_backref = weakref.ref(
+                            request
+                        )
+                        pass
+                    return old_drf_initial(self, request, *args, **kwargs)
+
+                APIView.initial = sentry_patched_drf_initial
+
+
 def _make_event_processor(weak_request, integration):
     # type: (Callable[[], WSGIRequest], DjangoIntegration) -> Callable
     def event_processor(event, hint):


Test patch:
diff --git a/tests/integrations/django/myapp/settings.py b/tests/integrations/django/myapp/settings.py
--- a/tests/integrations/django/myapp/settings.py
+++ b/tests/integrations/django/myapp/settings.py
@@ -10,6 +10,15 @@
 https://docs.djangoproject.com/en/2.0/ref/settings/
 """
 
+
+# We shouldn't access settings while setting up integrations. Initialize SDK
+# here to provoke any errors that might occur.
+import sentry_sdk
+from sentry_sdk.integrations.django import DjangoIntegration
+
+sentry_sdk.init(integrations=[DjangoIntegration()])
+
+
 import os
 
 try:
diff --git a/tests/integrations/django/test_basic.py b/tests/integrations/django/test_basic.py
--- a/tests/integrations/django/test_basic.py
+++ b/tests/integrations/django/test_basic.py
@@ -179,7 +179,7 @@ def test_sql_queries(sentry_init, capture_events):
 
     event, = events
 
-    crumb, = event["breadcrumbs"]
+    crumb = event["breadcrumbs"][-1]
 
     assert crumb["message"] == """SELECT count(*) FROM people_person WHERE foo = 123"""
 
@@ -297,7 +297,7 @@ def test_sql_queries_large_params(sentry_init, capture_events):
 
     event, = events
 
-    crumb, = event["breadcrumbs"]
+    crumb = event["breadcrumbs"][-1]
     assert crumb["message"] == (
         "SELECT count(*) FROM people_person WHERE foo = '%s... and bar IS NULL"
         % ("x" * 124,)