Repository: scoutapp/scout_apm_python

Issue:
scout breaks caching for django
We gave scout a try and looked promising but then we quickly had to disable it on production.

Issue is that scout is adding a `Vary: Cookie` header which breaks caching



This normally happens when some code in django accesses the request.user object. I'm assuming scout is trying to add some meta data and accesses it causing this issue.

We've run into this problem ourselves in the past and the way around is it to look for the internal cached user on the request object. Like this:

```
    # going request.user will generate cookie vary headers, but since
    # we aren't changing the output based on this we want to see the user
    # without adding the header, so look for the lazy user
    if request and hasattr(request, '_cached_user'):
        # noinspection PyProtectedMember
        user = request._cached_user
```

I think if replaced the check for user here

https://github.com/scoutapp/scout_apm_python/blob/master/src/scout_apm/django/middleware.py#L139





Discussion:

Resolving patch:
diff --git a/src/scout_apm/django/middleware.py b/src/scout_apm/django/middleware.py
--- a/src/scout_apm/django/middleware.py
+++ b/src/scout_apm/django/middleware.py
@@ -136,7 +136,11 @@ def track_request_view_data(request, tracked_request):
         except Exception:
             pass
 
-    user = getattr(request, "user", None)
+    # Django's request.user caches in this attribute on first access. We only
+    # want to track the user if the application code has touched request.user
+    # because touching it causes session access, which adds "Cookie" to the
+    # "Vary" header.
+    user = getattr(request, "_cached_user", None)
     if user is not None:
         try:
             tracked_request.tag("username", user.get_username())
@@ -209,6 +213,7 @@ def __call__(self, request):
         tracked_request.start_span(operation="Unknown", should_capture_backtrace=False)
         try:
             response = self.get_response(request)
+            track_request_view_data(request, tracked_request)
             if 500 <= response.status_code <= 599:
                 tracked_request.tag("error", "true")
             return response
@@ -224,8 +229,6 @@ def process_view(self, request, view_func, view_args, view_kwargs):
         tracked_request = TrackedRequest.instance()
         tracked_request.is_real_request = True
 
-        track_request_view_data(request, tracked_request)
-
         span = tracked_request.current_span()
         if span is not None:
             span.operation = get_operation_name(request)
@@ -288,8 +291,6 @@ def process_view(self, request, view_func, view_func_args, view_func_kwargs):
 
         tracked_request.is_real_request = True
 
-        track_request_view_data(request, tracked_request)
-
         span = tracked_request.start_span(
             operation=get_operation_name(request), should_capture_backtrace=False
         )
@@ -304,6 +305,8 @@ def process_response(self, request, response):
             # don't do anything
             return response
 
+        track_request_view_data(request, tracked_request)
+
         # Only stop span if we started, but presume we are balanced, i.e. that
         # custom instrumentation within the application is not causing errors
         span = getattr(request, "_scout_view_span", None)


Test patch:
diff --git a/tests/integration/django_app.py b/tests/integration/django_app.py
--- a/tests/integration/django_app.py
+++ b/tests/integration/django_app.py
@@ -91,6 +91,10 @@ def get(self, request):
         return HttpResponse("Hello getter")
 
 
+def get_username(request):
+    return HttpResponse(request.user.username)
+
+
 def sql(request):
     with connection.cursor() as cursor:
         cursor.execute("CREATE TABLE IF NOT EXISTS test(item)")
@@ -218,6 +222,7 @@ def urlpatterns():
             path("crash/", crash),
             path("return-error/", return_error),
             path("cbv/", CbvView.as_view()),
+            path("get-username/", get_username),
             path("sql/", sql),
             path("sql-kwargs/", sql_kwargs),
             path("sql-type-errors/", sql_type_errors),
@@ -239,6 +244,7 @@ def urlpatterns():
             url(r"^crash/$", crash),
             url(r"^return-error/$", return_error),
             url(r"^cbv/$", CbvView.as_view()),
+            url(r"^get-username/$", get_username),
             url(r"^sql/$", sql),
             url(r"^sql-kwargs/$", sql_kwargs),
             url(r"^sql-type-errors/$", sql_type_errors),
diff --git a/tests/integration/test_django.py b/tests/integration/test_django.py
--- a/tests/integration/test_django.py
+++ b/tests/integration/test_django.py
@@ -553,101 +553,61 @@ def test_no_monitor_server_error(tracked_requests):
     assert tracked_requests == []
 
 
-def fake_authentication_middleware(get_response):
-    def middleware(request):
-        # Mock the User instance to avoid a dependency on django.contrib.auth.
-        request.user = mock.Mock()
-        request.user.get_username.return_value = "scout"
-        return get_response(request)
-
-    return middleware
-
-
-@skip_unless_new_style_middleware
-def test_username(tracked_requests):
-    new_middleware = (
-        settings.MIDDLEWARE[:-1]
-        + [__name__ + ".fake_authentication_middleware"]
-        + settings.MIDDLEWARE[-1:]
-    )
-    with app_with_scout(MIDDLEWARE=new_middleware) as app:
-        response = TestApp(app).get("/hello/")
+def test_username_anonymous(tracked_request):
+    with app_with_scout() as app:
+        response = TestApp(app).get("/get-username/")
 
     assert response.status_int == 200
-    assert len(tracked_requests) == 1
-    tracked_request = tracked_requests[0]
-    assert tracked_request.tags["username"] == "scout"
+    assert tracked_request.tags["username"] == ""
+    # The view touched request.user, which touched request.session, causing
+    # SessionMiddleware to insert the Vary header
+    assert response.headers["Vary"] == "Cookie"
 
 
-def crashy_authentication_middleware(get_response):
-    def middleware(request):
-        # Mock the User instance to avoid a dependency on django.contrib.auth.
-        request.user = mock.Mock()
-        request.user.get_username.side_effect = ValueError
-        return get_response(request)
+def test_username_logged_in(tracked_requests):
+    with app_with_scout() as app:
+        admin_user = make_admin_user()
+        test_app = TestApp(app)
 
-    return middleware
+        # Log in via admin form
+        login_response = test_app.get("/admin/login/")
+        assert login_response.status_int == 200
+        form = login_response.form
+        form["username"] = admin_user.username
+        form["password"] = admin_user.testing_password
+        form.submit()
 
-
-@skip_unless_new_style_middleware
-def test_username_exception(tracked_requests):
-    new_middleware = (
-        settings.MIDDLEWARE[:-1]
-        + [__name__ + ".crashy_authentication_middleware"]
-        + settings.MIDDLEWARE[-1:]
-    )
-    with app_with_scout(MIDDLEWARE=new_middleware) as app:
-        response = TestApp(app).get("/")
+        response = test_app.get("/get-username/")
 
     assert response.status_int == 200
-    assert len(tracked_requests) == 1
-    tracked_request = tracked_requests[0]
-    assert "username" not in tracked_request.tags
-
-
-class FakeAuthenticationMiddleware(object):
-    def process_request(self, request):
-        # Mock the User instance to avoid a dependency on django.contrib.auth.
-        request.user = mock.Mock()
-        request.user.get_username.return_value = "scout"
+    assert len(tracked_requests) == 3
+    tracked_request = tracked_requests[-1]
+    assert tracked_request.tags["username"] == admin_user.username
+    # The view touched request.user, which touched request.session, causing
+    # SessionMiddleware to insert the Vary header
+    assert response.headers["Vary"] == "Cookie"
 
 
-@skip_unless_old_style_middleware
-def test_old_style_username(tracked_requests):
-    new_middleware = (
-        settings.MIDDLEWARE_CLASSES[:-1]
-        + [__name__ + ".FakeAuthenticationMiddleware"]
-        + settings.MIDDLEWARE_CLASSES[-1:]
-    )
-    with app_with_scout(MIDDLEWARE_CLASSES=new_middleware) as app:
+def test_username_not_tagged_when_user_not_accessed(tracked_request):
+    with app_with_scout() as app:
         response = TestApp(app).get("/")
 
-    assert response.status_int == 200
-    assert len(tracked_requests) == 1
-    tracked_request = tracked_requests[0]
-    assert tracked_request.tags["username"] == "scout"
-
+    assert "Vary" not in response.headers
+    assert "username" not in tracked_request.tags
 
-class CrashyAuthenticationMiddleware(object):
-    def process_request(self, request):
-        # Mock the User instance to avoid a dependency on django.contrib.auth.
-        request.user = mock.Mock()
-        request.user.get_username.side_effect = ValueError
 
+def test_username_exception(tracked_request):
+    with app_with_scout() as app:
+        from django.contrib.auth.models import AnonymousUser
 
-@skip_unless_old_style_middleware
-def test_old_style_username_exception(tracked_requests):
-    new_middleware = (
-        settings.MIDDLEWARE_CLASSES[:-1]
-        + [__name__ + ".CrashyAuthenticationMiddleware"]
-        + settings.MIDDLEWARE_CLASSES[-1:]
-    )
-    with app_with_scout(MIDDLEWARE_CLASSES=new_middleware) as app:
-        response = TestApp(app).get("/")
+        with mock.patch.object(
+            AnonymousUser,
+            "get_username",
+            mock.Mock(side_effect=ValueError("Test Exception")),
+        ):
+            response = TestApp(app).get("/get-username/")
 
     assert response.status_int == 200
-    assert len(tracked_requests) == 1
-    tracked_request = tracked_requests[0]
     assert "username" not in tracked_request.tags