Repository: scoutapp/scout_apm_python
Pull request #463 (2020-01-31 11:39:47)

Remove fakeredis
It's incompatible with the latest redis-py 3.4.0: https://github.com/jamesls/fakeredis/issues/260

We only used it for a few tests. It's easier to simply use the Redis server we already run during normal tests.


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 33f217c..c6ff089 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,11 @@
 - Improved Falcon integration: tracing is automatically disabled if
   installation fails, and callable class-based responders are supported.
 
+### Fixed
+
+- Fix Jinja2 asynchronous rendering instrumentation for Jinja2 2.11.0+
+  ([PR #462](https://github.com/scoutapp/scout_apm_python/pull/462)).
+
 ## [2.10.0] 2020-01-02
 
 ### Added
diff --git a/src/scout_apm/instruments/jinja2.py b/src/scout_apm/instruments/jinja2.py
index 879be20..27119cc 100644
--- a/src/scout_apm/instruments/jinja2.py
+++ b/src/scout_apm/instruments/jinja2.py
@@ -2,11 +2,17 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import logging
+import sys
 
 import wrapt
 
 from scout_apm.core.tracked_request import TrackedRequest
 
+try:
+    from jinja2 import Environment
+except ImportError:  # pragma: no cover
+    Environment = None
+
 try:
     from jinja2 import Template
 except ImportError:  # pragma: no cover
@@ -22,13 +28,14 @@ except ImportError:
 logger = logging.getLogger(__name__)
 
 
+have_patched_environment_init = False
 have_patched_template_render = False
 have_patched_template_render_async = False
 
 
 def ensure_installed():
+    global have_patched_environment_init
     global have_patched_template_render
-    global have_patched_template_render_async
 
     logger.info("Ensuring Jinja2 instrumentation is installed.")
 
@@ -36,27 +43,27 @@ def ensure_installed():
         logger.info("Unable to import jinja2.Template")
         return
 
-    if not have_patched_template_render:
+    if not have_patched_environment_init:
         try:
-            Template.render = wrapped_render(Template.render)
+            Environment.__init__ = wrapped_environment_init(Environment.__init__)
         except Exception as exc:
             logger.warning(
-                "Unable to instrument jinja2.Template.render: %r", exc, exc_info=exc
+                "Unable to instrument jinja2.Environment.__init__: %r",
+                exc,
+                exc_info=exc,
             )
         else:
-            have_patched_template_render = True
+            have_patched_environment_init = True
 
-    if not have_patched_template_render_async and wrapped_render_async is not None:
+    if not have_patched_template_render:
         try:
-            Template.render_async = wrapped_render_async(Template.render_async)
+            Template.render = wrapped_render(Template.render)
         except Exception as exc:
             logger.warning(
-                "Unable to instrument jinja2.Template.render_async: %r",
-                exc,
-                exc_info=exc,
+                "Unable to instrument jinja2.Template.render: %r", exc, exc_info=exc
             )
         else:
-            have_patched_template_render_async = True
+            have_patched_template_render = True
 
 
 @wrapt.decorator
@@ -68,3 +75,33 @@ def wrapped_render(wrapped, instance, args, kwargs):
         return wrapped(*args, **kwargs)
     finally:
         tracked_request.stop_span()
+
+
+@wrapt.decorator
+def wrapped_environment_init(wrapped, instance, args, kwargs):
+    """
+    Delayed wrapping of render_async(), since Template won't have this method
+    until after jinja2.asyncsupport is imported, which since Jinja2 2.11.0 is
+    done conditionally in Environment.__init__:
+    https://github.com/pallets/jinja/issues/765
+    """
+    global have_patched_template_render_async
+    result = wrapped(*args, **kwargs)
+
+    if (
+        wrapped_render_async is not None
+        and not have_patched_template_render_async
+        and "jinja2.asyncsupport" in sys.modules
+    ):
+        try:
+            Template.render_async = wrapped_render_async(Template.render_async)
+        except Exception as exc:
+            logger.warning(
+                "Unable to instrument jinja2.Template.render_async: %r",
+                exc,
+                exc_info=exc,
+            )
+        else:
+            have_patched_template_render_async = True
+
+    return result
diff --git a/tox.ini b/tox.ini
index e98eaf8..2937571 100644
--- a/tox.ini
+++ b/tox.ini
@@ -47,7 +47,6 @@ deps =
     django30: djangorestframework
     dramatiq>=1.0.0 ; python_version >= "3.5"
     elasticsearch
-    fakeredis
     falcon
     flask
     flask-sqlalchemy


Test patch:
diff --git a/tests/integration/instruments/test_jinja2.py b/tests/integration/instruments/test_jinja2.py
index 374876e..386bf38 100644
--- a/tests/integration/instruments/test_jinja2.py
+++ b/tests/integration/instruments/test_jinja2.py
@@ -12,13 +12,16 @@ from tests.compat import mock
 
 @contextmanager
 def mock_not_attempted():
+    not_environment = mock.patch(
+        "scout_apm.instruments.jinja2.have_patched_environment_init", new=False
+    )
     not_render = mock.patch(
         "scout_apm.instruments.jinja2.have_patched_template_render", new=False
     )
     not_render_async = mock.patch(
         "scout_apm.instruments.jinja2.have_patched_template_render_async", new=False
     )
-    with not_render, not_render_async:
+    with not_environment, not_render, not_render_async:
         yield
 
 
diff --git a/tests/integration/instruments/test_jinja2_py36plus.py b/tests/integration/instruments/test_jinja2_py36plus.py
index a955d85..af520ab 100644
--- a/tests/integration/instruments/test_jinja2_py36plus.py
+++ b/tests/integration/instruments/test_jinja2_py36plus.py
@@ -18,18 +18,15 @@ def test_ensure_installed_fail_no_render_async_attribute(caplog):
 
         ensure_installed()
 
-    assert len(caplog.record_tuples) == 2
-    assert caplog.record_tuples[0] == (
-        "scout_apm.instruments.jinja2",
-        logging.INFO,
-        "Ensuring Jinja2 instrumentation is installed.",
-    )
-    logger, level, message = caplog.record_tuples[1]
-    assert logger == "scout_apm.instruments.jinja2"
-    assert level == logging.WARNING
-    assert message.startswith(
-        "Unable to instrument jinja2.Template.render_async: AttributeError"
-    )
+        jinja2.Environment()
+
+    assert caplog.record_tuples == [
+        (
+            "scout_apm.instruments.jinja2",
+            logging.INFO,
+            "Ensuring Jinja2 instrumentation is installed.",
+        )
+    ]
 
 
 @async_test
diff --git a/tests/integration/test_rq.py b/tests/integration/test_rq.py
index 57bf98d..b14c05b 100644
--- a/tests/integration/test_rq.py
+++ b/tests/integration/test_rq.py
@@ -1,15 +1,28 @@
 # coding=utf-8
 from __future__ import absolute_import, division, print_function, unicode_literals
 
+import os
 from collections import namedtuple
 from contextlib import contextmanager
 
-from fakeredis import FakeStrictRedis
+import pytest
+import redis
 from rq import Queue
 
 import scout_apm.rq
 from scout_apm.api import Config
 from scout_apm.compat import kwargs_only, string_type
+from scout_apm.instruments.redis import ensure_installed
+
+
+@pytest.fixture(scope="module")
+def redis_conn():
+    # Copied from test_redis.py
+    ensure_installed()
+    # e.g. export REDIS_URL="redis://localhost:6379/0"
+    if "REDIS_URL" not in os.environ:
+        raise pytest.skip("Redis isn't available")
+    yield redis.Redis.from_url(os.environ["REDIS_URL"])
 
 
 def hello():
@@ -22,7 +35,7 @@ def fail():
 
 @contextmanager
 @kwargs_only
-def app_with_scout(scout_config=None):
+def app_with_scout(redis_conn, scout_config=None):
     """
     Context manager that configures a Huey app with Scout installed.
     """
@@ -39,7 +52,7 @@ def app_with_scout(scout_config=None):
     # Setup according to https://docs.scoutapm.com/#rq
     # Using job_class argument to Queue
     Config.set(**scout_config)
-    queue = Queue(name="myqueue", connection=FakeStrictRedis())
+    queue = Queue(name="myqueue", connection=redis_conn)
     worker = scout_apm.rq.SimpleWorker([queue], connection=queue.connection)
 
     App = namedtuple("App", ["queue", "worker"])
@@ -49,8 +62,8 @@ def app_with_scout(scout_config=None):
         Config.reset_all()
 
 
-def test_hello(tracked_requests):
-    with app_with_scout() as app:
+def test_hello(redis_conn, tracked_requests):
+    with app_with_scout(redis_conn=redis_conn) as app:
         job = app.queue.enqueue(hello)
         app.worker.work(burst=True)
 
@@ -70,8 +83,8 @@ def test_hello(tracked_requests):
     )
 
 
-def test_fail(tracked_requests):
-    with app_with_scout() as app:
+def test_fail(redis_conn, tracked_requests):
+    with app_with_scout(redis_conn=redis_conn) as app:
         app.queue.enqueue(fail)
         app.worker.work(burst=True)
 
@@ -89,8 +102,8 @@ def test_fail(tracked_requests):
     )
 
 
-def test_no_monitor(tracked_requests):
-    with app_with_scout(scout_config={"monitor": False}) as app:
+def test_no_monitor(redis_conn, tracked_requests):
+    with app_with_scout(redis_conn=redis_conn, scout_config={"monitor": False}) as app:
         job = app.queue.enqueue(hello)
         app.worker.work(burst=True)