Repository: scoutapp/scout_apm_python

Issue:
Instrument CherryPy
CherryPy was reported used by 3% of developers in the [Python developers survey 2018](https://www.jetbrains.com/research/python-developers-survey-2018/). We should support it.


Discussion:
3% in 2019 survey too: https://www.jetbrains.com/lp/devecosystem-2019/python/

Resolving patch:
diff --git a/src/scout_apm/cherrypy.py b/src/scout_apm/cherrypy.py
new file mode 100644
--- /dev/null
+++ b/src/scout_apm/cherrypy.py
@@ -0,0 +1,118 @@
+# coding=utf-8
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import cherrypy
+from cherrypy.lib.encoding import ResponseEncoder
+from cherrypy.process import plugins
+
+import scout_apm.core
+from scout_apm.compat import parse_qsl
+from scout_apm.core.tracked_request import TrackedRequest
+from scout_apm.core.web_requests import (
+    create_filtered_path,
+    ignore_path,
+    track_amazon_request_queue_time,
+    track_request_queue_time,
+)
+
+
+class ScoutPlugin(plugins.SimplePlugin):
+    def __init__(self, bus):
+        super(ScoutPlugin, self).__init__(bus)
+        installed = scout_apm.core.install()
+        self._do_nothing = not installed
+
+    def before_request(self):
+        if self._do_nothing:
+            return
+        request = cherrypy.request
+        tracked_request = TrackedRequest.instance()
+        tracked_request.is_real_request = True
+        request._scout_tracked_request = tracked_request
+
+        # Can't name operation until after request, when routing has been done
+        request._scout_controller_span = tracked_request.start_span(
+            "Controller/Unknown"
+        )
+
+    def after_request(self):
+        if self._do_nothing:
+            return
+        request = cherrypy.request
+        tracked_request = getattr(request, "_scout_tracked_request", None)
+        if tracked_request is None:
+            return
+
+        # Rename controller span now routing has been done
+        operation_name = get_operation_name(request)
+        if operation_name is not None:
+            request._scout_controller_span.operation = operation_name
+
+        # Grab general request data now it has been parsed
+        path = request.path_info
+        # Parse params ourselves because we want only GET params but CherryPy
+        # parses POST params (nearly always sensitive) into the same dict.
+        params = parse_qsl(request.query_string)
+        tracked_request.tag("path", create_filtered_path(path, params))
+        if ignore_path(path):
+            tracked_request.tag("ignore_transaction", True)
+
+        # Determine a remote IP to associate with the request. The value is
+        # spoofable by the requester so this is not suitable to use in any
+        # security sensitive context.
+        user_ip = (
+            request.headers.get("x-forwarded-for", "").split(",")[0]
+            or request.headers.get("client-ip", "").split(",")[0]
+            or (request.remote.ip or None)
+        )
+        tracked_request.tag("user_ip", user_ip)
+
+        queue_time = request.headers.get("x-queue-start", "") or request.headers.get(
+            "x-request-start", ""
+        )
+        tracked_queue_time = track_request_queue_time(queue_time, tracked_request)
+        if not tracked_queue_time:
+            amazon_queue_time = request.headers.get("x-amzn-trace-id", "")
+            track_amazon_request_queue_time(amazon_queue_time, tracked_request)
+
+        response = cherrypy.response
+        status = response.status
+        if isinstance(status, int):
+            status_int = status
+        else:
+            status_first = status.split(" ", 1)[0]
+            try:
+                status_int = int(status_first)
+            except ValueError:
+                # Assume OK
+                status_int = 200
+        if 500 <= status_int <= 599:
+            tracked_request.tag("error", "true")
+        elif status_int == 404:
+            tracked_request.is_real_request = False
+
+        tracked_request.stop_span()
+
+
+def get_operation_name(request):
+    handler = request.handler
+    if handler is None:
+        return None
+
+    if isinstance(handler, ResponseEncoder):
+        real_handler = handler.oldhandler
+    else:
+        real_handler = handler
+
+    # Unwrap HandlerWrapperTool classes
+    while hasattr(real_handler, "callable"):
+        real_handler = real_handler.callable
+
+    # Not a method? Not from an exposed view then
+    if not hasattr(real_handler, "__self__"):
+        return None
+
+    real_handler_cls = real_handler.__self__.__class__
+    return "Controller/{}.{}.{}".format(
+        real_handler_cls.__module__, real_handler_cls.__name__, real_handler.__name__,
+    )
diff --git a/src/scout_apm/compat.py b/src/scout_apm/compat.py
--- a/src/scout_apm/compat.py
+++ b/src/scout_apm/compat.py
@@ -57,9 +57,10 @@ def text(value, encoding="utf-8", errors="strict"):
 
 
 if sys.version_info >= (3, 0):
-    from urllib.parse import urlencode
+    from urllib.parse import parse_qsl, urlencode
 else:
     from urllib import urlencode
+    from urlparse import parse_qsl
 
 
 def kwargs_only(func):
@@ -98,6 +99,7 @@ def wrapper(*args, **kwargs):
     "ContextDecorator",
     "datetime_to_timestamp",
     "kwargs_only",
+    "parse_qsl",
     "queue",
     "string_type",
     "text",
diff --git a/src/scout_apm/core/web_requests.py b/src/scout_apm/core/web_requests.py
--- a/src/scout_apm/core/web_requests.py
+++ b/src/scout_apm/core/web_requests.py
@@ -3,7 +3,7 @@
 
 import time
 
-from scout_apm.compat import datetime_to_timestamp, urlencode
+from scout_apm.compat import datetime_to_timestamp, parse_qsl, urlencode
 from scout_apm.core.config import scout_config
 
 # Originally derived from:
@@ -177,8 +177,6 @@ def asgi_track_request_data(scope, tracked_request):
     """
     Track request data from an ASGI HTTP or Websocket scope.
     """
-    from urllib.parse import parse_qsl  # Inner import due to Python 3+
-
     path = scope.get("root_path", "") + scope["path"]
     query_params = parse_qsl(scope.get("query_string", b"").decode("utf-8"))
     tracked_request.tag("path", create_filtered_path(path, query_params))


Test patch:
diff --git a/tests/integration/test_cherrypy.py b/tests/integration/test_cherrypy.py
new file mode 100644
--- /dev/null
+++ b/tests/integration/test_cherrypy.py
@@ -0,0 +1,204 @@
+# coding=utf-8
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import datetime as dt
+from contextlib import contextmanager
+
+import cherrypy
+from webtest import TestApp
+
+from scout_apm.api import Config
+from scout_apm.cherrypy import ScoutPlugin
+from scout_apm.compat import datetime_to_timestamp
+from tests.integration.util import (
+    parametrize_filtered_params,
+    parametrize_queue_time_header_name,
+    parametrize_user_ip_headers,
+)
+
+
+@contextmanager
+def app_with_scout(scout_config=None):
+    """
+    Context manager that configures and installs the Scout plugin for CherryPy.
+    """
+    if scout_config is None:
+        scout_config = {}
+
+    scout_config["core_agent_launch"] = False
+    scout_config.setdefault("monitor", True)
+    Config.set(**scout_config)
+
+    class Views(object):
+        @cherrypy.expose
+        def index(self, **params):  # Take all params so CherryPy doesn't 404
+            return "Welcome home."
+
+        @cherrypy.expose
+        def hello(self):
+            return "Hello World!"
+
+        @cherrypy.expose
+        def crash(self):
+            raise ValueError("BØØM!")  # non-ASCII
+
+        @cherrypy.expose
+        def return_error(self):
+            cherrypy.response.status = 503
+            return "Something went wrong"
+
+    app = cherrypy.Application(Views(), "/", config=None)
+
+    # Setup according to https://docs.scoutapm.com/#cherrypy
+    plugin = ScoutPlugin(cherrypy.engine)
+    plugin.subscribe()
+
+    try:
+        yield app
+    finally:
+        plugin.unsubscribe()
+        Config.reset_all()
+
+
+def test_home(tracked_requests):
+    with app_with_scout() as app:
+        response = TestApp(app).get("/")
+
+    assert response.status_int == 200
+    assert response.text == "Welcome home."
+    assert len(tracked_requests) == 1
+    tracked_request = tracked_requests[0]
+    assert len(tracked_request.complete_spans) == 1
+    assert tracked_request.tags["path"] == "/"
+    span = tracked_request.complete_spans[0]
+    assert span.operation == "Controller/tests.integration.test_cherrypy.Views.index"
+
+
+def test_home_ignored(tracked_requests):
+    with app_with_scout(scout_config={"ignore": "/"}) as app:
+        response = TestApp(app).get("/")
+
+    assert response.status_int == 200
+    assert response.text == "Welcome home."
+    assert tracked_requests == []
+
+
+@parametrize_filtered_params
+def test_filtered_params(params, expected_path, tracked_requests):
+    with app_with_scout() as app:
+        TestApp(app).get("/", params=params)
+
+    assert tracked_requests[0].tags["path"] == expected_path
+
+
+@parametrize_user_ip_headers
+def test_user_ip(headers, client_address, expected, tracked_requests):
+    with app_with_scout() as app:
+        TestApp(app).get(
+            "/",
+            headers=headers,
+            extra_environ=(
+                {str("REMOTE_ADDR"): client_address}
+                if client_address is not None
+                else {}
+            ),
+        )
+
+    tracked_request = tracked_requests[0]
+    assert tracked_request.tags["user_ip"] == expected
+
+
+@parametrize_queue_time_header_name
+def test_queue_time(header_name, tracked_requests):
+    # Not testing floats due to Python 2/3 rounding differences
+    queue_start = int(datetime_to_timestamp(dt.datetime.utcnow())) - 2
+    with app_with_scout() as app:
+        response = TestApp(app).get(
+            "/", headers={header_name: str("t=") + str(queue_start)}
+        )
+
+    assert response.status_int == 200
+    assert len(tracked_requests) == 1
+    queue_time_ns = tracked_requests[0].tags["scout.queue_time_ns"]
+    # Upper bound assumes we didn't take more than 2s to run this test...
+    assert queue_time_ns >= 2000000000 and queue_time_ns < 4000000000
+
+
+def test_amazon_queue_time(tracked_requests):
+    queue_start = int(datetime_to_timestamp(dt.datetime.utcnow())) - 2
+    with app_with_scout() as app:
+        response = TestApp(app).get(
+            "/",
+            headers={
+                "X-Amzn-Trace-Id": str(
+                    "Self=1-{}-12456789abcdef012345678".format(queue_start)
+                )
+            },
+        )
+
+    assert response.status_int == 200
+    assert len(tracked_requests) == 1
+    queue_time_ns = tracked_requests[0].tags["scout.queue_time_ns"]
+    # Upper bound assumes we didn't take more than 2s to run this test...
+    assert queue_time_ns >= 2000000000 and queue_time_ns < 4000000000
+
+
+def test_hello(tracked_requests):
+    with app_with_scout() as app:
+        response = TestApp(app).get("/hello/")
+
+    assert response.status_int == 200
+    assert response.text == "Hello World!"
+    assert len(tracked_requests) == 1
+    tracked_request = tracked_requests[0]
+    assert len(tracked_request.complete_spans) == 1
+    assert tracked_request.tags["path"] == "/hello/"
+    span = tracked_request.complete_spans[0]
+    assert span.operation == "Controller/tests.integration.test_cherrypy.Views.hello"
+
+
+def test_server_error(tracked_requests):
+    with app_with_scout() as app:
+        response = TestApp(app).get("/crash/", expect_errors=True)
+
+    assert response.status_int == 500
+    assert len(tracked_requests) == 1
+    tracked_request = tracked_requests[0]
+    assert tracked_request.tags["path"] == "/crash/"
+    assert tracked_request.tags["error"] == "true"
+    assert len(tracked_request.complete_spans) == 1
+    span = tracked_request.complete_spans[0]
+    assert span.operation == "Controller/tests.integration.test_cherrypy.Views.crash"
+
+
+def test_return_error(tracked_requests):
+    with app_with_scout() as app:
+        response = TestApp(app).get("/return-error/", expect_errors=True)
+
+    assert response.status_int == 503
+    assert len(tracked_requests) == 1
+    tracked_request = tracked_requests[0]
+    assert tracked_request.tags["path"] == "/return-error/"
+    assert tracked_request.tags["error"] == "true"
+    assert len(tracked_request.complete_spans) == 1
+    span = tracked_request.complete_spans[0]
+    assert (
+        span.operation
+        == "Controller/tests.integration.test_cherrypy.Views.return_error"
+    )
+
+
+def test_not_found(tracked_requests):
+    with app_with_scout() as app:
+        response = TestApp(app).get("/not-found/", expect_errors=True)
+
+    assert response.status_int == 404
+    assert tracked_requests == []
+
+
+def test_no_monitor(tracked_requests):
+    with app_with_scout(scout_config={"monitor": False}) as app:
+        response = TestApp(app).get("/")
+
+    assert response.status_int == 200
+    assert tracked_requests == []