Repository: elastic/apm-agent-python

Issue:
Support for google API client (via httplib2)
**Describe the solution you'd like**: I would like http requests done by `httplib2` captured the same as from the other http libraries.

**Additional context**: `httplib2` library is used in the [google API client](https://github.com/googleapis/google-api-python-client) which would be extremely useful to see as spans.

I would be happy to work on a PR if it would be accepted.

Thank you!


Discussion:
Hey @HonzaKral! Yes, that seems like a sizeable gap in our instrumentation story! A PR would be great! The [`urllib3`](https://github.com/elastic/apm-agent-python/blob/15278433c97d9c81c5ad1dcdf57d27da847be912/elasticapm/instrumentation/packages/urllib3.py) instrumentation and [tests](https://github.com/elastic/apm-agent-python/blob/15278433c97d9c81c5ad1dcdf57d27da847be912/tests/instrumentation/urllib3_tests.py) could serve as a starting point. There's also quite a bit of [boilerplate](https://github.com/elastic/apm-agent-python/blob/master/CONTRIBUTING.md#adding-new-instrumentations-to-the-matrix-build)  required to add a new instrumentation to the test matrix :see_no_evil: (feel free to leave that part to us :D ).

Resolving patch:
diff --git a/elasticapm/instrumentation/packages/httplib2.py b/elasticapm/instrumentation/packages/httplib2.py
new file mode 100644
--- /dev/null
+++ b/elasticapm/instrumentation/packages/httplib2.py
@@ -0,0 +1,118 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2019, Elasticsearch BV
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+#  * Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+#  * Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from
+#    this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+#  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+#  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+#  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+#  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+#  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from elasticapm.conf import constants
+from elasticapm.instrumentation.packages.base import AbstractInstrumentedModule
+from elasticapm.traces import DroppedSpan, capture_span, execution_context
+from elasticapm.utils import get_host_from_url, sanitize_url, url_to_destination
+from elasticapm.utils.disttracing import TracingOptions
+
+
+class Httplib2Instrumentation(AbstractInstrumentedModule):
+    name = "httplib2"
+
+    instrument_list = [("httplib2", "Http.request")]
+
+    # https://httplib2.readthedocs.io/en/latest/libhttplib2.html#httplib2.Http.request
+    args_list = ("url", "method", "body", "headers")
+
+    def _ensure_headers_in_kwargs(self, args, kwargs):
+        # strip out trailing None positional arguments
+        if len(args) > 2:
+            for _ in range(2, len(self.args_list)):
+                if args[-1] is not None:
+                    break
+                args = args[:-1]
+
+        # grab args passed in as positional
+        params = dict(zip(self.args_list, args))
+
+        # fall back to kwargs for args not supplied as positional
+        for arg in self.args_list:
+            if arg in params:
+                continue
+            if arg in kwargs:
+                params[arg] = kwargs[arg]
+
+        if "headers" not in params:
+            kwargs["headers"] = params["headers"] = {}
+
+        return args, kwargs, params
+
+    def call(self, module, method, wrapped, instance, args, kwargs):
+        args, kwargs, params = self._ensure_headers_in_kwargs(args, kwargs)
+
+        signature = params.get("method", "GET").upper()
+        signature += " " + get_host_from_url(params["url"])
+        url = sanitize_url(params["url"])
+        destination = url_to_destination(url)
+        transaction = execution_context.get_transaction()
+        with capture_span(
+            signature,
+            span_type="external",
+            span_subtype="http",
+            extra={"http": {"url": url}, "destination": destination},
+            leaf=True,
+        ) as span:
+            # if httplib2 has been called in a leaf span, this span might be a DroppedSpan.
+            leaf_span = span
+            while isinstance(leaf_span, DroppedSpan):
+                leaf_span = leaf_span.parent
+
+            # It's possible that there are only dropped spans, e.g. if we started dropping spans.
+            # In this case, the transaction.id is used
+            parent_id = leaf_span.id if leaf_span else transaction.id
+            trace_parent = transaction.trace_parent.copy_from(
+                span_id=parent_id, trace_options=TracingOptions(recorded=True)
+            )
+            self._set_disttracing_headers(params["headers"], trace_parent, transaction)
+
+            response, content = wrapped(*args, **kwargs)
+            if span.context:
+                span.context["http"]["status_code"] = response.status
+            span.set_success() if response.status < 400 else span.set_failure()
+            return response, content
+
+    def mutate_unsampled_call_args(self, module, method, wrapped, instance, args, kwargs, transaction):
+        # since we don't have a span, we set the span id to the transaction id
+        trace_parent = transaction.trace_parent.copy_from(
+            span_id=transaction.id, trace_options=TracingOptions(recorded=False)
+        )
+        args, kwargs, params = self._ensure_headers_in_kwargs(args, kwargs)
+        self._set_disttracing_headers(params["headers"], trace_parent, transaction)
+        return args, kwargs
+
+    def _set_disttracing_headers(self, headers, trace_parent, transaction):
+        trace_parent_str = trace_parent.to_string()
+        headers[constants.TRACEPARENT_HEADER_NAME] = trace_parent_str
+        if transaction.tracer.config.use_elastic_traceparent_header:
+            headers[constants.TRACEPARENT_LEGACY_HEADER_NAME] = trace_parent_str
+        if trace_parent.tracestate:
+            headers[constants.TRACESTATE_HEADER_NAME] = trace_parent.tracestate
diff --git a/elasticapm/instrumentation/register.py b/elasticapm/instrumentation/register.py
--- a/elasticapm/instrumentation/register.py
+++ b/elasticapm/instrumentation/register.py
@@ -63,6 +63,7 @@
     "elasticapm.instrumentation.packages.graphql.GraphQLExecutorInstrumentation",
     "elasticapm.instrumentation.packages.graphql.GraphQLBackendInstrumentation",
     "elasticapm.instrumentation.packages.httpcore.HTTPCoreInstrumentation",
+    "elasticapm.instrumentation.packages.httplib2.Httplib2Instrumentation",
 }
 
 if sys.version_info >= (3, 7):


Test patch:
diff --git a/tests/instrumentation/httplib2_tests.py b/tests/instrumentation/httplib2_tests.py
new file mode 100644
--- /dev/null
+++ b/tests/instrumentation/httplib2_tests.py
@@ -0,0 +1,120 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2019, Elasticsearch BV
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+#  * Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+#  * Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from
+#    this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+#  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+#  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+#  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+#  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+#  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import pytest  # isort:skip
+
+pytest.importorskip("httplib2")  # isort:skip
+
+import httplib2
+
+from elasticapm.conf import constants
+from elasticapm.conf.constants import TRANSACTION
+from elasticapm.traces import capture_span
+from elasticapm.utils import compat
+from elasticapm.utils.disttracing import TraceParent
+
+pytestmark = pytest.mark.httplib2
+
+
+@pytest.mark.parametrize("args", [(), ("GET",), ("GET", None, {}), ("GET", None, None)])
+def test_httplib2_instrumentation(instrument, elasticapm_client, waiting_httpserver, args):
+    waiting_httpserver.serve_content("")
+    url = waiting_httpserver.url + "/hello_world"
+    parsed_url = compat.urlparse.urlparse(url)
+    elasticapm_client.begin_transaction("transaction.test")
+    with capture_span("test_request", "test"):
+        httplib2.Http().request(url, *args)
+    elasticapm_client.end_transaction("MyView")
+
+    transactions = elasticapm_client.events[TRANSACTION]
+    spans = elasticapm_client.spans_for_transaction(transactions[0])
+    assert spans[0]["name"].startswith("GET 127.0.0.1:")
+    assert spans[0]["type"] == "external"
+    assert spans[0]["subtype"] == "http"
+    assert url == spans[0]["context"]["http"]["url"]
+    assert 200 == spans[0]["context"]["http"]["status_code"]
+    assert spans[0]["context"]["destination"]["service"] == {
+        "name": "http://127.0.0.1:%d" % parsed_url.port,
+        "resource": "127.0.0.1:%d" % parsed_url.port,
+        "type": "external",
+    }
+    assert spans[0]["outcome"] == "success"
+
+    assert constants.TRACEPARENT_HEADER_NAME in waiting_httpserver.requests[0].headers
+    trace_parent = TraceParent.from_string(
+        waiting_httpserver.requests[0].headers[constants.TRACEPARENT_HEADER_NAME],
+        tracestate_string=waiting_httpserver.requests[0].headers[constants.TRACESTATE_HEADER_NAME],
+    )
+    assert trace_parent.trace_id == transactions[0]["trace_id"]
+    # Check that sample_rate was correctly placed in the tracestate
+    assert constants.TRACESTATE.SAMPLE_RATE in trace_parent.tracestate_dict
+
+
+@pytest.mark.parametrize("status_code", [400, 500])
+def test_httplib2_instrumentation_error(instrument, elasticapm_client, waiting_httpserver, status_code):
+    waiting_httpserver.serve_content("", code=status_code)
+    url = waiting_httpserver.url + "/hello_world"
+    parsed_url = compat.urlparse.urlparse(url)
+    elasticapm_client.begin_transaction("transaction.test")
+    with capture_span("test_request", "test"):
+        httplib2.Http().request(url, "GET")
+    elasticapm_client.end_transaction("MyView")
+
+    transactions = elasticapm_client.events[TRANSACTION]
+    spans = elasticapm_client.spans_for_transaction(transactions[0])
+    assert spans[0]["name"].startswith("GET 127.0.0.1:")
+    assert spans[0]["type"] == "external"
+    assert spans[0]["subtype"] == "http"
+    assert url == spans[0]["context"]["http"]["url"]
+    assert status_code == spans[0]["context"]["http"]["status_code"]
+    assert spans[0]["context"]["destination"]["service"] == {
+        "name": "http://127.0.0.1:%d" % parsed_url.port,
+        "resource": "127.0.0.1:%d" % parsed_url.port,
+        "type": "external",
+    }
+    assert spans[0]["outcome"] == "failure"
+
+
+@pytest.mark.parametrize(
+    "is_sampled", [pytest.param(True, id="is_sampled-True"), pytest.param(False, id="is_sampled-False")]
+)
+def test_tracestate_propagation(instrument, elasticapm_client, waiting_httpserver, is_sampled):
+    traceparent = TraceParent.from_string(
+        "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-03", "foo=bar,baz=bazzinga"
+    )
+
+    waiting_httpserver.serve_content("")
+    url = waiting_httpserver.url + "/hello_world"
+    transaction_object = elasticapm_client.begin_transaction("transaction", trace_parent=traceparent)
+    transaction_object.is_sampled = is_sampled
+    httplib2.Http().request(url, "GET")
+    elasticapm_client.end_transaction("MyView")
+    headers = waiting_httpserver.requests[0].headers
+    assert headers[constants.TRACESTATE_HEADER_NAME] == "foo=bar,baz=bazzinga"
diff --git a/tests/requirements/reqs-httplib2-newest.txt b/tests/requirements/reqs-httplib2-newest.txt
new file mode 100644
--- /dev/null
+++ b/tests/requirements/reqs-httplib2-newest.txt
@@ -0,0 +1,2 @@
+httplib2>=0.18
+-r reqs-base.txt
diff --git a/tests/scripts/envs/httplib2.sh b/tests/scripts/envs/httplib2.sh
new file mode 100644
--- /dev/null
+++ b/tests/scripts/envs/httplib2.sh
@@ -0,0 +1 @@
+export PYTEST_MARKER="-m httplib2"