Repository: elastic/apm-agent-python

Issue:
Add instrumentation for pymemcache (memcache)
Recently released v3.0 https://github.com/pinterest/pymemcache

Similar to python-memcached in that its pure python, but with more features and, apparently, better perf.



Discussion:
Thanks for the request!

Resolving patch:
diff --git a/elasticapm/instrumentation/packages/pymemcache.py b/elasticapm/instrumentation/packages/pymemcache.py
new file mode 100644
--- /dev/null
+++ b/elasticapm/instrumentation/packages/pymemcache.py
@@ -0,0 +1,110 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2019, Elasticsearch BV
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+#  * Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+#  * Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from
+#    this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+#  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+#  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+#  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+#  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+#  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from elasticapm.instrumentation.packages.base import AbstractInstrumentedModule
+from elasticapm.traces import capture_span
+
+
+class PyMemcacheInstrumentation(AbstractInstrumentedModule):
+    name = "pymemcache"
+
+    method_list = [
+        "add",
+        "append",
+        "cas",
+        "decr",
+        "delete",
+        "delete_many",
+        "delete_multi",
+        "flush_all",
+        "get",
+        "get_many",
+        "get_multi",
+        "gets",
+        "gets_many",
+        "incr",
+        "prepend",
+        "quit",
+        "replace",
+        "set",
+        "set_many",
+        "set_multi",
+        "stats",
+        "touch",
+    ]
+
+    def get_instrument_list(self):
+        return (
+            [("pymemcache.client.base", "Client." + method) for method in self.method_list]
+            + [("pymemcache.client.base", "PooledClient." + method) for method in self.method_list]
+            + [("pymemcache.client.hash", "HashClient." + method) for method in self.method_list]
+        )
+
+    def call(self, module, method, wrapped, instance, args, kwargs):
+        name = self.get_wrapped_name(wrapped, instance, method)
+
+        # Since HashClient uses Client/PooledClient for the actual calls, we
+        # don't need to get address/port info for that class
+        address, port = None, None
+        if getattr(instance, "server", None):
+            if isinstance(instance.server, (list, tuple)):
+                # Address/port are a tuple
+                address, port = instance.server
+            else:
+                # Server is a UNIX domain socket
+                address = instance.server
+        destination = {
+            "address": address,
+            "port": port,
+            "service": {"name": "memcached", "resource": "memcached", "type": "cache"},
+        }
+
+        if "PooledClient" in name:
+            # PooledClient calls out to Client for the "work", but only once,
+            # so we don't care about the "duplicate" spans from Client in that
+            # case
+            with capture_span(
+                name,
+                span_type="cache",
+                span_subtype="memcached",
+                span_action="query",
+                extra={"destination": destination},
+                leaf=True,
+            ):
+                return wrapped(*args, **kwargs)
+        else:
+            with capture_span(
+                name,
+                span_type="cache",
+                span_subtype="memcached",
+                span_action="query",
+                extra={"destination": destination},
+            ):
+                return wrapped(*args, **kwargs)
diff --git a/elasticapm/instrumentation/register.py b/elasticapm/instrumentation/register.py
--- a/elasticapm/instrumentation/register.py
+++ b/elasticapm/instrumentation/register.py
@@ -45,6 +45,7 @@
     "elasticapm.instrumentation.packages.pymongo.PyMongoBulkInstrumentation",
     "elasticapm.instrumentation.packages.pymongo.PyMongoCursorInstrumentation",
     "elasticapm.instrumentation.packages.python_memcached.PythonMemcachedInstrumentation",
+    "elasticapm.instrumentation.packages.pymemcache.PyMemcacheInstrumentation",
     "elasticapm.instrumentation.packages.redis.RedisInstrumentation",
     "elasticapm.instrumentation.packages.redis.RedisPipelineInstrumentation",
     "elasticapm.instrumentation.packages.redis.RedisConnectionInstrumentation",


Test patch:
diff --git a/tests/instrumentation/pymemcache_tests.py b/tests/instrumentation/pymemcache_tests.py
new file mode 100644
--- /dev/null
+++ b/tests/instrumentation/pymemcache_tests.py
@@ -0,0 +1,147 @@
+#  BSD 3-Clause License
+#
+#  Copyright (c) 2019, Elasticsearch BV
+#  All rights reserved.
+#
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions are met:
+#
+#  * Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+#  * Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+#  * Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from
+#    this software without specific prior written permission.
+#
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+#  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+#  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+#  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+#  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+#  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+#  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+
+import pytest
+
+from elasticapm.conf.constants import TRANSACTION
+from elasticapm.traces import capture_span
+
+pymemcache = pytest.importorskip("pymemcache")  # isort:skip
+
+
+pytestmark = [pytest.mark.pymemcache]
+
+
+if "MEMCACHED_HOST" not in os.environ:
+    pytestmark.append(pytest.mark.skip("Skipping pymemcache tests, no MEMCACHED_HOST environment variable set"))
+
+
+@pytest.mark.integrationtest
+def test_pymemcache_client(instrument, elasticapm_client):
+    elasticapm_client.begin_transaction("transaction.test")
+    host = os.environ.get("MEMCACHED_HOST", "localhost")
+    with capture_span("test_pymemcache", "test"):
+        conn = pymemcache.client.base.Client((host, 11211))
+        conn.set("mykey", "a")
+        assert b"a" == conn.get("mykey")
+        assert {"mykey": b"a"} == conn.get_many(["mykey", "myotherkey"])
+    elasticapm_client.end_transaction("BillingView")
+
+    transactions = elasticapm_client.events[TRANSACTION]
+    spans = elasticapm_client.spans_for_transaction(transactions[0])
+
+    expected_signatures = {"test_pymemcache", "Client.set", "Client.get", "Client.get_many"}
+
+    assert {t["name"] for t in spans} == expected_signatures
+
+    assert spans[0]["name"] == "Client.set"
+    assert spans[0]["type"] == "cache"
+    assert spans[0]["subtype"] == "memcached"
+    assert spans[0]["action"] == "query"
+    assert spans[0]["parent_id"] == spans[3]["id"]
+
+    assert spans[1]["name"] == "Client.get"
+    assert spans[1]["type"] == "cache"
+    assert spans[1]["subtype"] == "memcached"
+    assert spans[1]["action"] == "query"
+    assert spans[1]["parent_id"] == spans[3]["id"]
+    assert spans[1]["context"]["destination"] == {
+        "address": host,
+        "port": 11211,
+        "service": {"name": "memcached", "resource": "memcached", "type": "cache"},
+    }
+
+    assert spans[2]["name"] == "Client.get_many"
+    assert spans[2]["type"] == "cache"
+    assert spans[2]["subtype"] == "memcached"
+    assert spans[2]["action"] == "query"
+    assert spans[2]["parent_id"] == spans[3]["id"]
+
+    assert spans[3]["name"] == "test_pymemcache"
+    assert spans[3]["type"] == "test"
+
+    assert len(spans) == 4
+
+
+@pytest.mark.integrationtest
+def test_pymemcache_pooled_client(instrument, elasticapm_client):
+    elasticapm_client.begin_transaction("transaction.test")
+    host = os.environ.get("MEMCACHED_HOST", "localhost")
+    with capture_span("test_pymemcache", "test"):
+        conn = pymemcache.client.base.PooledClient((host, 11211))
+        conn.set("mykey", "a")
+        assert b"a" == conn.get("mykey")
+        assert {"mykey": b"a"} == conn.get_many(["mykey", "myotherkey"])
+    elasticapm_client.end_transaction("BillingView")
+
+    transactions = elasticapm_client.events[TRANSACTION]
+    spans = elasticapm_client.spans_for_transaction(transactions[0])
+
+    expected_signatures = {
+        "test_pymemcache",
+        "PooledClient.set",
+        "PooledClient.get",
+        "PooledClient.get_many",
+    }
+
+    assert {t["name"] for t in spans} == expected_signatures
+
+    assert len(spans) == 4
+
+
+@pytest.mark.integrationtest
+def test_pymemcache_hash_client(instrument, elasticapm_client):
+    elasticapm_client.begin_transaction("transaction.test")
+    host = os.environ.get("MEMCACHED_HOST", "localhost")
+    with capture_span("test_pymemcache", "test"):
+        conn = pymemcache.client.hash.HashClient([(host, 11211)])
+        conn.set("mykey", "a")
+        assert b"a" == conn.get("mykey")
+        assert {"mykey": b"a"} == conn.get_many(["mykey", "myotherkey"])
+    elasticapm_client.end_transaction("BillingView")
+
+    transactions = elasticapm_client.events[TRANSACTION]
+    spans = elasticapm_client.spans_for_transaction(transactions[0])
+
+    expected_signatures = {
+        "test_pymemcache",
+        "HashClient.set",
+        "HashClient.get",
+        "HashClient.get_many",
+        "Client.set",
+        "Client.get",
+        "Client.get_many",
+    }
+
+    assert {t["name"] for t in spans} == expected_signatures
+
+    assert len(spans) == 7
diff --git a/tests/instrumentation/python_memcached_tests.py b/tests/instrumentation/python_memcached_tests.py
--- a/tests/instrumentation/python_memcached_tests.py
+++ b/tests/instrumentation/python_memcached_tests.py
@@ -42,7 +42,7 @@
 
 
 if "MEMCACHED_HOST" not in os.environ:
-    pytestmark.append(pytest.mark.skip("Skipping memchached tests, no MEMCACHED_HOST environment variable set"))
+    pytestmark.append(pytest.mark.skip("Skipping memcached tests, no MEMCACHED_HOST environment variable set"))
 
 
 @pytest.mark.integrationtest
diff --git a/tests/requirements/reqs-pymemcache-3.0.txt b/tests/requirements/reqs-pymemcache-3.0.txt
new file mode 100644
--- /dev/null
+++ b/tests/requirements/reqs-pymemcache-3.0.txt
@@ -0,0 +1,2 @@
+pymemcache==3.0.0
+-r reqs-base.txt
diff --git a/tests/requirements/reqs-pymemcache-newest.txt b/tests/requirements/reqs-pymemcache-newest.txt
new file mode 100644
--- /dev/null
+++ b/tests/requirements/reqs-pymemcache-newest.txt
@@ -0,0 +1,2 @@
+pymemcache>=3.1.0
+-r reqs-base.txt
diff --git a/tests/scripts/envs/pymemcache.sh b/tests/scripts/envs/pymemcache.sh
new file mode 100644
--- /dev/null
+++ b/tests/scripts/envs/pymemcache.sh
@@ -0,0 +1,3 @@
+export PYTEST_MARKER="-m pymemcache"
+export DOCKER_DEPS="memcached"
+export MEMCACHED_HOST="memcached"