Repository: elastic/apm-agent-python

Issue:
Update elasticsearch-py instrumentation for 8.0
While figuring out how to test my fix for https://github.com/elastic/apm-agent-python/issues/1615, I realized that with 8.0, `elasticsearch-py` has moved its transport and connection code to a new project, `elastic-transport`.

I also realized we are not testing against elasticsearch-py 8.x. 😅 

Everything may still work with the [stub file provided](https://github.com/elastic/elasticsearch-py/blob/main/elasticsearch/transport.py). But since importing that file is deprecated, we should add an instrumentation that targets `elastic-transport` directly.

/cc @sethmlarson @miriam-eid 


Discussion:

Resolving patch:
diff --git a/elasticapm/instrumentation/packages/asyncio/elasticsearch.py b/elasticapm/instrumentation/packages/asyncio/elasticsearch.py
--- a/elasticapm/instrumentation/packages/asyncio/elasticsearch.py
+++ b/elasticapm/instrumentation/packages/asyncio/elasticsearch.py
@@ -40,10 +40,18 @@
 class ElasticSearchAsyncConnection(ElasticsearchConnectionInstrumentation, AsyncAbstractInstrumentedModule):
     name = "elasticsearch_connection"
 
-    instrument_list = [
-        ("elasticsearch_async.connection", "AIOHttpConnection.perform_request"),
-        ("elasticsearch._async.http_aiohttp", "AIOHttpConnection.perform_request"),
-    ]
+    def get_instrument_list(self):
+        try:
+            import elastic_transport  # noqa: F401
+
+            return [
+                ("elastic_transport._node._http_aiohttp", "AiohttpHttpNode.perform_request"),
+            ]
+        except ImportError:
+            return [
+                ("elasticsearch_async.connection", "AIOHttpConnection.perform_request"),
+                ("elasticsearch._async.http_aiohttp", "AIOHttpConnection.perform_request"),
+            ]
 
     async def call(self, module, method, wrapped, instance, args, kwargs):
         span = execution_context.get_span()
@@ -52,11 +60,15 @@ async def call(self, module, method, wrapped, instance, args, kwargs):
 
         self._update_context_by_request_data(span.context, instance, args, kwargs)
 
-        status_code, headers, raw_data = await wrapped(*args, **kwargs)
+        result = await wrapped(*args, **kwargs)
+        if hasattr(result, "meta"):  # elasticsearch-py 8.x+
+            status_code = result.meta.status
+        else:
+            status_code = result[0]
 
         span.context["http"] = {"status_code": status_code}
 
-        return status_code, headers, raw_data
+        return result
 
 
 class ElasticsearchAsyncTransportInstrumentation(
@@ -68,6 +80,18 @@ class ElasticsearchAsyncTransportInstrumentation(
         ("elasticsearch._async.transport", "AsyncTransport.perform_request"),
     ]
 
+    def get_instrument_list(self):
+        try:
+            import elastic_transport  # noqa: F401
+
+            return [
+                ("elastic_transport", "AsyncTransport.perform_request"),
+            ]
+        except ImportError:
+            return [
+                ("elasticsearch._async.transport", "AsyncTransport.perform_request"),
+            ]
+
     async def call(self, module, method, wrapped, instance, args, kwargs):
         async with elasticapm.async_capture_span(
             self._get_signature(args, kwargs),
@@ -80,7 +104,8 @@ async def call(self, module, method, wrapped, instance, args, kwargs):
         ) as span:
             result_data = await wrapped(*args, **kwargs)
 
-            if isinstance(result_data, dict) and "hits" in result_data:
-                span.context["db"]["rows_affected"] = result_data["hits"]["total"]["value"]
+            hits = self._get_hits(result_data)
+            if hits:
+                span.context["db"]["rows_affected"] = hits
 
             return result_data
diff --git a/elasticapm/instrumentation/packages/elasticsearch.py b/elasticapm/instrumentation/packages/elasticsearch.py
--- a/elasticapm/instrumentation/packages/elasticsearch.py
+++ b/elasticapm/instrumentation/packages/elasticsearch.py
@@ -31,6 +31,8 @@
 from __future__ import absolute_import
 
 import re
+from typing import Optional
+from urllib.parse import parse_qs, urlparse
 
 import elasticapm
 from elasticapm.instrumentation.packages.base import AbstractInstrumentedModule
@@ -45,10 +47,19 @@
 class ElasticsearchConnectionInstrumentation(AbstractInstrumentedModule):
     name = "elasticsearch_connection"
 
-    instrument_list = [
-        ("elasticsearch.connection.http_urllib3", "Urllib3HttpConnection.perform_request"),
-        ("elasticsearch.connection.http_requests", "RequestsHttpConnection.perform_request"),
-    ]
+    def get_instrument_list(self):
+        try:
+            import elastic_transport  # noqa: F401
+
+            return [
+                ("elastic_transport._node._http_urllib3", "Urllib3HttpNode.perform_request"),
+                ("elastic_transport._node._http_requests", "RequestsHttpNode.perform_request"),
+            ]
+        except ImportError:
+            return [
+                ("elasticsearch.connection.http_urllib3", "Urllib3HttpConnection.perform_request"),
+                ("elasticsearch.connection.http_requests", "RequestsHttpConnection.perform_request"),
+            ]
 
     def call(self, module, method, wrapped, instance, args, kwargs):
         span = execution_context.get_span()
@@ -57,11 +68,14 @@ def call(self, module, method, wrapped, instance, args, kwargs):
 
         self._update_context_by_request_data(span.context, instance, args, kwargs)
 
-        status_code, headers, raw_data = wrapped(*args, **kwargs)
-
+        result = wrapped(*args, **kwargs)
+        if hasattr(result, "meta"):  # elasticsearch-py 8.x+
+            status_code = result.meta.status
+        else:
+            status_code = result[0]
         span.context["http"] = {"status_code": status_code}
 
-        return status_code, headers, raw_data
+        return result
 
     def _update_context_by_request_data(self, context, instance, args, kwargs):
         args_len = len(args)
@@ -69,6 +83,10 @@ def _update_context_by_request_data(self, context, instance, args, kwargs):
         params = args[2] if args_len > 2 else kwargs.get("params")
         body_serialized = args[3] if args_len > 3 else kwargs.get("body")
 
+        if "?" in url and not params:
+            url, qs = url.split("?", 1)
+            params = {k: v[0] for k, v in parse_qs(qs).items()}
+
         should_capture_body = bool(should_capture_body_re.search(url))
 
         context["db"] = {"type": "elasticsearch"}
@@ -78,9 +96,12 @@ def _update_context_by_request_data(self, context, instance, args, kwargs):
             # but not in others. We simply capture both if they are there so the
             # user can see it.
             if params and "q" in params:
-                # 'q' is already encoded to a byte string at this point
-                # we assume utf8, which is the default
-                query.append("q=" + params["q"].decode("utf-8", errors="replace"))
+                # 'q' may already be encoded to a byte string at this point.
+                # We assume utf8, which is the default
+                q = params["q"]
+                if isinstance(q, bytes):
+                    q = q.decode("utf-8", errors="replace")
+                query.append("q=" + q)
             if body_serialized:
                 if isinstance(body_serialized, bytes):
                     query.append(body_serialized.decode("utf-8", errors="replace"))
@@ -89,17 +110,40 @@ def _update_context_by_request_data(self, context, instance, args, kwargs):
             if query:
                 context["db"]["statement"] = "\n\n".join(query)
 
-        context["destination"] = {
-            "address": instance.host,
-        }
+        # ES5: `host` is URL, no `port` attribute
+        # ES6, ES7: `host` URL, `hostname` is host, `port` is port
+        # ES8: `host` is hostname, no `hostname` attribute, `port` is `port`
+        if not hasattr(instance, "port"):
+            # ES5, parse hostname and port from URL stored in `host`
+            parsed_url = urlparse(instance.host)
+            host = parsed_url.hostname
+            port = parsed_url.port
+        elif not hasattr(instance, "hostname"):
+            # ES8 (and up, one can hope)
+            host = instance.host
+            port = instance.port
+        else:
+            # ES6, ES7
+            host = instance.hostname
+            port = instance.port
+
+        context["destination"] = {"address": host, "port": port}
 
 
 class ElasticsearchTransportInstrumentation(AbstractInstrumentedModule):
     name = "elasticsearch_connection"
 
-    instrument_list = [
-        ("elasticsearch.transport", "Transport.perform_request"),
-    ]
+    def get_instrument_list(self):
+        try:
+            import elastic_transport  # noqa: F401
+
+            return [
+                ("elastic_transport", "Transport.perform_request"),
+            ]
+        except ImportError:
+            return [
+                ("elasticsearch.transport", "Transport.perform_request"),
+            ]
 
     def call(self, module, method, wrapped, instance, args, kwargs):
         with elasticapm.capture_span(
@@ -113,10 +157,9 @@ def call(self, module, method, wrapped, instance, args, kwargs):
         ) as span:
             result_data = wrapped(*args, **kwargs)
 
-            try:
-                span.context["db"]["rows_affected"] = result_data["hits"]["total"]["value"]
-            except (KeyError, TypeError):
-                pass
+            hits = self._get_hits(result_data)
+            if hits:
+                span.context["db"]["rows_affected"] = hits
 
             return result_data
 
@@ -124,5 +167,16 @@ def _get_signature(self, args, kwargs):
         args_len = len(args)
         http_method = args[0] if args_len else kwargs.get("method")
         http_path = args[1] if args_len > 1 else kwargs.get("url")
+        http_path = http_path.split("?", 1)[0]  # we don't want to capture a potential query string in the span name
 
         return "ES %s %s" % (http_method, http_path)
+
+    def _get_hits(self, result) -> Optional[int]:
+        if getattr(result, "body", None) and "hits" in result.body:  # ES >= 8
+            return result.body["hits"]["total"]["value"]
+        elif isinstance(result, dict) and "hits" in result:
+            return (
+                result["hits"]["total"]["value"]
+                if isinstance(result["hits"]["total"], dict)
+                else result["hits"]["total"]
+            )


Test patch:
diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
--- a/tests/docker-compose.yml
+++ b/tests/docker-compose.yml
@@ -66,7 +66,7 @@ services:
       - pyesdata6:/usr/share/elasticsearch/data
 
   elasticsearch7:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.9.0
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9200"]
     environment:
@@ -78,6 +78,22 @@ services:
     volumes:
       - pyesdata7:/usr/share/elasticsearch/data
 
+  elasticsearch8:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9200"]
+    ports:
+      - "9200:9200"
+    environment:
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "network.host=_site_"
+      - "transport.host=127.0.0.1"
+      - "http.host=0.0.0.0"
+      - "xpack.security.enabled=false"
+      - "action.destructive_requires_name=false" # allow for easy cleanup by calling DELETE *
+    volumes:
+      - pyesdata8:/usr/share/elasticsearch/data
+
   elasticsearch5:
     image: docker.elastic.co/elasticsearch/elasticsearch:5.6.16
     healthcheck:
@@ -164,6 +180,7 @@ services:
   run_tests:
     image: apm-agent-python:${PYTHON_VERSION}
     environment:
+      ES_8_URL: 'http://elasticsearch8:9200'
       ES_7_URL: 'http://elasticsearch7:9200'
       ES_6_URL: 'http://elasticsearch6:9200'
       ES_5_URL: 'http://elasticsearch5:9200'
@@ -183,6 +200,8 @@ volumes:
     driver: local
   pyesdata7:
     driver: local
+  pyesdata8:
+    driver: local
   pyesdata6:
     driver: local
   pyesdata5:
diff --git a/tests/instrumentation/asyncio_tests/async_elasticsearch_client_tests.py b/tests/instrumentation/asyncio_tests/async_elasticsearch_client_tests.py
--- a/tests/instrumentation/asyncio_tests/async_elasticsearch_client_tests.py
+++ b/tests/instrumentation/asyncio_tests/async_elasticsearch_client_tests.py
@@ -30,10 +30,13 @@
 
 import pytest  # isort:skip
 
+from tests.instrumentation.elasticsearch_tests import get_kwargs
+
 pytest.importorskip("elasticsearch._async")  # isort:skip
 
 import json
 import os
+import urllib.parse
 
 from elasticsearch import VERSION as ES_VERSION
 from elasticsearch import AsyncElasticsearch
@@ -64,6 +67,7 @@ async def test_ping(instrument, elasticapm_client, async_elasticsearch):
     elasticapm_client.begin_transaction("test")
     result = await async_elasticsearch.ping()
     elasticapm_client.end_transaction("test", "OK")
+    parsed_url = urllib.parse.urlparse(os.environ["ES_URL"])
 
     transaction = elasticapm_client.events[TRANSACTION][0]
     spans = elasticapm_client.spans_for_transaction(transaction)
@@ -74,6 +78,12 @@ async def test_ping(instrument, elasticapm_client, async_elasticsearch):
     assert span["subtype"] == "elasticsearch"
     assert span["action"] == "query"
     assert span["sync"] is False
+    assert span["context"]["destination"] == {
+        "address": parsed_url.hostname,
+        "port": parsed_url.port,
+        "service": {"name": "", "resource": "elasticsearch", "type": ""},
+    }
+    assert span["context"]["http"]["status_code"] == 200
 
 
 async def test_info(instrument, elasticapm_client, async_elasticsearch):
@@ -96,21 +106,29 @@ async def test_info(instrument, elasticapm_client, async_elasticsearch):
 
 async def test_create(instrument, elasticapm_client, async_elasticsearch):
     elasticapm_client.begin_transaction("test")
+    responses = []
+    iid = lambda: str(len(responses) + 1)
     if ES_VERSION[0] < 5:
-        r1 = await async_elasticsearch.create("tweets", document_type, {"user": "kimchy", "text": "hola"}, 1)
+        responses.append(
+            await async_elasticsearch.create("tweets", document_type, {"user": "kimchy", "text": "hola"}, iid())
+        )
     elif ES_VERSION[0] < 7:
-        r1 = await async_elasticsearch.create("tweets", document_type, 1, body={"user": "kimchy", "text": "hola"})
+        responses.append(
+            await async_elasticsearch.create("tweets", document_type, iid(), body={"user": "kimchy", "text": "hola"})
+        )
+    elif ES_VERSION[0] < 8:
+        responses.append(await async_elasticsearch.create("tweets", iid(), body={"user": "kimchy", "text": "hola"}))
     else:
-        r1 = await async_elasticsearch.create("tweets", 1, body={"user": "kimchy", "text": "hola"})
-    r2 = await async_elasticsearch.create(
-        index="tweets", doc_type=document_type, id=2, body={"user": "kimchy", "text": "hola"}, refresh=True
+        pass  # elasticsearch-py 8+ doesn't support positional arguments
+    responses.append(
+        await async_elasticsearch.create(index="tweets", id=iid(), **get_kwargs({"user": "kimchy", "text": "hola"}))
     )
     elasticapm_client.end_transaction("test", "OK")
 
     transaction = elasticapm_client.events[TRANSACTION][0]
 
     spans = elasticapm_client.spans_for_transaction(transaction)
-    assert len(spans) == 2
+    assert len(spans) == len(responses)
 
     for i, span in enumerate(spans):
         if ES_VERSION[0] >= 5:
@@ -130,7 +148,7 @@ async def test_create(instrument, elasticapm_client, async_elasticsearch):
 
 async def test_search_body(instrument, elasticapm_client, async_elasticsearch):
     await async_elasticsearch.create(
-        index="tweets", doc_type=document_type, id=1, body={"user": "kimchy", "text": "hola", "userid": 1}, refresh=True
+        index="tweets", id="1", refresh=True, **get_kwargs({"user": "kimchy", "text": "hola", "userid": 1})
     )
     elasticapm_client.begin_transaction("test")
     search_query = {"query": {"term": {"user": "kimchy"}}, "sort": ["userid"]}
@@ -161,7 +179,7 @@ async def test_search_body(instrument, elasticapm_client, async_elasticsearch):
 
 async def test_count_body(instrument, elasticapm_client, async_elasticsearch):
     await async_elasticsearch.create(
-        index="tweets", doc_type=document_type, id=1, body={"user": "kimchy", "text": "hola"}, refresh=True
+        index="tweets", id="1", refresh=True, **get_kwargs({"user": "kimchy", "text": "hola"})
     )
     elasticapm_client.begin_transaction("test")
     search_query = {"query": {"term": {"user": "kimchy"}}}
diff --git a/tests/instrumentation/elasticsearch_tests.py b/tests/instrumentation/elasticsearch_tests.py
--- a/tests/instrumentation/elasticsearch_tests.py
+++ b/tests/instrumentation/elasticsearch_tests.py
@@ -34,6 +34,7 @@
 
 import json
 import os
+import urllib.parse
 
 from elasticsearch import VERSION as ES_VERSION
 from elasticsearch import Elasticsearch
@@ -51,6 +52,17 @@
 document_type = "_doc" if ES_VERSION[0] >= 6 else "doc"
 
 
+def get_kwargs(document=None, document_kwarg_name="document"):
+    if ES_VERSION[0] < 6:
+        return {"doc_type": "doc", "body": document} if document else {"doc_type": "doc"}
+    if ES_VERSION[0] < 7:
+        return {"doc_type": "_doc", "body": document} if document else {"doc_type": "_doc"}
+    elif ES_VERSION[0] < 8:
+        return {"body": document} if document else {}
+    else:
+        return {document_kwarg_name: document} if document else {}
+
+
 class NumberObj:
     def __init__(self, value):
         self.value = value
@@ -84,6 +96,7 @@ def dumps(self, obj):
 def elasticsearch(request):
     """Elasticsearch client fixture."""
     client = Elasticsearch(hosts=os.environ["ES_URL"], serializer=SpecialEncoder())
+    client.indices.delete(index="*")
     try:
         yield client
     finally:
@@ -95,6 +108,7 @@ def test_ping(instrument, elasticapm_client, elasticsearch):
     elasticapm_client.begin_transaction("test")
     result = elasticsearch.ping()
     elasticapm_client.end_transaction("test", "OK")
+    parsed_url = urllib.parse.urlparse(os.environ["ES_URL"])
 
     transaction = elasticapm_client.events[TRANSACTION][0]
     spans = elasticapm_client.spans_for_transaction(transaction)
@@ -105,7 +119,8 @@ def test_ping(instrument, elasticapm_client, elasticsearch):
     assert span["subtype"] == "elasticsearch"
     assert span["action"] == "query"
     assert span["context"]["destination"] == {
-        "address": os.environ["ES_URL"],
+        "address": parsed_url.hostname,
+        "port": parsed_url.port,
         "service": {"name": "", "resource": "elasticsearch", "type": ""},
     }
     assert span["context"]["http"]["status_code"] == 200
@@ -132,12 +147,12 @@ def test_info(instrument, elasticapm_client, elasticsearch):
 @pytest.mark.integrationtest
 def test_create(instrument, elasticapm_client, elasticsearch):
     elasticapm_client.begin_transaction("test")
-    if ES_VERSION[0] < 7:
-        r1 = elasticsearch.create(index="tweets", doc_type=document_type, id=1, body={"user": "kimchy", "text": "hola"})
-    else:
-        r1 = elasticsearch.create(index="tweets", id=1, body={"user": "kimchy", "text": "hola"})
-    r2 = elasticsearch.create(
-        index="tweets", doc_type=document_type, id=2, body={"user": "kimchy", "text": "hola"}, refresh=True
+    elasticsearch.create(index="tweets", id="1", **get_kwargs({"user": "kimchy", "text": "hola"}))
+    elasticsearch.create(
+        index="tweets",
+        id="2",
+        refresh=True,
+        **get_kwargs({"user": "kimchy", "text": "hola"}),
     )
     elasticapm_client.end_transaction("test", "OK")
 
@@ -151,6 +166,7 @@ def test_create(instrument, elasticapm_client, elasticsearch):
             assert span["name"] in (
                 "ES PUT /tweets/%s/%d/_create" % (document_type, i + 1),
                 "ES PUT /tweets/_create/%d" % (i + 1),
+                "ES PUT /tweets/_create/%d?refresh=true" % (i + 1),
             )
         else:
             assert span["name"] == "ES PUT /tweets/%s/%d" % (document_type, i + 1)
@@ -165,13 +181,8 @@ def test_create(instrument, elasticapm_client, elasticsearch):
 @pytest.mark.integrationtest
 def test_index(instrument, elasticapm_client, elasticsearch):
     elasticapm_client.begin_transaction("test")
-    if ES_VERSION[0] < 7:
-        r1 = elasticsearch.index("tweets", document_type, {"user": "kimchy", "text": "hola"})
-    else:
-        r1 = elasticsearch.index(index="tweets", body={"user": "kimchy", "text": "hola"})
-    r2 = elasticsearch.index(
-        index="tweets", doc_type=document_type, body={"user": "kimchy", "text": "hola"}, refresh=True
-    )
+    r1 = elasticsearch.index(index="tweets", **get_kwargs({"user": "kimchy", "text": "hola"}))
+    r2 = elasticsearch.index(index="tweets", refresh=True, **get_kwargs({"user": "kimchy", "text": "hola"}))
     elasticapm_client.end_transaction("test", "OK")
 
     transaction = elasticapm_client.events[TRANSACTION][0]
@@ -180,7 +191,7 @@ def test_index(instrument, elasticapm_client, elasticsearch):
     assert len(spans) == 2
 
     for span in spans:
-        assert span["name"] == "ES POST /tweets/%s" % document_type
+        assert span["name"] in ("ES POST /tweets/%s" % document_type, "ES POST /tweets/_doc?refresh=true")
         assert span["type"] == "db"
         assert span["subtype"] == "elasticsearch"
         assert span["action"] == "query"
@@ -191,11 +202,9 @@ def test_index(instrument, elasticapm_client, elasticsearch):
 
 @pytest.mark.integrationtest
 def test_exists(instrument, elasticapm_client, elasticsearch):
-    elasticsearch.create(
-        index="tweets", doc_type=document_type, id=1, body={"user": "kimchy", "text": "hola"}, refresh=True
-    )
+    elasticsearch.create(index="tweets", id="1", refresh=True, **get_kwargs({"user": "kimchy", "text": "hola"}))
     elasticapm_client.begin_transaction("test")
-    result = elasticsearch.exists(id=1, index="tweets", doc_type=document_type)
+    result = elasticsearch.exists(id="1", index="tweets", **get_kwargs())
     elasticapm_client.end_transaction("test", "OK")
 
     transaction = elasticapm_client.events[TRANSACTION][0]
@@ -214,15 +223,13 @@ def test_exists(instrument, elasticapm_client, elasticsearch):
 @pytest.mark.skipif(ES_VERSION[0] < 5, reason="unsupported method")
 @pytest.mark.integrationtest
 def test_exists_source(instrument, elasticapm_client, elasticsearch):
-    elasticsearch.create(
-        index="tweets", doc_type=document_type, id=1, body={"user": "kimchy", "text": "hola"}, refresh=True
-    )
+    elasticsearch.create(index="tweets", id="1", refresh=True, **get_kwargs({"user": "kimchy", "text": "hola"}))
     elasticapm_client.begin_transaction("test")
     if ES_VERSION[0] < 7:
         assert elasticsearch.exists_source("tweets", document_type, 1) is True
     else:
-        assert elasticsearch.exists_source(index="tweets", id=1, doc_type=document_type) is True
-    assert elasticsearch.exists_source(index="tweets", doc_type=document_type, id=1) is True
+        assert bool(elasticsearch.exists_source(index="tweets", id="1", **get_kwargs())) is True
+    assert bool(elasticsearch.exists_source(index="tweets", id="1", **get_kwargs())) is True
     elasticapm_client.end_transaction("test", "OK")
 
     transaction = elasticapm_client.events[TRANSACTION][0]
@@ -231,7 +238,7 @@ def test_exists_source(instrument, elasticapm_client, elasticsearch):
     assert len(spans) == 2
 
     for span in spans:
-        assert span["name"] == "ES HEAD /tweets/%s/1/_source" % document_type
+        assert span["name"] in ("ES HEAD /tweets/%s/1/_source" % document_type, "ES HEAD /tweets/_source/1")
         assert span["type"] == "db"
         assert span["subtype"] == "elasticsearch"
         assert span["action"] == "query"
@@ -242,17 +249,15 @@ def test_exists_source(instrument, elasticapm_client, elasticsearch):
 
 @pytest.mark.integrationtest
 def test_get(instrument, elasticapm_client, elasticsearch):
-    elasticsearch.create(
-        index="tweets", doc_type=document_type, id=1, body={"user": "kimchy", "text": "hola"}, refresh=True
-    )
+    elasticsearch.create(index="tweets", id="1", refresh=True, **get_kwargs({"user": "kimchy", "text": "hola"}))
     elasticapm_client.begin_transaction("test")
     # this is a fun one. Order pre-6x was (index, id, doc_type), changed to (index, doc_type, id) in 6.x, and reverted
     # to (index, id, doc_type) in 7.x. OK then.
     if ES_VERSION[0] == 6:
         r1 = elasticsearch.get("tweets", document_type, 1)
     else:
-        r1 = elasticsearch.get(index="tweets", id=1, doc_type=document_type)
-    r2 = elasticsearch.get(index="tweets", doc_type=document_type, id=1)
+        r1 = elasticsearch.get(index="tweets", id="1", **get_kwargs())
+    r2 = elasticsearch.get(index="tweets", id="1", **get_kwargs())
     elasticapm_client.end_transaction("test", "OK")
 
     transaction = elasticapm_client.events[TRANSACTION][0]
@@ -274,15 +279,13 @@ def test_get(instrument, elasticapm_client, elasticsearch):
 
 @pytest.mark.integrationtest
 def test_get_source(instrument, elasticapm_client, elasticsearch):
-    elasticsearch.create(
-        index="tweets", doc_type=document_type, id=1, body={"user": "kimchy", "text": "hola"}, refresh=True
-    )
+    elasticsearch.create(index="tweets", refresh=True, id="1", **get_kwargs({"user": "kimchy", "text": "hola"}))
     elasticapm_client.begin_transaction("test")
     if ES_VERSION[0] < 7:
         r1 = elasticsearch.get_source("tweets", document_type, 1)
     else:
-        r1 = elasticsearch.get_source(index="tweets", id=1, doc_type=document_type)
-    r2 = elasticsearch.get_source(index="tweets", doc_type=document_type, id=1)
+        r1 = elasticsearch.get_source(index="tweets", id="1", **get_kwargs())
+    r2 = elasticsearch.get_source(index="tweets", id="1", **get_kwargs())
     elasticapm_client.end_transaction("test", "OK")
 
     transaction = elasticapm_client.events[TRANSACTION][0]
@@ -294,7 +297,7 @@ def test_get_source(instrument, elasticapm_client, elasticsearch):
     assert len(spans) == 2
 
     for span in spans:
-        assert span["name"] == "ES GET /tweets/%s/1/_source" % document_type
+        assert span["name"] in ("ES GET /tweets/%s/1/_source" % document_type, "ES GET /tweets/_source/1")
         assert span["type"] == "db"
         assert span["subtype"] == "elasticsearch"
         assert span["action"] == "query"
@@ -305,23 +308,19 @@ def test_get_source(instrument, elasticapm_client, elasticsearch):
 
 @pytest.mark.integrationtest
 def test_update_document(instrument, elasticapm_client, elasticsearch):
-    elasticsearch.create(
-        index="tweets", doc_type=document_type, id=1, body={"user": "kimchy", "text": "hola"}, refresh=True
-    )
+    elasticsearch.create(index="tweets", id="1", refresh=True, **get_kwargs({"user": "kimchy", "text": "hola"}))
     elasticapm_client.begin_transaction("test")
-    r1 = elasticsearch.update(
-        index="tweets", id=1, doc_type=document_type, body={"doc": {"text": "adios"}}, refresh=True
-    )
+    r1 = elasticsearch.update(index="tweets", id=1, body={"doc": {"text": "adios"}}, refresh=True, **get_kwargs())
     elasticapm_client.end_transaction("test", "OK")
 
     transaction = elasticapm_client.events[TRANSACTION][0]
-    r2 = elasticsearch.get(index="tweets", doc_type=document_type, id=1)
+    r2 = elasticsearch.get(index="tweets", id="1", **get_kwargs())
     assert r2["_source"] == {"user": "kimchy", "text": "adios"}
     spans = elasticapm_client.spans_for_transaction(transaction)
     assert len(spans) == 1
 
     span = spans[0]
-    assert span["name"] == "ES POST /tweets/%s/1/_update" % document_type
+    assert span["name"] in ("ES POST /tweets/_update/1", "ES POST /tweets/%s/1/_update" % document_type)
     assert span["type"] == "db"
     assert span["subtype"] == "elasticsearch"
     assert span["action"] == "query"
@@ -333,7 +332,7 @@ def test_update_document(instrument, elasticapm_client, elasticsearch):
 @pytest.mark.integrationtest
 def test_search_body(instrument, elasticapm_client, elasticsearch):
     elasticsearch.create(
-        index="tweets", doc_type=document_type, id=1, body={"user": "kimchy", "text": "hola", "userid": 1}, refresh=True
+        index="tweets", id="1", refresh=True, **get_kwargs({"user": "kimchy", "text": "hola", "userid": 1})
     )
     elasticapm_client.begin_transaction("test")
     search_query = {"query": {"term": {"user": "kimchy"}}, "sort": ["userid"]}
@@ -356,16 +355,14 @@ def test_search_body(instrument, elasticapm_client, elasticsearch):
     ) or json.loads(span["context"]["db"]["statement"]) == json.loads(
         '{"query": {"term": {"user": "kimchy"}}, "sort": ["userid"]}'
     )
-    if ES_VERSION[0] >= 7:
+    if ES_VERSION[0] >= 6:
         assert span["context"]["db"]["rows_affected"] == 1
     assert span["context"]["http"]["status_code"] == 200
 
 
 @pytest.mark.integrationtest
 def test_search_querystring(instrument, elasticapm_client, elasticsearch):
-    elasticsearch.create(
-        index="tweets", doc_type=document_type, id=1, body={"user": "kimchy", "text": "hola"}, refresh=True
-    )
+    elasticsearch.create(index="tweets", id="1", refresh=True, **get_kwargs({"user": "kimchy", "text": "hola"}))
     elasticapm_client.begin_transaction("test")
     search_query = "user:kimchy"
     result = elasticsearch.search(q=search_query, index="tweets")
@@ -384,16 +381,14 @@ def test_search_querystring(instrument, elasticapm_client, elasticsearch):
     assert span["action"] == "query"
     assert span["context"]["db"]["type"] == "elasticsearch"
     assert span["context"]["db"]["statement"] == "q=user:kimchy"
-    if ES_VERSION[0] >= 7:
+    if ES_VERSION[0] >= 6:
         assert span["context"]["db"]["rows_affected"] == 1
     assert span["context"]["http"]["status_code"] == 200
 
 
 @pytest.mark.integrationtest
 def test_search_both(instrument, elasticapm_client, elasticsearch):
-    elasticsearch.create(
-        index="tweets", doc_type=document_type, id=1, body={"user": "kimchy", "text": "hola"}, refresh=True
-    )
+    elasticsearch.create(index="tweets", id="1", refresh=True, **get_kwargs({"user": "kimchy", "text": "hola"}))
     elasticapm_client.begin_transaction("test")
     search_querystring = "text:hola"
     search_query = {"query": {"term": {"user": "kimchy"}}}
@@ -419,9 +414,7 @@ def test_search_both(instrument, elasticapm_client, elasticsearch):
 
 @pytest.mark.integrationtest
 def test_count_body(instrument, elasticapm_client, elasticsearch):
-    elasticsearch.create(
-        index="tweets", doc_type=document_type, id=1, body={"user": "kimchy", "text": "hola"}, refresh=True
-    )
+    elasticsearch.create(index="tweets", id="1", refresh=True, **get_kwargs({"user": "kimchy", "text": "hola"}))
     elasticapm_client.begin_transaction("test")
     search_query = {"query": {"term": {"user": "kimchy"}}}
     result = elasticsearch.count(body=search_query)
@@ -446,9 +439,7 @@ def test_count_body(instrument, elasticapm_client, elasticsearch):
 
 @pytest.mark.integrationtest
 def test_count_querystring(instrument, elasticapm_client, elasticsearch):
-    elasticsearch.create(
-        index="tweets", doc_type=document_type, id=1, body={"user": "kimchy", "text": "hola"}, refresh=True
-    )
+    elasticsearch.create(index="tweets", id="1", refresh=True, **get_kwargs({"user": "kimchy", "text": "hola"}))
     elasticapm_client.begin_transaction("test")
     search_query = "user:kimchy"
     result = elasticsearch.count(q=search_query, index="tweets")
@@ -472,11 +463,9 @@ def test_count_querystring(instrument, elasticapm_client, elasticsearch):
 
 @pytest.mark.integrationtest
 def test_delete(instrument, elasticapm_client, elasticsearch):
-    elasticsearch.create(
-        index="tweets", doc_type=document_type, id=1, body={"user": "kimchy", "text": "hola"}, refresh=True
-    )
+    elasticsearch.create(index="tweets", id="1", refresh=True, **get_kwargs({"user": "kimchy", "text": "hola"}))
     elasticapm_client.begin_transaction("test")
-    result = elasticsearch.delete(id=1, index="tweets", doc_type=document_type)
+    result = elasticsearch.delete(id="1", index="tweets", **get_kwargs())
     elasticapm_client.end_transaction("test", "OK")
 
     transaction = elasticapm_client.events[TRANSACTION][0]
@@ -493,8 +482,8 @@ def test_delete(instrument, elasticapm_client, elasticsearch):
 
 @pytest.mark.integrationtest
 def test_multiple_indexes(instrument, elasticapm_client, elasticsearch):
-    elasticsearch.create(index="tweets", doc_type="users", id=1, body={"user": "kimchy", "text": "hola"}, refresh=True)
-    elasticsearch.create(index="snaps", doc_type="posts", id=1, body={"user": "kimchy", "text": "hola"}, refresh=True)
+    elasticsearch.create(index="tweets", id="1", refresh=True, **get_kwargs({"user": "kimchy", "text": "hola"}))
+    elasticsearch.create(index="snaps", id="1", refresh=True, **get_kwargs({"user": "kimchy", "text": "hola"}))
     elasticapm_client.begin_transaction("test")
     result = elasticsearch.search(index=["tweets", "snaps"], q="user:kimchy")
     elasticapm_client.end_transaction("test", "OK")
diff --git a/tests/requirements/reqs-elasticsearch-8.txt b/tests/requirements/reqs-elasticsearch-8.txt
new file mode 100644
--- /dev/null
+++ b/tests/requirements/reqs-elasticsearch-8.txt
@@ -0,0 +1,3 @@
+elasticsearch>=8.0,<9.0
+aiohttp ; python_version >= '3.6'
+-r reqs-base.txt
diff --git a/tests/scripts/envs/elasticsearch-8.sh b/tests/scripts/envs/elasticsearch-8.sh
new file mode 100644
--- /dev/null
+++ b/tests/scripts/envs/elasticsearch-8.sh
@@ -0,0 +1,5 @@
+export PYTEST_MARKER="-m elasticsearch"
+export ES_URL="http://elasticsearch8:9200"
+export DOCKER_DEPS="elasticsearch8"
+export WAIT_FOR_HOST="elasticsearch8"
+export WAIT_FOR_PORT=9200