Repository: elastic/apm-agent-python

Issue:
httpx instrumentation: IndexError: tuple index out of range
**Describe the bug**
In some cases `httpx` passes `request` param to its `httpx.Client.send()` as keyword argument rather than positional one, like e.g. when using `httpx.Client.stream()` in a context manager, which is unacceptable by `elastic-apm`.
**Please note same happens when using `httpx.AsyncClient.stream()` instead of sync `httpx.Client.stream()`**.

**Current workaround**
Disable `httpx` instrumenting (set environment variable `SKIP_INSTRUMENT_HTTPX="true"`) prior `elasticapm.instrument()` is called.

**To Reproduce**
```python3
import elasticapm
import httpx


def main():
    elasticapm.instrument()

    client = httpx.Client()
    elastic_apm_client = elasticapm.Client()

    elastic_apm_client.begin_transaction("httpx-context-manager-client-streaming")
    try:
        with client.stream("GET", url="https://example.com") as response:
            assert response
    finally:
        client.close()
        elastic_apm_client.end_transaction("httpx-context-manager-client-streaming")


if __name__ == "__main__":
    main()
```

**Expected result**
No exception is raised.

**Actual result**
```python3
Traceback (most recent call last):
  File "<...>/test.py", line 21, in <module>
    main()
  File "<...>/test.py", line 13, in main
    with client.stream("GET", url="https://example.com") as response:
  File "<...>/venv/lib/python3.8/site-packages/httpx/_client.py", line 1799, in __enter__
    self.response = self.client.send(
  File "<...>/venv/lib/python3.8/site-packages/elasticapm/instrumentation/packages/base.py", line 210, in call_if_sampling
    return self.call(module, method, wrapped, instance, args, kwargs)
  File "<...>/venv/lib/python3.8/site-packages/elasticapm/instrumentation/packages/httpx.py", line 42, in call
    request = kwargs.get("request", args[0])
IndexError: tuple index out of range
```

<br/>
<details>
<summary><b>See similar results for <code>httpx.AsyncClient.stream()</code></b></summary>

**To Reproduce**
```python3
import asyncio

import elasticapm
import httpx


async def main():
    elasticapm.instrument()

    client = httpx.AsyncClient()
    elastic_apm_client = elasticapm.Client()

    elastic_apm_client.begin_transaction("httpx-async-context-manager-client-streaming")
    try:
        async with client.stream("GET", url="https://example.com") as response:
            assert response
    finally:
        await client.aclose()
        elastic_apm_client.end_transaction(
            "httpx-async-context-manager-client-streaming",
        )


if __name__ == "__main__":
    asyncio.run(main())
```

**Actual result**
```python3
Traceback (most recent call last):
  File "<...>/test.py", line 23, in <module>
    asyncio.run(main())
  File "/usr/local/lib/python3.8/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/usr/local/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
    return future.result()
  File "<...>/test.py", line 15, in main
    async with client.stream("GET", url="https://example.com") as response:
  File "<...>/venv/lib/python3.8/site-packages/httpx/_client.py", line 1821, in __aenter__
    self.response = await self.client.send(
  File "<...>/venv/lib/python3.8/site-packages/elasticapm/instrumentation/packages/asyncio/httpx.py", line 42, in call
    request = kwargs.get("request", args[0])
IndexError: tuple index out of range
```
</details>
<br/>

**Environment**
- OS: Ubuntu 18.04.5 LTS
- Python version: 3.8.9
- Framework and version: `httpx==0.17.1` (also reproducible with current latest `httpx==0.19.0`)
- APM Server version: --
- Agent version: `elasticapm==6.4.0`



Discussion:

Resolving patch:
diff --git a/elasticapm/instrumentation/packages/asyncio/httpx.py b/elasticapm/instrumentation/packages/asyncio/httpx.py
--- a/elasticapm/instrumentation/packages/asyncio/httpx.py
+++ b/elasticapm/instrumentation/packages/asyncio/httpx.py
@@ -39,7 +39,7 @@ class HttpxAsyncClientInstrumentation(AsyncAbstractInstrumentedModule):
     instrument_list = [("httpx", "AsyncClient.send")]
 
     async def call(self, module, method, wrapped, instance, args, kwargs):
-        request = kwargs.get("request", args[0])
+        request = kwargs.get("request") or args[0]
 
         request_method = request.method.upper()
         url = str(request.url)
diff --git a/elasticapm/instrumentation/packages/httpx.py b/elasticapm/instrumentation/packages/httpx.py
--- a/elasticapm/instrumentation/packages/httpx.py
+++ b/elasticapm/instrumentation/packages/httpx.py
@@ -39,7 +39,7 @@ class HttpxClientInstrumentation(AbstractInstrumentedModule):
     instrument_list = [("httpx", "Client.send")]
 
     def call(self, module, method, wrapped, instance, args, kwargs):
-        request = kwargs.get("request", args[0])
+        request = kwargs.get("request") or args[0]
 
         request_method = request.method.upper()
         url = str(request.url)


Test patch:
diff --git a/tests/instrumentation/asyncio_tests/httpx_tests.py b/tests/instrumentation/asyncio_tests/httpx_tests.py
--- a/tests/instrumentation/asyncio_tests/httpx_tests.py
+++ b/tests/instrumentation/asyncio_tests/httpx_tests.py
@@ -168,3 +168,22 @@ async def test_httpx_error(instrument, elasticapm_client, waiting_httpserver, st
     assert spans[0]["context"]["http"]["url"] == url
     assert spans[0]["context"]["http"]["status_code"] == status_code
     assert spans[0]["outcome"] == "failure"
+
+
+async def test_httpx_streaming(instrument, elasticapm_client, waiting_httpserver):
+    # client.stream passes the request as a keyword argument to the instrumented method.
+    # This helps test that we can handle it both as an arg and a kwarg
+    # see https://github.com/elastic/apm-agent-python/issues/1336
+    elasticapm_client.begin_transaction("httpx-context-manager-client-streaming")
+    client = httpx.AsyncClient()
+    try:
+        async with client.stream("GET", url=waiting_httpserver.url) as response:
+            assert response
+    finally:
+        await client.aclose()
+        elasticapm_client.end_transaction("httpx-context-manager-client-streaming")
+
+    transactions = elasticapm_client.events[TRANSACTION]
+    span = elasticapm_client.spans_for_transaction(transactions[0])[0]
+    assert span["type"] == "external"
+    assert span["subtype"] == "http"
diff --git a/tests/instrumentation/httpx_tests.py b/tests/instrumentation/httpx_tests.py
--- a/tests/instrumentation/httpx_tests.py
+++ b/tests/instrumentation/httpx_tests.py
@@ -167,3 +167,22 @@ def test_httpx_error(instrument, elasticapm_client, waiting_httpserver, status_c
     assert spans[0]["context"]["http"]["url"] == url
     assert spans[0]["context"]["http"]["status_code"] == status_code
     assert spans[0]["outcome"] == "failure"
+
+
+def test_httpx_streaming(instrument, elasticapm_client, waiting_httpserver):
+    # client.stream passes the request as a keyword argument to the instrumented method.
+    # This helps test that we can handle it both as an arg and a kwarg
+    # see https://github.com/elastic/apm-agent-python/issues/1336
+    elasticapm_client.begin_transaction("httpx-context-manager-client-streaming")
+    client = httpx.Client()
+    try:
+        with client.stream("GET", url=waiting_httpserver.url) as response:
+            assert response
+    finally:
+        client.close()
+        elasticapm_client.end_transaction("httpx-context-manager-client-streaming")
+
+    transactions = elasticapm_client.events[TRANSACTION]
+    span = elasticapm_client.spans_for_transaction(transactions[0])[0]
+    assert span["type"] == "external"
+    assert span["subtype"] == "http"