Repository: elastic/apm-agent-python

Issue:
urllib3 v2.0 compatibility
urllib3 v2.0 came out during PyConUS 2023 and is resulting in hanging tests on windows, and test failures on Linux. We need to update our code/tests to ensure compatibility for this new version.


Discussion:
Hello fellow Elasticians :wave: 

We just released [urllib3 2.0.1](https://github.com/urllib3/urllib3/releases/tag/2.0.1) that includes https://github.com/urllib3/urllib3/pull/2992. We have extensive tests for cert pinning, but Tornado (our test server) can accept multiple connections at a time unlike pytest-localserver. So we did not see the resource leak that caused the deadlock in your case, sorry about that.

Would you mind giving 2.0.1 a try?

Resolving patch:
diff --git a/elasticapm/instrumentation/packages/urllib3.py b/elasticapm/instrumentation/packages/urllib3.py
--- a/elasticapm/instrumentation/packages/urllib3.py
+++ b/elasticapm/instrumentation/packages/urllib3.py
@@ -61,7 +61,12 @@ def update_headers(args, kwargs, instance, transaction, trace_parent):
     :param trace_parent: the TraceParent object
     :return: an (args, kwargs) tuple
     """
-    if len(args) >= 4 and args[3]:
+    from urllib3._version import __version__ as urllib3_version
+
+    if urllib3_version.startswith("2") and len(args) >= 5 and args[4]:
+        headers = args[4].copy()
+        args = tuple(itertools.chain((args[:4]), (headers,), args[5:]))
+    elif len(args) >= 4 and args[3]:
         headers = args[3].copy()
         args = tuple(itertools.chain((args[:3]), (headers,), args[4:]))
     elif "headers" in kwargs and kwargs["headers"]:


Test patch:
diff --git a/tests/client/client_tests.py b/tests/client/client_tests.py
--- a/tests/client/client_tests.py
+++ b/tests/client/client_tests.py
@@ -530,7 +530,7 @@ def test_check_server_version(elasticapm_client):
     indirect=["elasticapm_client"],
 )
 def test_user_agent(elasticapm_client, expected):
-    assert elasticapm_client.get_user_agent() == "apm-agent-python/unknown (myapp{})".format(expected)
+    assert elasticapm_client.get_user_agent() == "apm-agent-python/{} (myapp{})".format(elasticapm.VERSION, expected)
 
 
 def test_label_without_client():
diff --git a/tests/client/exception_tests.py b/tests/client/exception_tests.py
--- a/tests/client/exception_tests.py
+++ b/tests/client/exception_tests.py
@@ -122,7 +122,12 @@ def test_message_event(elasticapm_client):
     assert "stacktrace" in event["log"]
     # check that only frames from `tests` module are not marked as library frames
     for frame in event["log"]["stacktrace"]:
-        assert frame["library_frame"] or frame["module"].startswith(("tests", "__main__")), (
+        # vscode's python extension runs tests using libraries from the
+        # extension's dir, which results in false positives here. This is where
+        # runpy, _pydevd, and debugpy come from.
+        assert frame["library_frame"] or frame["module"].startswith(
+            ("tests", "__main__", "runpy", "_pydevd", "debugpy")
+        ), (
             frame["module"],
             frame["abs_path"],
         )
diff --git a/tests/instrumentation/urllib3_tests.py b/tests/instrumentation/urllib3_tests.py
--- a/tests/instrumentation/urllib3_tests.py
+++ b/tests/instrumentation/urllib3_tests.py
@@ -31,6 +31,7 @@
 
 import pytest
 import urllib3
+from urllib3._version import __version__ as urllib3_version
 
 from elasticapm.conf import constants
 from elasticapm.conf.constants import SPAN, TRANSACTION
@@ -273,7 +274,10 @@ def test_instance_headers_are_respected(
         headers={"instance": "true"} if instance_headers else None,
     )
     if header_arg:
-        args = ("GET", url, None, {"args": "true"})
+        if urllib3_version.startswith("2"):
+            args = ("GET", url, None, None, {"args": "true"})
+        else:
+            args = ("GET", url, None, {"args": "true"})
     else:
         args = ("GET", url)
     if header_kwarg:
diff --git a/tests/requirements/reqs-base.txt b/tests/requirements/reqs-base.txt
--- a/tests/requirements/reqs-base.txt
+++ b/tests/requirements/reqs-base.txt
@@ -18,7 +18,7 @@ jsonschema==3.2.0 ; python_version == '3.6'
 jsonschema==4.17.3 ; python_version > '3.6'
 
 
-urllib3<2.0
+urllib3!=2.0.0,<3.0.0
 certifi
 Logbook
 mock