Repository: twisted/twisted

Issue:
remove all references to `urllib2` and `cookielib`
It's been a decade or so since these modules were deprecated, and it would be good to clean up both our docs and our implementation to stop referring to them.


Discussion:

Resolving patch:
diff --git a/docs/web/howto/listings/client/cookies.py b/docs/web/howto/listings/client/cookies.py
--- a/docs/web/howto/listings/client/cookies.py
+++ b/docs/web/howto/listings/client/cookies.py
@@ -1,5 +1,7 @@
+from http.cookiejar import CookieJar
+
 from twisted.internet import reactor
-from twisted.python import compat, log
+from twisted.python import log
 from twisted.web.client import Agent, CookieAgent
 
 
@@ -12,7 +14,7 @@ def displayCookies(response, cookieJar):
 
 
 def main():
-    cookieJar = compat.cookielib.CookieJar()
+    cookieJar = CookieJar()
     agent = CookieAgent(Agent(reactor), cookieJar)
 
     d = agent.request(b"GET", b"http://httpbin.org/cookies/set?some=data")
diff --git a/src/twisted/web/client.py b/src/twisted/web/client.py
--- a/src/twisted/web/client.py
+++ b/src/twisted/web/client.py
@@ -6,13 +6,16 @@
 HTTP client.
 """
 
+from __future__ import annotations
 
 import collections
 import os
 import warnings
 import zlib
+from dataclasses import dataclass
 from functools import wraps
-from typing import Iterable
+from http.cookiejar import CookieJar
+from typing import TYPE_CHECKING, Iterable, Optional
 from urllib.parse import urldefrag, urljoin, urlunparse as _urlunparse
 
 from zope.interface import implementer
@@ -21,6 +24,7 @@
 
 from twisted.internet import defer, protocol, task
 from twisted.internet.abstract import isIPv6Address
+from twisted.internet.defer import Deferred
 from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
 from twisted.internet.interfaces import IOpenSSLContextFactory, IProtocol
 from twisted.logger import Logger
@@ -43,6 +47,20 @@
     IResponse,
 )
 
+# For the purpose of type-checking we want our faked-out types to be identical to the types they are replacing.
+# For the purpose of the impementation, we want to start
+# with a blank slate so that we don't accidentally use
+# any of the real implementation.
+
+if TYPE_CHECKING:
+    from email.message import EmailMessage as _InfoType
+    from http.client import HTTPResponse as _ResponseBase
+    from urllib.request import Request as _RequestBase
+else:
+    _RequestBase = object
+    _ResponseBase = object
+    _InfoType = object
+
 
 def urlunparse(parts):
     result = _urlunparse(tuple(p.decode("charmap") for p in parts))
@@ -1200,36 +1218,38 @@ def request(self, method, uri, headers=None, bodyProducer=None):
         )
 
 
-class _FakeUrllib2Request:
+class _FakeStdlibRequest(_RequestBase):
     """
-    A fake C{urllib2.Request} object for C{cookielib} to work with.
+    A fake L{urllib.request.Request} object for L{cookiejar} to work with.
 
-    @see: U{http://docs.python.org/library/urllib2.html#request-objects}
+    @see: U{urllib.request.Request
+        <https://docs.python.org/3/library/urllib.request.html#urllib.request.Request>}
 
-    @type uri: native L{str}
     @ivar uri: Request URI.
 
-    @type headers: L{twisted.web.http_headers.Headers}
     @ivar headers: Request headers.
 
-    @type type: native L{str}
     @ivar type: The scheme of the URI.
 
-    @type host: native L{str}
     @ivar host: The host[:port] of the URI.
 
     @since: 11.1
     """
 
-    def __init__(self, uri):
+    uri: str
+    type: str
+    host: str
+    # The received headers managed using Twisted API.
+    _twistedHeaders: Headers
+
+    def __init__(self, uri: bytes) -> None:
         """
-        Create a fake Urllib2 request.
+        Create a fake  request.
 
         @param uri: Request URI.
-        @type uri: L{bytes}
         """
         self.uri = nativeString(uri)
-        self.headers = Headers()
+        self._twistedHeaders = Headers()
 
         _uri = URI.fromBytes(uri)
         self.type = nativeString(_uri.scheme)
@@ -1240,19 +1260,19 @@ def __init__(self, uri):
             self.host += ":" + str(_uri.port)
 
         self.origin_req_host = nativeString(_uri.host)
-        self.unverifiable = lambda _: False
+        self.unverifiable = False
 
     def has_header(self, header):
-        return self.headers.hasHeader(networkString(header))
+        return self._twistedHeaders.hasHeader(networkString(header))
 
     def add_unredirected_header(self, name, value):
-        self.headers.addRawHeader(networkString(name), networkString(value))
+        self._twistedHeaders.addRawHeader(networkString(name), networkString(value))
 
     def get_full_url(self):
         return self.uri
 
     def get_header(self, name, default=None):
-        headers = self.headers.getRawHeaders(networkString(name), default)
+        headers = self._twistedHeaders.getRawHeaders(networkString(name), default)
         if headers is not None:
             headers = [nativeString(x) for x in headers]
             return headers[0]
@@ -1269,62 +1289,68 @@ def is_unverifiable(self):
         return False
 
 
-class _FakeUrllib2Response:
+@dataclass
+class _FakeUrllibResponseInfo(_InfoType):
+    response: IResponse
+
+    def get_all(self, name: str, default: bytes) -> list[str]:  # type:ignore[override]
+        headers = self.response.headers.getRawHeaders(networkString(name), default)
+        h = [nativeString(x) for x in headers]
+        return h
+
+
+class _FakeStdlibResponse(_ResponseBase):
     """
-    A fake C{urllib2.Response} object for C{cookielib} to work with.
+    A fake L{urllib.response.Response} object for L{http.cookiejar} to work
+    with.
 
-    @type response: C{twisted.web.iweb.IResponse}
     @ivar response: Underlying Twisted Web response.
 
     @since: 11.1
     """
 
-    def __init__(self, response):
-        self.response = response
+    response: IResponse
 
-    def info(self):
-        class _Meta:
-            def getheaders(zelf, name):
-                # PY2
-                headers = self.response.headers.getRawHeaders(name, [])
-                return headers
-
-            def get_all(zelf, name, default):
-                # PY3
-                headers = self.response.headers.getRawHeaders(
-                    networkString(name), default
-                )
-                h = [nativeString(x) for x in headers]
-                return h
+    def __init__(self, response: IResponse) -> None:
+        self.response = response
 
-        return _Meta()
+    def info(self) -> _InfoType:
+        result = _FakeUrllibResponseInfo(self.response)
+        return result
 
 
 @implementer(IAgent)
 class CookieAgent:
     """
-    L{CookieAgent} extends the basic L{Agent} to add RFC-compliant
-    handling of HTTP cookies.  Cookies are written to and extracted
-    from a C{cookielib.CookieJar} instance.
+    L{CookieAgent} extends the basic L{Agent} to add RFC-compliant handling of
+    HTTP cookies.  Cookies are written to and extracted from a L{CookieJar}
+    instance.
 
     The same cookie jar instance will be used for any requests through this
     agent, mutating it whenever a I{Set-Cookie} header appears in a response.
 
-    @type _agent: L{twisted.web.client.Agent}
     @ivar _agent: Underlying Twisted Web agent to issue requests through.
 
-    @type cookieJar: C{cookielib.CookieJar}
     @ivar cookieJar: Initialized cookie jar to read cookies from and store
         cookies to.
 
     @since: 11.1
     """
 
-    def __init__(self, agent, cookieJar):
+    _agent: IAgent
+    cookieJar: CookieJar
+
+    def __init__(self, agent: IAgent, cookieJar: CookieJar) -> None:
         self._agent = agent
         self.cookieJar = cookieJar
 
-    def request(self, method, uri, headers=None, bodyProducer=None):
+    def request(
+        self,
+        method: bytes,
+        uri: bytes,
+        headers: Optional[Headers] = None,
+        bodyProducer: Optional[IBodyProducer] = None,
+    ) -> Deferred[IResponse]:
         """
         Issue a new request to the wrapped L{Agent}.
 
@@ -1337,33 +1363,33 @@ def request(self, method, uri, headers=None, bodyProducer=None):
 
         @see: L{Agent.request}
         """
-        if headers is None:
-            headers = Headers()
-        lastRequest = _FakeUrllib2Request(uri)
+        actualHeaders = headers if headers is not None else Headers()
+        lastRequest = _FakeStdlibRequest(uri)
         # Setting a cookie header explicitly will disable automatic request
         # cookies.
-        if not headers.hasHeader(b"cookie"):
+        if not actualHeaders.hasHeader(b"cookie"):
             self.cookieJar.add_cookie_header(lastRequest)
             cookieHeader = lastRequest.get_header("Cookie", None)
             if cookieHeader is not None:
-                headers = headers.copy()
-                headers.addRawHeader(b"cookie", networkString(cookieHeader))
+                actualHeaders = actualHeaders.copy()
+                actualHeaders.addRawHeader(b"cookie", networkString(cookieHeader))
 
-        d = self._agent.request(method, uri, headers, bodyProducer)
-        d.addCallback(self._extractCookies, lastRequest)
-        return d
+        return self._agent.request(
+            method, uri, actualHeaders, bodyProducer
+        ).addCallback(self._extractCookies, lastRequest)
 
-    def _extractCookies(self, response, request):
+    def _extractCookies(
+        self, response: IResponse, request: _FakeStdlibRequest
+    ) -> IResponse:
         """
         Extract response cookies and store them in the cookie jar.
 
-        @type response: L{twisted.web.iweb.IResponse}
-        @param response: Twisted Web response.
+        @param response: the Twisted Web response that we are processing.
 
-        @param request: A urllib2 compatible request object.
+        @param request: A L{_FakeStdlibRequest} wrapping our Twisted request,
+            for L{CookieJar} to extract cookies from.
         """
-        resp = _FakeUrllib2Response(response)
-        self.cookieJar.extract_cookies(resp, request)
+        self.cookieJar.extract_cookies(_FakeStdlibResponse(response), request)
         return response
 
 


Test patch:
diff --git a/src/twisted/web/test/test_agent.py b/src/twisted/web/test/test_agent.py
--- a/src/twisted/web/test/test_agent.py
+++ b/src/twisted/web/test/test_agent.py
@@ -4,6 +4,9 @@
 """
 Tests for L{twisted.web.client.Agent} and related new client APIs.
 """
+
+from __future__ import annotations
+
 import zlib
 from http.cookiejar import CookieJar
 from io import BytesIO
@@ -1281,7 +1284,7 @@ def test_responseIncludesRequest(self):
         self.assertIsInstance(req, Request)
 
         resp = client.Response._construct(
-            (b"HTTP", 1, 1), 200, b"OK", client.Headers({}), None, req
+            (b"HTTP", 1, 1), 200, b"OK", Headers({}), None, req
         )
         res.callback(resp)
 
@@ -1313,7 +1316,7 @@ def test_requestMissingAbsoluteURI(self):
         """
         L{Request.absoluteURI} is L{None} if L{Request._parsedURI} is L{None}.
         """
-        request = client.Request(b"FOO", b"/", client.Headers(), None)
+        request = client.Request(b"FOO", b"/", Headers(), None)
         self.assertIdentical(request.absoluteURI, None)
 
     def test_endpointFactory(self):
@@ -1368,7 +1371,7 @@ def attemptRequestWithMaliciousMethod(self, method):
         """
         agent = client.Agent(self.createReactor())
         uri = b"http://twisted.invalid"
-        agent.request(method, uri, client.Headers(), None)
+        agent.request(method, uri, Headers(), None)
 
 
 class AgentURIInjectionTests(
@@ -1388,7 +1391,7 @@ def attemptRequestWithMaliciousURI(self, uri):
         """
         agent = client.Agent(self.createReactor())
         method = b"GET"
-        agent.request(method, uri, client.Headers(), None)
+        agent.request(method, uri, Headers(), None)
 
 
 @skipIf(not sslPresent, "SSL not present, cannot run SSL tests.")
@@ -1795,9 +1798,7 @@ def newProtocol():
             return defer.succeed(protocol)
 
         bodyProducer = object()
-        request = client.Request(
-            b"FOO", b"/", client.Headers(), bodyProducer, persistent=True
-        )
+        request = client.Request(b"FOO", b"/", Headers(), bodyProducer, persistent=True)
         newProtocol()
         protocol = protocols[0]
         retrier = client._RetryingHTTP11ClientProtocol(protocol, newProtocol)
@@ -1936,32 +1937,36 @@ class CookieTestsMixin:
     Mixin for unit tests dealing with cookies.
     """
 
-    def addCookies(self, cookieJar, uri, cookies):
+    def addCookies(
+        self, cookieJar: CookieJar, uri: bytes, cookies: list[bytes]
+    ) -> tuple[client._FakeStdlibRequest, client._FakeStdlibResponse]:
         """
         Add a cookie to a cookie jar.
         """
-        response = client._FakeUrllib2Response(
+        response = client._FakeStdlibResponse(
             client.Response(
                 (b"HTTP", 1, 1),
                 200,
                 b"OK",
-                client.Headers({b"Set-Cookie": cookies}),
+                Headers({b"Set-Cookie": cookies}),
                 None,
             )
         )
-        request = client._FakeUrllib2Request(uri)
+        request = client._FakeStdlibRequest(uri)
         cookieJar.extract_cookies(response, request)
         return request, response
 
 
 class CookieJarTests(TestCase, CookieTestsMixin):
     """
-    Tests for L{twisted.web.client._FakeUrllib2Response} and
-    L{twisted.web.client._FakeUrllib2Request}'s interactions with
-    L{CookieJar} instances.
+    Tests for L{twisted.web.client._FakeStdlibResponse} and
+    L{twisted.web.client._FakeStdlibRequest}'s interactions with L{CookieJar}
+    instances.
     """
 
-    def makeCookieJar(self):
+    def makeCookieJar(
+        self,
+    ) -> tuple[CookieJar, tuple[client._FakeStdlibRequest, client._FakeStdlibResponse]]:
         """
         @return: a L{CookieJar} with some sample cookies
         """
@@ -1973,10 +1978,11 @@ def makeCookieJar(self):
         )
         return cookieJar, reqres
 
-    def test_extractCookies(self):
+    def test_extractCookies(self) -> None:
         """
-        L{CookieJar.extract_cookies} extracts cookie information from
-        fake urllib2 response instances.
+        L{CookieJar.extract_cookies} extracts cookie information from our
+        stdlib-compatibility wrappers, L{client._FakeStdlibRequest} and
+        L{client._FakeStdlibResponse}.
         """
         jar = self.makeCookieJar()[0]
         cookies = {c.name: c for c in jar}
@@ -1997,17 +2003,20 @@ def test_extractCookies(self):
         self.assertEqual(cookie.comment, "goodbye")
         self.assertIdentical(cookie.get_nonstandard_attr("cow"), None)
 
-    def test_sendCookie(self):
+    def test_sendCookie(self) -> None:
         """
-        L{CookieJar.add_cookie_header} adds a cookie header to a fake
-        urllib2 request instance.
+        L{CookieJar.add_cookie_header} adds a cookie header to a Twisted
+        request via our L{client._FakeStdlibRequest} wrapper.
         """
         jar, (request, response) = self.makeCookieJar()
 
         self.assertIdentical(request.get_header("Cookie", None), None)
 
         jar.add_cookie_header(request)
-        self.assertEqual(request.get_header("Cookie", None), "foo=1; bar=2")
+        self.assertEqual(
+            list(request._twistedHeaders.getAllRawHeaders()),
+            [(b"Cookie", [b"foo=1; bar=2"])],
+        )
 
 
 class CookieAgentTests(
@@ -2057,7 +2066,7 @@ def _checkCookie(ignored):
             (b"HTTP", 1, 1),
             200,
             b"OK",
-            client.Headers(
+            Headers(
                 {
                     b"Set-Cookie": [
                         b"foo=1",
@@ -2070,6 +2079,26 @@ def _checkCookie(ignored):
 
         return d
 
+    def test_leaveExistingCookieHeader(self) -> None:
+        """
+        L{CookieAgent.request} will not insert a C{'Cookie'} header into the
+        L{Request} object when there is already a C{'Cookie'} header in the
+        request headers parameter.
+        """
+        uri = b"http://example.com:1234/foo?bar"
+        cookie = b"foo=1"
+
+        cookieJar = CookieJar()
+        self.addCookies(cookieJar, uri, [cookie])
+        self.assertEqual(len(list(cookieJar)), 1)
+
+        agent = self.buildAgentForWrapperTest(self.reactor)
+        cookieAgent = client.CookieAgent(agent, cookieJar)
+        cookieAgent.request(b"GET", uri, Headers({"cookie": ["already-set"]}))
+
+        req, res = self.protocol.requests.pop()
+        self.assertEqual(req.headers.getRawHeaders(b"cookie"), [b"already-set"])
+
     def test_requestWithCookie(self):
         """
         L{CookieAgent.request} inserts a C{'Cookie'} header into the L{Request}