Repository: urllib3/urllib3

Issue:
HTTPResponse.getheaders() is not backwards compatible
The `getheaders()` method of the `HTTPResponse` class returns the dict-like `HTTPHeaderDict`, while the httplib version of that method returns a list of (header, value) tuples. This means that code iterating over the response as in `for header, value in response.getheaders()` will not work with urllib3 response objects unless `.items()` is called on the returned `HTTPHeaderDict` object. A comment near the `getheaders()` method indicates it's supposed to be backwards compatible.


Discussion:
You're right, this comment and interface aren't correct. If we were to change this behavior I'm not sure how many people would be impacted but at a minimum that comment either needs to be removed or this interface fixed.

I think the safest bet is to just remove the comment about compatibility, but it still feels kinda bad to pretend to be the same as an `HTTPResponse` object. Thoughts from other @urllib3/maintainers?

Resolving patch:
diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py
--- a/src/urllib3/connectionpool.py
+++ b/src/urllib3/connectionpool.py
@@ -35,7 +35,7 @@
 from .packages import six
 from .packages.ssl_match_hostname import CertificateError
 from .request import RequestMethods
-from .response import HTTPResponse
+from .response import BaseHTTPResponse, HTTPResponse
 from .util.connection import is_connection_dropped
 from .util.proxy import connection_requires_http_tunnel
 from .util.queue import LifoQueue
@@ -484,7 +484,7 @@ def urlopen(
         chunked=False,
         body_pos=None,
         **response_kw,
-    ):
+    ) -> BaseHTTPResponse:
         """
         Get a connection from the pool and perform an HTTP request. This is the
         lowest level call for making a request, so you'll need to specify all
diff --git a/src/urllib3/request.py b/src/urllib3/request.py
--- a/src/urllib3/request.py
+++ b/src/urllib3/request.py
@@ -1,6 +1,7 @@
 from urllib.parse import urlencode
 
 from .filepost import encode_multipart_formdata
+from .response import BaseHTTPResponse
 
 __all__ = ["RequestMethods"]
 
@@ -48,13 +49,15 @@ def urlopen(
         encode_multipart=True,
         multipart_boundary=None,
         **kw
-    ):  # Abstract
+    ) -> BaseHTTPResponse:  # Abstract
         raise NotImplementedError(
             "Classes extending RequestMethods must implement "
             "their own ``urlopen`` method."
         )
 
-    def request(self, method, url, fields=None, headers=None, **urlopen_kw):
+    def request(
+        self, method, url, fields=None, headers=None, **urlopen_kw
+    ) -> BaseHTTPResponse:
         """
         Make a request using :meth:`urlopen` with the appropriate encoding of
         ``fields`` based on the ``method`` used.
@@ -78,7 +81,9 @@ def request(self, method, url, fields=None, headers=None, **urlopen_kw):
                 method, url, fields=fields, headers=headers, **urlopen_kw
             )
 
-    def request_encode_url(self, method, url, fields=None, headers=None, **urlopen_kw):
+    def request_encode_url(
+        self, method, url, fields=None, headers=None, **urlopen_kw
+    ) -> BaseHTTPResponse:
         """
         Make a request using :meth:`urlopen` with the ``fields`` encoded in
         the url. This is useful for request methods like GET, HEAD, DELETE, etc.
@@ -103,7 +108,7 @@ def request_encode_body(
         encode_multipart=True,
         multipart_boundary=None,
         **urlopen_kw
-    ):
+    ) -> BaseHTTPResponse:
         """
         Make a request using :meth:`urlopen` with the ``fields`` encoded in
         the body. This is useful for request methods like POST, PUT, PATCH, etc.
diff --git a/src/urllib3/response.py b/src/urllib3/response.py
--- a/src/urllib3/response.py
+++ b/src/urllib3/response.py
@@ -1,5 +1,6 @@
 import io
 import logging
+import typing
 import zlib
 from contextlib import contextmanager
 from socket import error as SocketError
@@ -29,15 +30,20 @@
 log = logging.getLogger(__name__)
 
 
-class DeflateDecoder:
+class ContentDecoder:
+    def decompress(self, data: bytes) -> bytes:
+        raise NotImplementedError()
+
+    def flush(self) -> bytes:
+        raise NotImplementedError()
+
+
+class DeflateDecoder(ContentDecoder):
     def __init__(self):
         self._first_try = True
         self._data = b""
         self._obj = zlib.decompressobj()
 
-    def __getattr__(self, name):
-        return getattr(self._obj, name)
-
     def decompress(self, data):
         if not data:
             return data
@@ -60,6 +66,9 @@ def decompress(self, data):
             finally:
                 self._data = None
 
+    def flush(self) -> bytes:
+        return self._obj.flush()
+
 
 class GzipDecoderState:
 
@@ -68,14 +77,11 @@ class GzipDecoderState:
     SWALLOW_DATA = 2
 
 
-class GzipDecoder:
+class GzipDecoder(ContentDecoder):
     def __init__(self):
         self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
         self._state = GzipDecoderState.FIRST_MEMBER
 
-    def __getattr__(self, name):
-        return getattr(self._obj, name)
-
     def decompress(self, data):
         ret = bytearray()
         if self._state == GzipDecoderState.SWALLOW_DATA or not data:
@@ -97,10 +103,13 @@ def decompress(self, data):
             self._state = GzipDecoderState.OTHER_MEMBERS
             self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
 
+    def flush(self) -> bytes:
+        return self._obj.flush()
+
 
 if brotli is not None:
 
-    class BrotliDecoder:
+    class BrotliDecoder(ContentDecoder):
         # Supports both 'brotlipy' and 'Brotli' packages
         # since they share an import name. The top branches
         # are for 'brotlipy' and bottom branches for 'Brotli'
@@ -117,7 +126,7 @@ def flush(self):
             return b""
 
 
-class MultiDecoder:
+class MultiDecoder(ContentDecoder):
     """
     From RFC7231:
         If one or more encodings have been applied to a representation, the
@@ -138,7 +147,7 @@ def decompress(self, data):
         return data
 
 
-def _get_decoder(mode):
+def _get_decoder(mode: str) -> ContentDecoder:
     if "," in mode:
         return MultiDecoder(mode)
 
@@ -151,7 +160,180 @@ def _get_decoder(mode):
     return DeflateDecoder()
 
 
-class HTTPResponse(io.IOBase):
+class BaseHTTPResponse(io.IOBase):
+    CONTENT_DECODERS = ["gzip", "deflate"]
+    if brotli is not None:
+        CONTENT_DECODERS += ["br"]
+    REDIRECT_STATUSES = [301, 302, 303, 307, 308]
+
+    DECODER_ERROR_CLASSES = (IOError, zlib.error)
+    if brotli is not None:
+        DECODER_ERROR_CLASSES += (brotli.error,)
+
+    def __init__(
+        self,
+        *,
+        headers: typing.Optional[typing.Mapping[typing.AnyStr, typing.AnyStr]] = None,
+        status: int,
+        version: int,
+        reason: str,
+        decode_content: bool,
+    ) -> None:
+        if isinstance(headers, HTTPHeaderDict):
+            self.headers = headers
+        else:
+            self.headers = HTTPHeaderDict(headers)
+        self.status = status
+        self.version = version
+        self.reason = reason
+        self.decode_content = decode_content
+
+        self.chunked = False
+        tr_enc = self.headers.get("transfer-encoding", "").lower()
+        # Don't incur the penalty of creating a list and then discarding it
+        encodings = (enc.strip() for enc in tr_enc.split(","))
+        if "chunked" in encodings:
+            self.chunked = True
+
+        self._decoder: typing.Optional[ContentDecoder] = None
+
+    def get_redirect_location(self) -> typing.Optional[typing.Union[bool, str]]:
+        """
+        Should we redirect and where to?
+
+        :returns: Truthy redirect location string if we got a redirect status
+            code and valid location. ``None`` if redirect status and no
+            location. ``False`` if not a redirect status code.
+        """
+        if self.status in self.REDIRECT_STATUSES:
+            return self.headers.get("location")
+        return False
+
+    @property
+    def data(self) -> bytes:
+        raise NotImplementedError()
+
+    @property
+    def url(self) -> str:
+        raise NotImplementedError()
+
+    @property
+    def closed(self) -> bool:
+        raise NotImplementedError()
+
+    @property
+    def connection(self):
+        raise NotImplementedError()
+
+    def stream(
+        self, amt: int = 2 ** 16, decode_content: typing.Optional[bool] = None
+    ) -> typing.Generator[bytes, None, None]:
+        raise NotImplementedError()
+
+    def read(
+        self,
+        amt: typing.Optional[int] = None,
+        decode_content: typing.Optional[bool] = None,
+        cache_content: bool = False,
+    ) -> bytes:
+        raise NotImplementedError()
+
+    def read_chunked(
+        self,
+        amt: typing.Optional[int] = None,
+        decode_content: typing.Optional[bool] = None,
+    ) -> bytes:
+        raise NotImplementedError()
+
+    def release_conn(self) -> None:
+        raise NotImplementedError()
+
+    def drain_conn(self) -> None:
+        raise NotImplementedError()
+
+    def close(self) -> None:
+        raise NotImplementedError()
+
+    def _init_decoder(self) -> None:
+        """
+        Set-up the _decoder attribute if necessary.
+        """
+        # Note: content-encoding value should be case-insensitive, per RFC 7230
+        # Section 3.2
+        content_encoding = self.headers.get("content-encoding", "").lower()
+        if self._decoder is None:
+            if content_encoding in self.CONTENT_DECODERS:
+                self._decoder = _get_decoder(content_encoding)
+            elif "," in content_encoding:
+                encodings = [
+                    e.strip()
+                    for e in content_encoding.split(",")
+                    if e.strip() in self.CONTENT_DECODERS
+                ]
+                if len(encodings):
+                    self._decoder = _get_decoder(content_encoding)
+
+    def _decode(self, data: bytes, decode_content: bool, flush_decoder: bool) -> bytes:
+        """
+        Decode the data passed in and potentially flush the decoder.
+        """
+        if not decode_content:
+            return data
+
+        try:
+            if self._decoder:
+                data = self._decoder.decompress(data)
+        except self.DECODER_ERROR_CLASSES as e:
+            content_encoding = self.headers.get("content-encoding", "").lower()
+            raise DecodeError(
+                "Received response with content-encoding: %s, but "
+                "failed to decode it." % content_encoding,
+                e,
+            )
+        if flush_decoder:
+            data += self._flush_decoder()
+
+        return data
+
+    def _flush_decoder(self) -> bytes:
+        """
+        Flushes the decoder. Should only be called if the decoder is actually
+        being used.
+        """
+        if self._decoder:
+            return self._decoder.decompress(b"") + self._decoder.flush()
+        return b""
+
+    # Compatibility methods for `io` module
+    def readable(self) -> bool:
+        return True
+
+    def readinto(self, b: bytearray) -> int:
+        temp = self.read(len(b))
+        if len(temp) == 0:
+            return 0
+        else:
+            b[: len(temp)] = temp
+            return len(temp)
+
+    # Compatibility methods for http.client.HTTPResponse
+    def getheaders(self) -> typing.List[typing.Tuple[str, str]]:
+        return list(self.headers.items())
+
+    def getheader(
+        self, name: str, default: typing.Optional[str] = None
+    ) -> typing.Optional[str]:
+        return self.headers.get(name, default)
+
+    # Compatibility method for http.cookiejar
+    def info(self):
+        return self.headers
+
+    def geturl(self):
+        return self.url
+
+
+class HTTPResponse(BaseHTTPResponse):
     """
     HTTP Response container.
 
@@ -184,11 +366,6 @@ class is also compatible with the Python standard library's :mod:`io`
         value of Content-Length header, if present. Otherwise, raise error.
     """
 
-    CONTENT_DECODERS = ["gzip", "deflate"]
-    if brotli is not None:
-        CONTENT_DECODERS += ["br"]
-    REDIRECT_STATUSES = [301, 302, 303, 307, 308]
-
     def __init__(
         self,
         body="",
@@ -208,20 +385,18 @@ def __init__(
         request_url=None,
         auto_close=True,
     ):
+        super().__init__(
+            headers=headers,
+            status=status,
+            version=version,
+            reason=reason,
+            decode_content=decode_content,
+        )
 
-        if isinstance(headers, HTTPHeaderDict):
-            self.headers = headers
-        else:
-            self.headers = HTTPHeaderDict(headers)
-        self.status = status
-        self.version = version
-        self.reason = reason
-        self.decode_content = decode_content
         self.retries = retries
         self.enforce_content_length = enforce_content_length
         self.auto_close = auto_close
 
-        self._decoder = None
         self._body = None
         self._fp = None
         self._original_response = original_response
@@ -239,13 +414,7 @@ def __init__(
             self._fp = body
 
         # Are we using the chunked-style of transfer encoding?
-        self.chunked = False
         self.chunk_left = None
-        tr_enc = self.headers.get("transfer-encoding", "").lower()
-        # Don't incur the penalty of creating a list and then discarding it
-        encodings = (enc.strip() for enc in tr_enc.split(","))
-        if "chunked" in encodings:
-            self.chunked = True
 
         # Determine length of response
         self.length_remaining = self._init_length(request_method)
@@ -254,19 +423,6 @@ def __init__(
         if preload_content and not self._body:
             self._body = self.read(decode_content=decode_content)
 
-    def get_redirect_location(self):
-        """
-        Should we redirect and where to?
-
-        :returns: Truthy redirect location string if we got a redirect status
-            code and valid location. ``None`` if redirect status and no
-            location. ``False`` if not a redirect status code.
-        """
-        if self.status in self.REDIRECT_STATUSES:
-            return self.headers.get("location")
-
-        return False
-
     def release_conn(self):
         if not self._pool or not self._connection:
             return
@@ -361,62 +517,6 @@ def _init_length(self, request_method):
 
         return length
 
-    def _init_decoder(self):
-        """
-        Set-up the _decoder attribute if necessary.
-        """
-        # Note: content-encoding value should be case-insensitive, per RFC 7230
-        # Section 3.2
-        content_encoding = self.headers.get("content-encoding", "").lower()
-        if self._decoder is None:
-            if content_encoding in self.CONTENT_DECODERS:
-                self._decoder = _get_decoder(content_encoding)
-            elif "," in content_encoding:
-                encodings = [
-                    e.strip()
-                    for e in content_encoding.split(",")
-                    if e.strip() in self.CONTENT_DECODERS
-                ]
-                if len(encodings):
-                    self._decoder = _get_decoder(content_encoding)
-
-    DECODER_ERROR_CLASSES = (IOError, zlib.error)
-    if brotli is not None:
-        DECODER_ERROR_CLASSES += (brotli.error,)
-
-    def _decode(self, data, decode_content, flush_decoder):
-        """
-        Decode the data passed in and potentially flush the decoder.
-        """
-        if not decode_content:
-            return data
-
-        try:
-            if self._decoder:
-                data = self._decoder.decompress(data)
-        except self.DECODER_ERROR_CLASSES as e:
-            content_encoding = self.headers.get("content-encoding", "").lower()
-            raise DecodeError(
-                "Received response with content-encoding: %s, but "
-                "failed to decode it." % content_encoding,
-                e,
-            )
-        if flush_decoder:
-            data += self._flush_decoder()
-
-        return data
-
-    def _flush_decoder(self):
-        """
-        Flushes the decoder. Should only be called if the decoder is actually
-        being used.
-        """
-        if self._decoder:
-            buf = self._decoder.decompress(b"")
-            return buf + self._decoder.flush()
-
-        return b""
-
     @contextmanager
     def _error_catcher(self):
         """
@@ -597,17 +697,6 @@ def from_httplib(ResponseCls, r, **response_kw):
         )
         return resp
 
-    # Backwards-compatibility methods for http.client.HTTPResponse
-    def getheaders(self):
-        return self.headers
-
-    def getheader(self, name, default=None):
-        return self.headers.get(name, default)
-
-    # Backwards compatibility for http.cookiejar
-    def info(self):
-        return self.headers
-
     # Overrides from io.IOBase
     def close(self):
         if not self.closed:
@@ -651,19 +740,6 @@ def flush(self):
         ):
             return self._fp.flush()
 
-    def readable(self):
-        # This method is required for `io` module compatibility.
-        return True
-
-    def readinto(self, b):
-        # This method is required for `io` module compatibility.
-        temp = self.read(len(b))
-        if len(temp) == 0:
-            return 0
-        else:
-            b[: len(temp)] = temp
-            return len(temp)
-
     def supports_chunked_reads(self):
         """
         Checks if the underlying file-like object looks like a
@@ -779,7 +855,8 @@ def read_chunked(self, amt=None, decode_content=None):
             if self._original_response:
                 self._original_response.close()
 
-    def geturl(self):
+    @property
+    def url(self) -> str:
         """
         Returns the URL that was the source of this response.
         If the request that generated this response redirected, this method


Test patch:
diff --git a/test/test_response.py b/test/test_response.py
--- a/test/test_response.py
+++ b/test/test_response.py
@@ -55,7 +55,7 @@ class TestLegacyResponse:
     def test_getheaders(self):
         headers = {"host": "example.com"}
         r = HTTPResponse(headers=headers)
-        assert r.getheaders() == headers
+        assert r.getheaders() == [("host", "example.com")]
 
     def test_getheader(self):
         headers = {"host": "example.com"}