Repository: urllib3/urllib3

Issue:
Shouldn't urllib3 verify SSL/TLS certificates by default ?
The README says:

> ... urllib3 brings many critical features that are missing from the Python standard libraries: 
> ...
> Client-side SSL/TLS verification.
> ...

and only in https://urllib3.readthedocs.io/en/latest/user-guide.html#certificate-verification it's explained that 

> By default, urllib3 does not verify HTTPS requests.

Because since [pep-0476](https://www.python.org/dev/peps/pep-0476) (ie python >= 2.7.9 or > 3.4 ) standard libraries verifies SSL certificates by default, nowadays projects switching from urllib2 or httplib to urllib3 without explicitly enabling certificate validation would loose the feature of certificate validation that was provided by standard library.

Of course, this would be an incompatible breaking change, like it was for python at the time, but is it really still a sane default to not verify certificates nowadays ?


Discussion:
I think if we were to "do-it-all-over" we'd have TLS verification on by default but to preserve backwards compatibility we'll probably keep the current behavior until v2. Currently a warning is emitted when using an `HTTPSConnection` without TLS verification.
For the records, see https://github.com/SeleniumHQ/selenium/issues/6534 for a case where switching from httplib to urllib3 accidentally disabled certificate validation ( the change introducing the issue https://github.com/SeleniumHQ/selenium/commit/8d98f3223c3757bac24f779f5d8c1e9e1e8edcfb )
I would personally be totally okay with us changing this behavior in a minor release (Python did it in a patch release, hilariously). I would want some level of consensus with the maintainers of this library and Requests.
Knowing Python did this change in a patch makes me feel more comfortable upgrading to verify by default. I have a feeling many of our downstreams already implement this anyways. 
@theacodes, I believe Requests already has verifying on by default, so I’m fully in support of this. I’m assuming this isn’t an api change beyond changing the default value for the function signature?
Correct.

On Tue, Oct 16, 2018 at 9:41 AM Nate Prewitt <notifications@github.com>
wrote:

> @theacodes <https://github.com/theacodes>, I believe Requests already has
> verifying on by default, so I’m fully in support of this. I’m assuming this
> isn’t an api change beyond changing the default value for the function
> signature?
>
> —
> You are receiving this because you were mentioned.
>
>
> Reply to this email directly, view it on GitHub
> <https://github.com/urllib3/urllib3/issues/1454#issuecomment-430309401>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AAPUc1wfMmXZ8f9lCErmvkgp0dKnQpmGks5ulgwugaJpZM4XdWzL>
> .
>

I'd say this is probably ready to go then :)

Resolving patch:
diff --git a/src/urllib3/connection.py b/src/urllib3/connection.py
--- a/src/urllib3/connection.py
+++ b/src/urllib3/connection.py
@@ -282,15 +282,13 @@ def set_cert(self, key_file=None, cert_file=None,
         """
         This method should only be called once, before the connection is used.
         """
-        # If cert_reqs is not provided, we can try to guess. If the user gave
-        # us a cert database, we assume they want to use it: otherwise, if
-        # they gave us an SSL Context object we should use whatever is set for
-        # it.
+        # If cert_reqs is not provided we'll assume CERT_REQUIRED unless we also
+        # have an SSLContext object in which case we'll use its verify_mode.
         if cert_reqs is None:
-            if ca_certs or ca_cert_dir:
-                cert_reqs = 'CERT_REQUIRED'
-            elif self.ssl_context is not None:
+            if self.ssl_context is not None:
                 cert_reqs = self.ssl_context.verify_mode
+            else:
+                cert_reqs = resolve_cert_reqs(None)
 
         self.key_file = key_file
         self.cert_file = cert_file
diff --git a/src/urllib3/connectionpool.py b/src/urllib3/connectionpool.py
--- a/src/urllib3/connectionpool.py
+++ b/src/urllib3/connectionpool.py
@@ -767,9 +767,6 @@ def __init__(self, host, port=None,
                                     block, headers, retries, _proxy, _proxy_headers,
                                     **conn_kw)
 
-        if ca_certs and cert_reqs is None:
-            cert_reqs = 'CERT_REQUIRED'
-
         self.key_file = key_file
         self.cert_file = cert_file
         self.cert_reqs = cert_reqs
diff --git a/src/urllib3/util/ssl_.py b/src/urllib3/util/ssl_.py
--- a/src/urllib3/util/ssl_.py
+++ b/src/urllib3/util/ssl_.py
@@ -54,7 +54,7 @@ def _const_compare_digest_backport(a, b):
 
 try:  # Test for SSL features
     import ssl
-    from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23
+    from ssl import wrap_socket, CERT_REQUIRED, PROTOCOL_SSLv23
     from ssl import HAS_SNI  # Has SNI?
 except ImportError:
     pass
@@ -189,7 +189,7 @@ def resolve_cert_reqs(candidate):
     constant which can directly be passed to wrap_socket.
     """
     if candidate is None:
-        return CERT_NONE
+        return CERT_REQUIRED
 
     if isinstance(candidate, str):
         res = getattr(ssl, candidate, None)


Test patch:
diff --git a/test/contrib/test_socks.py b/test/contrib/test_socks.py
--- a/test/contrib/test_socks.py
+++ b/test/contrib/test_socks.py
@@ -4,7 +4,7 @@
 from urllib3.contrib import socks
 from urllib3.exceptions import ConnectTimeoutError, NewConnectionError
 
-from dummyserver.server import DEFAULT_CERTS
+from dummyserver.server import DEFAULT_CERTS, DEFAULT_CA
 from dummyserver.testcase import IPV4SocketDummyServerTestCase
 
 import pytest
@@ -715,7 +715,7 @@ def request_handler(listener):
 
         self._start_server(request_handler)
         proxy_url = "socks5h://%s:%s" % (self.host, self.port)
-        pm = socks.SOCKSProxyManager(proxy_url)
+        pm = socks.SOCKSProxyManager(proxy_url, ca_certs=DEFAULT_CA)
         self.addCleanup(pm.clear)
         response = pm.request('GET', 'https://localhost')
 
diff --git a/test/test_connectionpool.py b/test/test_connectionpool.py
--- a/test/test_connectionpool.py
+++ b/test/test_connectionpool.py
@@ -1,5 +1,6 @@
 from __future__ import absolute_import
 
+import ssl
 import pytest
 
 from urllib3.connectionpool import (
@@ -350,7 +351,7 @@ def test_absolute_url(self):
     def test_ca_certs_default_cert_required(self):
         with connection_from_url('https://google.com:80', ca_certs=DEFAULT_CA) as pool:
             conn = pool._get_conn()
-            assert conn.cert_reqs == 'CERT_REQUIRED'
+            assert conn.cert_reqs == ssl.CERT_REQUIRED
 
     def test_cleanup_on_extreme_connection_error(self):
         """
diff --git a/test/test_util.py b/test/test_util.py
--- a/test/test_util.py
+++ b/test/test_util.py
@@ -473,7 +473,7 @@ def test_timeout_elapsed(self, current_time):
         assert timeout.get_connect_duration() == 37
 
     @pytest.mark.parametrize('candidate, requirements', [
-        (None, ssl.CERT_NONE),
+        (None, ssl.CERT_REQUIRED),
         (ssl.CERT_NONE, ssl.CERT_NONE),
         (ssl.CERT_REQUIRED, ssl.CERT_REQUIRED),
         ('REQUIRED', ssl.CERT_REQUIRED),
diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py
--- a/test/with_dummyserver/test_https.py
+++ b/test/with_dummyserver/test_https.py
@@ -31,7 +31,6 @@
 from urllib3 import HTTPSConnectionPool
 from urllib3.connection import (
     VerifiedHTTPSConnection,
-    UnverifiedHTTPSConnection,
     RECENT_DATE,
 )
 from urllib3.exceptions import (
@@ -60,7 +59,7 @@
 
 class TestHTTPS(HTTPSDummyServerTestCase):
     def setUp(self):
-        self._pool = HTTPSConnectionPool(self.host, self.port)
+        self._pool = HTTPSConnectionPool(self.host, self.port, ca_certs=DEFAULT_CA)
         self.addCleanup(self._pool.close)
 
     def test_simple(self):
@@ -69,7 +68,7 @@ def test_simple(self):
 
     @fails_on_travis_gce
     def test_dotted_fqdn(self):
-        pool = HTTPSConnectionPool(self.host + '.', self.port)
+        pool = HTTPSConnectionPool(self.host + '.', self.port, ca_certs=DEFAULT_CA)
         r = pool.request('GET', '/')
         self.assertEqual(r.status, 200, r.data)
 
@@ -85,7 +84,8 @@ def test_client_intermediate(self):
         )
         https_pool = HTTPSConnectionPool(self.host, self.port,
                                          key_file=client_key,
-                                         cert_file=client_cert)
+                                         cert_file=client_cert,
+                                         ca_certs=DEFAULT_CA)
         r = https_pool.request('GET', '/certificate')
         subject = json.loads(r.data.decode('utf-8'))
         assert subject['organizationalUnitName'].startswith(
@@ -98,7 +98,8 @@ def test_client_no_intermediate(self):
         )
         https_pool = HTTPSConnectionPool(self.host, self.port,
                                          cert_file=client_cert,
-                                         key_file=client_key)
+                                         key_file=client_key,
+                                         ca_certs=DEFAULT_CA)
         try:
             https_pool.request('GET', '/certificate', retries=False)
         except SSLError as e:
@@ -121,6 +122,7 @@ def test_client_key_password(self):
             PASSWORD_CLIENT_KEYFILE,
         )
         https_pool = HTTPSConnectionPool(self.host, self.port,
+                                         ca_certs=DEFAULT_CA,
                                          key_file=client_key,
                                          cert_file=client_cert,
                                          key_password="letmein")
@@ -310,8 +312,7 @@ def test_no_ssl(self):
 
     def test_unverified_ssl(self):
         """ Test that bare HTTPSConnection can connect, make requests """
-        pool = HTTPSConnectionPool(self.host, self.port)
-        pool.ConnectionCls = UnverifiedHTTPSConnection
+        pool = HTTPSConnectionPool(self.host, self.port, cert_reqs=ssl.CERT_NONE)
         self.addCleanup(pool.close)
 
         with mock.patch('warnings.warn') as warn:
@@ -551,15 +552,14 @@ def new_pool(timeout, cert_reqs='CERT_REQUIRED'):
     def test_enhanced_ssl_connection(self):
         fingerprint = '92:81:FE:85:F7:0C:26:60:EC:D6:B3:BF:93:CF:F9:71:CC:07:7D:0A'
 
-        conn = VerifiedHTTPSConnection(self.host, self.port)
-        self.addCleanup(conn.close)
         https_pool = HTTPSConnectionPool(self.host, self.port,
                                          cert_reqs='CERT_REQUIRED',
                                          ca_certs=DEFAULT_CA,
                                          assert_fingerprint=fingerprint)
         self.addCleanup(https_pool.close)
 
-        https_pool._make_request(conn, 'GET', '/')
+        r = https_pool.request('GET', '/')
+        assert r.status == 200
 
     @onlyPy279OrNewer
     def test_ssl_correct_system_time(self):
@@ -610,8 +610,8 @@ def test_discards_connection_on_sslerror(self):
 
     def test_set_cert_default_cert_required(self):
         conn = VerifiedHTTPSConnection(self.host, self.port)
-        conn.set_cert(ca_certs=DEFAULT_CA)
-        self.assertEqual(conn.cert_reqs, 'CERT_REQUIRED')
+        conn.set_cert()
+        self.assertEqual(conn.cert_reqs, ssl.CERT_REQUIRED)
 
 
 class TestHTTPS_NoSAN(HTTPSDummyServerTestCase):
diff --git a/test/with_dummyserver/test_no_ssl.py b/test/with_dummyserver/test_no_ssl.py
--- a/test/with_dummyserver/test_no_ssl.py
+++ b/test/with_dummyserver/test_no_ssl.py
@@ -21,7 +21,7 @@ def test_simple(self):
 
 class TestHTTPSWithoutSSL(HTTPSDummyServerTestCase, TestWithoutSSL):
     def test_simple(self):
-        pool = urllib3.HTTPSConnectionPool(self.host, self.port)
+        pool = urllib3.HTTPSConnectionPool(self.host, self.port, cert_reqs="NONE")
         self.addCleanup(pool.close)
         try:
             pool.request('GET', '/')
diff --git a/test/with_dummyserver/test_proxy_poolmanager.py b/test/with_dummyserver/test_proxy_poolmanager.py
--- a/test/with_dummyserver/test_proxy_poolmanager.py
+++ b/test/with_dummyserver/test_proxy_poolmanager.py
@@ -28,7 +28,7 @@ def setUp(self):
         self.proxy_url = 'http://%s:%d' % (self.proxy_host, self.proxy_port)
 
     def test_basic_proxy(self):
-        http = proxy_from_url(self.proxy_url)
+        http = proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA)
         self.addCleanup(http.clear)
 
         r = http.request('GET', '%s/' % self.http_url)
@@ -66,7 +66,7 @@ def test_proxy_conn_fail(self):
             self.assertEqual(type(e.reason), ProxyError)
 
     def test_oldapi(self):
-        http = ProxyManager(connection_from_url(self.proxy_url))
+        http = ProxyManager(connection_from_url(self.proxy_url), ca_certs=DEFAULT_CA)
         self.addCleanup(http.clear)
 
         r = http.request('GET', '%s/' % self.http_url)
@@ -146,7 +146,7 @@ def test_cross_host_redirect(self):
         self.assertNotEqual(r._pool.host, self.http_host_alt)
 
     def test_cross_protocol_redirect(self):
-        http = proxy_from_url(self.proxy_url)
+        http = proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA)
         self.addCleanup(http.clear)
 
         cross_protocol_location = '%s/echo?a=b' % self.https_url
@@ -166,7 +166,8 @@ def test_cross_protocol_redirect(self):
 
     def test_headers(self):
         http = proxy_from_url(self.proxy_url, headers={'Foo': 'bar'},
-                              proxy_headers={'Hickory': 'dickory'})
+                              proxy_headers={'Hickory': 'dickory'},
+                              ca_certs=DEFAULT_CA)
         self.addCleanup(http.clear)
 
         r = http.request_encode_url('GET', '%s/headers' % self.http_url)
@@ -190,13 +191,6 @@ def test_headers(self):
         self.assertEqual(returned_headers.get('Host'),
                          '%s:%s' % (self.https_host, self.https_port))
 
-        r = http.request_encode_url('GET', '%s/headers' % self.https_url_alt)
-        returned_headers = json.loads(r.data.decode())
-        self.assertEqual(returned_headers.get('Foo'), 'bar')
-        self.assertIsNone(returned_headers.get('Hickory'))
-        self.assertEqual(returned_headers.get('Host'),
-                         '%s:%s' % (self.https_host_alt, self.https_port))
-
         r = http.request_encode_body('POST', '%s/headers' % self.http_url)
         returned_headers = json.loads(r.data.decode())
         self.assertEqual(returned_headers.get('Foo'), 'bar')
@@ -254,7 +248,7 @@ def test_headerdict(self):
         self.assertEqual(returned_headers.get('Baz'), 'quux')
 
     def test_proxy_pooling(self):
-        http = proxy_from_url(self.proxy_url)
+        http = proxy_from_url(self.proxy_url, cert_reqs='NONE')
         self.addCleanup(http.clear)
 
         for x in range(2):
@@ -320,7 +314,7 @@ def test_https_proxy_pool_timeout(self):
 
     def test_scheme_host_case_insensitive(self):
         """Assert that upper-case schemes and hosts are normalized."""
-        http = proxy_from_url(self.proxy_url.upper())
+        http = proxy_from_url(self.proxy_url.upper(), ca_certs=DEFAULT_CA)
         self.addCleanup(http.clear)
 
         r = http.request('GET', '%s/' % self.http_url.upper())
@@ -342,7 +336,7 @@ def setUp(self):
         self.proxy_url = 'http://[%s]:%d' % (self.proxy_host, self.proxy_port)
 
     def test_basic_ipv6_proxy(self):
-        http = proxy_from_url(self.proxy_url)
+        http = proxy_from_url(self.proxy_url, ca_certs=DEFAULT_CA)
         self.addCleanup(http.clear)
 
         r = http.request('GET', '%s/' % self.http_url)
diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py
--- a/test/with_dummyserver/test_socketlevel.py
+++ b/test/with_dummyserver/test_socketlevel.py
@@ -972,7 +972,7 @@ def echo_socket_handler(listener):
         self._start_server(echo_socket_handler)
         base_url = 'http://%s:%d' % (self.host, self.port)
 
-        proxy = proxy_from_url(base_url)
+        proxy = proxy_from_url(base_url, ca_certs=DEFAULT_CA)
         self.addCleanup(proxy.clear)
 
         url = 'https://{0}'.format(self.host)
@@ -1016,7 +1016,7 @@ def echo_socket_handler(listener):
         self._start_server(echo_socket_handler)
         base_url = 'http://%s:%d' % (self.host, self.port)
 
-        proxy = proxy_from_url(base_url)
+        proxy = proxy_from_url(base_url, cert_reqs='NONE')
         self.addCleanup(proxy.clear)
 
         url = 'https://[{0}]'.format(ipv6_addr)
@@ -1070,8 +1070,7 @@ def socket_handler(listener):
             ssl_sock = ssl.wrap_socket(sock,
                                        server_side=True,
                                        keyfile=DEFAULT_CERTS['keyfile'],
-                                       certfile=DEFAULT_CERTS['certfile'],
-                                       ca_certs=DEFAULT_CA)
+                                       certfile=DEFAULT_CERTS['certfile'])
 
             buf = b''
             while not buf.endswith(b'\r\n\r\n'):
@@ -1090,7 +1089,7 @@ def socket_handler(listener):
             ssl_sock.close()
 
         self._start_server(socket_handler)
-        pool = HTTPSConnectionPool(self.host, self.port)
+        pool = HTTPSConnectionPool(self.host, self.port, ca_certs=DEFAULT_CA)
         self.addCleanup(pool.close)
 
         response = pool.urlopen('GET', '/', retries=0, preload_content=False,
@@ -1180,7 +1179,7 @@ def socket_handler(listener):
 
         self._start_server(socket_handler)
 
-        pool = HTTPSConnectionPool(self.host, self.port)
+        pool = HTTPSConnectionPool(self.host, self.port, ca_certs=DEFAULT_CA)
         self.addCleanup(pool.close)
         response = pool.urlopen('GET', '/', retries=1)
         self.assertEqual(response.data, b'Success')