Repository: dask/distributed

Issue:
Mismatched TLS certs result in opaque error message
**What happened**:

When trying to connect a client to a scheduler that uses TLS if the credentials are wrong the error message is pretty long and dense.

```python
from dask.distributed import LocalCluster, Client, Security

# Create two security credentials which are intentionally mismatched
cluster_security = Security.temporary()
client_security = Security.temporary()

cluster = LocalCluster(security=cluster_security)
client = Client(cluster, security=client_security)
```

```python-traceback
distributed.comm.tcp - WARNING - Listener on 'tls://127.0.0.1:51494': TLS handshake failed with remote 'tls://127.0.0.1:51513': [SSL: TLSV1_ALERT_UNKNOWN_CA] tlsv1 alert unknown ca (_ssl.c:1125)

---------------------------------------------------------------------------
SSLCertVerificationError                  Traceback (most recent call last)
~/Projects/dask/distributed/distributed/comm/tcp.py in connect(self, address, deserialize, **connection_args)
    375         try:
--> 376             stream = await self.client.connect(
    377                 ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs

~/miniconda3/envs/dask/lib/python3.8/site-packages/tornado/tcpclient.py in connect(self, host, port, af, ssl_options, max_buffer_size, source_ip, source_port, timeout)
    287             else:
--> 288                 stream = await stream.start_tls(
    289                     False, ssl_options=ssl_options, server_hostname=host

~/miniconda3/envs/dask/lib/python3.8/site-packages/tornado/iostream.py in _do_ssl_handshake(self)
   1390             self._handshake_writing = False
-> 1391             self.socket.do_handshake()
   1392         except ssl.SSLError as err:

~/miniconda3/envs/dask/lib/python3.8/ssl.py in do_handshake(self, block)
   1308                 self.settimeout(None)
-> 1309             self._sslobj.do_handshake()
   1310         finally:

SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate (_ssl.c:1125)

The above exception was the direct cause of the following exception:

FatalCommClosedError                      Traceback (most recent call last)
<ipython-input-1-3970b06f9445> in <module>
      6 
      7 cluster = LocalCluster(security=cluster_security)
----> 8 client = Client(cluster, security=client_security)

~/Projects/dask/distributed/distributed/client.py in __init__(self, address, loop, timeout, set_as_default, scheduler_file, security, asynchronous, name, heartbeat_interval, serializers, deserializers, extensions, direct_to_workers, connection_limit, **kwargs)
    755             ext(self)
    756 
--> 757         self.start(timeout=timeout)
    758         Client._instances.add(self)
    759 

~/Projects/dask/distributed/distributed/client.py in start(self, **kwargs)
   1001             self._started = asyncio.ensure_future(self._start(**kwargs))
   1002         else:
-> 1003             sync(self.loop, self._start, **kwargs)
   1004 
   1005     def __await__(self):

~/Projects/dask/distributed/distributed/utils.py in sync(loop, func, callback_timeout, *args, **kwargs)
    324     if error[0]:
    325         typ, exc, tb = error[0]
--> 326         raise exc.with_traceback(tb)
    327     else:
    328         return result[0]

~/Projects/dask/distributed/distributed/utils.py in f()
    307             if callback_timeout is not None:
    308                 future = asyncio.wait_for(future, callback_timeout)
--> 309             result[0] = yield future
    310         except Exception:
    311             error[0] = sys.exc_info()

~/miniconda3/envs/dask/lib/python3.8/site-packages/tornado/gen.py in run(self)
    760 
    761                     try:
--> 762                         value = future.result()
    763                     except Exception:
    764                         exc_info = sys.exc_info()

~/Projects/dask/distributed/distributed/client.py in _start(self, timeout, **kwargs)
   1091 
   1092         try:
-> 1093             await self._ensure_connected(timeout=timeout)
   1094         except (OSError, ImportError):
   1095             await self._close()

~/Projects/dask/distributed/distributed/client.py in _ensure_connected(self, timeout)
   1148 
   1149         try:
-> 1150             comm = await connect(
   1151                 self.scheduler.address, timeout=timeout, **self.connection_args
   1152             )

~/Projects/dask/distributed/distributed/comm/core.py in connect(addr, timeout, deserialize, handshake_overrides, **connection_args)
    284     while time_left() > 0:
    285         try:
--> 286             comm = await asyncio.wait_for(
    287                 connector.connect(loc, deserialize=deserialize, **connection_args),
    288                 timeout=min(intermediate_cap, time_left()),

~/miniconda3/envs/dask/lib/python3.8/asyncio/tasks.py in wait_for(fut, timeout, loop)
    492 
    493         if fut.done():
--> 494             return fut.result()
    495         else:
    496             fut.remove_done_callback(cb)

~/Projects/dask/distributed/distributed/comm/tcp.py in connect(self, address, deserialize, **connection_args)
    388             convert_stream_closed_error(self, e)
    389         except SSLError as err:
--> 390             raise FatalCommClosedError() from err
    391 
    392         local_address = self.prefix + get_stream_address(stream)

FatalCommClosedError: 
```

**What you expected to happen**:

The connection fails with a simple and clear exception.

- Dask version: 2021.6.2
- Python version: 3.8
- Operating System: macos
- Install method (conda, pip, source): source



Discussion:

Resolving patch:
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -5,7 +5,7 @@
 import struct
 import sys
 import weakref
-from ssl import SSLError
+from ssl import SSLCertVerificationError, SSLError
 
 from tornado import gen
 
@@ -386,6 +386,11 @@ async def connect(self, address, deserialize=True, **connection_args):
         except StreamClosedError as e:
             # The socket connect() call failed
             convert_stream_closed_error(self, e)
+        except SSLCertVerificationError as err:
+            raise FatalCommClosedError(
+                "TLS certificate does not match. Check your security settings. "
+                "More info at https://distributed.dask.org/en/latest/tls.html"
+            ) from err
         except SSLError as err:
             raise FatalCommClosedError() from err
 


Test patch: