Repository: dask/distributed

Issue:
Add a configuration entry for TCP socket backlog
This may help write tests which would stress saturating the connection accept queue.


Discussion:

Resolving patch:
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -37,6 +37,8 @@ def get_total_physical_memory():
 
 MAX_BUFFER_SIZE = get_total_physical_memory()
 
+DEFAULT_BACKLOG = 2048
+
 
 def set_tcp_timeout(stream):
     """
@@ -337,6 +339,7 @@ def start(self):
         self.tcp_server = TCPServer(max_buffer_size=MAX_BUFFER_SIZE,
                                     **self.server_args)
         self.tcp_server.handle_stream = self._handle_stream
+        backlog = int(config.get('socket-backlog', DEFAULT_BACKLOG))
         for i in range(5):
             try:
                 # When shuffling data between workers, there can
@@ -344,7 +347,7 @@ def start(self):
                 # on a single worker socket, make sure the backlog
                 # is large enough not to lose any.
                 sockets = netutil.bind_sockets(self.port, address=self.ip,
-                                               backlog=2048)
+                                               backlog=backlog)
             except EnvironmentError as e:
                 # EADDRINUSE can happen sporadically when trying to bind
                 # to an ephemeral port


Test patch:
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -16,7 +16,7 @@
 from distributed.metrics import time
 from distributed.utils import All
 from distributed.utils_test import (gen_cluster, cluster, inc, slowinc, loop,
-        slowadd, slow, slowsum)
+        slowadd, slow, slowsum, bump_rlimit)
 from distributed.client import _wait
 from tornado import gen
 
@@ -170,14 +170,7 @@ def test_stress_communication(c, s, *workers):
     da = pytest.importorskip('dask.array')
     # Test consumes many file descriptors and can hang if the limit is too low
     resource = pytest.importorskip('resource')
-    try:
-        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
-        lim = 8192
-        if soft < lim:
-            resource.setrlimit(resource.RLIMIT_NOFILE, (lim, max(hard, lim)))
-    except Exception as e:
-        pytest.skip("file descriptor limit too low and can't be increased :"
-                    + str(e))
+    bump_rlimit(resource.RLIMIT_NOFILE, 8192)
 
     n = 20
     xs = [da.random.random((100, 100), chunks=(5, 5)) for i in range(n)]
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -938,6 +938,7 @@ def get_server_ssl_context(certfile='tls-cert.pem', keyfile='tls-key.pem',
     ctx.load_cert_chain(get_cert(certfile), get_cert(keyfile))
     return ctx
 
+
 def get_client_ssl_context(certfile='tls-cert.pem', keyfile='tls-key.pem',
                            ca_file='tls-ca-cert.pem'):
     ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH,
@@ -946,3 +947,15 @@ def get_client_ssl_context(certfile='tls-cert.pem', keyfile='tls-key.pem',
     ctx.verify_mode = ssl.CERT_REQUIRED
     ctx.load_cert_chain(get_cert(certfile), get_cert(keyfile))
     return ctx
+
+
+def bump_rlimit(limit, desired):
+    resource = pytest.importorskip('resource')
+    try:
+        soft, hard = resource.getrlimit(limit)
+        if soft < desired:
+            resource.setrlimit(limit,
+                               (desired, max(hard, desired)))
+    except Exception as e:
+        pytest.skip("rlimit too low (%s) and can't be increased: %s"
+                    % (soft, e))