Repository: dask/distributed

Issue:
Add preload scripts to configuration
Users can provide a `--preload` script to their dask-worker or dask-scheduler commands to launch custom code during execution.  This is pragmatic when you want to extend Dask with plugins, but somewhat annoying to do every time.

We might add two entries to the configuration `distribued.scheduler.preloads` and `distributed.worker.preloads` that are lists of scripts or imports to pass to the Worker and Scheduler on startup.  This should be the default within the `worker.py::WorkerBase` and `scheduler.py::Scheduler` constructors.

This was first suggested, I think, by @lesteve 


Discussion:

Resolving patch:
diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -1,6 +1,7 @@
 from __future__ import print_function, division, absolute_import
 
 import atexit
+import dask
 import logging
 import os
 import shutil
@@ -58,7 +59,7 @@
               "cluster is on a shared network file system.")
 @click.option('--local-directory', default='', type=str,
               help="Directory to place scheduler files")
-@click.option('--preload', type=str, multiple=True, is_eager=True,
+@click.option('--preload', type=str, multiple=True, is_eager=True, default='',
               help='Module that should be loaded by the scheduler process  '
                    'like "foo.bar" or "/path/to/foo.py".')
 @click.argument('preload_argv', nargs=-1,
@@ -125,6 +126,10 @@ def del_pid_file():
                           scheduler_file=scheduler_file,
                           security=sec)
     scheduler.start(addr)
+    if not preload:
+        preload = dask.config.get('distributed.scheduler.preload')
+    if not preload_argv:
+        preload_argv = dask.config.get('distributed.scheduler.preload-argv')
     preload_modules(preload, parameter=scheduler, file_dir=local_directory, argv=preload_argv)
 
     logger.info('Local Directory: %26s', local_directory)
diff --git a/distributed/worker.py b/distributed/worker.py
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -87,7 +87,7 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
                  services=None, service_ports=None, name=None,
                  reconnect=True, memory_limit='auto',
                  executor=None, resources=None, silence_logs=None,
-                 death_timeout=None, preload=(), preload_argv=[], security=None,
+                 death_timeout=None, preload=None, preload_argv=None, security=None,
                  contact_address=None, memory_monitor_interval='200ms',
                  extensions=None, metrics=None, **kwargs):
 
@@ -108,7 +108,11 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
         self.available_resources = (resources or {}).copy()
         self.death_timeout = death_timeout
         self.preload = preload
-        self.preload_argv = preload_argv,
+        if self.preload is None:
+            self.preload = dask.config.get('distributed.worker.preload')
+        self.preload_argv = preload_argv
+        if self.preload_argv is None:
+            self.preload_argv = dask.config.get('distributed.worker.preload-argv')
         self.contact_address = contact_address
         self.memory_monitor_interval = parse_timedelta(memory_monitor_interval, default='ms')
         self.extensions = dict()


Test patch: