Repository: DataBiosphere/toil

Issue:
Change dockerCall to use the docker-py API.
dockerCall in Toil as it is, is just a wrapper for calling docker via the terminal with minimal documentation.  This issue proposes a move to implement the docker-py API, a well-documented, standardized, and more natural interface for python to interact with docker.


Discussion:

Resolving patch:
diff --git a/docker/Dockerfile.py b/docker/Dockerfile.py
--- a/docker/Dockerfile.py
+++ b/docker/Dockerfile.py
@@ -91,7 +91,7 @@ def heredoc(s):
         && ln -s /home/s3am/bin/s3am /usr/local/bin/
 
     # Install statically linked version of docker client
-    RUN curl https://get.docker.com/builds/Linux/x86_64/docker-1.12.3.tgz \
+    RUN curl https://get.docker.com/builds/Linux/x86_64/docker-1.13.1.tgz \
          | tar -xvzf - --transform='s,[^/]*/,,g' -C /usr/local/bin/ \
          && chmod u+x /usr/local/bin/docker
 
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -35,7 +35,8 @@ def runSetup():
             'bd2k-python-lib>=1.14a1.dev35',
             'dill==0.2.5',
             'six>=1.10.0',
-            'future'],
+            'future',
+            'docker==2.5.1'],
         extras_require={
             'mesos': [
                 'psutil==3.0.1'],
diff --git a/src/toil/lib/docker.py b/src/toil/lib/docker.py
--- a/src/toil/lib/docker.py
+++ b/src/toil/lib/docker.py
@@ -1,280 +1,361 @@
-"""
-    Module for calling Docker. Assumes `docker` is on the PATH.
-
-    Contains two user-facing functions: dockerCall and dockerCheckOutput
-
-    Uses Toil's defer functionality to ensure containers are shutdown even in case of job or pipeline failure
-
-    Example of using dockerCall in a Toil pipeline to index a FASTA file with SAMtools:
-        def toil_job(job):
-            work_dir = job.fileStore.getLocalTempDir()
-            path = job.fileStore.readGlobalFile(ref_id, os.path.join(work_dir, 'ref.fasta')
-            parameters = ['faidx', path]
-            dockerCall(job, tool='quay.io/ucgc_cgl/samtools:latest', work_dir=work_dir, parameters=parameters)
-"""
-from builtins import str
-from builtins import map
-import base64
+from __future__ import absolute_import
 import logging
 import os
 import pipes
-import uuid
+import subprocess
+import docker
+import base64
+import time
+import requests
+
+from docker.errors import create_api_error_from_http_exception
+from docker.errors import ContainerError
+from docker.errors import ImageNotFound
+from docker.errors import APIError
+from docker.errors import NotFound
+from docker.errors import DockerException
+from docker.utils.types import LogConfig
+from docker.api.container import ContainerApiMixin
 
-from bd2k.util.exceptions import require
 from bd2k.util.retry import retry
+from docker import client
+from pwd import getpwuid
 
-from toil.lib import *
+from toil.lib import dockerPredicate
+from toil.lib import FORGO
+from toil.lib import STOP
+from toil.lib import RM
+from toil.test import timeLimit
 
 _logger = logging.getLogger(__name__)
 
 
+def dockerCheckOutput(job, *args, **kwargs):
+    """
+    Runs dockerCall().
+
+    Provided for backwards compatibility with a previous implementation that
+    used 'subprocess.check_call()' and 'subprocess.check_output()'.  This has
+    since been supplanted and all dockerCall() runs now return the same output
+    so there is no need for a separate dockerCheckOutput() function that calls
+    'subprocess.check_output()'.
+
+    See dockerCall().
+    """
+    return dockerCall(job=job, *args, **kwargs)
+
 def dockerCall(job,
                tool,
                parameters=None,
                workDir=None,
                dockerParameters=None,
-               outfile=None,
-               defer=None):
+               deferParam=None,
+               containerName=None,
+               entrypoint=['/bin/bash', '-c'],
+               detach=False,
+               stderr=None,
+               dataDir='/data',
+               logDriver=None,
+               removeOnExit=False,
+               userPrivilege=None,
+               environment=None,
+               **kwargs):
     """
-    Throws CalledProcessorError if the Docker invocation returns a non-zero exit code
-    This function blocks until the subprocess call to Docker returns
+    A toil wrapper for the python docker API.
 
-    :param toil.Job.job job: The Job instance for the calling function.
-    :param str tool: Name of the Docker image to be used (e.g. quay.io/ucsc_cgl/samtools:latest).
-    :param list[str] parameters: Command line arguments to be passed to the tool.
-           If list of lists: list[list[str]], then treat as successive commands chained with pipe.
-    :param str workDir: Directory to mount into the container via `-v`. Destination convention is /data
-    :param list[str] dockerParameters: Parameters to pass to Docker. Default parameters are `--rm`,
-            `--log-driver none`, and the mountpoint `-v work_dir:/data` where /data is the destination convention.
-             These defaults are removed if docker_parmaters is passed, so be sure to pass them if they are desired.
-    :param file outfile: Pipe output of Docker call to file handle
-    :param int defer: What action should be taken on the container upon job completion?
-           FORGO (0) will leave the container untouched.
-           STOP (1) will attempt to stop the container with `docker stop` (useful for debugging).
-           RM (2) will stop the container and then forcefully remove it from the system
-           using `docker rm -f`. This is the default behavior if defer is set to None.
-    """
-    _docker(job, tool=tool, parameters=parameters, workDir=workDir, dockerParameters=dockerParameters,
-            outfile=outfile, checkOutput=False, defer=defer)
+    Docker API Docs: https://docker-py.readthedocs.io/en/stable/index.html
+    Docker API Code: https://github.com/docker/docker-py
 
+    This implements docker's python API within toil so that calls are run as
+    jobs, with the intention that failed/orphaned docker jobs be handled
+    appropriately.
 
-def dockerCheckOutput(job,
-                      tool,
-                      parameters=None,
-                      workDir=None,
-                      dockerParameters=None,
-                      defer=None):
-    """
-    Returns the stdout from the Docker invocation (via subprocess.check_output)
-    Throws CalledProcessorError if the Docker invocation returns a non-zero exit code
-    This function blocks until the subprocess call to Docker returns
+    Example of using dockerCall in toil to index a FASTA file with SAMtools:
+
+    def toil_job(job):
+        workDir = job.fileStore.getLocalTempDir()
+        path = job.fileStore.readGlobalFile(ref_id,
+                                            os.path.join(work_dir, 'ref.fasta')
+        parameters = ['faidx', path]
+        dockerCall(job,
+                   tool='quay.io/ucgc_cgl/samtools:latest',
+                   workDir=workDir,
+                   parameters=parameters)
 
     :param toil.Job.job job: The Job instance for the calling function.
-    :param str tool: Name of the Docker image to be used (e.g. quay.io/ucsc_cgl/samtools:latest).
-    :param list[str] parameters: Command line arguments to be passed to the tool.
-           If list of lists: list[list[str]], then treat as successive commands chained with pipe.
-    :param str workDir: Directory to mount into the container via `-v`. Destination convention is /data
-    :param list[str] dockerParameters: Parameters to pass to Docker. Default parameters are `--rm`,
-            `--log-driver none`, and the mountpoint `-v work_dir:/data` where /data is the destination convention.
-             These defaults are removed if docker_parmaters is passed, so be sure to pass them if they are desired.
-    :param int defer: What action should be taken on the container upon job completion?
-           FORGO (0) will leave the container untouched.
-           STOP (1) will attempt to stop the container with `docker stop` (useful for debugging).
-           RM (2) will stop the container and then forcefully remove it from the system
-           using `docker rm -f`. This is the default behavior if defer is set to None.
-    :returns: Stdout from the docker call
-    :rtype: str
-    """
-    return _docker(job, tool=tool, parameters=parameters, workDir=workDir,
-                   dockerParameters=dockerParameters, checkOutput=True, defer=defer)
-
-
-def _docker(job,
-            tool,
-            parameters=None,
-            workDir=None,
-            dockerParameters=None,
-            outfile=None,
-            checkOutput=False,
-            defer=None):
-    """
-    :param toil.Job.job job: The Job instance for the calling function.
-    :param str tool: Name of the Docker image to be used (e.g. quay.io/ucsc_cgl/samtools).
-    :param list[str] parameters: Command line arguments to be passed to the tool.
-           If list of lists: list[list[str]], then treat as successive commands chained with pipe.
-    :param str workDir: Directory to mount into the container via `-v`. Destination convention is /data
-    :param list[str] dockerParameters: Parameters to pass to Docker. Default parameters are `--rm`,
-            `--log-driver none`, and the mountpoint `-v work_dir:/data` where /data is the destination convention.
-             These defaults are removed if docker_parmaters is passed, so be sure to pass them if they are desired.
-    :param file outfile: Pipe output of Docker call to file handle
-    :param bool checkOutput: When True, this function returns docker's output.
-    :param int defer: What action should be taken on the container upon job completion?
-           FORGO (0) will leave the container untouched.
-           STOP (1) will attempt to stop the container with `docker stop` (useful for debugging).
-           RM (2) will stop the container and then forcefully remove it from the system
-           using `docker rm -f`. This is the default behavior if defer is set to None.
+    :param str tool: Name of the Docker image to be used.
+                     (e.g. 'quay.io/ucsc_cgl/samtools:latest')
+    :param list[str] parameters: A list of string elements.  If there are
+                                 multiple elements, these will be joined with
+                                 spaces.  This handling of multiple elements
+                                 provides backwards compatibility with previous
+                                 versions which called docker using
+                                 subprocess.check_call().
+                                 **If list of lists: list[list[str]], then treat
+                                 as successive commands chained with pipe.
+    :param str workDir: The working directory where output files are deposited.
+    :param list[str] dockerParameters: Deprecated.  These parameters are bash
+                                       options supplied through check_call such
+                                       as '--rm' and '-d', and are no longer
+                                       used.  This is currently handled for a
+                                       limited number of options with a function
+                                       that converts to the docker API format.
+    :param int deferParam: Action to take on the container upon job completion.
+           FORGO (0) leaves the container untouched and running.
+           STOP (1) Sends SIGTERM, then SIGKILL if necessary to the container.
+           RM (2) Immediately send SIGKILL to the container. This is the default
+           behavior if defer is set to None.
+    :param str containerName: The name of the container.
+    :param str entrypoint: Prepends commands sent to the container.  See:
+                      https://docker-py.readthedocs.io/en/stable/containers.html
+    :param bool detach: Run the container in detached mode. (equivalent to '-d')
+    :param bool stderr: Return stderr after running the container or not.
+    :param str dataDir: The path in the container where the data is held.
+    :param str logDriver: Specify the logs to return from the container.  See:
+                      https://docker-py.readthedocs.io/en/stable/containers.html
+    :param bool removeOnExit: Remove the container on exit or not.
+    :param str userPrivilege: The container will be run with the privileges of
+                              the user specified.  Can be an actual name, such
+                              as 'root' or 'lifeisaboutfishtacos', or it can be
+                              the uid or gid of the user ('0' is root; '1000' is
+                              an example of a less privileged uid or gid), or a
+                              complement of the uid:gid (RECOMMENDED), such as
+                              '0:0' (root user : root group) or '1000:1000'
+                              (some other user : some other user group).
+    :param environment: Allows one to set environment variables inside of the
+                        container, such as:
+    :param kwargs: Additional keyword arguments supplied to the docker API's
+                   run command.  The list is 75 keywords total, for examples
+                   and full documentation see:
+
+                   https://docker-py.readthedocs.io/en/stable/containers.html
+
+                   * NOTE: A number of the above are redundant with kwargs,
+                           mostly due to wanting to supply nonstandard defaults
+                           for toil's dockerCall method in order to maintain
+                           backwards compatibility with existing toil workflows.
+                           Try to use the params provided before using kwargs.
+                           For example, don't use 'tool=XXX' AND 'image=XXX', or
+                           the image used will be submitted twice.
     """
-    if parameters is None:
-        parameters = []
+    dockerApiKeywords = [containerName,
+                         workDir,
+                         dataDir,
+                         removeOnExit,
+                         detach,
+                         environment,
+                         logDriver]
+    # dockerParameters is deprecated, but if used will use some of the
+    # commandline inputs to define variables.  This is limited to a small subset
+    # of supported commandline inputs only intended for backwards compatibility.
+    if dockerParameters:
+        apiParameters = getDeprecatedCmdlineParams(dockerParameters)
+        for k, v in apiParameters:
+            for keyword in dockerApiKeywords:
+                if k == str(keyword):
+                    keyword = v
+
+    # make certain that files have the correct permissions
+    thisUser = os.getuid()
+    thisGroup = os.getgid()
+    if userPrivilege is None:
+        userPrivilege = str(thisUser) + ":" + str(thisGroup)
+
+    if containerName is None:
+        containerName = _getContainerName(job)
+
     if workDir is None:
         workDir = os.getcwd()
 
-    # Setup the outgoing subprocess call for docker
-    baseDockerCall = ['docker', 'run']
-    if dockerParameters:
-        baseDockerCall += dockerParameters
-    else:
-        baseDockerCall += ['--rm', '--log-driver', 'none', '-v',
-                           os.path.abspath(workDir) + ':/data']
-
-    # Ensure the user has passed a valid value for defer
-    require(defer in (None, FORGO, STOP, RM),
-            'Please provide a valid value for defer.')
+    if parameters is None:
+        parameters = []
 
-    # Get container name which is needed for _dockerKill
-    try:
-        if any('--name' in x for x in baseDockerCall):
-            if any('--name=' in x for x in baseDockerCall):
-                containerName = [x.split('=')[1] for x in baseDockerCall if '--name' in x][0]
-            else:
-                containerName = baseDockerCall[baseDockerCall.index('--name') + 1]
-        else:
-            containerName = _getContainerName(job)
-            baseDockerCall.extend(['--name', containerName])
-    except ValueError:
-        containerName = _getContainerName(job)
-        baseDockerCall.extend(['--name', containerName])
-    except IndexError:
-        raise RuntimeError("Couldn't parse Docker's `--name=` option, check parameters: " + str(dockerParameters))
-
-    # Defer the container on-exit action
-    if '--rm' in baseDockerCall and defer is None:
-        defer = RM
-    if '--rm' in baseDockerCall and defer is not RM:
-        _logger.warn('--rm being passed to docker call but defer not set to dockerCall.RM, defer set to: ' + str(defer))
-    job.defer(_dockerKill, containerName, action=defer)
-    # Defer the permission fixing function which will run after this job concludes.
-    # We call this explicitly later on in this function, but we defer it as well to handle unexpected job failure.
-    job.defer(_fixPermissions, tool, workDir)
-
-    # Make subprocess call
-
-    # If parameters is list of lists, treat each list as separate command and chain with pipes
+    # If 'parameters' is a list of lists, treat each list as a separate command
+    # and chain with pipes.
     if len(parameters) > 0 and type(parameters[0]) is list:
-        # When piping, all arguments now get merged into a single string to bash -c.
-        # We try to support spaces in paths by wrapping them all in quotes first.
-        chain_params = [' '.join(p) for p in [list(map(pipes.quote, q)) for q in parameters]]
-        # Use bash's set -eo pipefail to detect and abort on a failure in any command in the chain
-        call = baseDockerCall + ['--entrypoint', '/bin/bash',  tool, '-c',
-                                 'set -eo pipefail && {}'.format(' | '.join(chain_params))]
-    else:
-        call = baseDockerCall + [tool] + parameters
-    _logger.info("Calling docker with " + repr(call))
-
-    params = {}
-    if outfile:
-        params['stdout'] = outfile
-    if checkOutput:
-        callMethod = subprocess.check_output
+        chain_params = \
+            [' '.join((pipes.quote(arg) for arg in command)) \
+             for command in parameters]
+        command = ' | '.join(chain_params)
+        pipe_prefix = "set -eo pipefail && "
+        command = [pipe_prefix + command]
+        _logger.debug("Calling docker with: " + repr(command))
+
+    # If 'parameters' is a normal list, join all elements into a single string
+    # element.
+    # Example: ['echo','the', 'Oread'] becomes: ['echo the Oread']
+    # Note that this is still a list, and the docker API prefers this as best
+    # practice:
+    # http://docker-py.readthedocs.io/en/stable/containers.html
+    elif len(parameters) > 0 and type(parameters) is list:
+        command = ' '.join(parameters)
+        _logger.debug("Calling docker with: " + repr(command))
+
+    # If the 'parameters' lists are empty, they are respecified as None, which
+    # tells the API to simply create and run the container
     else:
-        callMethod = subprocess.check_call
+        entrypoint = None
+        command = None
+
+    workDir = os.path.abspath(workDir)
 
-    for attempt in retry(predicate=dockerPredicate):
-        with attempt:
-            out = callMethod(call, **params)
+    # Ensure the user has passed a valid value for deferParam
+    assert deferParam in (None, FORGO, STOP, RM), \
+        'Please provide a valid value for deferParam.'
 
-    _fixPermissions(tool=tool, workDir=workDir)
-    return out
+    client = docker.from_env(version='auto')
 
+    if deferParam == STOP:
+        job.defer(_dockerStop, containerName, client)
 
-def _dockerKill(containerName, action):
+    if deferParam == FORGO:
+        removeOnExit = False
+    elif deferParam == RM:
+        removeOnExit = True
+        job.defer(_dockerKill, containerName, client)
+    elif removeOnExit is True:
+        job.defer(_dockerKill, containerName, client)
+
+    try:
+        out = client.containers.run(image=tool,
+                                    command=command,
+                                    working_dir=workDir,
+                                    entrypoint=entrypoint,
+                                    name=containerName,
+                                    detach=detach,
+                                    volumes=
+                                     {workDir: {'bind': dataDir, 'mode': 'rw'}},
+                                    auto_remove=removeOnExit,
+                                    remove=removeOnExit,
+                                    log_config=logDriver,
+                                    user=userPrivilege,
+                                    environment=environment,
+                                    **kwargs)
+        return out
+    # If the container exits with a non-zero exit code and detach is False.
+    except ContainerError:
+        _logger.error("Docker had non-zero exit.  Check your command: " + \
+                      repr(command))
+        raise
+    except ImageNotFound:
+        _logger.error("Docker image not found.")
+        raise
+    except requests.exceptions.HTTPError as e:
+        _logger.error("The server returned an error.")
+        raise create_api_error_from_http_exception(e)
+    except:
+        raise
+
+def _dockerKill(container_name, client, gentleKill=False):
     """
-    Kills the specified container.
-    :param str containerName: The name of the container created by docker_call
-    :param int action: What action should be taken on the container?  See `defer=` in
-           :func:`docker_call`
+    Immediately kills a container.  Equivalent to "docker kill":
+    https://docs.docker.com/engine/reference/commandline/kill/
+
+    :param container_name: Name of the container being killed.
+    :param client: The docker API client object to call.
     """
-    running = _containerIsRunning(containerName)
-    if running is None:
-        # This means that the container doesn't exist.  We will see this if the container was run
-        # with --rm and has already exited before this call.
-        _logger.info('The container with name "%s" appears to have already been removed.  Nothing to '
-                  'do.', containerName)
-    else:
-        if action in (None, FORGO):
-            _logger.info('The container with name %s continues to exist as we were asked to forgo a '
-                      'post-job action on it.', containerName)
-        else:
-            _logger.info('The container with name %s exists. Running user-specified defer functions.',
-                         containerName)
-            if running and action >= STOP:
-                _logger.info('Stopping container "%s".', containerName)
-                for attempt in retry(predicate=dockerPredicate):
-                    with attempt:
-                        subprocess.check_call(['docker', 'stop', containerName])
+    try:
+        this_container = client.containers.get(container_name)
+        while this_container.status == 'running':
+            if gentleKill is False:
+                client.containers.get(container_name).kill()
             else:
-                _logger.info('The container "%s" was not found to be running.', containerName)
-            if action >= RM:
-                # If the container was run with --rm, then stop will most likely remove the
-                # container.  We first check if it is running then remove it.
-                running = _containerIsRunning(containerName)
-                if running is not None:
-                    _logger.info('Removing container "%s".', containerName)
-                    for attempt in retry(predicate=dockerPredicate):
-                        with attempt:
-                            subprocess.check_call(['docker', 'rm', '-f', containerName])
-                else:
-                    _logger.info('The container "%s" was not found on the system.  Nothing to remove.',
-                                 containerName)
-
-def _fixPermissions(tool, workDir):
+                client.containers.get(container_name).stop()
+            this_container = client.containers.get(container_name)
+    except NotFound:
+        _logger.debug("Attempted to stop container, but container != exist: ",
+                      container_name)
+    except requests.exceptions.HTTPError as e:
+        _logger.debug("Attempted to stop container, but server gave an error: ",
+                      container_name)
+        raise create_api_error_from_http_exception(e)
+
+def _dockerStop(container_name, client):
     """
-    Fix permission of a mounted Docker directory by reusing the tool to change ownership.
-    Docker natively runs as a root inside the container, and files written to the
-    mounted directory are implicitly owned by root.
+    Gracefully kills a container.  Equivalent to "docker stop":
+    https://docs.docker.com/engine/reference/commandline/stop/
 
-    :param list baseDockerCall: Docker run parameters
-    :param str tool: Name of tool
-    :param str workDir: Path of work directory to recursively chown
+    :param container_name: Name of the container being stopped.
+    :param client: The docker API client object to call.
     """
-    if os.geteuid() == 0:
-        # we're running as root so this chown is redundant
-        return
-
-    baseDockerCall = ['docker', 'run', '--log-driver=none',
-                      '-v', os.path.abspath(workDir) + ':/data', '--rm', '--entrypoint=chown']
-    stat = os.stat(workDir)
-    command = baseDockerCall + [tool] + ['-R', '{}:{}'.format(stat.st_uid, stat.st_gid), '/data']
-    for attempt in retry(predicate=dockerPredicate):
-        with attempt:
-            subprocess.check_call(command)
-
-
-def _getContainerName(job):
-    return '--'.join([str(job),
-                      base64.b64encode(os.urandom(9), '-_')]).replace("'", '').replace('_', '')
-
+    _dockerKill(container_name, client, gentleKill=True)
 
 def _containerIsRunning(container_name):
     """
     Checks whether the container is running or not.
+
     :param container_name: Name of the container being checked.
-    :returns: True if running, False if not running, None if the container doesn't exist.
-    :rtype: bool
+    :returns: True if status is 'running', False if status is anything else,
+    and None if the container does not exist.
     """
+    client = docker.from_env(version='auto')
     try:
-        for attempt in retry(predicate=dockerPredicate):
-            with attempt:
-                output = subprocess.check_output(['docker', 'inspect', '--format', '{{.State.Running}}',
-                                                  container_name]).strip()
-    except subprocess.CalledProcessError:
-        # This will be raised if the container didn't exist.
-        _logger.debug("'docker inspect' failed. Assuming container %s doesn't exist.", container_name,
-                   exc_info=True)
+        this_container = client.containers.get(container_name)
+        if this_container.status == 'running':
+            return True
+        else:
+            # this_container.status == 'exited', 'restarting', or 'paused'
+            return False
+    except NotFound:
         return None
-    if output == 'true':
-        return True
-    elif output == 'false':
-        return False
-    else:
-        raise RuntimeError("Got unexpected value for State.Running (%s)" % output)
+    except requests.exceptions.HTTPError as e:
+        _logger.debug("Server error attempting to call container: ",
+                      container_name)
+        raise create_api_error_from_http_exception(e)
+
+def _getContainerName(job):
+    """Create a random string including the job name, and return it."""
+    return '--'.join([str(job),
+                      base64.b64encode(os.urandom(9), '-_')])\
+                      .replace("'", '').replace('"', '').replace('_', '')
+
+def getDeprecatedCmdlineParams(cmdline_params):
+    """
+    Supports a minimal number of key terms for backwards compatibility with the
+    arguments formerly supplied to check_call as a commandline array of strings.
+    This is not intended to be maintained or developed beyond those needs, as
+    the dockerCall use of the python docker API keywords is strongly encouraged.
+
+    :param cmdline_params: An array of strings intended for use by check_call.
+                           e.g. ['--rm', '-d']
+    :returns: A dictionary of terms compatible with the python docker API.
+    """
+    cmdline_dict = {}
+    skip_to_name = 0
+    skip_to_volume = 0
+    for term in cmdline_params:
+        if term == '--rm':
+            cmdline_dict['removeOnExit'] = True
+
+        if term == '-d':
+            cmdline_dict['detach'] = True
+
+        if term.beginswith('--logdriver'):
+            log_config = term.split('=')[-1]
+            cmdline_dict['log_config'] = log_config
+
+        if skip_to_name == 1:
+            skip_to_name = 0
+            cmdline_dict['name'] = term
+
+        if term == '--name':
+            skip_to_name = 1
+
+        if skip_to_volume == 1:
+            skip_to_volume = 0
+            workDir = term.split(':')[0]
+            dataDir = term.split(':')[-1]
+            cmdline_dict['workDir'] = workDir
+            cmdline_dict['dataDir'] = dataDir
+
+        if term == '-v':
+            skip_to_volume = 1
+
+        if skip_to_envar == 1:
+            skip_to_envar = 0
+            cmdline_dict['environment'] = [term]
+
+        if term == '--env':
+            skip_to_envar = 1
+
+    return cmdline_dict


Test patch:
diff --git a/src/toil/test/lib/dockerTest.py b/src/toil/test/lib/dockerTest.py
--- a/src/toil/test/lib/dockerTest.py
+++ b/src/toil/test/lib/dockerTest.py
@@ -1,226 +1,333 @@
+from __future__ import absolute_import
 import logging
 import signal
 import time
+import os
 import uuid
+import docker
 from threading import Thread
-from subprocess import CalledProcessError
+from docker.errors import ContainerError
 
-import os
-from pwd import getpwuid
 from bd2k.util.files import mkdir_p
 from toil.job import Job
 from toil.leader import FailedJobsException
-from toil.lib.docker import dockerCall, dockerCheckOutput, _containerIsRunning, _dockerKill, STOP, FORGO, RM
 from toil.test import ToilTest, slow
+from toil.lib import FORGO, STOP, RM
+from toil.lib.docker import dockerCall, dockerCheckOutput, \
+                            _containerIsRunning, _dockerKill
 
-_log = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
 
 class DockerTest(ToilTest):
     """
     Tests dockerCall and ensures no containers are left around.
 
-
-    When running tests you may optionally set the TOIL_TEST_TEMP environment variable to the path
-    of a directory where you want temporary test files be placed. The directory will be created
-    if it doesn't exist. The path may be relative in which case it will be assumed to be relative
-    to the project root. If TOIL_TEST_TEMP is not defined, temporary files and directories will
-    be created in the system's default location for such files and any temporary files or
-    directories left over from tests will be removed automatically removed during tear down.
+    When running tests you may optionally set the TOIL_TEST_TEMP environment
+    variable to the path of a directory where you want temporary test files be
+    placed. The directory will be created if it doesn't exist. The path may be
+    relative in which case it will be assumed to be relative to the project
+    root. If TOIL_TEST_TEMP is not defined, temporary files and directories will
+    be created in the system's default location for such files and any temporary
+    files or directories left over from tests will be removed automatically
+    removed during tear down.
     Otherwise, left-over files will not be removed.
     """
     def setUp(self):
         self.tempDir = self._createTempDir(purpose='tempDir')
+        self.dockerTestLogLevel = 'INFO'
 
-    @slow
-    def testDockerClean(self, caching=True):
+    def testDockerClean(self,
+                        disableCaching=True,
+                        detached=True,
+                        rm=True,
+                        deferParam=None):
         """
-        Run the test container that creates a file in the work dir, and sleeps for 5 minutes.  Ensure
-        that the calling job gets SIGKILLed after a minute, leaving behind the spooky/ghost/zombie
-        container. Ensure that the container is killed on batch system shutdown (through the defer
-        mechanism).
-        This inherently also tests _docker
-        :returns: None
+        Run the test container that creates a file in the work dir, and sleeps
+        for 5 minutes.
+
+        Ensure that the calling job gets SIGKILLed after a minute, leaving
+        behind the spooky/ghost/zombie container. Ensure that the container is
+        killed on batch system shutdown (through the deferParam mechanism).
         """
-        # We need to test the behaviour of `defer` with `rm` and `detached`. We do not look at the case
-        # where `rm` and `detached` are both True.  This is the truth table for the different
-        # combinations at the end of the test. R = Running, X = Does not exist, E = Exists but not
-        # running.
+
+        # We need to test the behaviour of `deferParam` with `rm` and
+        # `detached`. We do not look at the case where `rm` and `detached` are
+        # both True.  This is the truth table for the different combinations at
+        # the end of the test. R = Running, X = Does not exist, E = Exists but
+        # not running.
         #              None     FORGO     STOP    RM
         #    rm        X         R         X      X
         # detached     R         R         E      X
         #  Neither     R         R         E      X
-        assert os.getuid() != 0, "Cannot test this if the user is root."
+
         data_dir = os.path.join(self.tempDir, 'data')
         work_dir = os.path.join(self.tempDir, 'working')
-        test_file = os.path.join(data_dir, 'test.txt')
+        test_file = os.path.join(work_dir, 'test.txt')
+
         mkdir_p(data_dir)
         mkdir_p(work_dir)
-        options = Job.Runner.getDefaultOptions(os.path.join(self.tempDir, 'jobstore'))
-        options.logLevel = 'INFO'
+
+        options = Job.Runner.getDefaultOptions(os.path.join(self.tempDir,
+                                                            'jobstore'))
+        options.logLevel = self.dockerTestLogLevel
         options.workDir = work_dir
         options.clean = 'always'
-        if not caching:
-            options.disableCaching = True
-        for rm in (True, False):
-            for detached in (True, False):
-                if detached and rm:
-                    continue
-                for defer in (FORGO, STOP, RM, None):
-                    # Not using base64 logic here since it might create a name starting with a `-`.
-                    container_name = uuid.uuid4().hex
-                    A = Job.wrapJobFn(_testDockerCleanFn, data_dir, detached, rm, defer,
-                                      container_name)
-                    try:
-                        Job.Runner.startToil(A, options)
-                    except FailedJobsException:
-                        # The file created by spooky_container would remain in the directory, and since
-                        # it was created inside the container, it would have had uid and gid == 0 (root)
-                        # upon creation. If the defer mechanism worked, it should now be non-zero and we
-                        # check for that.
-                        file_stats = os.stat(test_file)
-                        assert file_stats.st_gid != 0
-                        assert file_stats.st_uid != 0
-                        if (rm and defer != FORGO) or defer == RM:
-                            # These containers should not exist
-                            assert _containerIsRunning(container_name) is None, \
-                                'Container was not removed.'
-                        elif defer == STOP:
-                            # These containers should exist but be non-running
-                            assert _containerIsRunning(container_name) == False, \
-                                'Container was not stopped.'
-                        else:
-                            # These containers will be running
-                            assert _containerIsRunning(container_name) == True, \
-                                'Container was not running.'
-                    finally:
-                        # Prepare for the next test.
-                        _dockerKill(container_name, RM)
-                        os.remove(test_file)
-
-    def testDockerPipeChain(self, caching=True):
+        options.disableCaching = disableCaching
+
+        # No base64 logic since it might create a name starting with a `-`.
+        container_name = uuid.uuid4().hex
+        A = Job.wrapJobFn(_testDockerCleanFn,
+                          work_dir,
+                          detached,
+                          rm,
+                          deferParam,
+                          container_name)
+        try:
+            Job.Runner.startToil(A, options)
+        except FailedJobsException:
+            # The file created by spooky_container would remain in the directory
+            # and since it was created inside the container, it would have had
+            # uid and gid == 0 (root) which may cause problems when docker
+            # attempts to clean up the jobstore.
+            file_stats = os.stat(test_file)
+            assert file_stats.st_gid != 0
+            assert file_stats.st_uid != 0
+
+            if (rm and (deferParam != FORGO)) or deferParam == RM:
+                # These containers should not exist
+                assert _containerIsRunning(container_name) is None, \
+                    'Container was not removed.'
+
+            elif deferParam == STOP:
+                # These containers should exist but be non-running
+                assert _containerIsRunning(container_name) == False, \
+                    'Container was not stopped.'
+
+            else:
+                # These containers will be running
+                assert _containerIsRunning(container_name) == True, \
+                    'Container was not running.'
+        client = docker.from_env(version='auto')
+        _dockerKill(container_name, client)
+        try:
+            os.remove(test_file)
+        except:
+            pass
+
+    def testDockerClean_CRx_FORGO(self):
+        self.testDockerClean(disableCaching=True, detached=False, rm=True,
+                             deferParam=FORGO)
+
+    def testDockerClean_CRx_STOP(self):
+        self.testDockerClean(disableCaching=True, detached=False, rm=True,
+                             deferParam=STOP)
+
+    def testDockerClean_CRx_RM(self):
+        self.testDockerClean(disableCaching=True, detached=False, rm=True,
+                             deferParam=RM)
+
+    def testDockerClean_CRx_None(self):
+        self.testDockerClean(disableCaching=True, detached=False, rm=True,
+                             deferParam=None)
+
+    @slow
+    def testDockerClean_CxD_FORGO(self):
+        self.testDockerClean(disableCaching=True, detached=True, rm=False,
+                             deferParam=FORGO)
+
+    @slow
+    def testDockerClean_CxD_STOP(self):
+        self.testDockerClean(disableCaching=True, detached=True, rm=False,
+                             deferParam=STOP)
+
+    @slow
+    def testDockerClean_CxD_RM(self):
+        self.testDockerClean(disableCaching=True, detached=True, rm=False,
+                             deferParam=RM)
+
+    @slow
+    def testDockerClean_CxD_None(self):
+        self.testDockerClean(disableCaching=True, detached=True, rm=False,
+                             deferParam=None)
+
+    @slow
+    def testDockerClean_Cxx_FORGO(self):
+        self.testDockerClean(disableCaching=True, detached=False, rm=False,
+                             deferParam=FORGO)
+
+    @slow
+    def testDockerClean_Cxx_STOP(self):
+        self.testDockerClean(disableCaching=True, detached=False, rm=False,
+                             deferParam=STOP)
+
+    @slow
+    def testDockerClean_Cxx_RM(self):
+        self.testDockerClean(disableCaching=True, detached=False, rm=False,
+                             deferParam=RM)
+
+    @slow
+    def testDockerClean_Cxx_None(self):
+        self.testDockerClean(disableCaching=True, detached=False, rm=False,
+                             deferParam=None)
+
+    @slow
+    def testDockerClean_xRx_FORGO(self):
+        self.testDockerClean(disableCaching=False, detached=False, rm=True,
+                             deferParam=FORGO)
+
+    @slow
+    def testDockerClean_xRx_STOP(self):
+        self.testDockerClean(disableCaching=False, detached=False, rm=True,
+                             deferParam=STOP)
+
+    @slow
+    def testDockerClean_xRx_RM(self):
+        self.testDockerClean(disableCaching=False, detached=False, rm=True,
+                             deferParam=RM)
+
+    @slow
+    def testDockerClean_xRx_None(self):
+        self.testDockerClean(disableCaching=False, detached=False, rm=True,
+                             deferParam=None)
+
+    @slow
+    def testDockerClean_xxD_FORGO(self):
+        self.testDockerClean(disableCaching=False, detached=True, rm=False,
+                             deferParam=FORGO)
+
+    @slow
+    def testDockerClean_xxD_STOP(self):
+        self.testDockerClean(disableCaching=False, detached=True, rm=False,
+                             deferParam=STOP)
+
+    @slow
+    def testDockerClean_xxD_RM(self):
+        self.testDockerClean(disableCaching=False, detached=True, rm=False,
+                             deferParam=RM)
+
+    @slow
+    def testDockerClean_xxD_None(self):
+        self.testDockerClean(disableCaching=False, detached=True, rm=False,
+                             deferParam=None)
+
+    @slow
+    def testDockerClean_xxx_FORGO(self):
+        self.testDockerClean(disableCaching=False, detached=False, rm=False,
+                             deferParam=FORGO)
+
+    @slow
+    def testDockerClean_xxx_STOP(self):
+        self.testDockerClean(disableCaching=False, detached=False, rm=False,
+                             deferParam=STOP)
+
+    @slow
+    def testDockerClean_xxx_RM(self):
+        self.testDockerClean(disableCaching=False, detached=False, rm=False,
+                             deferParam=RM)
+
+    @slow
+    def testDockerClean_xxx_None(self):
+        self.testDockerClean(disableCaching=False, detached=False, rm=False,
+                             deferParam=None)
+
+    def testDockerPipeChain(self, disableCaching=True):
         """
-        Test for piping API for dockerCall().  Using this API (activated when list of
-        argument lists is given as parameters), commands a piped together into a chain
+        Test for piping API for dockerCall().  Using this API (activated when
+        list of argument lists is given as parameters), commands a piped
+        together into a chain.
         ex:  parameters=[ ['printf', 'x\n y\n'], ['wc', '-l'] ] should execute:
         printf 'x\n y\n' | wc -l
         """
-        options = Job.Runner.getDefaultOptions(os.path.join(self.tempDir, 'jobstore'))
-        options.logLevel = 'INFO'
+        options = Job.Runner.getDefaultOptions(os.path.join(self.tempDir,
+                                                            'jobstore'))
+        options.logLevel = self.dockerTestLogLevel
         options.workDir = self.tempDir
         options.clean = 'always'
-        if not caching:
-            options.disableCaching = True
+        options.caching = disableCaching
         A = Job.wrapJobFn(_testDockerPipeChainFn)
         rv = Job.Runner.startToil(A, options)
         assert rv.strip() == '2'
 
-    def testDockerPipeChainErrorDetection(self, caching=True):
+    def testDockerPipeChainErrorDetection(self, disableCaching=True):
         """
-        By default, executing cmd1 | cmd2 | ... | cmdN, will only return an error
-        if cmdN fails.  This can lead to all manor of errors being silently missed.
-        This tests to make sure that the piping API for dockerCall() throws an exception
-        if non-last commands in the chain fail.
+        By default, executing cmd1 | cmd2 | ... | cmdN, will only return an
+        error if cmdN fails.  This can lead to all manor of errors being
+        silently missed.  This tests to make sure that the piping API for
+        dockerCall() throws an exception if non-last commands in the chain fail.
         """
-        options = Job.Runner.getDefaultOptions(os.path.join(self.tempDir, 'jobstore'))
-        options.logLevel = 'INFO'
+        options = Job.Runner.getDefaultOptions(os.path.join(self.tempDir,
+                                                            'jobstore'))
+        options.logLevel = self.dockerTestLogLevel
         options.workDir = self.tempDir
         options.clean = 'always'
-        if not caching:
-            options.disableCaching = True
+        options.caching = disableCaching
         A = Job.wrapJobFn(_testDockerPipeChainErrorFn)
         rv = Job.Runner.startToil(A, options)
         assert rv == True
 
-    def testDockerPermissions(self, caching=True):
-        options = Job.Runner.getDefaultOptions(os.path.join(self.tempDir, 'jobstore'))
-        options.logLevel = 'INFO'
-        options.workDir = self.tempDir
-        options.clean = 'always'
-        if not caching:
-            options.disableCaching = True
-        A = Job.wrapJobFn(_testDockerPermissions)
-        Job.Runner.startToil(A, options)
-
-    def testDockerPermissionsNonCaching(self):
-        self.testDockerPermissions(caching=False)
-
     @slow
     def testNonCachingDockerChain(self):
-        self.testDockerPipeChain(caching=False)
+        self.testDockerPipeChain(disableCaching=False)
 
+    @slow
     def testNonCachingDockerChainErrorDetection(self):
-        self.testDockerPipeChainErrorDetection(caching=False)
+        self.testDockerPipeChainErrorDetection(disableCaching=False)
 
-    def testNonCachingDockerClean(self):
-        self.testDockerClean(caching=False)
-
-
-def _testDockerCleanFn(job, workDir, detached=None, rm=None, defer=None, containerName=None):
+def _testDockerCleanFn(job,
+                       workDir,
+                       detached=None,
+                       rm=None,
+                       deferParam=None,
+                       containerName=None):
     """
-    Test function for test docker_clean.  Runs a container with given flags and then dies leaving
-    behind a zombie container
+    Test function for test docker_clean.  Runs a container with given flags and
+    then dies leaving behind a zombie container.
     :param toil.job.Job job: job
     :param workDir: See `work_dir=` in :func:`dockerCall`
     :param bool rm: See `rm=` in :func:`dockerCall`
     :param bool detached: See `detached=` in :func:`dockerCall`
-    :param int defer: See `defer=` in :func:`dockerCall`
+    :param int deferParam: See `deferParam=` in :func:`dockerCall`
     :param str containerName: See `container_name=` in :func:`dockerCall`
-    :return:
     """
-    dockerParameters = ['--log-driver=none', '-v', os.path.abspath(workDir) + ':/data',
-                        '--name', containerName]
-    if detached:
-        dockerParameters.append('-d')
-    if rm:
-        dockerParameters.append('--rm')
-
     def killSelf():
         test_file = os.path.join(workDir, 'test.txt')
-        # This will kill the worker once we are sure the docker container started
+        # Kill the worker once we are sure the docker container is started
         while not os.path.exists(test_file):
-            _log.debug('Waiting on the file created by spooky_container.')
+            _logger.debug('Waiting on the file created by spooky_container.')
             time.sleep(1)
         # By the time we reach here, we are sure the container is running.
-        os.kill(os.getpid(), signal.SIGKILL)  # signal.SIGINT)
+        time.sleep(1)
+        os.kill(os.getpid(), signal.SIGKILL)
+
     t = Thread(target=killSelf)
     # Make it a daemon thread so that thread failure doesn't hang tests.
     t.daemon = True
     t.start()
-    dockerCall(job, tool='quay.io/ucsc_cgl/spooky_test', workDir=workDir, defer=defer, dockerParameters=dockerParameters)
-
+    dockerCall(job,
+               tool='quay.io/ucsc_cgl/spooky_test',
+               workDir=workDir,
+               deferParam=deferParam,
+               containerName=containerName,
+               detach=detached,
+               removeOnExit=rm,
+               privileged=True)
 
 def _testDockerPipeChainFn(job):
-    """
-    Return the result of simple pipe chain.  Should be 2
-    """
+    """Return the result of a simple pipe chain.  Should be 2."""
     parameters = [ ['printf', 'x\n y\n'], ['wc', '-l'] ]
-    return dockerCheckOutput(job, tool='quay.io/ucsc_cgl/spooky_test', parameters=parameters)
+    return dockerCheckOutput(job,
+                             tool='quay.io/ucsc_cgl/spooky_test',
+                             parameters=parameters,
+                             privileged=True)
 
 def _testDockerPipeChainErrorFn(job):
-    """
-    Return True if the command exit 1 | wc -l raises a CalledProcessError when run through 
-    the docker interface
-    """
+    """Return True if the command exit 1 | wc -l raises a ContainerError."""
     parameters = [ ['exit', '1'], ['wc', '-l'] ]
     try:
-        return dockerCheckOutput(job, tool='quay.io/ucsc_cgl/spooky_test', parameters=parameters)
-    except CalledProcessError:
+        dockerCheckOutput(job,
+                          tool='quay.io/ucsc_cgl/spooky_test',
+                          parameters=parameters)
+    except ContainerError:
         return True
     return False
-
-def _testDockerPermissions(job):
-    testDir = job.fileStore.getLocalTempDir()
-    dockerCall(job, tool='ubuntu', workDir=testDir, parameters=[['touch', '/data/test.txt']])
-    outFile = os.path.join(testDir, 'test.txt')
-    assert os.path.exists(outFile)
-    assert not ownerName(outFile) == "root"
-
-
-def ownerName(filename):
-    """
-    Determines a given file's owner
-    :param str filename: path to a file
-    :return: name of filename's owner
-    """
-    return getpwuid(os.stat(filename).st_uid).pw_name