Repository: DataBiosphere/toil

Issue:
Exited Docker containers are not cleaned up
I ran this command a bunch in `toil-vg` commit b3c290f7143376978e35e98ed524ca613941de01 or so:

```
make test tests=src/toil_vg/test/test_vg.py::VGCGLTest::test_10_construct_multiple_contigs
```

Now I have a lot of "Exited" Docker containers on my machine:

```
$ docker ps -a | wc -l
1950
```

It looks like when Toil runs a Docker container, it does not delete the finished container. It could be because `toil-vg` always uses `detach=True` with `apiDockerCall`.

Toil should delete the job's containers after the job is finished, or, better yet, always do the equivalent of `docker run --rm` when running Docker containers.

┆Issue is synchronized with this [Jira Task](https://ucsc-cgl.atlassian.net/browse/TOIL-830)
┆Issue Number: TOIL-830



Discussion:
@adamnovak It looks like we would just need to set `auto_remove=True` in `apiDockerCall()` when `detach=True` (and `remove=True, auto_remove=False` when `detach=False`) to properly remove the containers after they exit. Are you suggesting that this should be the default behavior? We might want to be consistent with docker (i.e.: only remove the container if specified by the --rm flag). Though, it would also make sense to clean up everything after a Toil job is finished.
That sounds about right. I would suggest that removing be the default behavior, even though it's not the default with the Docker command on the command line. It's really the only behavior that makes sense for running containerized commands as part of a workflow, unless you want to break into a dead container to debug something somehow.

Resolving patch:
diff --git a/src/toil/lib/docker.py b/src/toil/lib/docker.py
--- a/src/toil/lib/docker.py
+++ b/src/toil/lib/docker.py
@@ -91,8 +91,8 @@ def toil_job(job):
                       working_dir=working_dir,
                       parameters=parameters)
 
-    Note that when run with detatch=False, or with detatch=True and stdout=True
-    or stderr=True, this is a blocking call. When run with detatch=True and
+    Note that when run with detach=False, or with detach=True and stdout=True
+    or stderr=True, this is a blocking call. When run with detach=True and
     without output capture, the container is started and returned without
     waiting for it to finish.
 
@@ -112,7 +112,7 @@ def toil_job(job):
            FORGO (0) leaves the container untouched and running.
            STOP (1) Sends SIGTERM, then SIGKILL if necessary to the container.
            RM (2) Immediately send SIGKILL to the container. This is the default
-           behavior if defer is set to None.
+           behavior if deferParam is set to None.
     :param str name: The name/ID of the container.
     :param str entrypoint: Prepends commands sent to the container.  See:
                       https://docker-py.readthedocs.io/en/stable/containers.html
@@ -157,9 +157,9 @@ def toil_job(job):
                    https://docker-py.readthedocs.io/en/stable/containers.html
 
     :returns: Returns the standard output/standard error text, as requested, when
-              detatch=False. Returns the underlying
+              detach=False. Returns the underlying
               docker.models.containers.Container object from the Docker API when
-              detatch=True.
+              detach=True.
     """
 
     # make certain that files have the correct permissions
@@ -221,16 +221,19 @@ def toil_job(job):
 
     client = docker.from_env(version='auto', timeout=timeout)
 
+    if deferParam is None:
+        deferParam = RM
+
     if deferParam == STOP:
         job.defer(dockerStop, containerName)
 
     if deferParam == FORGO:
-        remove = False
+        # Leave the container untouched and running
+        pass
     elif deferParam == RM:
-        remove = True
-        job.defer(dockerKill, containerName)
-    elif remove is True:
-        job.defer(dockerKill, containerName)
+        job.defer(dockerKill, containerName, remove=True)
+    elif remove:
+        job.defer(dockerKill, containerName, remove=True)
 
     if auto_remove is None:
         auto_remove = remove
@@ -304,11 +307,11 @@ def toil_job(job):
             if stdout or stderr:
                 if streamfile is None:
                     streamfile = 'output.log'
-                for line in container.logs(stdout=stdout, stderr=stderr, stream=True):
+                with open(streamfile, 'wb') as f:
                     # stream=True makes this loop blocking; we will loop until
                     # the container stops and there is no more output.
-                    with open(streamfile, 'w') as f:
-                        f.write(line)
+                    for line in container.logs(stdout=stdout, stderr=stderr, stream=True):
+                        f.write(line.encode() if isinstance(line, str) else line)
 
             # If we didn't capture output, the caller will need to .wait() on
             # the container to know when it is done. Even if we did capture
@@ -326,12 +329,17 @@ def toil_job(job):
         raise create_api_error_from_http_exception(e)
 
 
-def dockerKill(container_name, gentleKill=False, timeout=365 * 24 * 60 * 60):
+def dockerKill(container_name: str,
+               gentleKill: bool = False,
+               remove: bool = False,
+               timeout: int = 365 * 24 * 60 * 60) -> None:
     """
     Immediately kills a container.  Equivalent to "docker kill":
     https://docs.docker.com/engine/reference/commandline/kill/
+
     :param container_name: Name of the container being killed.
-    :param client: The docker API client object to call.
+    :param gentleKill: If True, trigger a graceful shutdown.
+    :param remove: If True, remove the container after it exits.
     :param int timeout: Use the given timeout in seconds for interactions with
                         the Docker daemon. Note that the underlying docker module is
                         not always able to abort ongoing reads and writes in order
@@ -347,36 +355,38 @@ def dockerKill(container_name, gentleKill=False, timeout=365 * 24 * 60 * 60):
             else:
                 client.containers.get(container_name).stop()
             this_container = client.containers.get(container_name)
+        if remove:
+            this_container.remove()
     except NotFound:
-        logger.debug("Attempted to stop container, but container != exist: ",
-                     container_name)
+        logger.debug(f"Attempted to stop container ({container_name}), but container != exist.")
     except requests.exceptions.HTTPError as e:
-        logger.debug("Attempted to stop container, but server gave an error: ",
-                     container_name)
+        logger.debug(f"Attempted to stop container ({container_name}), but server gave an error:")
         raise create_api_error_from_http_exception(e)
 
 
-def dockerStop(container_name):
+def dockerStop(container_name: str, remove: bool = False) -> None:
     """
     Gracefully kills a container.  Equivalent to "docker stop":
     https://docs.docker.com/engine/reference/commandline/stop/
+
     :param container_name: Name of the container being stopped.
-    :param client: The docker API client object to call.
+    :param remove: If True, remove the container after it exits.
     """
-    dockerKill(container_name, gentleKill=True)
+    dockerKill(container_name, gentleKill=True, remove=remove)
 
 
-def containerIsRunning(container_name, timeout=365 * 24 * 60 * 60):
+def containerIsRunning(container_name: str, timeout: int = 365 * 24 * 60 * 60):
     """
     Checks whether the container is running or not.
+
     :param container_name: Name of the container being checked.
-    :returns: True if status is 'running', False if status is anything else,
-    and None if the container does not exist.
     :param int timeout: Use the given timeout in seconds for interactions with
                         the Docker daemon. Note that the underlying docker module is
                         not always able to abort ongoing reads and writes in order
                         to respect the timeout. Defaults to 1 year (i.e. wait
                         essentially indefinitely).
+    :returns: True if status is 'running', False if status is anything else,
+    and None if the container does not exist.
     """
     client = docker.from_env(version='auto', timeout=timeout)
     try:


Test patch:
diff --git a/src/toil/test/lib/dockerTest.py b/src/toil/test/lib/dockerTest.py
--- a/src/toil/test/lib/dockerTest.py
+++ b/src/toil/test/lib/dockerTest.py
@@ -18,8 +18,8 @@
 import uuid
 from threading import Thread
 
-import docker
 from docker.errors import ContainerError
+from toil.common import Toil
 from toil.job import Job
 from toil.leader import FailedJobsException
 from toil.lib.docker import (FORGO,
@@ -70,8 +70,8 @@ def testDockerClean(self,
         # not running.
         #              None     FORGO     STOP    RM
         #    rm        X         R         X      X
-        # detached     R         R         E      X
-        #  Neither     R         R         E      X
+        # detached     X         R         E      X
+        #  Neither     X         R         E      X
 
         data_dir = os.path.join(self.tempDir, 'data')
         working_dir = os.path.join(self.tempDir, 'working')
@@ -85,6 +85,7 @@ def testDockerClean(self,
         options.logLevel = self.dockerTestLogLevel
         options.workDir = working_dir
         options.clean = 'always'
+        options.retryCount = 0  # we're expecting the job to fail so don't retry!
         options.disableCaching = disableCaching
 
         # No base64 logic since it might create a name starting with a `-`.
@@ -96,7 +97,8 @@ def testDockerClean(self,
                           deferParam,
                           container_name)
         try:
-            Job.Runner.startToil(A, options)
+            with Toil(options) as toil:
+                toil.start(A)
         except FailedJobsException:
             # The file created by spooky_container would remain in the directory
             # and since it was created inside the container, it would have had
@@ -106,7 +108,7 @@ def testDockerClean(self,
             assert file_stats.st_gid != 0
             assert file_stats.st_uid != 0
 
-            if (rm and (deferParam != FORGO)) or deferParam == RM:
+            if (rm and (deferParam != FORGO)) or deferParam == RM or deferParam is None:
                 # These containers should not exist
                 assert containerIsRunning(container_name) is None, \
                     'Container was not removed.'
@@ -120,12 +122,13 @@ def testDockerClean(self,
                 # These containers will be running
                 assert containerIsRunning(container_name) == True, \
                     'Container was not running.'
-        client = docker.from_env(version='auto')
-        dockerKill(container_name, client)
-        try:
-            os.remove(test_file)
-        except:
-            pass
+        finally:
+            # Clean up
+            try:
+                dockerKill(container_name, remove=True)
+                os.remove(test_file)
+            except:
+                pass
 
     def testDockerClean_CRx_FORGO(self):
         self.testDockerClean(disableCaching=True, detached=False, rm=True,