Repository: docker/docker-py

Issue:
containers.list() fails if a container is removed while it's being called
I'm getting the following error in an application that uses docker-py with fairly heavy container churn. 

```
Traceback (most recent call last):
  File "executor.py", line 44, in event_loop
    num_executors = len(get_executor_containers())
  File "executor.py", line 186, in get_executor_containers
    running_containers = client.containers.list(all=True)
  File "/home/ben/envs/executor/lib/python3.6/site-packages/docker/models/containers.py", line 767, in list
    return [self.get(r['Id']) for r in resp]
  File "/home/ben/envs/executor/lib/python3.6/site-packages/docker/models/containers.py", line 767, in <listcomp>
    return [self.get(r['Id']) for r in resp]
  File "/home/ben/envs/executor/lib/python3.6/site-packages/docker/models/containers.py", line 720, in get
    resp = self.client.api.inspect_container(container_id)
  File "/home/ben/envs/executor/lib/python3.6/site-packages/docker/utils/decorators.py", line 21, in wrapped
    return f(self, resource_id, *args, **kwargs)
  File "/home/ben/envs/executor/lib/python3.6/site-packages/docker/api/container.py", line 745, in inspect_container
    self._get(self._url("/containers/{0}/json", container)), True
  File "/home/ben/envs/executor/lib/python3.6/site-packages/docker/api/client.py", line 220, in _result
    self._raise_for_status(response)
  File "/home/ben/envs/executor/lib/python3.6/site-packages/docker/api/client.py", line 216, in _raise_for_status
    raise create_api_error_from_http_exception(e)
  File "/home/ben/envs/executor/lib/python3.6/site-packages/docker/errors.py", line 30, in create_api_error_from_http_exception
    raise cls(e, response=response, explanation=explanation)
docker.errors.NotFound: 404 Client Error: Not Found for url: http+docker://localunixsocket/v1.24/containers/762ad9c915018c1d38c0ae71ab44ce69b3750b70b0524d0ec6e01dd62bd7fe48/json ("No such container: 762ad9c915018c1d38c0ae71ab44ce69b3750b70b0524d0ec6e01dd62bd7fe48")
```

It seems to me that list() gets all containers from the api, then *while* it's iterating through that list, a container in the list is removed. This causes the call to list() to fail with a NotFound exception.

I think it could be fixed by replacing models/containers.py:767
```python
return [self.get(r['Id']) for r in resp]
```
with
```python
containers = []
for r in resp:
    try:
        containers.append(self.get(r['id']))
    except NotFound: # a container may have been removed while iterating
        pass
return containers
```
but I'm not sure if that would cause unforeseen issues.

I'm seeing this with a few (3 or 4) containers running at a time, but I think the issue would be much worse if you had many containers running at once.

## Misc info
```
→ pip freeze | grep docker && python --version && docker version

docker==2.1.0
docker-pycreds==0.2.1
Python 3.6.0
Client:
 Version:      1.13.1
 API version:  1.26
 Go version:   go1.7.5
 Git commit:   092cba3727
 Built:        Sun Feb 12 02:40:56 2017
 OS/Arch:      linux/amd64

Server:
 Version:      1.13.1
 API version:  1.26 (minimum version 1.12)
 Go version:   go1.7.5
 Git commit:   092cba3727
 Built:        Sun Feb 12 02:40:56 2017
 OS/Arch:      linux/amd64
 Experimental: false
```

```
→ uname -a
Linux jubal 4.9.11-1-ARCH #1 SMP PREEMPT Sun Feb 19 13:45:52 UTC 2017 x86_64 GNU/Linux
```
```
→ cat /etc/issue
Arch Linux \r (\l)
```


Discussion:
I'm running into the same problem. 
I have also hit this issue. 

Resolving patch:
diff --git a/docker/models/containers.py b/docker/models/containers.py
--- a/docker/models/containers.py
+++ b/docker/models/containers.py
@@ -6,7 +6,7 @@
 from ..constants import DEFAULT_DATA_CHUNK_SIZE
 from ..errors import (
     ContainerError, DockerException, ImageNotFound,
-    create_unexpected_kwargs_error
+    NotFound, create_unexpected_kwargs_error
 )
 from ..types import HostConfig
 from ..utils import version_gte
@@ -844,7 +844,7 @@ def get(self, container_id):
         return self.prepare_model(resp)
 
     def list(self, all=False, before=None, filters=None, limit=-1, since=None,
-             sparse=False):
+             sparse=False, ignore_removed=False):
         """
         List containers. Similar to the ``docker ps`` command.
 
@@ -882,6 +882,10 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None,
                 information, but guaranteed not to block. Use
                 :py:meth:`Container.reload` on resulting objects to retrieve
                 all attributes. Default: ``False``
+            ignore_removed (bool): Ignore failures due to missing containers
+                when attempting to inspect containers from the original list.
+                Set to ``True`` if race conditions are likely. Has no effect
+                if ``sparse=True``. Default: ``False``
 
         Returns:
             (list of :py:class:`Container`)
@@ -896,7 +900,15 @@ def list(self, all=False, before=None, filters=None, limit=-1, since=None,
         if sparse:
             return [self.prepare_model(r) for r in resp]
         else:
-            return [self.get(r['Id']) for r in resp]
+            containers = []
+            for r in resp:
+                try:
+                    containers.append(self.get(r['Id']))
+                # a container may have been removed while iterating
+                except NotFound:
+                    if not ignore_removed:
+                        raise
+            return containers
 
     def prune(self, filters=None):
         return self.client.api.prune_containers(filters=filters)


Test patch:
diff --git a/tests/unit/fake_api_client.py b/tests/unit/fake_api_client.py
--- a/tests/unit/fake_api_client.py
+++ b/tests/unit/fake_api_client.py
@@ -20,15 +20,18 @@ def _mock_call(self, *args, **kwargs):
         return ret
 
 
-def make_fake_api_client():
+def make_fake_api_client(overrides=None):
     """
     Returns non-complete fake APIClient.
 
     This returns most of the default cases correctly, but most arguments that
     change behaviour will not work.
     """
+
+    if overrides is None:
+        overrides = {}
     api_client = docker.APIClient()
-    mock_client = CopyReturnMagicMock(**{
+    mock_attrs = {
         'build.return_value': fake_api.FAKE_IMAGE_ID,
         'commit.return_value': fake_api.post_fake_commit()[1],
         'containers.return_value': fake_api.get_fake_containers()[1],
@@ -47,15 +50,18 @@ def make_fake_api_client():
         'networks.return_value': fake_api.get_fake_network_list()[1],
         'start.return_value': None,
         'wait.return_value': {'StatusCode': 0},
-    })
+    }
+    mock_attrs.update(overrides)
+    mock_client = CopyReturnMagicMock(**mock_attrs)
+
     mock_client._version = docker.constants.DEFAULT_DOCKER_API_VERSION
     return mock_client
 
 
-def make_fake_client():
+def make_fake_client(overrides=None):
     """
     Returns a Client with a fake APIClient.
     """
     client = docker.DockerClient()
-    client.api = make_fake_api_client()
+    client.api = make_fake_api_client(overrides)
     return client
diff --git a/tests/unit/models_containers_test.py b/tests/unit/models_containers_test.py
--- a/tests/unit/models_containers_test.py
+++ b/tests/unit/models_containers_test.py
@@ -359,6 +359,18 @@ def test_list(self):
         assert isinstance(containers[0], Container)
         assert containers[0].id == FAKE_CONTAINER_ID
 
+    def test_list_ignore_removed(self):
+        def side_effect(*args, **kwargs):
+            raise docker.errors.NotFound('Container not found')
+        client = make_fake_client({
+            'inspect_container.side_effect': side_effect
+        })
+
+        with pytest.raises(docker.errors.NotFound):
+            client.containers.list(all=True, ignore_removed=False)
+
+        assert client.containers.list(all=True, ignore_removed=True) == []
+
 
 class ContainerTest(unittest.TestCase):
     def test_name(self):