Repository: containrrr/watchtower (go)

Issue:
Compatibility issue with docker compose abort-on-container-exit flag
### Describe the bug

Watchtower causes an issue for me in combination with the `--abort-on-container-exit` flag for dockers `compose up` subcommand.

Docker compose removed the container on its own after the container was stopped. This leads to an error when watchtower attempts to remove the container.

`Error: No such container: 425173...`

The error is thrown by this [ContainerRemove](https://github.com/containrrr/watchtower/blob/d744c3488667d6cc4bb8258bb4b351bfd80e1602/pkg/container/client.go#L194) call.

### Steps to reproduce

1. Run container with `docker compose up --abort-on-container-exit`
2. Update image with watchtower


### Expected behavior

I expect for watchtower to be able to skip the container removal, either automatically if the container is already gone or with an explicit option.

### Screenshots

_No response_

### Environment

- Debian 11 amd64
- Docker 20.10.21


### Your logs

```text
...
Nov 20 23:09:37 docker[17234]: watchtower  | time="2022-11-20T23:09:37+01:00" level=debug msg="Found a match"
Nov 20 23:09:37 docker[17234]: watchtower  | time="2022-11-20T23:09:37+01:00" level=debug msg="No pull needed. Skipping image."
Nov 20 23:09:37 docker[17234]: watchtower  | time="2022-11-20T23:09:37+01:00" level=debug msg="No new images found for /traefik"
Nov 20 23:09:37 docker[17234]: watchtower  | time="2022-11-20T23:09:37+01:00" level=debug msg="This is the watchtower container /watchtower"
Nov 20 23:09:37 docker[17234]: watchtower  | time="2022-11-20T23:09:37+01:00" level=info msg="Stopping /website (425173d9c76c) with SIGTERM"
Nov 20 23:09:38 docker[17234]: watchtower  | time="2022-11-20T23:09:38+01:00" level=debug msg="Removing container 425173d9c76c"
Nov 20 23:09:38 docker[17234]: watchtower  | time="2022-11-20T23:09:38+01:00" level=error msg="Error: No such container: 425173d9c76c6f794674ac37f4999ce0817967ec6819c2ba21c61572bed4f383"
Nov 20 23:09:38 docker[17234]: watchtower  | time="2022-11-20T23:09:38+01:00" level=info msg="Session done" Failed=1 Scanned=5 Updated=0 notify=no
```


Pull request description:
feat: ignore removal error due to non-existing containers
This PR allows watchtower to ignore a failure to remove containers that doesn't exist. This can be useful when the start of the containers is managed by docker compose or an external system such as systemd.

Watchtower causes an issue for me in combination with the `--abort-on-container-exit` flag for dockers `compose up` subcommand.

Docker compose removed the container on its own after the container was stopped. This leads to an error when watchtower attempts to remove the container.

`Error: No such container: 425173...`

The error is thrown by this [ContainerRemove](https://github.com/containrrr/watchtower/blob/d744c3488667d6cc4bb8258bb4b351bfd80e1602/pkg/container/client.go#L194) call.

Fixes #1480 

Patch:
diff --git a/pkg/container/client.go b/pkg/container/client.go
index f534bd0..7447828 100644
--- a/pkg/container/client.go
+++ b/pkg/container/client.go
@@ -192,6 +192,10 @@ func (client dockerClient) StopContainer(c Container, timeout time.Duration) err
 		log.Debugf("Removing container %s", shortID)
 
 		if err := client.api.ContainerRemove(bg, idStr, types.ContainerRemoveOptions{Force: true, RemoveVolumes: client.RemoveVolumes}); err != nil {
+			if sdkClient.IsErrNotFound(err) {
+				log.Debugf("Container %s not found, skipping removal.", shortID)
+				return nil
+			}
 			return err
 		}
 	}
diff --git a/pkg/container/mocks/ApiServer.go b/pkg/container/mocks/ApiServer.go
index 20610cd..1a05355 100644
--- a/pkg/container/mocks/ApiServer.go
+++ b/pkg/container/mocks/ApiServer.go
@@ -3,14 +3,15 @@ package mocks
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
-	O "github.com/onsi/gomega"
-	"github.com/onsi/gomega/ghttp"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path/filepath"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	O "github.com/onsi/gomega"
+	"github.com/onsi/gomega/ghttp"
 )
 
 func getMockJSONFile(relPath string) ([]byte, error) {
@@ -42,14 +43,14 @@ func respondWithJSONFile(relPath string, statusCode int, optionalHeader ...http.
 func GetContainerHandlers(containerFiles ...string) []http.HandlerFunc {
 	handlers := make([]http.HandlerFunc, 0, len(containerFiles)*2)
 	for _, file := range containerFiles {
-		handlers = append(handlers, getContainerHandler(file))
+		handlers = append(handlers, getContainerFileHandler(file))
 
 		// Also append the image request since that will be called for every container
 		if file == "running" {
 			// The "running" container is the only one using image02
-			handlers = append(handlers, getImageHandler(1))
+			handlers = append(handlers, getImageFileHandler(1))
 		} else {
-			handlers = append(handlers, getImageHandler(0))
+			handlers = append(handlers, getImageFileHandler(0))
 		}
 	}
 	return handlers
@@ -75,15 +76,36 @@ var imageIds = []string{
 	"sha256:19d07168491a3f9e2798a9bed96544e34d57ddc4757a4ac5bb199dea896c87fd",
 }
 
-func getContainerHandler(file string) http.HandlerFunc {
+func getContainerFileHandler(file string) http.HandlerFunc {
 	id, ok := containerFileIds[file]
 	failTestUnless(ok)
-	return ghttp.CombineHandlers(
-		ghttp.VerifyRequest("GET", O.HaveSuffix("/containers/%v/json", id)),
+	return getContainerHandler(
+		id,
 		RespondWithJSONFile(fmt.Sprintf("./mocks/data/container_%v.json", file), http.StatusOK),
 	)
 }
 
+func getContainerHandler(containerId string, responseHandler http.HandlerFunc) http.HandlerFunc {
+	return ghttp.CombineHandlers(
+		ghttp.VerifyRequest("GET", O.HaveSuffix("/containers/%v/json", containerId)),
+		responseHandler,
+	)
+}
+
+// GetContainerHandler mocks the GET containers/{id}/json endpoint
+func GetContainerHandler(containerID string, containerInfo *types.ContainerJSON) http.HandlerFunc {
+	responseHandler := containerNotFoundResponse(containerID)
+	if containerInfo != nil {
+		responseHandler = ghttp.RespondWithJSONEncoded(http.StatusOK, containerInfo)
+	}
+	return getContainerHandler(containerID, responseHandler)
+}
+
+// GetImageHandler mocks the GET images/{id}/json endpoint
+func GetImageHandler(imageInfo *types.ImageInspect) http.HandlerFunc {
+	return getImageHandler(imageInfo.ID, ghttp.RespondWithJSONEncoded(http.StatusOK, imageInfo))
+}
+
 // ListContainersHandler mocks the GET containers/json endpoint, filtering the returned containers based on statuses
 func ListContainersHandler(statuses ...string) http.HandlerFunc {
 	filterArgs := createFilterArgs(statuses)
@@ -116,9 +138,15 @@ func respondWithFilteredContainers(filters filters.Args) http.HandlerFunc {
 	return ghttp.RespondWithJSONEncoded(http.StatusOK, filteredContainers)
 }
 
-func getImageHandler(index int) http.HandlerFunc {
+func getImageHandler(imageId string, responseHandler http.HandlerFunc) http.HandlerFunc {
 	return ghttp.CombineHandlers(
-		ghttp.VerifyRequest("GET", O.HaveSuffix("/images/%v/json", imageIds[index])),
+		ghttp.VerifyRequest("GET", O.HaveSuffix("/images/%s/json", imageId)),
+		responseHandler,
+	)
+}
+
+func getImageFileHandler(index int) http.HandlerFunc {
+	return getImageHandler(imageIds[index],
 		RespondWithJSONFile(fmt.Sprintf("./mocks/data/image%02d.json", index+1), http.StatusOK),
 	)
 }
@@ -126,3 +154,40 @@ func getImageHandler(index int) http.HandlerFunc {
 func failTestUnless(ok bool) {
 	O.ExpectWithOffset(2, ok).To(O.BeTrue(), "test setup failed")
 }
+
+// KillContainerHandler mocks the POST containers/{id}/kill endpoint
+func KillContainerHandler(containerID string, found FoundStatus) http.HandlerFunc {
+	responseHandler := noContentStatusResponse
+	if !found {
+		responseHandler = containerNotFoundResponse(containerID)
+	}
+	return ghttp.CombineHandlers(
+		ghttp.VerifyRequest("POST", O.HaveSuffix("containers/%s/kill", containerID)),
+		responseHandler,
+	)
+}
+
+// RemoveContainerHandler mocks the DELETE containers/{id} endpoint
+func RemoveContainerHandler(containerID string, found FoundStatus) http.HandlerFunc {
+	responseHandler := noContentStatusResponse
+	if !found {
+		responseHandler = containerNotFoundResponse(containerID)
+	}
+	return ghttp.CombineHandlers(
+		ghttp.VerifyRequest("DELETE", O.HaveSuffix("containers/%s", containerID)),
+		responseHandler,
+	)
+}
+
+func containerNotFoundResponse(containerID string) http.HandlerFunc {
+	return ghttp.RespondWithJSONEncoded(http.StatusNotFound, struct{ message string }{message: "No such container: " + containerID})
+}
+
+var noContentStatusResponse = ghttp.RespondWith(http.StatusNoContent, nil)
+
+type FoundStatus bool
+
+const (
+	Found   FoundStatus = true
+	Missing FoundStatus = false
+)


Test patch:
diff --git a/pkg/container/client_test.go b/pkg/container/client_test.go
index 6ccc93c..1100ac9 100644
--- a/pkg/container/client_test.go
+++ b/pkg/container/client_test.go
@@ -1,6 +1,8 @@
 package container
 
 import (
+	"time"
+
 	"github.com/containrrr/watchtower/pkg/container/mocks"
 	"github.com/containrrr/watchtower/pkg/filters"
 	t "github.com/containrrr/watchtower/pkg/types"
@@ -69,6 +71,38 @@ var _ = Describe("the client", func() {
 			})
 		})
 	})
+	When("removing a running container", func() {
+		When("the container still exist after stopping", func() {
+			It("should attempt to remove the container", func() {
+				container := *MockContainer(WithContainerState(types.ContainerState{Running: true}))
+				containerStopped := *MockContainer(WithContainerState(types.ContainerState{Running: false}))
+
+				cid := container.ContainerInfo().ID
+				mockServer.AppendHandlers(
+					mocks.KillContainerHandler(cid, mocks.Found),
+					mocks.GetContainerHandler(cid, containerStopped.ContainerInfo()),
+					mocks.RemoveContainerHandler(cid, mocks.Found),
+					mocks.GetContainerHandler(cid, nil),
+				)
+
+				Expect(dockerClient{api: docker}.StopContainer(container, time.Minute)).To(Succeed())
+			})
+		})
+		When("the container does not exist after stopping", func() {
+			It("should not cause an error", func() {
+				container := *MockContainer(WithContainerState(types.ContainerState{Running: true}))
+
+				cid := container.ContainerInfo().ID
+				mockServer.AppendHandlers(
+					mocks.KillContainerHandler(cid, mocks.Found),
+					mocks.GetContainerHandler(cid, nil),
+					mocks.RemoveContainerHandler(cid, mocks.Missing),
+				)
+
+				Expect(dockerClient{api: docker}.StopContainer(container, time.Minute)).To(Succeed())
+			})
+		})
+	})
 	When("listing containers", func() {
 		When("no filter is provided", func() {
 			It("should return all available containers", func() {