Repository: moby/moby (language: go)

Issue:
live-restore in combination with docker compose is broken with docker-ce version 20.10.19 and newer
### Description

Hello,

we are heavily using the live-restore feature in our production environments and facing automatic deployment issues through our ci. In our environment all Operating System Packages are updated automatically. When the docker engine gets updated and someone restarts a container in a running compose stack a docker compose down -v --remove-orphans does not work.

Update:
This happens particularly to named volumes with local driver and binded directory.

docker-compose.yml example
```
version: '2.4'
services:
  app:
    image: appimage
    networks:
      app-network:
        aliases:
          - app
    restart: "unless-stopped"
    volumes:
      - log-app:/log
     
volumes:
  log-app:
    driver: local
    driver_opts:
      type: none
      device: /srv/docker/stack/prod/log/app
      o: bind
```

Operating System: Debian GNU/Linux 11.5 (Bullseye)

### Reproduce

```
root@host:/srv/docker/stack/prod# su - dockerdeploy
dockerdeploy@host:~# cd /srv/docker/stack/prod
dockerdeploy@host:/srv/docker/stack/prod# docker compose up -d
dockerdeploy@host:/srv/docker/stack/prod# exit
root@host:/srv/docker/stack/prod# systemctl restart docker.service
root@host:/srv/docker/stack/prod# su - dockerdeploy
dockerdeploy@host:~# cd /srv/docker/stack/prod
dockerdeploy@host:/srv/docker/stack/prod$ docker compose restart app
[+] Running 1/1
 ⠿ Container app  Started
dockerdeploy@host:/srv/docker/stack/prod$ docker compose down -v --remove-orphans
[+] Running 21/37
.....
Error response from daemon: remove prod_log-app: volume has active mounts
```

### Expected behavior

docker compose down -v --remove-orphans should reliably tear down the stack

### docker version

```bash
Client: Docker Engine - Community
 Version:           20.10.21
 API version:       1.41
 Go version:        go1.18.7
 Git commit:        baeda1f
 Built:             Tue Oct 25 18:02:28 2022
 OS/Arch:           linux/amd64
 Context:           default
 Experimental:      true

Server: Docker Engine - Community
 Engine:
  Version:          20.10.21
  API version:      1.41 (minimum version 1.12)
  Go version:       go1.18.7
  Git commit:       3056208
  Built:            Tue Oct 25 18:00:19 2022
  OS/Arch:          linux/amd64
  Experimental:     false
 containerd:
  Version:          1.6.9
  GitCommit:        1c90a442489720eec95342e1789ee8a5e1b9536f
 runc:
  Version:          1.1.4
  GitCommit:        v1.1.4-0-g5fd4c4d
 docker-init:
  Version:          0.19.0
  GitCommit:        de40ad0
```


### docker info

```bash
Client:
 Context:    default
 Debug Mode: false
 Plugins:
  app: Docker App (Docker Inc., v0.9.1-beta3)
  buildx: Docker Buildx (Docker Inc., v0.9.1-docker)

Server:
 Containers: 34
  Running: 34
  Paused: 0
  Stopped: 0
 Images: 28
 Server Version: 20.10.21
 Storage Driver: overlay2
  Backing Filesystem: extfs
  Supports d_type: true
  Native Overlay Diff: true
  userxattr: false
 Logging Driver: json-file
 Cgroup Driver: systemd
 Cgroup Version: 2
 Plugins:
  Volume: local
  Network: bridge host ipvlan macvlan null overlay
  Log: awslogs fluentd gcplogs gelf journald json-file local logentries splunk syslog
 Swarm: inactive
 Runtimes: io.containerd.runc.v2 io.containerd.runtime.v1.linux runc
 Default Runtime: runc
 Init Binary: docker-init
 containerd version: 1c90a442489720eec95342e1789ee8a5e1b9536f
 runc version: v1.1.4-0-g5fd4c4d
 init version: de40ad0
 Security Options:
  apparmor
  seccomp
   Profile: default
  cgroupns
 Kernel Version: 5.10.0-19-amd64
 Operating System: Debian GNU/Linux 11 (bullseye)
 OSType: linux
 Architecture: x86_64
 CPUs: 60
 Total Memory: 157.2GiB
 Name: prod
 ID: XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX:XXXX
 Docker Root Dir: /var/lib/docker
 Debug Mode: false
 Registry: https://index.docker.io/v1/
 Labels:
 Experimental: false
 Insecure Registries:
  127.0.0.0/8
 Live Restore Enabled: true
```


### Additional Info

Docker Compose version v2.12.2

### daemon.json
```
{
  "experimental": true,
  "fixed-cidr-v6": "fd00:dead:beef:c0::/80",
  "ipv6": true,
  "ip6tables": true,
  "live-restore": true,
  "mtu": 1400
}


Discussion / hints:
This is most likely related to this change https://github.com/moby/moby/commit/66ddb7f91c7a0ea4daa5cb96e8701a9378fe22ec (https://github.com/moby/moby/pull/44238)

Poking around things I don't really see anything wrong in the code.
Are you sure something else doesn't have the volume mounted?

It seems like you were exploiting a bug in live-restore before that has now been fixed that allowed volumes to be removed even though they were being referenced.
as i mentioned in my detailed report version 20.10.18 was fine. i updated my report with a sample compose file, with this you can easily reproduce this behaviour. as a workaround for us we do not use named volumes with bind mounted directories anymore.
Can you still reproduce on 23.0?
I can confirm that this is still an issue on 24.0.2.  Reproduced with this setup.

Versions
```
$ docker version
Client: Docker Engine - Community
 Version:           24.0.2
 API version:       1.43
 Go version:        go1.20.4
 Git commit:        cb74dfc
 Built:             Thu May 25 21:52:22 2023
 OS/Arch:           linux/amd64
 Context:           default

Server: Docker Engine - Community
 Engine:
  Version:          24.0.2
  API version:      1.43 (minimum version 1.12)
  Go version:       go1.20.4
  Git commit:       659604f
  Built:            Thu May 25 21:52:22 2023
  OS/Arch:          linux/amd64
  Experimental:     true
 containerd:
  Version:          1.6.20
  GitCommit:        2806fc1057397dbaeefbea0e4e17bddfbd388f38
 runc:
  Version:          1.1.5
  GitCommit:        v1.1.5-0-gf19387a
 docker-init:
  Version:          0.19.0
  GitCommit:        de40ad0
```
Docker compose version
```
$ docker compose version
Docker Compose version v2.18.1
```
OS
```
$ lsb_release -a
No LSB modules are available.
Distributor ID:	Ubuntu
Description:	Ubuntu 20.04.6 LTS
Release:	20.04
Codename:	focal
```

Compose file
```
$ cat docker-compose.yaml 
services:
  app:
    image: ubuntu:focal
    network_mode: none
    command: "sleep infinity"
    volumes:
      - trouble:/trouble

volumes:
  trouble:
    driver_opts:
      type: none
      device: ${PWD}
      o: bind
```
Start up the stack, auto-creating the volume
```
$ docker compose up -d 
[+] Building 0.0s (0/0)                                                         
[+] Running 2/2
 ✔ Volume "moby-44422_trouble"  Created                                    0.0s 
 ✔ Container moby-44422-app-1   Started                                    0.4s 

$ docker compose ps
NAME                IMAGE               COMMAND             SERVICE             CREATED             STATUS              PORTS
moby-44422-app-1    ubuntu:focal        "sleep infinity"    app                 12 seconds ago      Up 11 seconds       
```
Restart docker daemon -- *note* that this bug exists across any restart, updating the docker daemon is just one possible trigger of a restart.
```
$ sudo systemctl restart docker
```
Bring down the container stack leaving volumes so we can see their state before trying to delete them.  *note* that running `docker compose down -v` here also triggers the same error as described in the original bug report.
```
$ docker compose down
[+] Running 1/1
 ✔ Container moby-44422-app-1  Removed                                    10.2s 
```
Current status of the volume
```
$ docker volume inspect moby-44422_trouble 
[
    {
        "CreatedAt": "2023-06-13T22:38:59Z",
        "Driver": "local",
        "Labels": {
            "com.docker.compose.project": "moby-44422",
            "com.docker.compose.version": "2.18.1",
            "com.docker.compose.volume": "trouble"
        },
        "Mountpoint": "/var/lib/docker/volumes/moby-44422_trouble/_data",
        "Name": "moby-44422_trouble",
        "Options": {
            "device": "/home/XXXXXXXXX/moby-44422",
            "o": "bind",
            "type": "none"
        },
        "Scope": "local"
    }
]
```
Now try to delete the volume
```
$ docker volume rm moby-44422_trouble 
Error response from daemon: remove moby-44422_trouble: volume has active mounts
```
*This failure should not happen.  This appears to be a bug in the internal reference counting in the docker daemon.*

Restart the docker daemon again to try to clear up the error in reference counting.
```
$ sudo systemctl restart docker
```
Now notice that we can delete the volume with no error.
```
$ docker volume rm moby-44422_trouble 
moby-44422_trouble
```

I am unable to reproduce the issue, either with live-restore or not.
This is 100% reproducible for me.  I've hit this issue over many versions of docker and on many similarly configured ubuntu systems.  This burns me every time the docker daemon is restarted under my running stack and then I try to bring down the stack with `docker compose down -v`.

I also notice that I have similar settings to @fichte in my `/etc/docker/daemon.json`.  Here's mine:
```
$ cat /etc/docker/daemon.json
{
		"experimental": true,
		"ip6tables": true,
		"live-restore": true,
		"ipv6": true,
		"fixed-cidr-v6": "fd00::/80"
}
```
@cpuguy83 did you run your test with the same config?  Maybe `experimental` or the various `ipv6` config options somehow make a difference to the outcome for volumes.

I just ran a few tests while reducing my docker daemon config file.  I can 100% reproduce this with only this in my config file
```
$ cat /etc/docker/daemon.json
{
		"live-restore": true
}
```

I am running a dockerd from the dev branch.
I'll try this tomorrow on a released version.

Also it is hitting me now that this error is coming from the (local) volume driver which is refcounting mounts and not from the volume service which handles refcounting objects referencing a volume.
Originally I was thinking the error was from the latter but that is incorrect.
Looking through the code a bit, if I had to guess this is the cause: https://github.com/moby/moby/blob/a9788886e6f9c94f93cab701eb33e189fd95b2ec/daemon/start.go#L253-L257
Or more specifically that in live-restore `BaseFS` is not set.
And sure enough I can reproduce this with the graphdriver backend, does not reproduce with containerd snapshotters.
Even worse, it is calling `container.UnmountVolumes` so somewhere the count is just getting off.
So this is happening because the active mount count is not initialized for live-restored containers.
When unmount is called its decrementing the count but the count is a uint64 so it overflows.
It also seems something on master has changed since I'd built it before and it is now broken with containerd snapshotters as well.
I'm not sure what that would be exactly.
Thank you very much for digging into this issue.
I have a working patch for this, it needs tests and I'd like to do a bisect to see exactly the commit that caused the issue so it's going to take a bit of time to get this in, but I at least see what's happening.

Fix patch:
diff --git a/daemon/mounts.go b/daemon/mounts.go
index 383a38e7ebe2b..ad637df03d073 100644
--- a/daemon/mounts.go
+++ b/daemon/mounts.go
@@ -5,16 +5,28 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/containerd/containerd/log"
 	mounttypes "github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/container"
 	volumesservice "github.com/docker/docker/volume/service"
+	"github.com/sirupsen/logrus"
 )
 
 func (daemon *Daemon) prepareMountPoints(container *container.Container) error {
+	alive := container.IsRunning()
 	for _, config := range container.MountPoints {
 		if err := daemon.lazyInitializeVolume(container.ID, config); err != nil {
 			return err
 		}
+		if alive {
+			log.G(context.TODO()).WithFields(logrus.Fields{
+				"container": container.ID,
+				"volume":    config.Volume.Name(),
+			}).Debug("Live-restoring volume for alive container")
+			if err := config.LiveRestore(context.TODO()); err != nil {
+				return err
+			}
+		}
 	}
 	return nil
 }
diff --git a/daemon/volumes.go b/daemon/volumes.go
index 9d1ebee19c3fc..a0667843f1142 100644
--- a/daemon/volumes.go
+++ b/daemon/volumes.go
@@ -21,6 +21,8 @@ import (
 	"github.com/pkg/errors"
 )
 
+var _ volume.LiveRestorer = (*volumeWrapper)(nil)
+
 type mounts []container.Mount
 
 // Len returns the number of mounts. Used in sorting.
@@ -257,6 +259,7 @@ func (daemon *Daemon) VolumesService() *service.VolumesService {
 type volumeMounter interface {
 	Mount(ctx context.Context, v *volumetypes.Volume, ref string) (string, error)
 	Unmount(ctx context.Context, v *volumetypes.Volume, ref string) error
+	LiveRestoreVolume(ctx context.Context, v *volumetypes.Volume, ref string) error
 }
 
 type volumeWrapper struct {
@@ -291,3 +294,7 @@ func (v *volumeWrapper) CreatedAt() (time.Time, error) {
 func (v *volumeWrapper) Status() map[string]interface{} {
 	return v.v.Status
 }
+
+func (v *volumeWrapper) LiveRestoreVolume(ctx context.Context, ref string) error {
+	return v.s.LiveRestoreVolume(ctx, v.v, ref)
+}
diff --git a/volume/local/local.go b/volume/local/local.go
index 306e974e571b6..d3dcc59947bfb 100644
--- a/volume/local/local.go
+++ b/volume/local/local.go
@@ -19,6 +19,7 @@ import (
 	"github.com/docker/docker/quota"
 	"github.com/docker/docker/volume"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 const (
@@ -36,6 +37,8 @@ var (
 	// This name is used to create the bind directory, so we need to avoid characters that
 	// would make the path to escape the root directory.
 	volumeNameRegex = names.RestrictedNamePattern
+
+	_ volume.LiveRestorer = (*localVolume)(nil)
 )
 
 type activeMount struct {
@@ -297,14 +300,17 @@ func (v *localVolume) CachedPath() string {
 func (v *localVolume) Mount(id string) (string, error) {
 	v.m.Lock()
 	defer v.m.Unlock()
+	logger := log.G(context.TODO()).WithField("volume", v.name)
 	if v.needsMount() {
 		if !v.active.mounted {
+			logger.Debug("Mounting volume")
 			if err := v.mount(); err != nil {
 				return "", errdefs.System(err)
 			}
 			v.active.mounted = true
 		}
 		v.active.count++
+		logger.WithField("active mounts", v.active).Debug("Decremented active mount count")
 	}
 	if err := v.postMount(); err != nil {
 		return "", err
@@ -317,6 +323,7 @@ func (v *localVolume) Mount(id string) (string, error) {
 func (v *localVolume) Unmount(id string) error {
 	v.m.Lock()
 	defer v.m.Unlock()
+	logger := log.G(context.TODO()).WithField("volume", v.name)
 
 	// Always decrement the count, even if the unmount fails
 	// Essentially docker doesn't care if this fails, it will send an error, but
@@ -324,12 +331,14 @@ func (v *localVolume) Unmount(id string) error {
 	// this volume can never be removed until a daemon restart occurs.
 	if v.needsMount() {
 		v.active.count--
+		logger.WithField("active mounts", v.active).Debug("Decremented active mount count")
 	}
 
 	if v.active.count > 0 {
 		return nil
 	}
 
+	logger.Debug("Unmounting volume")
 	return v.unmount()
 }
 
@@ -370,6 +379,24 @@ func (v *localVolume) saveOpts() error {
 	return nil
 }
 
+// LiveRestoreVolume restores reference counts for mounts
+// It is assumed that the volume is already mounted since this is only called for active, live-restored containers.
+func (v *localVolume) LiveRestoreVolume(ctx context.Context, _ string) error {
+	v.m.Lock()
+	defer v.m.Unlock()
+
+	if !v.needsMount() {
+		return nil
+	}
+	v.active.count++
+	v.active.mounted = true
+	log.G(ctx).WithFields(logrus.Fields{
+		"volume":        v.name,
+		"active mounts": v.active,
+	}).Debugf("Live restored volume")
+	return nil
+}
+
 // getAddress finds out address/hostname from options
 func getAddress(opts string) string {
 	for _, opt := range strings.Split(opts, ",") {
diff --git a/volume/mounts/mounts.go b/volume/mounts/mounts.go
index c441e51ed9018..bc90bb9def9de 100644
--- a/volume/mounts/mounts.go
+++ b/volume/mounts/mounts.go
@@ -1,17 +1,20 @@
 package mounts // import "github.com/docker/docker/volume/mounts"
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
 	"syscall"
 
+	"github.com/containerd/containerd/log"
 	mounttypes "github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/docker/volume"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 // MountPoint is the intersection point between a volume and a container. It
@@ -164,6 +167,32 @@ func (m *MountPoint) Setup(mountLabel string, rootIDs idtools.Identity, checkFun
 	return m.Source, nil
 }
 
+func (m *MountPoint) LiveRestore(ctx context.Context) error {
+	if m.Volume == nil {
+		logrus.Debug("No volume to restore")
+		return nil
+	}
+
+	lrv, ok := m.Volume.(volume.LiveRestorer)
+	if !ok {
+		log.G(ctx).WithField("volume", m.Volume.Name()).Debugf("Volume does not support live restore: %T", m.Volume)
+		return nil
+	}
+
+	id := m.ID
+	if id == "" {
+		id = stringid.GenerateRandomID()
+	}
+
+	if err := lrv.LiveRestoreVolume(ctx, id); err != nil {
+		return errors.Wrapf(err, "error while restoring volume '%s'", m.Source)
+	}
+
+	m.ID = id
+	m.active++
+	return nil
+}
+
 // Path returns the path of a volume in a mount point.
 func (m *MountPoint) Path() string {
 	if m.Volume != nil {
diff --git a/volume/service/service.go b/volume/service/service.go
index cfac31823b235..13cadf53769cd 100644
--- a/volume/service/service.go
+++ b/volume/service/service.go
@@ -274,3 +274,18 @@ func (s *VolumesService) List(ctx context.Context, filter filters.Args) (volumes
 func (s *VolumesService) Shutdown() error {
 	return s.vs.Shutdown()
 }
+
+// LiveRestoreVolume passes through the LiveRestoreVolume call to the volume if it is implemented
+// otherwise it is a no-op.
+func (s *VolumesService) LiveRestoreVolume(ctx context.Context, vol *volumetypes.Volume, ref string) error {
+	v, err := s.vs.Get(ctx, vol.Name, opts.WithGetDriver(vol.Driver))
+	if err != nil {
+		return err
+	}
+	rlv, ok := v.(volume.LiveRestorer)
+	if !ok {
+		log.G(ctx).WithField("volume", vol.Name).Debugf("volume does not implement LiveRestoreVolume: %T", v)
+		return nil
+	}
+	return rlv.LiveRestoreVolume(ctx, ref)
+}
diff --git a/volume/service/store.go b/volume/service/store.go
index a7b8a707d9858..9ff8f53235999 100644
--- a/volume/service/store.go
+++ b/volume/service/store.go
@@ -24,6 +24,8 @@ const (
 	volumeDataDir = "volumes"
 )
 
+var _ volume.LiveRestorer = (*volumeWrapper)(nil)
+
 type volumeWrapper struct {
 	volume.Volume
 	labels  map[string]string
@@ -67,6 +69,13 @@ func (v volumeWrapper) CachedPath() string {
 	return v.Volume.Path()
 }
 
+func (v volumeWrapper) LiveRestoreVolume(ctx context.Context, ref string) error {
+	if vv, ok := v.Volume.(volume.LiveRestorer); ok {
+		return vv.LiveRestoreVolume(ctx, ref)
+	}
+	return nil
+}
+
 // StoreOpt sets options for a VolumeStore
 type StoreOpt func(store *VolumeStore) error
 
diff --git a/volume/volume.go b/volume/volume.go
index 61c8243979f11..2dcbdebe16f78 100644
--- a/volume/volume.go
+++ b/volume/volume.go
@@ -1,6 +1,7 @@
 package volume // import "github.com/docker/docker/volume"
 
 import (
+	"context"
 	"time"
 )
 
@@ -60,6 +61,15 @@ type Volume interface {
 	Status() map[string]interface{}
 }
 
+// LiveRestorer is an optional interface that can be implemented by a volume driver
+// It is used to restore any resources that are necessary for a volume to be used by a live-restored container
+type LiveRestorer interface {
+	// LiveRestoreVolume allows a volume driver which implements this interface to restore any necessary resources (such as reference counting)
+	// This is called only after the daemon is restarted with live-restored containers
+	// It is called once per live-restored container.
+	LiveRestoreVolume(_ context.Context, ref string) error
+}
+
 // DetailedVolume wraps a Volume with user-defined labels, options, and cluster scope (e.g., `local` or `global`)
 type DetailedVolume interface {
 	Labels() map[string]string


Test patch:
diff --git a/integration/daemon/daemon_test.go b/integration/daemon/daemon_test.go
index 7ecb2c0aa4793..9dcaed374757b 100644
--- a/integration/daemon/daemon_test.go
+++ b/integration/daemon/daemon_test.go
@@ -400,6 +400,42 @@ func testLiveRestoreVolumeReferences(t *testing.T) {
 		runTest(t, "on-failure")
 		runTest(t, "no")
 	})
+
+	// Make sure that the local volume driver's mount ref count is restored
+	// Addresses https://github.com/moby/moby/issues/44422
+	t.Run("local volume with mount options", func(t *testing.T) {
+		v, err := c.VolumeCreate(ctx, volume.CreateOptions{
+			Driver: "local",
+			Name:   "test-live-restore-volume-references-local",
+			DriverOpts: map[string]string{
+				"type":   "tmpfs",
+				"device": "tmpfs",
+			},
+		})
+		assert.NilError(t, err)
+		m := mount.Mount{
+			Type:   mount.TypeVolume,
+			Source: v.Name,
+			Target: "/foo",
+		}
+		cID := container.Run(ctx, t, c, container.WithMount(m), container.WithCmd("top"))
+		defer c.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{Force: true})
+
+		d.Restart(t, "--live-restore", "--iptables=false")
+
+		// Try to remove the volume
+		// This should fail since its used by a container
+		err = c.VolumeRemove(ctx, v.Name, false)
+		assert.ErrorContains(t, err, "volume is in use")
+
+		// Remove that container which should free the references in the volume
+		err = c.ContainerRemove(ctx, cID, types.ContainerRemoveOptions{Force: true})
+		assert.NilError(t, err)
+
+		// Now we should be able to remove the volume
+		err = c.VolumeRemove(ctx, v.Name, false)
+		assert.NilError(t, err)
+	})
 }
 
 func TestDaemonDefaultBridgeWithFixedCidrButNoBip(t *testing.T) {