Repository: threefoldtech/zos
Pull request #883 (2020-08-05 11:36:26)

Adds cleanup of provisiond unused resources


Patch:
diff --git a/cmds/provisiond/main.go b/cmds/provisiond/main.go
index c84edfe1..3e46c6d9 100644
--- a/cmds/provisiond/main.go
+++ b/cmds/provisiond/main.go
@@ -136,6 +136,7 @@ func main() {
 		Feedback:       explorer.NewFeedback(e, primitives.ResultToSchemaType),
 		Signer:         identity,
 		Statser:        statser,
+		ZbusCl:         zbusCl,
 	})
 
 	server.Register(zbus.ObjectID{Name: module, Version: "0.0.1"}, pkg.ProvisionMonitor(engine))
diff --git a/go.mod b/go.mod
index 24f0dd1a..c9eefb9e 100644
--- a/go.mod
+++ b/go.mod
@@ -60,7 +60,7 @@ require (
 	github.com/stretchr/testify v1.6.1
 	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
 	github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae
-	github.com/threefoldtech/tfexplorer v0.4.1-0.20200925140208-64f1857969f1
+	github.com/threefoldtech/tfexplorer v0.4.1-0.20200930141942-5d5dac62dca0
 	github.com/threefoldtech/zbus v0.1.3
 	github.com/urfave/cli v1.22.4
 	github.com/vishvananda/netlink v1.1.0
@@ -71,10 +71,12 @@ require (
 	github.com/yggdrasil-network/yggdrasil-go v0.3.15-0.20200526002434-ed3bf5ef0736
 	go.etcd.io/bbolt v1.3.4 // indirect
 	golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4
+	golang.org/x/net v0.0.0-20200301022130-244492dfa37a
 	golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20191219145116-fa6499c8e75f
 	google.golang.org/appengine v1.6.5 // indirect
 	google.golang.org/grpc v1.29.1 // indirect
+	gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5
 	gopkg.in/yaml.v2 v2.2.7
 	gotest.tools v2.2.0+incompatible
 	gotest.tools/v3 v3.0.2 // indirect
diff --git a/go.sum b/go.sum
index 406709c7..e86c4b12 100644
--- a/go.sum
+++ b/go.sum
@@ -588,12 +588,18 @@ github.com/threefoldtech/tfexplorer v0.3.2-0.20200824142536-b4319e0a006d h1:/4U3
 github.com/threefoldtech/tfexplorer v0.3.2-0.20200824142536-b4319e0a006d/go.mod h1:oFtPBaRLW4c6YuEfTrWYzAGw+wb6I4+og133zjDOYP0=
 github.com/threefoldtech/tfexplorer v0.3.2-0.20200918135528-58e333897454 h1:MUtGnBw3P5Zp6CS26dOw6/WksisommGCd4hqP4G4D3w=
 github.com/threefoldtech/tfexplorer v0.3.2-0.20200918135528-58e333897454/go.mod h1:fA2oR3D6Wre9tXSiWFiapeuUI5UA9aXP9fejr6hDEYM=
+github.com/threefoldtech/tfexplorer v0.4.0 h1:Km5kQEOe/MFSqR7iSZ/oCSQG16+F1oknCOTT+/kpg5k=
+github.com/threefoldtech/tfexplorer v0.4.0/go.mod h1:wxnWTecsczkzkiJoZB19tt49l8CWkTNLD2VkKBK6mSc=
 github.com/threefoldtech/tfexplorer v0.4.1-0.20200923122946-a429c33ade21 h1:p3Hpq3bmBNKDaMCA7gn8ltMk84ZAU9OurMUF1K93Kqk=
 github.com/threefoldtech/tfexplorer v0.4.1-0.20200923122946-a429c33ade21/go.mod h1:wxnWTecsczkzkiJoZB19tt49l8CWkTNLD2VkKBK6mSc=
 github.com/threefoldtech/tfexplorer v0.4.1-0.20200924232024-7ce1840cdaa6 h1:RGOwrJjhM4QFVpejELSs4fe+OM5UNtbkvZNRpNeA3QQ=
 github.com/threefoldtech/tfexplorer v0.4.1-0.20200924232024-7ce1840cdaa6/go.mod h1:wxnWTecsczkzkiJoZB19tt49l8CWkTNLD2VkKBK6mSc=
 github.com/threefoldtech/tfexplorer v0.4.1-0.20200925140208-64f1857969f1 h1:PYrmO+AmHUBpHvjEAqNtRY4HCztwMeVVQK7pUPmwBcg=
 github.com/threefoldtech/tfexplorer v0.4.1-0.20200925140208-64f1857969f1/go.mod h1:0261YnFlaQNDv1hNzdJc/2DuYq9B9qe9D6QnZuZOCh0=
+github.com/threefoldtech/tfexplorer v0.4.1-0.20200929100544-617163959db1 h1:rHXWnxE139JwaXQ+U4Uyv0bbZ9sS2U4oentTxqFnfBc=
+github.com/threefoldtech/tfexplorer v0.4.1-0.20200929100544-617163959db1/go.mod h1:0261YnFlaQNDv1hNzdJc/2DuYq9B9qe9D6QnZuZOCh0=
+github.com/threefoldtech/tfexplorer v0.4.1-0.20200930141942-5d5dac62dca0 h1:K7XPmiDWMpSUxDQ8aPbhh03ZZAdqCMtaN+GEWDWOvyc=
+github.com/threefoldtech/tfexplorer v0.4.1-0.20200930141942-5d5dac62dca0/go.mod h1:0261YnFlaQNDv1hNzdJc/2DuYq9B9qe9D6QnZuZOCh0=
 github.com/threefoldtech/zbus v0.1.3 h1:18DnIzximRbATle5ZdZz0i84n/bCYB8k/gkhr2dXayc=
 github.com/threefoldtech/zbus v0.1.3/go.mod h1:ZtiRpcqzEBJetVQDsEbw0p48h/AF3O1kf0tvd30I0BU=
 github.com/threefoldtech/zos v0.2.4-rc2/go.mod h1:7A2oflcmSVsHFC4slOcydWgJyFBMFMH9wsaTRv+CnTA=
@@ -829,6 +835,8 @@ gopkg.in/gavv/httpexpect.v1 v1.0.0-20170111145843-40724cf1e4a0/go.mod h1:WtiW9ZA
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
 gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw=
 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5 h1:E846t8CnR+lv5nE+VuiKTDG/v1U2stad0QzddfJC7kY=
+gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5/go.mod h1:hiOFpYm0ZJbusNj2ywpbrXowU3G8U6GIQzqn2mw1UIE=
 gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
diff --git a/pkg/container.go b/pkg/container.go
index 6dea13bd..759a5088 100644
--- a/pkg/container.go
+++ b/pkg/container.go
@@ -65,7 +65,11 @@ type ContainerModule interface {
 	// data: Container info
 	Run(ns string, data Container) (ContainerID, error)
 
-	//List all the existing container IDs from a certain namespace ns
+	// ListNS list the name of all the container namespaces
+	ListNS() ([]string, error)
+
+	// List all the existing container IDs from a certain namespace ns
+	// if ns is empty, then the container IDs from all existing namespaces will be return
 	List(ns string) ([]ContainerID, error)
 
 	// Inspect, return information about the container, given its container id
diff --git a/pkg/container/container.go b/pkg/container/container.go
index 0d692215..3bc1cc9a 100644
--- a/pkg/container/container.go
+++ b/pkg/container/container.go
@@ -288,7 +288,18 @@ func (c *containerModule) Inspect(ns string, id pkg.ContainerID) (result pkg.Con
 	return
 }
 
-// List returns running containers IDs for a specific namespace
+// ListNS list the name of all the container namespaces
+func (c *containerModule) ListNS() ([]string, error) {
+	client, err := containerd.New(c.containerd)
+	if err != nil {
+		return nil, err
+	}
+
+	ctx := context.Background()
+	return client.NamespaceService().List(ctx)
+}
+
+// List all the existing container IDs from a certain namespace ns
 func (c *containerModule) List(ns string) ([]pkg.ContainerID, error) {
 	client, err := containerd.New(c.containerd)
 	if err != nil {
diff --git a/pkg/flist/cleanup.go b/pkg/flist/cleanup.go
index 2d287bfe..09d3fc2f 100644
--- a/pkg/flist/cleanup.go
+++ b/pkg/flist/cleanup.go
@@ -78,13 +78,13 @@ func (f *flistModule) cleanupMount(name string) error {
 		log.Warn().Err(err).Str("path", path).Msg("fail to unmount flist")
 	}
 
-	volume, err := f.storage.Path(name)
+	fs, err := f.storage.Path(name)
 	if err != nil {
 		log.Warn().Err(err).Str("subvolume", name).Msg("subvolume does not exist")
 		return nil
 	}
 
-	ro := filepath.Join(volume, "ro")
+	ro := filepath.Join(fs.Path, "ro")
 	if err := syscall.Unmount(ro, syscall.MNT_DETACH); err != nil {
 		log.Warn().Err(err).Str("path", ro).Msg("fail to unmount ro layer")
 	}
diff --git a/pkg/flist/flist.go b/pkg/flist/flist.go
index 6c0f9e78..8d284e79 100644
--- a/pkg/flist/flist.go
+++ b/pkg/flist/flist.go
@@ -196,7 +196,7 @@ func (f *flistModule) mount(name, url, storage string, opts pkg.MountOptions) (s
 		return "", err
 	}
 
-	var backend string
+	var backend pkg.Filesystem
 	var newAllocation bool
 	var args []string
 	if !opts.ReadOnly {
@@ -218,8 +218,8 @@ func (f *flistModule) mount(name, url, storage string, opts pkg.MountOptions) (s
 		}
 	}
 
-	if len(backend) != 0 {
-		args = append(args, "-backend", backend)
+	if len(backend.Path) != 0 {
+		args = append(args, "-backend", backend.Path)
 		// in case of an error (mount is never fully completed)
 		// we need to deallocate the filesystem
 		defer func() {
diff --git a/pkg/provision/cleanup.go b/pkg/provision/cleanup.go
new file mode 100644
index 00000000..5d09c430
--- /dev/null
+++ b/pkg/provision/cleanup.go
@@ -0,0 +1,180 @@
+package provision
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+	"github.com/threefoldtech/tfexplorer/client"
+	"github.com/threefoldtech/tfexplorer/models/generated/workloads"
+	"github.com/threefoldtech/tfexplorer/schema"
+	"github.com/threefoldtech/zbus"
+	"github.com/threefoldtech/zos/pkg"
+	"github.com/threefoldtech/zos/pkg/app"
+	"github.com/threefoldtech/zos/pkg/provision/common"
+	"github.com/threefoldtech/zos/pkg/stubs"
+	"github.com/threefoldtech/zos/pkg/zdb"
+	"golang.org/x/net/context"
+)
+
+// CleanupResources cleans up unused resources
+func CleanupResources(ctx context.Context, zbus zbus.Client) error {
+	explorer, err := app.ExplorerClient()
+	if err != nil {
+		return err
+	}
+	storaged := stubs.NewStorageModuleStub(zbus)
+
+	toSave, err := checkContainers(ctx, zbus)
+	if err != nil {
+		return errors.Wrap(err, "failed to check containers")
+	}
+
+	// ListFilesystems do not return the special cache and vdisk filesystem
+	// so we are safe to process everything that is returned
+	fss, err := storaged.ListFilesystems()
+	if err != nil {
+		return err
+	}
+
+	for _, fs := range fss {
+		log.Info().Msgf("checking subvol %s", fs.Path)
+
+		// Now, is this subvol in one of the toSave ?
+		if _, ok := toSave[fs.Path]; ok {
+			log.Info().Msgf("skipping volume '%s' is used", fs.Path)
+			continue
+		}
+
+		// Is this subvol not in toSave?
+		// Check the explorer if it needs to be deleted
+		delete := checkReservationToDelete(fs, explorer)
+		if delete {
+			log.Info().Msgf("deleting subvolume %s", fs.Path)
+			if err := storaged.ReleaseFilesystem(fs.Name); err != nil {
+				log.Err(err).Msgf("failed to delete subvol '%s'", fs.Path)
+			}
+			continue
+		}
+		log.Info().Msgf("skipping subvolume %s", fs.Path)
+	}
+
+	return nil
+}
+
+func checkReservationToDelete(fs pkg.Filesystem, cl *client.Client) bool {
+	wid, err := WorkloadIDFromFilesystem(fs)
+	if err != nil {
+		log.Err(err).Msgf("failed to convert workload id %d", wid)
+		return false
+	}
+
+	log.Info().Msgf("checking explorer for reservation: %d", wid)
+	reservation, err := cl.Workloads.Get(schema.ID(wid))
+	if err != nil {
+		var hErr client.HTTPError
+		if ok := errors.As(err, &hErr); ok {
+			resp := hErr.Response()
+			// If reservation is not found it should be deleted
+			if resp.StatusCode == 404 {
+				return true
+			}
+		}
+		return false
+	}
+
+	if reservation.GetNextAction() == workloads.NextActionDelete {
+		log.Info().Msgf("workload %d has next action to delete", wid)
+		return true
+	}
+
+	return false
+}
+
+// checks running containers for subvolumes that might need to be saved because they are used
+// and subvolumes that might need to be deleted because they have no attached container anymore
+func checkContainers(ctx context.Context, zbus zbus.Client) (map[string]struct{}, error) {
+	toSave := make(map[string]struct{})
+
+	contd := stubs.NewContainerModuleStub(zbus)
+
+	cNamespaces, err := contd.ListNS()
+	if err != nil {
+		log.Err(err).Msgf("failed to list namespaces")
+		return nil, err
+	}
+
+	for _, ns := range cNamespaces {
+		containerIDs, err := contd.List(ns)
+		if err != nil {
+			log.Error().Err(err).Msg("failed to list container IDs")
+			return nil, err
+		}
+
+		for _, id := range containerIDs {
+			ctr, err := contd.Inspect(ns, id)
+			if err != nil {
+				log.Error().Err(err).Msgf("failed to inspect container %s", id)
+				return nil, err
+			}
+
+			log.Info().Msgf("container ID %s", id)
+			var zdbNamespaces []string
+			if ns == "zdb" {
+				zdbNamespaces, err = listNamespaces(string(id))
+				if err != nil {
+					log.Err(err).Msg("failed to list container namespaces")
+					continue
+				}
+			}
+
+			// avoid to remove any used subvolume used by flistd for root container fs
+			toSave[ctr.RootFS] = struct{}{}
+
+			for _, mnt := range ctr.Mounts {
+				// TODO: do we need this check ?
+				// if !strings.HasPrefix(mnt.Source, "/mnt/") {
+				// 	continue
+				// }
+				if len(zdbNamespaces) == 1 && zdbNamespaces[0] == "default" {
+					err := common.DeleteZdbContainer(id, zbus)
+					if err != nil {
+						log.Err(err).Msgf("failed to delete zdb container %s", string(id))
+						continue
+					}
+				} else {
+					toSave[mnt.Source] = struct{}{}
+				}
+
+			}
+		}
+	}
+
+	return toSave, nil
+}
+
+func socketDir(containerID string) string {
+	return fmt.Sprintf("/var/run/zdb_%s", containerID)
+}
+
+func initZdbConnection(id string) zdb.Client {
+	socket := fmt.Sprintf("unix://%s/zdb.sock", socketDir(id))
+	return zdb.New(socket)
+}
+
+func listNamespaces(containterID string) ([]string, error) {
+	zdbCl := initZdbConnection(containterID)
+	if err := zdbCl.Connect(); err != nil {
+		log.Err(err).Msgf("failed to connect to 0-db: %s", containterID)
+		return nil, err
+	}
+
+	zdbNamespaces, err := zdbCl.Namespaces()
+	if err != nil {
+		log.Err(err).Msg("failed to retrieve zdb namespaces")
+		return nil, err
+	}
+	defer zdbCl.Close()
+
+	return zdbNamespaces, nil
+}
diff --git a/pkg/provision/common/doc.go b/pkg/provision/common/doc.go
new file mode 100644
index 00000000..ae0ffa62
--- /dev/null
+++ b/pkg/provision/common/doc.go
@@ -0,0 +1,3 @@
+// Package common hold logic that is used by both the provision and primitive package
+// it purpose is mainly to avoid circular dependencies
+package common
diff --git a/pkg/provision/common/zdb.go b/pkg/provision/common/zdb.go
new file mode 100644
index 00000000..b16f2bb4
--- /dev/null
+++ b/pkg/provision/common/zdb.go
@@ -0,0 +1,38 @@
+package common
+
+import (
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/threefoldtech/zbus"
+	"github.com/threefoldtech/zos/pkg"
+	"github.com/threefoldtech/zos/pkg/stubs"
+)
+
+// DeleteZdbContainer removes a 0-DB container and cleanup all related resources (container, flist, network)
+func DeleteZdbContainer(containerID pkg.ContainerID, zbus zbus.Client) error {
+	container := stubs.NewContainerModuleStub(zbus)
+	flist := stubs.NewFlisterStub(zbus)
+
+	info, err := container.Inspect("zdb", containerID)
+	if err != nil && strings.Contains(err.Error(), "not found") {
+		return nil
+	} else if err != nil {
+		return errors.Wrapf(err, "failed to inspect container '%s'", containerID)
+	}
+
+	if err := container.Delete("zdb", containerID); err != nil {
+		return errors.Wrapf(err, "failed to delete container %s", containerID)
+	}
+
+	network := stubs.NewNetworkerStub(zbus)
+	if err := network.ZDBDestroy(info.Network.Namespace); err != nil {
+		return errors.Wrapf(err, "failed to destroy zdb network namespace")
+	}
+
+	if err := flist.Umount(info.RootFS); err != nil {
+		return errors.Wrapf(err, "failed to unmount flist at %s", info.RootFS)
+	}
+
+	return nil
+}
diff --git a/pkg/provision/engine.go b/pkg/provision/engine.go
index 7380f97b..3b491f58 100644
--- a/pkg/provision/engine.go
+++ b/pkg/provision/engine.go
@@ -10,8 +10,11 @@ import (
 	"time"
 
 	"github.com/jbenet/go-base58"
+	"github.com/threefoldtech/zbus"
 	"github.com/threefoldtech/zos/pkg"
 
+	"gopkg.in/robfig/cron.v2"
+
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 )
@@ -27,6 +30,7 @@ type Engine struct {
 	decomissioners map[ReservationType]DecomissionerFunc
 	signer         Signer
 	statser        Statser
+	zbusCl         zbus.Client
 }
 
 // EngineOps are the configuration of the engine
@@ -53,6 +57,8 @@ type EngineOps struct {
 	// are reserved on the system running the engine
 	// After each provision/decomission the engine sends statistics update to the staster
 	Statser Statser
+	// ZbusCl is a client to Zbus
+	ZbusCl zbus.Client
 }
 
 // New creates a new engine. Once started, the engine
@@ -71,16 +77,26 @@ func New(opts EngineOps) *Engine {
 		decomissioners: opts.Decomissioners,
 		signer:         opts.Signer,
 		statser:        opts.Statser,
+		zbusCl:         opts.ZbusCl,
 	}
 }
 
 // Run starts reader reservation from the Source and handle them
 func (e *Engine) Run(ctx context.Context) error {
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-
 	cReservation := e.source.Reservations(ctx)
 
+	isAllWorkloadsProcessed := false
+	// run a cron task that will fire the cleanup at midnight
+	cleanUp := make(chan struct{}, 2)
+	c := cron.New()
+	_, err := c.AddFunc("@midnight", func() {
+		cleanUp <- struct{}{}
+	})
+	if err != nil {
+		return fmt.Errorf("failed to setup cron task: %w", err)
+	}
+	defer c.Stop()
+
 	for {
 		select {
 		case <-ctx.Done():
@@ -93,6 +109,13 @@ func (e *Engine) Run(ctx context.Context) error {
 				return nil
 			}
 
+			if reservation.last {
+				isAllWorkloadsProcessed = true
+				// Trigger cleanup by sending a struct onto the channel
+				cleanUp <- struct{}{}
+				continue
+			}
+
 			expired := reservation.Expired()
 			slog := log.With().
 				Str("id", string(reservation.ID)).
@@ -105,13 +128,13 @@ func (e *Engine) Run(ctx context.Context) error {
 
 			if expired || reservation.ToDelete {
 				slog.Info().Msg("start decommissioning reservation")
-				if err := e.decommission(ctx, reservation); err != nil {
+				if err := e.decommission(ctx, &reservation.Reservation); err != nil {
 					log.Error().Err(err).Msgf("failed to decommission reservation %s", reservation.ID)
 					continue
 				}
 			} else {
 				slog.Info().Msg("start provisioning reservation")
-				if err := e.provision(ctx, reservation); err != nil {
+				if err := e.provision(ctx, &reservation.Reservation); err != nil {
 					log.Error().Err(err).Msgf("failed to provision reservation %s", reservation.ID)
 					continue
 				}
@@ -121,6 +144,18 @@ func (e *Engine) Run(ctx context.Context) error {
 				log.Error().Err(err).Msg("failed to updated the capacity counters")
 			}
 
+		case <-cleanUp:
+			if !isAllWorkloadsProcessed {
+				// only allow cleanup triggered by the cron to run once
+				// we are sure all the workloads from the cache/explorer have been processed
+				log.Info().Msg("all workloads not yet processed, delay cleanup")
+				continue
+			}
+			log.Info().Msg("start cleaning up resources")
+			if err := CleanupResources(ctx, e.zbusCl); err != nil {
+				log.Error().Err(err).Msg("failed to cleanup resources")
+				continue
+			}
 		}
 	}
 }
diff --git a/pkg/provision/filesystem.go b/pkg/provision/filesystem.go
new file mode 100644
index 00000000..3bc2e498
--- /dev/null
+++ b/pkg/provision/filesystem.go
@@ -0,0 +1,21 @@
+package provision
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/threefoldtech/zos/pkg"
+)
+
+// FilesystemName return a string to be used as filesystem name from
+// a reservation object
+func FilesystemName(r Reservation) string {
+	return r.ID
+}
+
+// WorkloadIDFromFilesystem parse a filesystem object and return the reservation ID that
+// created it
+func WorkloadIDFromFilesystem(fs pkg.Filesystem) (int64, error) {
+	sid := strings.Split(fs.Name, "-")[0]
+	return strconv.ParseInt(sid, 10, 64)
+}
diff --git a/pkg/provision/interface.go b/pkg/provision/interface.go
index 4607cf60..5d961203 100644
--- a/pkg/provision/interface.go
+++ b/pkg/provision/interface.go
@@ -12,7 +12,7 @@ import (
 // then reservations are applied to the node to deploy
 // a resource of the given Reservation.Type
 type ReservationSource interface {
-	Reservations(ctx context.Context) <-chan *Reservation
+	Reservations(ctx context.Context) <-chan *ReservationJob
 }
 
 // ProvisionerFunc is the function called by the Engine to provision a workload
diff --git a/pkg/provision/primitives/container.go b/pkg/provision/primitives/container.go
index 952eb05b..9b5ff5c3 100644
--- a/pkg/provision/primitives/container.go
+++ b/pkg/provision/primitives/container.go
@@ -244,7 +244,7 @@ func (p *Provisioner) containerProvisionImpl(ctx context.Context, reservation *p
 	}
 
 	var mnt string
-	mnt, err = flistClient.NamedMount(reservation.ID, config.FList, config.FlistStorage, rootfsMntOpt)
+	mnt, err = flistClient.NamedMount(provision.FilesystemName(*reservation), config.FList, config.FlistStorage, rootfsMntOpt)
 	if err != nil {
 		return ContainerResult{}, err
 	}
@@ -258,7 +258,7 @@ func (p *Provisioner) containerProvisionImpl(ctx context.Context, reservation *p
 		if err := os.MkdirAll(path.Join(mnt, mountpoint), 0755); err != nil {
 			return ContainerResult{}, err
 		}
-		var source string
+		var source pkg.Filesystem
 		source, err = storageClient.Path(mount.VolumeID)
 		if err != nil {
 			return ContainerResult{}, errors.Wrapf(err, "failed to get the mountpoint path of the volume %s", mount.VolumeID)
@@ -267,7 +267,7 @@ func (p *Provisioner) containerProvisionImpl(ctx context.Context, reservation *p
 		mounts = append(
 			mounts,
 			pkg.MountInfo{
-				Source: source,
+				Source: source.Path,
 				Target: mountpoint,
 			},
 		)
diff --git a/pkg/provision/primitives/kubernetes.go b/pkg/provision/primitives/kubernetes.go
index eebabcfb..e58e2382 100644
--- a/pkg/provision/primitives/kubernetes.go
+++ b/pkg/provision/primitives/kubernetes.go
@@ -101,7 +101,7 @@ func (p *Provisioner) kubernetesProvisionImpl(ctx context.Context, reservation *
 	}
 
 	var imagePath string
-	imagePath, err = flist.NamedMount(reservation.ID, k3osFlistURL, "", pkg.ReadOnlyMountOptions)
+	imagePath, err = flist.NamedMount(provision.FilesystemName(*reservation), k3osFlistURL, "", pkg.ReadOnlyMountOptions)
 	if err != nil {
 		return result, errors.Wrap(err, "could not mount k3os flist")
 	}
@@ -113,7 +113,7 @@ func (p *Provisioner) kubernetesProvisionImpl(ctx context.Context, reservation *
 	}()
 
 	var diskPath string
-	diskName := fmt.Sprintf("%s-%s", reservation.ID, "vda")
+	diskName := fmt.Sprintf("%s-%s", provision.FilesystemName(*reservation), "vda")
 	if storage.Exists(diskName) {
 		needsInstall = false
 		info, err := storage.Inspect(diskName)
diff --git a/pkg/provision/primitives/volume.go b/pkg/provision/primitives/volume.go
index 57ab51c6..4af72691 100644
--- a/pkg/provision/primitives/volume.go
+++ b/pkg/provision/primitives/volume.go
@@ -46,7 +46,7 @@ func (p *Provisioner) volumeProvisionImpl(ctx context.Context, reservation *prov
 		}, nil
 	}
 
-	_, err = storageClient.CreateFilesystem(reservation.ID, config.Size*gigabyte, config.Type)
+	_, err = storageClient.CreateFilesystem(provision.FilesystemName(*reservation), config.Size*gigabyte, config.Type)
 
 	return VolumeResult{
 		ID: reservation.ID,
diff --git a/pkg/provision/primitives/zdb.go b/pkg/provision/primitives/zdb.go
index 86ff3aba..e5939aed 100644
--- a/pkg/provision/primitives/zdb.go
+++ b/pkg/provision/primitives/zdb.go
@@ -13,6 +13,7 @@ import (
 	"github.com/cenkalti/backoff/v3"
 	"github.com/threefoldtech/zos/pkg/network/ifaceutil"
 	"github.com/threefoldtech/zos/pkg/provision"
+	"github.com/threefoldtech/zos/pkg/provision/common"
 	"github.com/threefoldtech/zos/pkg/zdb"
 
 	"github.com/pkg/errors"
@@ -416,31 +417,7 @@ func (p *Provisioner) zdbDecommission(ctx context.Context, reservation *provisio
 }
 
 func (p *Provisioner) deleteZdbContainer(containerID pkg.ContainerID) error {
-	container := stubs.NewContainerModuleStub(p.zbus)
-	flist := stubs.NewFlisterStub(p.zbus)
-	// networkMgr := stubs.NewNetworkerStub(p.zbus)
-
-	info, err := container.Inspect("zdb", containerID)
-	if err != nil && strings.Contains(err.Error(), "not found") {
-		return nil
-	} else if err != nil {
-		return errors.Wrapf(err, "failed to inspect container '%s'", containerID)
-	}
-
-	if err := container.Delete("zdb", containerID); err != nil {
-		return errors.Wrapf(err, "failed to delete container %s", containerID)
-	}
-
-	network := stubs.NewNetworkerStub(p.zbus)
-	if err := network.ZDBDestroy(info.Network.Namespace); err != nil {
-		return errors.Wrapf(err, "failed to destroy zdb network namespace")
-	}
-
-	if err := flist.Umount(info.RootFS); err != nil {
-		return errors.Wrapf(err, "failed to unmount flist at %s", info.RootFS)
-	}
-
-	return nil
+	return common.DeleteZdbContainer(containerID, p.zbus)
 }
 
 func socketDir(containerID pkg.ContainerID) string {
diff --git a/pkg/provision/source.go b/pkg/provision/source.go
index 86b6d9f3..683941b6 100644
--- a/pkg/provision/source.go
+++ b/pkg/provision/source.go
@@ -30,6 +30,13 @@ type ReservationPoller interface {
 	Poll(nodeID pkg.Identifier, from uint64) (reservations []*Reservation, lastID uint64, err error)
 }
 
+// ReservationJob wraps a reservation type and has
+// a boolean to indicate it is the last reservation
+type ReservationJob struct {
+	Reservation
+	last bool
+}
+
 // PollSource does a long poll on address to get new and to be deleted
 // reservations. the server should only return unique reservations
 // stall the connection as long as possible if no new reservations
@@ -48,21 +55,20 @@ type pollSource struct {
 	maxSleep time.Duration
 }
 
-func (s *pollSource) Reservations(ctx context.Context) <-chan *Reservation {
-	ch := make(chan *Reservation)
-	// on the first run we will get all the reservation
-	// ever made to this know, to make sure we provision
-	// everything at boot
-	// after that, we only ask for the new reservations
+func (s *pollSource) Reservations(ctx context.Context) <-chan *ReservationJob {
+	ch := make(chan *ReservationJob)
+	// On the first run we will get all the reservation ever made to this node to make sure we provision everything at boot.
+	// After that, we only ask for the new reservations.
 	go func() {
 		defer close(ch)
 		var next uint64
+		var previousLastID uint64
+		var triggerCleanup = true
 		on := time.Now()
 		log.Info().Msg("Started polling for reservations")
 		for {
 			time.Sleep(time.Until(on))
 			on = time.Now().Add(s.maxSleep)
-			log.Info().Uint64("next", next).Msg("Polling for reservations")
 
 			res, lastID, err := s.store.Poll(pkg.StrIdentifier(s.nodeID), next)
 			if err != nil && err != ErrPollEOS {
@@ -84,10 +90,27 @@ func (s *pollSource) Reservations(ctx context.Context) <-chan *Reservation {
 				return
 			default:
 				for _, r := range res {
-					ch <- r
+					reservation := ReservationJob{
+						*r,
+						false,
+					}
+					ch <- &reservation
+				}
+
+				// if the explorer return twice the same last ID
+				// it means we have processed all the existing reservation for now
+				// it is safe to trigger a cleanup
+				if triggerCleanup && previousLastID == lastID {
+					ch <- &ReservationJob{
+						Reservation: Reservation{},
+						last:        true,
+					}
+					triggerCleanup = false
 				}
 			}
 
+			previousLastID = lastID
+
 			if err == ErrPollEOS {
 				return
 			}
@@ -117,9 +140,9 @@ func NewDecommissionSource(store ReservationExpirer) ReservationSource {
 	}
 }
 
-func (s *decommissionSource) Reservations(ctx context.Context) <-chan *Reservation {
+func (s *decommissionSource) Reservations(ctx context.Context) <-chan *ReservationJob {
 	log.Info().Msg("start decommission source")
-	c := make(chan *Reservation)
+	c := make(chan *ReservationJob)
 
 	go func() {
 		defer close(c)
@@ -145,7 +168,12 @@ func (s *decommissionSource) Reservations(ctx context.Context) <-chan *Reservati
 						Time("created", r.Created).
 						Str("duration", fmt.Sprintf("%v", r.Duration)).
 						Msg("reservation expired")
-					c <- r
+
+					reservation := ReservationJob{
+						*r,
+						false,
+					}
+					c <- &reservation
 				}
 			}
 		}
@@ -165,14 +193,14 @@ func CombinedSource(sources ...ReservationSource) ReservationSource {
 	}
 }
 
-func (s *combinedSource) Reservations(ctx context.Context) <-chan *Reservation {
+func (s *combinedSource) Reservations(ctx context.Context) <-chan *ReservationJob {
 	var wg sync.WaitGroup
 
-	out := make(chan *Reservation)
+	out := make(chan *ReservationJob)
 
 	// Start an send goroutine for each input channel in cs. send
 	// copies values from c to out until c is closed, then calls wg.Done.
-	send := func(c <-chan *Reservation) {
+	send := func(c <-chan *ReservationJob) {
 		for n := range c {
 			out <- n
 		}
diff --git a/pkg/storage.go b/pkg/storage.go
index 2e6b3baf..d33a630a 100644
--- a/pkg/storage.go
+++ b/pkg/storage.go
@@ -112,6 +112,26 @@ type StoragePolicy struct {
 	MaxPools uint8
 }
 
+// Usage struct
+type Usage struct {
+	Size uint64
+	Used uint64
+}
+
+// Filesystem represents a storage space that can be used as a filesystem
+type Filesystem struct {
+	// Filesystem ID
+	ID int
+	// Path of the Filesystem
+	Path string
+	// Usage reports the current usage of the Filesystem
+	Usage Usage
+	// Name of the Filesystem
+	Name string
+	// FsType of the Filesystem
+	FsType string
+}
+
 // VolumeAllocater is the zbus interface of the storage module responsible
 // for volume allocation
 type VolumeAllocater interface {
@@ -121,7 +141,7 @@ type VolumeAllocater interface {
 	// more space is available in such a pool, `ErrNotEnoughSpace` is returned.
 	// It is up to the caller to handle such a situation and decide if he wants
 	// to try again on a different devicetype
-	CreateFilesystem(name string, size uint64, poolType DeviceType) (string, error)
+	CreateFilesystem(name string, size uint64, poolType DeviceType) (Filesystem, error)
 
 	// ReleaseFilesystem signals that the named filesystem is no longer needed.
 	// The filesystem will be unmounted and subsequently removed.
@@ -129,9 +149,22 @@ type VolumeAllocater interface {
 	// space which has been reserved for this filesystem will be reclaimed.
 	ReleaseFilesystem(name string) error
 
-	// Path return the path of the mountpoint of the named filesystem
-	// if no volume with name exists, an empty path and an error is returned
-	Path(name string) (path string, err error)
+	// ListFilesystems return all the filesystem managed by storeaged present on the nodes
+	// this can be an expensive call on server with a lot of disk, don't use it in a
+	// intensive loop
+	// Special filesystem like internal cache and vdisk are not return by this function
+	// to access them use the GetCacheFS or GetVdiskFS
+	ListFilesystems() ([]Filesystem, error)
+
+	// Path return the filesystem named name
+	// if no filesystem with this name exists, an error is returned
+	Path(name string) (Filesystem, error)
+
+	// GetCacheFS return the special filesystem used by 0-OS to store internal state and flist cache
+	GetCacheFS() (Filesystem, error)
+
+	// GetVdiskFS return the filesystem used to store the vdisk file for the VM module
+	GetVdiskFS() (Filesystem, error)
 }
 
 // VDisk info returned by a call to inspect
diff --git a/pkg/storage/disk.go b/pkg/storage/disk.go
index b4a516a1..49198fa1 100644
--- a/pkg/storage/disk.go
+++ b/pkg/storage/disk.go
@@ -11,6 +11,7 @@ import (
 )
 
 const (
+	// vdiskVolumeName is the name of the volume used to store vdisks
 	vdiskVolumeName = "vdisks"
 
 	mib = 1024 * 1024
@@ -22,16 +23,16 @@ type vdiskModule struct {
 
 // NewVDiskModule creates a new disk allocator
 func NewVDiskModule(v pkg.VolumeAllocater) (pkg.VDiskModule, error) {
-	path, err := v.Path(vdiskVolumeName)
+	fs, err := v.Path(vdiskVolumeName)
 	if errors.Is(err, os.ErrNotExist) {
-		path, err = v.CreateFilesystem(vdiskVolumeName, 0, pkg.SSDDevice)
+		fs, err = v.CreateFilesystem(vdiskVolumeName, 0, pkg.SSDDevice)
 	}
 
 	if err != nil {
 		return nil, err
 	}
 
-	return &vdiskModule{path: filepath.Clean(path)}, nil
+	return &vdiskModule{path: filepath.Clean(fs.Path)}, nil
 }
 
 // AllocateDisk with given size, return path to virtual disk (size in MB)
diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go
index c9d8b8d1..9e5e8d82 100644
--- a/pkg/storage/storage.go
+++ b/pkg/storage/storage.go
@@ -24,9 +24,10 @@ import (
 const (
 	// CacheTarget is the path where the cache disk is mounted
 	CacheTarget = "/var/cache"
-	cacheLabel  = "zos-cache"
-	gib         = 1024 * 1024 * 1024
-	cacheSize   = 100 * gib
+	// cacheLabel is the name of the cache
+	cacheLabel = "zos-cache"
+	gib        = 1024 * 1024 * 1024
+	cacheSize  = 100 * gib
 )
 
 var (
@@ -336,17 +337,32 @@ func (s *storageModule) shutdownUnusedPools() error {
 }
 
 // CreateFilesystem with the given size in a storage pool.
-func (s *storageModule) CreateFilesystem(name string, size uint64, poolType pkg.DeviceType) (string, error) {
+func (s *storageModule) CreateFilesystem(name string, size uint64, poolType pkg.DeviceType) (pkg.Filesystem, error) {
 	log.Info().Msgf("Creating new volume with size %d", size)
 	if strings.HasPrefix(name, "zdb") {
-		return "", fmt.Errorf("invalid volume name. zdb prefix is reserved")
+		return pkg.Filesystem{}, fmt.Errorf("invalid volume name. zdb prefix is reserved")
 	}
 
 	fs, err := s.createSubvol(size, name, poolType)
 	if err != nil {
-		return "", err
+		return pkg.Filesystem{}, err
 	}
-	return fs.Path(), nil
+
+	usage, err := fs.Usage()
+	if err != nil {
+		return pkg.Filesystem{}, err
+	}
+
+	return pkg.Filesystem{
+		ID:     fs.ID(),
+		FsType: fs.FsType(),
+		Name:   fs.Name(),
+		Path:   fs.Path(),
+		Usage: pkg.Usage{
+			Size: usage.Size,
+			Used: usage.Used,
+		},
+	}, nil
 }
 
 // ReleaseFilesystem with the given name, this will unmount and then delete
@@ -393,22 +409,89 @@ func (s *storageModule) ReleaseFilesystem(name string) error {
 	return nil
 }
 
+// ListFilesystems return all the filesystem managed by storeaged present on the nodes
+func (s *storageModule) ListFilesystems() ([]pkg.Filesystem, error) {
+	fss := make([]pkg.Filesystem, 0, 10)
+
+	for _, pool := range s.pools {
+		if _, mounted := pool.Mounted(); !mounted {
+			continue
+		}
+
+		volumes, err := pool.Volumes()
+		if err != nil {
+			return nil, err
+		}
+		for _, v := range volumes {
+
+			// Do not return "special" volumes here
+			// instead the GetCacheFS and GetVdiskFS to access them
+			if v.Name() == cacheLabel ||
+				v.Name() == vdiskVolumeName {
+				continue
+			}
+
+			usage, err := v.Usage()
+			if err != nil {
+				return nil, err
+			}
+
+			fss = append(fss, pkg.Filesystem{
+				ID:     v.ID(),
+				FsType: v.FsType(),
+				Name:   v.Name(),
+				Path:   v.Path(),
+				Usage: pkg.Usage{
+					Size: usage.Size,
+					Used: usage.Used,
+				},
+			})
+		}
+	}
+
+	return fss, nil
+}
+
 // Path return the path of the mountpoint of the named filesystem
 // if no volume with name exists, an empty path and an error is returned
-func (s *storageModule) Path(name string) (string, error) {
+func (s *storageModule) Path(name string) (pkg.Filesystem, error) {
 	for idx := range s.pools {
 		filesystems, err := s.pools[idx].Volumes()
 		if err != nil {
-			return "", err
+			return pkg.Filesystem{}, err
 		}
 		for jdx := range filesystems {
 			if filesystems[jdx].Name() == name {
-				return filesystems[jdx].Path(), nil
+				usage, err := filesystems[jdx].Usage()
+				if err != nil {
+					return pkg.Filesystem{}, err
+				}
+
+				return pkg.Filesystem{
+					ID:     filesystems[jdx].ID(),
+					FsType: filesystems[jdx].FsType(),
+					Name:   filesystems[jdx].Name(),
+					Path:   filesystems[jdx].Path(),
+					Usage: pkg.Usage{
+						Size: usage.Size,
+						Used: usage.Used,
+					},
+				}, nil
 			}
 		}
 	}
 
-	return "", errors.Wrapf(os.ErrNotExist, "subvolume '%s' not found", name)
+	return pkg.Filesystem{}, errors.Wrapf(os.ErrNotExist, "subvolume '%s' not found", name)
+}
+
+// GetCacheFS return the special filesystem used by 0-OS to store internal state and flist cache
+func (s *storageModule) GetCacheFS() (pkg.Filesystem, error) {
+	return s.Path(cacheLabel)
+}
+
+// GetVdiskFS return the filesystem used to store the vdisk file for the VM module
+func (s *storageModule) GetVdiskFS() (pkg.Filesystem, error) {
+	return s.Path(vdiskVolumeName)
 }
 
 // ensureCache creates a "cache" subvolume and mounts it in /var
diff --git a/pkg/stubs/container_stub.go b/pkg/stubs/container_stub.go
index 685a46f9..3151a0dc 100644
--- a/pkg/stubs/container_stub.go
+++ b/pkg/stubs/container_stub.go
@@ -67,6 +67,22 @@ func (s *ContainerModuleStub) List(arg0 string) (ret0 []pkg.ContainerID, ret1 er
 	return
 }
 
+func (s *ContainerModuleStub) ListNS() (ret0 []string, ret1 error) {
+	args := []interface{}{}
+	result, err := s.client.Request(s.module, s.object, "ListNS", args...)
+	if err != nil {
+		panic(err)
+	}
+	if err := result.Unmarshal(0, &ret0); err != nil {
+		panic(err)
+	}
+	ret1 = new(zbus.RemoteError)
+	if err := result.Unmarshal(1, &ret1); err != nil {
+		panic(err)
+	}
+	return
+}
+
 func (s *ContainerModuleStub) Run(arg0 string, arg1 pkg.Container) (ret0 pkg.ContainerID, ret1 error) {
 	args := []interface{}{arg0, arg1}
 	result, err := s.client.Request(s.module, s.object, "Run", args...)
diff --git a/pkg/stubs/storage_stub.go b/pkg/stubs/storage_stub.go
index fa73e21b..4b52f7e2 100644
--- a/pkg/stubs/storage_stub.go
+++ b/pkg/stubs/storage_stub.go
@@ -63,7 +63,7 @@ func (s *StorageModuleStub) BrokenPools() (ret0 []pkg.BrokenPool) {
 	return
 }
 
-func (s *StorageModuleStub) CreateFilesystem(arg0 string, arg1 uint64, arg2 pkg.DeviceType) (ret0 string, ret1 error) {
+func (s *StorageModuleStub) CreateFilesystem(arg0 string, arg1 uint64, arg2 pkg.DeviceType) (ret0 pkg.Filesystem, ret1 error) {
 	args := []interface{}{arg0, arg1, arg2}
 	result, err := s.client.Request(s.module, s.object, "CreateFilesystem", args...)
 	if err != nil {
@@ -95,6 +95,54 @@ func (s *StorageModuleStub) Find(arg0 string) (ret0 pkg.Allocation, ret1 error)
 	return
 }
 
+func (s *StorageModuleStub) GetCacheFS() (ret0 pkg.Filesystem, ret1 error) {
+	args := []interface{}{}
+	result, err := s.client.Request(s.module, s.object, "GetCacheFS", args...)
+	if err != nil {
+		panic(err)
+	}
+	if err := result.Unmarshal(0, &ret0); err != nil {
+		panic(err)
+	}
+	ret1 = new(zbus.RemoteError)
+	if err := result.Unmarshal(1, &ret1); err != nil {
+		panic(err)
+	}
+	return
+}
+
+func (s *StorageModuleStub) GetVdiskFS() (ret0 pkg.Filesystem, ret1 error) {
+	args := []interface{}{}
+	result, err := s.client.Request(s.module, s.object, "GetVdiskFS", args...)
+	if err != nil {
+		panic(err)
+	}
+	if err := result.Unmarshal(0, &ret0); err != nil {
+		panic(err)
+	}
+	ret1 = new(zbus.RemoteError)
+	if err := result.Unmarshal(1, &ret1); err != nil {
+		panic(err)
+	}
+	return
+}
+
+func (s *StorageModuleStub) ListFilesystems() (ret0 []pkg.Filesystem, ret1 error) {
+	args := []interface{}{}
+	result, err := s.client.Request(s.module, s.object, "ListFilesystems", args...)
+	if err != nil {
+		panic(err)
+	}
+	if err := result.Unmarshal(0, &ret0); err != nil {
+		panic(err)
+	}
+	ret1 = new(zbus.RemoteError)
+	if err := result.Unmarshal(1, &ret1); err != nil {
+		panic(err)
+	}
+	return
+}
+
 func (s *StorageModuleStub) Monitor(ctx context.Context) (<-chan pkg.PoolsStats, error) {
 	ch := make(chan pkg.PoolsStats)
 	recv, err := s.client.Stream(ctx, s.module, s.object, "Monitor")
@@ -114,7 +162,7 @@ func (s *StorageModuleStub) Monitor(ctx context.Context) (<-chan pkg.PoolsStats,
 	return ch, nil
 }
 
-func (s *StorageModuleStub) Path(arg0 string) (ret0 string, ret1 error) {
+func (s *StorageModuleStub) Path(arg0 string) (ret0 pkg.Filesystem, ret1 error) {
 	args := []interface{}{arg0}
 	result, err := s.client.Request(s.module, s.object, "Path", args...)
 	if err != nil {


Test patch:
diff --git a/pkg/flist/flist_test.go b/pkg/flist/flist_test.go
index fab7b3a0..335b9be0 100644
--- a/pkg/flist/flist_test.go
+++ b/pkg/flist/flist_test.go
@@ -31,9 +31,11 @@ type StorageMock struct {
 }
 
 // CreateFilesystem create filesystem mock
-func (s *StorageMock) CreateFilesystem(name string, size uint64, poolType pkg.DeviceType) (string, error) {
+func (s *StorageMock) CreateFilesystem(name string, size uint64, poolType pkg.DeviceType) (pkg.Filesystem, error) {
 	args := s.Called(name, size, poolType)
-	return args.String(0), args.Error(1)
+	return pkg.Filesystem{
+		Path: args.String(0),
+	}, args.Error(1)
 }
 
 // ReleaseFilesystem releases filesystem mock
@@ -42,10 +44,28 @@ func (s *StorageMock) ReleaseFilesystem(name string) error {
 	return args.Error(0)
 }
 
+// ListFilesystems list filesystem mock
+func (s *StorageMock) ListFilesystems() ([]pkg.Filesystem, error) {
+	args := s.Called()
+	return nil, args.Error(1)
+}
+
 // Path implements the pkg.StorageModules interfaces
-func (s *StorageMock) Path(name string) (path string, err error) {
+func (s *StorageMock) Path(name string) (pkg.Filesystem, error) {
 	args := s.Called(name)
-	return args.String(0), args.Error(1)
+	return pkg.Filesystem{
+		Path: args.String(0),
+	}, args.Error(1)
+}
+
+// GetCacheFS return the special filesystem used by 0-OS to store internal state and flist cache
+func (s *StorageMock) GetCacheFS() (pkg.Filesystem, error) {
+	return pkg.Filesystem{}, nil
+}
+
+// GetVdiskFS return the filesystem used to store the vdisk file for the VM module
+func (s *StorageMock) GetVdiskFS() (pkg.Filesystem, error) {
+	return pkg.Filesystem{}, nil
 }
 
 type testCommander struct {
diff --git a/pkg/provision/filesystem_test.go b/pkg/provision/filesystem_test.go
new file mode 100644
index 00000000..af0c2abd
--- /dev/null
+++ b/pkg/provision/filesystem_test.go
@@ -0,0 +1,24 @@
+package provision
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/threefoldtech/zos/pkg"
+	"gotest.tools/assert"
+)
+
+func TestFilesystemNameFromReservation(t *testing.T) {
+	r := Reservation{
+		ID: "123-1",
+	}
+
+	name := FilesystemName(r)
+	assert.Equal(t, r.ID, name)
+
+	id, err := WorkloadIDFromFilesystem(pkg.Filesystem{
+		Name: name,
+	})
+	require.NoError(t, err)
+	assert.Equal(t, int64(123), id)
+}
diff --git a/pkg/provision/source_test.go b/pkg/provision/source_test.go
index f87a5cc0..736f4774 100644
--- a/pkg/provision/source_test.go
+++ b/pkg/provision/source_test.go
@@ -34,7 +34,7 @@ func TestHTTPReservationSource(t *testing.T) {
 			&Reservation{ID: "1-2"},
 		}, 1, ErrPollEOS)
 
-	reservations := []*Reservation{}
+	reservations := []*ReservationJob{}
 	for res := range chn {
 		reservations = append(reservations, res)
 	}
@@ -68,7 +68,7 @@ func TestHTTPReservationSourceMultiple(t *testing.T) {
 			&Reservation{ID: "4-1"},
 		}, 6, ErrPollEOS)
 
-	reservations := []*Reservation{}
+	reservations := []*ReservationJob{}
 	for res := range chn {
 		reservations = append(reservations, res)
 	}
@@ -88,20 +88,20 @@ type TestTrackSource struct {
 
 func (s *TestTrackSource) Poll(nodeID pkg.Identifier, from uint64) ([]*Reservation, uint64, error) {
 	if s.ID == s.Max {
-		return nil, 0, ErrPollEOS
+		return nil, s.Max, ErrPollEOS
 	}
 
 	s.Calls = append(s.Calls, time.Now().Unix())
 
-	defer func() {
+	reservations := make([]*Reservation, s.Max)
+	for i := uint64(0); i < s.Max; i++ {
+		reservations[i] = &Reservation{
+			ID: fmt.Sprint(s.ID, "-", "0"),
+		}
 		s.ID++
-	}()
+	}
 
-	return []*Reservation{
-		&Reservation{
-			ID: fmt.Sprint(s.ID, "-", "0"),
-		},
-	}, s.ID, nil
+	return reservations, s.Max, nil
 }
 
 func TestHTTPReservationSourceSleep(t *testing.T) {
@@ -118,18 +118,23 @@ func TestHTTPReservationSourceSleep(t *testing.T) {
 
 	chn := source.Reservations(context.Background())
 
-	reservations := []*Reservation{}
+	reservations := []*ReservationJob{}
 	for res := range chn {
 		reservations = append(reservations, res)
 	}
 
-	require.Len(reservations, 4)
+	require.Len(reservations, 5)
 	require.Equal("0-0", reservations[0].ID)
+	require.False(reservations[0].last)
 	require.Equal("1-0", reservations[1].ID)
+	require.False(reservations[1].last)
 	require.Equal("2-0", reservations[2].ID)
+	require.False(reservations[2].last)
 	require.Equal("3-0", reservations[3].ID)
+	require.False(reservations[3].last)
+	require.True(reservations[4].last)
 
-	require.Len(store.Calls, 4)
+	require.Len(store.Calls, 1)
 	for i := 1; i < len(store.Calls); i++ {
 		require.Equal(int64(3), store.Calls[i]-store.Calls[i-1])
 	}