Repository: threefoldtech/zos (go)

Issue:
Reserved SSD figures inconsistent (testnet)
I tried to deploy a full VM on testnet node 198 with a 200gb disk. From the playground, this looks okay:

![image](https://user-images.githubusercontent.com/17682591/197902082-796e1cc8-6839-48e6-8396-9b18932ecbbb.png)

But deployment fails with this error:

```
error: failed to find a candidate to host vdisk of size '214748364800': not enough space left in pools of this type ssd
```

I checked the dashboard and grid proxy:

```
$ curl https://gridproxy.test.grid.tf/nodes/198
{"id":"0002224520-000001-be1d8","nodeId":198,"farmId":1,"twinId":474,"country":"Belgium","gridVersion":3,"city":"Unknown","uptime":18784800,"created":1647952446,"farmingPolicyId":1,"updatedAt":1666736886,"capacity":{"total_resources":{"cru":24,"sru":512110190592,"hru":9001778946048,"mru":202802905088},"used_resources":{"cru":4,"sru":270582939648,"hru":0,"mru":28870225101}},"location":{"country":"Belgium","city":"Unknown"},"publicConfig":{"domain":"","gw4":"","gw6":"","ipv4":"","ipv6":""},"status":"up","certificationType":"Diy","dedicated":false,"rentContractId":0,"rentedByTwinId":0}

total_sru - reserved_sru = 512110190592 - 270582939648 = 241527250944 = ~241gb
```

Then I queried the node directly over RMB:

```
{"ver":1,"uid":"","cmd":"zos.statistics.get","exp":3600,"dat":"","src":0,"dst":[474],"ret":"1d879d91-0074-4f3f-9152-9c3a234b6056","try":2,"shm":"","now":1666739902,"err":""}

[
  {
    ver: 1,
    uid: '474.1',
    cmd: 'zos.statistics.get',
    exp: 3600,
    try: 2,
    dat: '{"total":{"cru":24,"sru":512110190592,"hru":9001778946048,"mru":201863462912,"ipv4u":0},"used":{"cru":0,"sru":107374182400,"hru":0,"mru":31233567948,"ipv4u":1}}'
```

Then I tried finding all contracts for this node via GraphQL. Only one was active and using resources:

```
        "resourcesUsed": {
          "cru": "4",
          "hru": "0",
          "mru": "8589934592",
          "sru": "163208757248"
        },
        "nodeID": 198
```

It seems that:
1. Zos is only reporting its own reservation of SRU/MRU over RMB, not the reservation by workloads
2. This node is rejecting a workload it should have capacity to run

Pull request:
Fix statistics reporting
Fixes #1819

To reduce the time to provision a new deploment, the system (provisiond in particular) uses a "cached" version of "used" capacity of the system. On boot the daemon usually scan it's db and calculate the number of used capacity from the deployment information. This was done once at the service boot, then for each deploy/remove of a workload the in memory cache is updated. 

There was some edge cases where updating the cached/calculated used capacity in memory is not sufficient, and eventually with many operations the in memory (what the system think is used) is not in sync with what is actually used.

We avoided scanning the db and recalculate the theoretical used capacity to avoid slowing down deployment. since for historic reason we never deleted information or history of "decomissioned" deployments. This means each time the system need to find out the used capacity it will have to go through a lot of data and check if this deployment is still active or not. This is why we used to scan this one time and then cache the numbers. 

It turns out that keeping this historic information is not needed anymore, and we can simply get rid of that. Once a deployment is deleted, it's gone from the db forever.

It means calculating the theoretical used capacity from the the real deployment information is not a heavy operation anymore and can be done on request.

This PR does the following:
- Migration code to clean up old decomissioned deployments
- When a deployment is deleted, it's immediately removed from the db
- Statistics calculations is now based on the real data in the db not cached.
- Other fixes along the way

Patch:
diff --git a/cmds/modules/provisiond/main.go b/cmds/modules/provisiond/main.go
index aa303bed..048c27fa 100644
--- a/cmds/modules/provisiond/main.go
+++ b/cmds/modules/provisiond/main.go
@@ -175,6 +175,10 @@ func action(cli *cli.Context) error {
 		}
 	}
 
+	if err := store.CleanDeleted(); err != nil {
+		log.Error().Err(err).Msg("failed to purge deleted deployments history")
+	}
+
 	provisioners := primitives.NewPrimitivesProvisioner(cl)
 
 	cap, err := capacity.NewResourceOracle(stubs.NewStorageModuleStub(cl)).Total()
@@ -212,7 +216,7 @@ func action(cli *cli.Context) error {
 	// also does some checks on capacity
 	statistics := primitives.NewStatistics(
 		cap,
-		current,
+		store,
 		reserved,
 		provisioners,
 	)
@@ -401,7 +405,7 @@ func action(cli *cli.Context) error {
 	return nil
 }
 
-func getNodeReserved(cl zbus.Client, available gridtypes.Capacity) (counter primitives.Counters, err error) {
+func getNodeReserved(cl zbus.Client, available gridtypes.Capacity) (counter gridtypes.Capacity, err error) {
 	// fill in reserved storage
 	storage := stubs.NewStorageModuleStub(cl)
 	fs, err := storage.Cache(context.TODO())
@@ -409,14 +413,10 @@ func getNodeReserved(cl zbus.Client, available gridtypes.Capacity) (counter prim
 		return counter, err
 	}
 
-	counter.SRU.Increment(fs.Usage.Size)
-
-	// we reserve 10% of memory to ZOS itself, with a min of 2G
-	counter.MRU.Increment(
-		gridtypes.Max(
-			available.MRU*10/100,
-			2*gridtypes.Gigabyte,
-		),
+	counter.SRU += fs.Usage.Size
+	counter.MRU += gridtypes.Max(
+		available.MRU*10/100,
+		2*gridtypes.Gigabyte,
 	)
 
 	return
diff --git a/pkg/events/events.go b/pkg/events/events.go
index cc99b140..e3f5ba1f 100644
--- a/pkg/events/events.go
+++ b/pkg/events/events.go
@@ -8,6 +8,7 @@ import (
 	"time"
 
 	gsrpc "github.com/centrifuge/go-substrate-rpc-client/v4"
+	rpc "github.com/centrifuge/go-substrate-rpc-client/v4/gethrpc"
 	"github.com/centrifuge/go-substrate-rpc-client/v4/types"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
@@ -98,9 +99,17 @@ func (e *Events) eventsTo(cl *gsrpc.SubstrateAPI, meta *types.Metadata, block ty
 			return errors.Wrapf(err, "failed to get block hash '%d'", start)
 		}
 
+		//state.ErrUnknownBlock
 		changes, err := cl.RPC.State.QueryStorageAt([]types.StorageKey{key}, hash)
+		if err, ok := err.(rpc.Error); ok {
+			if err.ErrorCode() == -32000 { // block is too old not in archive anymore
+				log.Debug().Uint32("block", uint32(i)).Msg("block not available in archive anymore")
+				continue
+			}
+		}
+
 		if err != nil {
-			return err
+			return errors.Wrapf(err, "failed to get block with hash '%s'", hash.Hex())
 		}
 
 		e.process(changes, meta)
diff --git a/pkg/primitives/counters.go b/pkg/primitives/counters.go
deleted file mode 100644
index 34b442f2..00000000
--- a/pkg/primitives/counters.go
+++ /dev/null
@@ -1,70 +0,0 @@
-package primitives
-
-import (
-	"sync/atomic"
-
-	"github.com/threefoldtech/zos/pkg/gridtypes"
-)
-
-// AtomicUnit value for safe increment/decrement
-type AtomicUnit gridtypes.Unit
-
-// Increment counter atomically by one
-func (c *AtomicUnit) Increment(v gridtypes.Unit) gridtypes.Unit {
-	return gridtypes.Unit(atomic.AddUint64((*uint64)(c), uint64(v)))
-}
-
-// Decrement counter atomically by one
-func (c *AtomicUnit) Decrement(v gridtypes.Unit) gridtypes.Unit {
-	// spinlock until the decrement succeeds
-	for {
-		current := c.Current()
-		// make sure we don't decrement below 0
-		dec := v
-		if dec > current {
-			dec = current
-		}
-		// compute new value
-		n := current - dec
-		// only swap if `current`, and therefore the above calculations,
-		// are still valid
-		if atomic.CompareAndSwapUint64((*uint64)(c), uint64(current), uint64(n)) {
-			return gridtypes.Unit(n)
-		}
-	}
-}
-
-// Current returns the current value
-func (c *AtomicUnit) Current() gridtypes.Unit {
-	return gridtypes.Unit(atomic.LoadUint64((*uint64)(c)))
-}
-
-// Counters tracks the amount of primitives workload deployed and
-// the amount of resource unit used
-type Counters struct {
-	//types map[gridtypes.WorkloadType]AtomicValue
-
-	SRU  AtomicUnit // SSD storage in bytes
-	HRU  AtomicUnit // HDD storage in bytes
-	MRU  AtomicUnit // Memory storage in bytes
-	CRU  AtomicUnit // CPU count absolute
-	IPv4 AtomicUnit // IPv4 count absolute
-}
-
-// Increment is called by the provision.Engine when a reservation has been provisionned
-func (c *Counters) Increment(cap gridtypes.Capacity) {
-	c.CRU.Increment(gridtypes.Unit(cap.CRU))
-	c.MRU.Increment(cap.MRU)
-	c.SRU.Increment(cap.SRU)
-	c.HRU.Increment(cap.HRU)
-	c.IPv4.Increment(gridtypes.Unit(cap.IPV4U))
-}
-
-// Decrement is called by the provision.Engine when a reservation has been decommissioned
-func (c *Counters) Decrement(cap gridtypes.Capacity) {
-	c.CRU.Decrement(gridtypes.Unit(cap.CRU))
-	c.MRU.Decrement(cap.MRU)
-	c.SRU.Decrement(cap.SRU)
-	c.HRU.Decrement(cap.HRU)
-	c.IPv4.Decrement(gridtypes.Unit(cap.IPV4U))
-}
diff --git a/pkg/primitives/statistics.go b/pkg/primitives/statistics.go
index 5e53afe0..d6982463 100644
--- a/pkg/primitives/statistics.go
+++ b/pkg/primitives/statistics.go
@@ -40,8 +40,8 @@ var (
 type Statistics struct {
 	inner    provision.Provisioner
 	total    gridtypes.Capacity
-	counters Counters
-	reserved Counters
+	reserved gridtypes.Capacity
+	storage  provision.Storage
 	mem      gridtypes.Unit
 }
 
@@ -52,7 +52,7 @@ func roundTotalMemory(t gridtypes.Unit) gridtypes.Unit {
 
 // NewStatistics creates a new statistics provisioner interceptor.
 // Statistics provisioner keeps track of used capacity and update explorer when it changes
-func NewStatistics(total, initial gridtypes.Capacity, reserved Counters, inner provision.Provisioner) *Statistics {
+func NewStatistics(total gridtypes.Capacity, storage provision.Storage, reserved gridtypes.Capacity, inner provision.Provisioner) *Statistics {
 	vm, err := mem.VirtualMemory()
 	if err != nil {
 		panic(err)
@@ -60,36 +60,35 @@ func NewStatistics(total, initial gridtypes.Capacity, reserved Counters, inner p
 
 	total.MRU = roundTotalMemory(total.MRU)
 
-	log.Debug().Msgf("initial used capacity %+v", initial)
-	var counters Counters
-	counters.Increment(initial)
-
 	return &Statistics{
 		inner:    inner,
 		total:    total,
-		counters: counters,
 		reserved: reserved,
+		storage:  storage,
 		mem:      gridtypes.Unit(vm.Total),
 	}
 }
 
-// Current returns the current used capacity
-func (s *Statistics) Current() gridtypes.Capacity {
-	total, usable, err := s.getUsableMemoryBytes()
+func (s *Statistics) active() (gridtypes.Capacity, error) {
+	cap, _, err := s.storage.Capacity()
+	return cap, err
+}
+
+// Current returns the current used capacity including reserved capacity
+// used by the system
+func (s *Statistics) Current() (gridtypes.Capacity, error) {
+	cap, usable, err := s.getUsableMemoryBytes()
 
 	if err != nil {
-		panic("failed to get memory consumption")
+		return cap, err
 	}
 
-	used := total + s.reserved.MRU.Current() - usable
+	used := s.total.MRU - usable
 
-	return gridtypes.Capacity{
-		CRU:   uint64(s.counters.CRU.Current() + s.reserved.CRU.Current()),
-		MRU:   used,
-		HRU:   s.counters.HRU.Current() + s.reserved.HRU.Current(),
-		SRU:   s.counters.SRU.Current() + s.reserved.SRU.Current(),
-		IPV4U: uint64(s.counters.IPv4.Current()),
-	}
+	cap.Add(&s.reserved)
+	cap.MRU = used
+
+	return cap, nil
 }
 
 // Total returns the node total capacity
@@ -97,50 +96,44 @@ func (s *Statistics) Total() gridtypes.Capacity {
 	return s.total
 }
 
-// getUsableMemoryBytes returns the usable free memory. this takes
-// into account the system reserved, actual available memory and the theorytical max reserved memory
-// by the workloads
-func (s *Statistics) getUsableMemoryBytes() (gridtypes.Unit, gridtypes.Unit, error) {
-	m, err := mem.VirtualMemory()
+// getUsableMemoryBytes returns the used capacity by *reservations* and usable free memory. for the memory
+// it takes into account reserved memory for the system
+func (s *Statistics) getUsableMemoryBytes() (gridtypes.Capacity, gridtypes.Unit, error) {
+	// [                          ]
+	// [[R][ WL ]                 ]
+	// [[    actual    ]          ]
+
+	cap, err := s.active()
 	if err != nil {
-		return 0, 0, err
+		return cap, 0, err
 	}
 
-	reserved := s.reserved.MRU.Current()
-	// total is always the total - the reserved
-	total := s.total.MRU - reserved
-
-	// which means the available memory is always also
-	// is actual_available - reserved
-	var available gridtypes.Unit
-	if gridtypes.Unit(m.Available) > reserved {
-		available = gridtypes.Unit(m.Available) - reserved
+	m, err := mem.VirtualMemory()
+	if err != nil {
+		return cap, 0, err
 	}
 
-	// get reserved memory from current deployed workloads (theoretical max)
-	theoryticalUsed := s.counters.MRU.Current()
-	var theoryticalFree gridtypes.Unit // this is the (total - workload)
-	if total > theoryticalUsed {
-		theoryticalFree = total - theoryticalUsed
-	}
+	theoreticalUsed := cap.MRU + s.reserved.MRU
+	actualUsed := m.Total - m.Available
 
-	// usable is the min of actual available on the system or theoryticalFree
-	usable := gridtypes.Unit(math.Min(float64(theoryticalFree), float64(available)))
-	return total, usable, nil
+	used := gridtypes.Max(theoreticalUsed, gridtypes.Unit(actualUsed))
+
+	usable := gridtypes.Unit(m.Total) - used
+	return cap, usable, nil
 }
 
-func (s *Statistics) hasEnoughCapacity(required *gridtypes.Capacity) error {
+func (s *Statistics) hasEnoughCapacity(required *gridtypes.Capacity) (gridtypes.Capacity, error) {
 	// checks memory
-	_, usable, err := s.getUsableMemoryBytes()
+	used, usable, err := s.getUsableMemoryBytes()
 	if err != nil {
-		return errors.Wrap(err, "failed to get available memory")
+		return used, errors.Wrap(err, "failed to get available memory")
 	}
 	if required.MRU > usable {
-		return fmt.Errorf("cannot fulfil required memory size %d bytes out of usable %d bytes", required.MRU, usable)
+		return used, fmt.Errorf("cannot fulfil required memory size %d bytes out of usable %d bytes", required.MRU, usable)
 	}
 
 	//check other resources as well?
-	return nil
+	return used, nil
 }
 
 // Initialize implements provisioner interface
@@ -150,7 +143,6 @@ func (s *Statistics) Initialize(ctx context.Context) error {
 
 // Provision implements the provisioner interface
 func (s *Statistics) Provision(ctx context.Context, wl *gridtypes.WorkloadWithID) (result gridtypes.Result, err error) {
-	current := s.Current()
 	needed, err := wl.Capacity()
 	if err != nil {
 		return result, errors.Wrap(err, "failed to calculate workload needed capacity")
@@ -162,38 +154,18 @@ func (s *Statistics) Provision(ctx context.Context, wl *gridtypes.WorkloadWithID
 		needed.MRU += gridtypes.Min(needed.MRU*5/100, gridtypes.Gigabyte)
 	} // TODO: other types ?
 
-	if err := s.hasEnoughCapacity(&needed); err != nil {
+	current, err := s.hasEnoughCapacity(&needed)
+	if err != nil {
 		return result, errors.Wrap(err, "failed to satisfy required capacity")
 	}
 
 	ctx = context.WithValue(ctx, currentCapacityKey{}, current)
-	result, err = s.inner.Provision(ctx, wl)
-	if err != nil {
-		return result, err
-	}
-
-	if result.State.IsOkay() {
-		log.Debug().Str("type", wl.Type.String()).Str("id", wl.ID.String()).Msgf("incrmenting capacity +%+v", needed)
-		s.counters.Increment(needed)
-	}
-
-	return result, nil
+	return s.inner.Provision(ctx, wl)
 }
 
 // Decommission implements the decomission interface
 func (s *Statistics) Deprovision(ctx context.Context, wl *gridtypes.WorkloadWithID) error {
-	if err := s.inner.Deprovision(ctx, wl); err != nil {
-		return err
-	}
-	cap, err := wl.Capacity()
-	if err != nil {
-		log.Error().Err(err).Msg("failed to decrement statistics counter")
-		return nil
-	}
-
-	s.counters.Decrement(cap)
-
-	return nil
+	return s.inner.Deprovision(ctx, wl)
 }
 
 // Update implements the provisioner interface
@@ -232,12 +204,17 @@ func (s *statisticsMessageBus) setup(router rmb.Router) error {
 }
 
 func (s *statisticsMessageBus) getCounters(ctx context.Context, payload []byte) (interface{}, error) {
+
+	used, err := s.stats.Current()
+	if err != nil {
+		return nil, err
+	}
 	return struct {
 		Total gridtypes.Capacity `json:"total"`
 		Used  gridtypes.Capacity `json:"used"`
 	}{
 		Total: s.stats.Total(),
-		Used:  s.stats.Current(),
+		Used:  used,
 	}, nil
 }
 
@@ -257,15 +234,19 @@ func (s *statsStream) ReservedStream(ctx context.Context) <-chan gridtypes.Capac
 			select {
 			case <-ctx.Done():
 				return
-			case <-time.After(10 * time.Second):
-				ch <- s.stats.Current()
+			case <-time.After(2 * time.Minute):
+				used, err := s.stats.Current()
+				if err != nil {
+					log.Error().Err(err).Msg("failed to get used capacity")
+				}
+				ch <- used
 			}
 		}
 	}(ctx)
 	return ch
 }
 
-func (s *statsStream) Current() gridtypes.Capacity {
+func (s *statsStream) Current() (gridtypes.Capacity, error) {
 	return s.stats.Current()
 }
 
diff --git a/pkg/provision.go b/pkg/provision.go
index 4c95b9bc..11702482 100644
--- a/pkg/provision.go
+++ b/pkg/provision.go
@@ -16,6 +16,6 @@ type Provision interface {
 
 type Statistics interface {
 	ReservedStream(ctx context.Context) <-chan gridtypes.Capacity
-	Current() gridtypes.Capacity
+	Current() (gridtypes.Capacity, error)
 	Total() gridtypes.Capacity
 }
diff --git a/pkg/provision/engine.go b/pkg/provision/engine.go
index 6be122c4..1d66d2ca 100644
--- a/pkg/provision/engine.go
+++ b/pkg/provision/engine.go
@@ -725,6 +725,7 @@ func (e *NativeEngine) installWorkload(ctx context.Context, wl *gridtypes.Worklo
 		// workload already exist, so no need to create a new transaction
 		return nil
 	} else if err != nil {
+		result.Created = gridtypes.Now()
 		result.State = gridtypes.StateError
 		result.Error = err.Error()
 	}
@@ -817,17 +818,31 @@ func (e *NativeEngine) lockWorkload(ctx context.Context, wl *gridtypes.WorkloadW
 		wl.Workload.WithResults(result))
 }
 
-func (e *NativeEngine) uninstallDeployment(ctx context.Context, getter gridtypes.WorkloadGetter, reason string) {
+func (e *NativeEngine) uninstallDeployment(ctx context.Context, dl *gridtypes.Deployment, reason string) {
+	var errors bool
 	for i := len(e.order) - 1; i >= 0; i-- {
 		typ := e.order[i]
 
-		workloads := getter.ByType(typ)
+		workloads := dl.ByType(typ)
 		for _, wl := range workloads {
 			if err := e.uninstallWorkload(ctx, wl, reason); err != nil {
+				errors = true
 				log.Error().Err(err).Stringer("id", wl.ID).Msg("failed to un-install workload")
 			}
 		}
 	}
+
+	if errors {
+		return
+	}
+
+	if err := e.storage.Delete(dl.TwinID, dl.ContractID); err != nil {
+		log.Error().Err(err).
+			Uint32("twin", dl.TwinID).
+			Uint64("contract", dl.ContractID).
+			Msg("failed to delete deployment")
+	}
+
 }
 
 func (e *NativeEngine) installDeployment(ctx context.Context, getter gridtypes.WorkloadGetter) {
diff --git a/pkg/provision/interface.go b/pkg/provision/interface.go
index 34d8f6bb..ef77cbdc 100644
--- a/pkg/provision/interface.go
+++ b/pkg/provision/interface.go
@@ -62,7 +62,7 @@ var (
 	// it conflicts with another deployment
 	ErrDeploymentConflict = fmt.Errorf("conflict")
 	//ErrDeploymentNotExists returned if object not exists
-	ErrDeploymentNotExists = fmt.Errorf("workload does not exist")
+	ErrDeploymentNotExists = fmt.Errorf("deployment does not exist")
 	// ErrWorkloadNotExist returned by storage if workload does not exist
 	ErrWorkloadNotExist = fmt.Errorf("workload does not exist")
 	// ErrNoActionNeeded can be returned by any provision method to indicate that
@@ -122,6 +122,8 @@ type Storage interface {
 	Twins() ([]uint32, error)
 	// ByTwin return list of deployments for a twin
 	ByTwin(twin uint32) ([]uint64, error)
+	// return total capacity and active deployments
+	Capacity() (cap gridtypes.Capacity, active []gridtypes.Deployment, err error)
 }
 
 // Janitor interface
diff --git a/pkg/provision/provisiner.go b/pkg/provision/provisiner.go
index 212c8340..68fa6b94 100644
--- a/pkg/provision/provisiner.go
+++ b/pkg/provision/provisiner.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"time"
 
 	"github.com/pkg/errors"
 	"github.com/threefoldtech/zos/pkg/gridtypes"
@@ -228,7 +227,7 @@ func (p *mapProvisioner) CanUpdate(ctx context.Context, typ gridtypes.WorkloadTy
 }
 
 func setState(result *gridtypes.Result, err error) {
-	result.Created = gridtypes.Timestamp(time.Now().Unix())
+	result.Created = gridtypes.Now()
 	state := gridtypes.StateOk
 	str := ""
 
diff --git a/pkg/provision/storage/storage.go b/pkg/provision/storage/storage.go
index 3dbeb32d..ece74d21 100644
--- a/pkg/provision/storage/storage.go
+++ b/pkg/provision/storage/storage.go
@@ -182,7 +182,40 @@ func (b *MigrationStorage) Migrate(dl gridtypes.Deployment) error {
 }
 
 func (b *BoltStorage) Delete(twin uint32, deployment uint64) error {
-	panic("unimplemented")
+	return b.db.Update(func(t *bolt.Tx) error {
+		bucket := t.Bucket(b.u32(twin))
+		if bucket == nil {
+			return nil
+		}
+
+		if err := bucket.DeleteBucket(b.u64(deployment)); err != nil && !errors.Is(err, bolt.ErrBucketNotFound) {
+			return err
+		}
+		// if the twin now is empty then we can also delete the twin
+		curser := bucket.Cursor()
+		found := false
+		for k, v := curser.First(); k != nil; k, v = curser.Next() {
+			if v != nil {
+				// checking that it is a bucket
+				continue
+			}
+
+			if len(k) != 8 || string(k) == "global" {
+				// sanity check it's a valid uint32
+				continue
+			}
+
+			found = true
+			break
+		}
+
+		if !found {
+			// empty bucket
+			return t.DeleteBucket(b.u32(twin))
+		}
+
+		return nil
+	})
 }
 
 func (b *BoltStorage) Get(twin uint32, deployment uint64) (dl gridtypes.Deployment, err error) {
@@ -692,3 +725,50 @@ func (b *BoltStorage) Capacity() (cap gridtypes.Capacity, active []gridtypes.Dep
 func (b *BoltStorage) Close() error {
 	return b.db.Close()
 }
+
+// CleanDeletes is a cleaner method intended to clean up old "deleted" contracts
+// that has no active workloads anymore. We used to always leave the entire history
+// of all deployments that ever lived on the system. But we changed that so once
+// a deployment is deleted, it's deleted forever. Hence this code is only needed
+// temporary until it's available on all environments then can be dropped.
+func (b *BoltStorage) CleanDeleted() error {
+	twins, err := b.Twins()
+	if err != nil {
+		return err
+	}
+
+	for _, twin := range twins {
+		dls, err := b.ByTwin(twin)
+		if err != nil {
+			log.Error().Err(err).Uint32("twin", twin).Msg("failed to get twin deployments")
+			continue
+		}
+		for _, dl := range dls {
+			deployment, err := b.Get(twin, dl)
+			if err != nil {
+				log.Error().Err(err).Uint32("twin", twin).Uint64("deployment", dl).Msg("failed to get deployment")
+				continue
+			}
+
+			isActive := false
+			for _, wl := range deployment.Workloads {
+				if !wl.Result.State.IsOkay() {
+					continue
+				}
+
+				isActive = true
+				break
+			}
+
+			if isActive {
+				continue
+			}
+
+			if err := b.Delete(twin, dl); err != nil {
+				log.Error().Err(err).Uint32("twin", twin).Uint64("deployment", dl).Msg("failed to delete deployment")
+			}
+		}
+	}
+
+	return nil
+}


Test patch:
diff --git a/pkg/primitives/counters_test.go b/pkg/primitives/counters_test.go
deleted file mode 100644
index cd7e01e8..00000000
--- a/pkg/primitives/counters_test.go
+++ /dev/null
@@ -1,32 +0,0 @@
-package primitives
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/threefoldtech/zos/pkg/gridtypes"
-)
-
-func TestCounterDecrement(t *testing.T) {
-	t.Run("regular_decrement", func(t *testing.T) {
-		counter := AtomicUnit(0)
-		counter.Increment(5)
-		dec := counter.Decrement(3)
-		assert.Equal(t, dec, gridtypes.Unit(2))
-		assert.Equal(t, counter.Current(), gridtypes.Unit(2))
-	})
-	t.Run("decrement_to_zero", func(t *testing.T) {
-		counter := AtomicUnit(0)
-		counter.Increment(5)
-		dec := counter.Decrement(5)
-		assert.Equal(t, dec, gridtypes.Unit(0))
-		assert.Equal(t, counter.Current(), gridtypes.Unit(0))
-	})
-	t.Run("decrement_negative", func(t *testing.T) {
-		counter := AtomicUnit(0)
-		counter.Increment(5)
-		dec := counter.Decrement(7)
-		assert.Equal(t, dec, gridtypes.Unit(0))
-		assert.Equal(t, counter.Current(), gridtypes.Unit(0))
-	})
-}
diff --git a/pkg/provision/storage/storage_test.go b/pkg/provision/storage/storage_test.go
index 1f77a329..9f1e0ece 100644
--- a/pkg/provision/storage/storage_test.go
+++ b/pkg/provision/storage/storage_test.go
@@ -10,6 +10,7 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/boltdb/bolt"
 	"github.com/stretchr/testify/require"
 	"github.com/threefoldtech/zos/pkg/gridtypes"
 	"github.com/threefoldtech/zos/pkg/provision"
@@ -432,3 +433,97 @@ func TestMigrateUnsafe(t *testing.T) {
 	require.False(db.unsafe)
 	require.True(migration.unsafe.unsafe)
 }
+
+func TestDeleteDeployment(t *testing.T) {
+	require := require.New(t)
+	path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63()))
+	defer os.RemoveAll(path)
+
+	db, err := New(path)
+	require.NoError(err)
+
+	dl := gridtypes.Deployment{
+		Version:     1,
+		TwinID:      1,
+		ContractID:  10,
+		Description: "description",
+		Metadata:    "some metadata",
+		Workloads: []gridtypes.Workload{
+			{
+				Type: testType1,
+				Name: "vm1",
+			},
+			{
+				Type: testType2,
+				Name: "vm2",
+			},
+		},
+	}
+
+	err = db.Create(dl)
+	require.NoError(err)
+
+	err = db.Delete(1, 10)
+	require.NoError(err)
+
+	_, err = db.Get(1, 10)
+	require.ErrorIs(err, provision.ErrDeploymentNotExists)
+	deployments, err := db.ByTwin(1)
+	require.NoError(err)
+	require.Empty(deployments)
+
+	err = db.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket(db.u32(1))
+		if bucket == nil {
+			return nil
+		}
+		return fmt.Errorf("twin bucket was not deleted")
+	})
+	require.NoError(err)
+}
+
+func TestDeleteDeploymentMultiple(t *testing.T) {
+	require := require.New(t)
+	path := filepath.Join(os.TempDir(), fmt.Sprint(rand.Int63()))
+	defer os.RemoveAll(path)
+
+	db, err := New(path)
+	require.NoError(err)
+
+	dl := gridtypes.Deployment{
+		Version:     1,
+		TwinID:      1,
+		ContractID:  10,
+		Description: "description",
+		Metadata:    "some metadata",
+		Workloads: []gridtypes.Workload{
+			{
+				Type: testType1,
+				Name: "vm1",
+			},
+			{
+				Type: testType2,
+				Name: "vm2",
+			},
+		},
+	}
+
+	err = db.Create(dl)
+	require.NoError(err)
+
+	dl.ContractID = 20
+	err = db.Create(dl)
+	require.NoError(err)
+
+	err = db.Delete(1, 10)
+	require.NoError(err)
+
+	_, err = db.Get(1, 10)
+	require.ErrorIs(err, provision.ErrDeploymentNotExists)
+	deployments, err := db.ByTwin(1)
+	require.NoError(err)
+	require.Len(deployments, 1)
+
+	_, err = db.Get(1, 20)
+	require.NoError(err)
+}