Repository: threefoldtech/zos (go)

Issue:
ZOS only seems to be using the first SSD for deployments
When trying to deploy a vm with 2.5TB of SSD storage capacity (BSC full node), we seem to have stumbled upon an issue where it does not seem to be possible to use more then the first physical SSD in a server.

We mostly tested on mainnet node 293, 303 & 306. These have 7.68TB of SSD storage. When I deployed an ubuntu container (where you can see the host root size) it looked like these servers have 4x 2TB SSD's. What we tried:

- deploy one vm with 5 x 500 GB disks (so to use the block devices as one btrfs filesystem for 2.5TB of storage)
- tried the same with 5 x 400 & 600 GB disks
- one big root disk of 1.7TB and a second 1.7TB disk -> only root disk worked, a second disk not
- 2 separate vm's with each a root disk of 1.7TB -> first vm deployed, second not

We always get the same error back from terraform: 

`Error: error waiting deployment: workload 1 failed within deployment 1373 with error no space left on device`

Logs for node 293 during tests: https://mon.grid.tf/explore?orgId=1&left=%5B%221652281747501%22,%221652284900473%22,%22Loki%22,%7B%22refId%22:%22A%22,%22expr%22:%22%7Bnetwork%3D%5C%22production%5C%22,node%3D%5C%22293%5C%22%7D%22%7D%5D

Used flist: https://hub.grid.tf/tf-official-vms/ubuntu-20.04-lts.flist

Pull request:
Fix usage calculation to find disks candidate
After switching vdisks allocations from fallocate to truncate. to suport
ssd over-provisioning. we needed to change and fix how we calculate
used space on a disk as follows

- subvolume with quota limit is assumed to be full usage of the
  subvolume
- subvolume with no-quota limit, we sum total size of all files in that
  subvolume

to find if disk has enough space for allocation, we calculate it as
 used + (requested) < ssd.size * 2

Fixes #1693

Patch:
diff --git a/pkg/storage/disk.go b/pkg/storage/disk.go
index b7bf3961..6ef3618a 100644
--- a/pkg/storage/disk.go
+++ b/pkg/storage/disk.go
@@ -8,6 +8,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"syscall"
 
 	"github.com/g0rbe/go-chattr"
 	"github.com/pkg/errors"
@@ -177,6 +178,8 @@ func (s *Module) DiskCreate(name string, size gridtypes.Unit) (disk pkg.VDisk, e
 		}
 	}()
 
+	defer syscall.Sync()
+
 	var file *os.File
 	file, err = os.Create(path)
 	if err != nil {
diff --git a/pkg/storage/filesystem/btrfs.go b/pkg/storage/filesystem/btrfs.go
index 18bb8fb5..e2862c03 100644
--- a/pkg/storage/filesystem/btrfs.go
+++ b/pkg/storage/filesystem/btrfs.go
@@ -243,32 +243,23 @@ func (p *btrfsPool) Usage() (usage Usage, err error) {
 		return usage, err
 	}
 
-	du, err := p.utils.GetDiskUsage(context.Background(), mnt)
-	if err != nil {
-		return Usage{}, err
-	}
-
-	return Usage{Size: p.device.Size, Used: du.Data.Used}, nil
-}
-
-// Reserved is reserved size of the devices in bytes
-func (p *btrfsPool) Reserved() (uint64, error) {
-
 	volumes, err := p.Volumes()
+
 	if err != nil {
-		return 0, err
+		return usage, errors.Wrapf(err, "failed to list pool '%s' volumes", mnt)
 	}
 
-	var total uint64
+	var used uint64
 	for _, volume := range volumes {
 		usage, err := volume.Usage()
 		if err != nil {
-			return 0, err
+			return Usage{}, errors.Wrapf(err, "failed to calculate volume '%s' usage", volume.Path())
 		}
-		total += usage.Size
+
+		used += usage.Used
 	}
 
-	return total, nil
+	return Usage{Size: p.device.Size, Used: used}, nil
 }
 
 func (p *btrfsPool) maintenance() error {
@@ -369,19 +360,20 @@ func (v *btrfsVolume) Usage() (usage Usage, err error) {
 		return
 	}
 
-	size := group.MaxRfer
-	if size == 0 {
+	// used is basically amount of space reserved for this
+	// volume. We assume that's total usage of the volume
+	used := group.MaxRfer
+
+	if used == 0 {
 		// in case no limit is set on the subvolume, we assume
 		// it's size is the size of the files on that volumes
-		size, err = FilesUsage(v.Path())
+		used, err = FilesUsage(v.Path())
 		if err != nil {
 			return usage, errors.Wrap(err, "failed to get subvolume usage")
 		}
 	}
-	// otherwise, we return the size as maxrefer and usage as the rfer of the
-	// associated group
-	// todo: size should be the size of the pool, if maxrfer is 0
-	return Usage{Used: group.Rfer, Size: size}, nil
+
+	return Usage{Used: used, Size: group.MaxRfer}, nil
 }
 
 // Limit size of volume, setting size to 0 means unlimited
diff --git a/pkg/storage/filesystem/filesystem.go b/pkg/storage/filesystem/filesystem.go
index 6e899292..c527d4f5 100644
--- a/pkg/storage/filesystem/filesystem.go
+++ b/pkg/storage/filesystem/filesystem.go
@@ -38,8 +38,6 @@ type Pool interface {
 	Mount() (string, error)
 	// UnMount the pool
 	UnMount() error
-	// Reserved is reserved size of the devices in bytes
-	Reserved() (uint64, error)
 	// Volumes are all subvolumes of this volume
 	Volumes() ([]Volume, error)
 	// AddVolume adds a new subvolume with the given name
diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go
index 86b27314..bbf666c5 100644
--- a/pkg/storage/storage.go
+++ b/pkg/storage/storage.go
@@ -210,6 +210,16 @@ func (s *Module) initialize() error {
 				continue
 			}
 
+			// for development purposes only
+			if vm {
+				// force ssd device for vms
+				typ = zos.SSDDevice
+
+				if device.Path == "/dev/vdd" || device.Path == "/dev/vde" {
+					typ = zos.HDDDevice
+				}
+			}
+
 			log.Debug().Str("device", device.Path).Str("type", typ.String()).Msg("caching device type")
 			if err := s.cache.Set(device.Name(), typ); err != nil {
 				log.Error().Str("device", device.Path).Err(err).Msg("failed to cache device type")
@@ -218,15 +228,6 @@ func (s *Module) initialize() error {
 			log.Debug().Str("device", device.Path).Str("type", typ.String()).Msg("device type loaded from cache")
 		}
 
-		if vm {
-			// force ssd device for vms
-			typ = zos.SSDDevice
-
-			if device.Path == "/dev/vdd" || device.Path == "/dev/vde" {
-				typ = zos.HDDDevice
-			}
-		}
-
 		switch typ {
 		case zos.SSDDevice:
 			s.totalSSD += usage.Size
@@ -687,19 +688,14 @@ func (s *Module) checkForCandidates(size gridtypes.Unit, mounted bool) ([]candid
 			continue
 		}
 
-		reserved, err := pool.Reserved()
-		if err != nil {
-			log.Error().Err(err).Msgf("failed to get size of pool %s", pool.Name())
-			continue
-		}
-
 		log.Debug().
-			Uint64("max size", usage.Size).
-			Uint64("reserved", reserved).
-			Uint64("new size", reserved+uint64(size)).
+			Uint64("pool size", usage.Size).
+			Uint64("used", usage.Used).
+			Uint64("new used", usage.Used+uint64(size)).
 			Msgf("usage of pool %s", pool.Name())
+
 		// Make sure adding this filesystem would not bring us over the disk limit
-		if reserved+uint64(size) > usage.Size*SSDOverProvisionFactor {
+		if usage.Used+uint64(size) > usage.Size*SSDOverProvisionFactor {
 			log.Info().Msgf("Disk does not have enough space left to hold filesystem")
 
 			if !poolIsMounted && !mounted {
@@ -719,7 +715,7 @@ func (s *Module) checkForCandidates(size gridtypes.Unit, mounted bool) ([]candid
 
 		candidates = append(candidates, candidate{
 			Pool:      pool,
-			Available: usage.Size - (reserved + uint64(size)),
+			Available: usage.Size*SSDOverProvisionFactor - (usage.Used + uint64(size)),
 		})
 
 		// if we are looking for not mounted pools, break here
@@ -727,7 +723,6 @@ func (s *Module) checkForCandidates(size gridtypes.Unit, mounted bool) ([]candid
 			return candidates, nil
 		}
 	}
-
 	sort.Slice(candidates, func(i, j int) bool {
 		// reverse sorting so most available is at beginning
 		return candidates[i].Available > candidates[j].Available


Test patch:
diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go
index 5e3842f1..3a2b075f 100644
--- a/pkg/storage/storage_test.go
+++ b/pkg/storage/storage_test.go
@@ -44,10 +44,9 @@ func (p *testVolume) FsType() string {
 
 type testPool struct {
 	mock.Mock
-	name     string
-	usage    filesystem.Usage
-	reserved uint64
-	ptype    zos.DeviceType
+	name  string
+	usage filesystem.Usage
+	ptype zos.DeviceType
 }
 
 var _ filesystem.Pool = &testPool{}
@@ -96,10 +95,6 @@ func (p *testPool) Type() zos.DeviceType {
 	return p.ptype
 }
 
-func (p *testPool) Reserved() (uint64, error) {
-	return p.reserved, nil
-}
-
 func (p *testPool) Volumes() ([]filesystem.Volume, error) {
 	args := p.Called()
 	return args.Get(0).([]filesystem.Volume), args.Error(1)
@@ -127,18 +122,16 @@ func TestCreateSubvol(t *testing.T) {
 	require := require.New(t)
 
 	pool1 := &testPool{
-		name:     "pool-1",
-		reserved: 2000,
+		name: "pool-1",
 		usage: filesystem.Usage{
 			Size: 10000,
-			Used: 100,
+			Used: 300,
 		},
 		ptype: zos.SSDDevice,
 	}
 
 	pool2 := &testPool{
-		name:     "pool-2",
-		reserved: 1000,
+		name: "pool-2",
 		usage: filesystem.Usage{
 			Size: 10000,
 			Used: 100,
@@ -171,18 +164,16 @@ func TestCreateSubvolUnlimited(t *testing.T) {
 	require := require.New(t)
 
 	pool1 := &testPool{
-		name:     "pool-1",
-		reserved: 2000,
+		name: "pool-1",
 		usage: filesystem.Usage{
 			Size: 10000,
-			Used: 100,
+			Used: 200,
 		},
 		ptype: zos.SSDDevice,
 	}
 
 	pool2 := &testPool{
-		name:     "pool-2",
-		reserved: 1000,
+		name: "pool-2",
 		usage: filesystem.Usage{
 			Size: 10000,
 			Used: 100,
@@ -191,8 +182,7 @@ func TestCreateSubvolUnlimited(t *testing.T) {
 	}
 
 	pool3 := &testPool{
-		name:     "pool-3",
-		reserved: 0,
+		name: "pool-3",
 		usage: filesystem.Usage{
 			Size: 100000,
 			Used: 0,
@@ -229,8 +219,7 @@ func TestCreateSubvolNoSpaceLeft(t *testing.T) {
 	require := require.New(t)
 
 	pool1 := &testPool{
-		name:     "pool-1",
-		reserved: 2000,
+		name: "pool-1",
 		usage: filesystem.Usage{
 			Size: 10000,
 			Used: 100,
@@ -239,8 +228,7 @@ func TestCreateSubvolNoSpaceLeft(t *testing.T) {
 	}
 
 	pool2 := &testPool{
-		name:     "pool-2",
-		reserved: 1000,
+		name: "pool-2",
 		usage: filesystem.Usage{
 			Size: 10000,
 			Used: 100,
@@ -249,8 +237,7 @@ func TestCreateSubvolNoSpaceLeft(t *testing.T) {
 	}
 
 	pool3 := &testPool{
-		name:     "pool-3",
-		reserved: 0,
+		name: "pool-3",
 		usage: filesystem.Usage{
 			Size: 100000,
 			Used: 0,
@@ -279,8 +266,7 @@ func TestVDiskFindCandidatesHasEnoughSpace(t *testing.T) {
 	require := require.New(t)
 
 	pool1 := &testPool{
-		name:     "pool-1",
-		reserved: 2000,
+		name: "pool-1",
 		usage: filesystem.Usage{
 			Size: 10000,
 			Used: 100,
@@ -289,8 +275,7 @@ func TestVDiskFindCandidatesHasEnoughSpace(t *testing.T) {
 	}
 
 	pool2 := &testPool{
-		name:     "pool-2",
-		reserved: 1000,
+		name: "pool-2",
 		usage: filesystem.Usage{
 			Size: 10000,
 			Used: 100,
@@ -299,8 +284,7 @@ func TestVDiskFindCandidatesHasEnoughSpace(t *testing.T) {
 	}
 
 	pool3 := &testPool{
-		name:     "pool-3",
-		reserved: 0,
+		name: "pool-3",
 		usage: filesystem.Usage{
 			Size: 100000,
 			Used: 0,
@@ -334,8 +318,7 @@ func TestVDiskFindCandidatesNoSpace(t *testing.T) {
 	require := require.New(t)
 
 	pool1 := &testPool{
-		name:     "pool-1",
-		reserved: 2000,
+		name: "pool-1",
 		usage: filesystem.Usage{
 			Size: 5000,
 			Used: 100,
@@ -344,8 +327,7 @@ func TestVDiskFindCandidatesNoSpace(t *testing.T) {
 	}
 
 	pool2 := &testPool{
-		name:     "pool-2",
-		reserved: 1000,
+		name: "pool-2",
 		usage: filesystem.Usage{
 			Size: 5000,
 			Used: 100,
@@ -354,8 +336,7 @@ func TestVDiskFindCandidatesNoSpace(t *testing.T) {
 	}
 
 	pool3 := &testPool{
-		name:     "pool-3",
-		reserved: 0,
+		name: "pool-3",
 		usage: filesystem.Usage{
 			Size: 100000,
 			Used: 0,
@@ -391,8 +372,7 @@ func TestVDiskFindCandidatesOverProvision(t *testing.T) {
 	require := require.New(t)
 
 	pool1 := &testPool{
-		name:     "pool-1",
-		reserved: 2000,
+		name: "pool-1",
 		usage: filesystem.Usage{
 			Size: 5000,
 			Used: 100,