Repository: threefoldtech/zos
Pull request #1536 (2022-01-03 15:16:55)

SRU calculations
- Compute the minimum rootfs size based on CPU & Memory
- Allow SSD overload by factor of 2


Patch:
diff --git a/pkg/gridtypes/zos/zmachine.go b/pkg/gridtypes/zos/zmachine.go
index aeb26f2d..65c6fd7f 100644
--- a/pkg/gridtypes/zos/zmachine.go
+++ b/pkg/gridtypes/zos/zmachine.go
@@ -11,6 +11,10 @@ import (
 	"github.com/threefoldtech/zos/pkg/gridtypes"
 )
 
+const (
+	sizeInGBPerCU = 50 // GB
+)
+
 // MachineInterface structure
 type MachineInterface struct {
 	// Network name (znet name) to join
@@ -60,6 +64,10 @@ type MachineCapacity struct {
 	Memory gridtypes.Unit `json:"memory"`
 }
 
+func (c *MachineCapacity) String() string {
+	return fmt.Sprintf("cpu(%d)+mem(%d)", c.CPU, c.Memory)
+}
+
 // Challenge builder
 func (c *MachineCapacity) Challenge(w io.Writer) error {
 	if _, err := fmt.Fprintf(w, "%d", c.CPU); err != nil {
@@ -119,6 +127,22 @@ type ZMachine struct {
 	Env map[string]string `json:"env"`
 }
 
+func (m *ZMachine) MinRootSize() gridtypes.Unit {
+	// sru = (cpu * mem_in_gb) / 8
+	// each 1 SRU is 50GB of storage
+	su := gridtypes.Unit(m.ComputeCapacity.CPU) * m.ComputeCapacity.Memory / 8
+	return gridtypes.Unit(su * sizeInGBPerCU)
+}
+
+func (m *ZMachine) RootSize() gridtypes.Unit {
+	min := m.MinRootSize()
+	if m.Size > min {
+		return m.Size
+	}
+
+	return min
+}
+
 // Valid implementation
 func (v ZMachine) Valid(getter gridtypes.WorkloadGetter) error {
 	if len(v.Network.Interfaces) != 1 {
@@ -136,8 +160,9 @@ func (v ZMachine) Valid(getter gridtypes.WorkloadGetter) error {
 	if v.ComputeCapacity.Memory < 250*gridtypes.Megabyte {
 		return fmt.Errorf("mem capacity can't be less that 250M")
 	}
-	if v.Size != 0 && v.Size < 250*gridtypes.Megabyte {
-		return fmt.Errorf("disk size can't be less that 250M")
+	minRoot := v.MinRootSize()
+	if v.Size != 0 && v.Size < minRoot {
+		return fmt.Errorf("disk size can't be less that %d. Set to 0 for minimum", minRoot)
 	}
 	if !v.Network.PublicIP.IsEmpty() {
 		wl, err := getter.Get(v.Network.PublicIP)
@@ -182,14 +207,10 @@ func (v ZMachine) Valid(getter gridtypes.WorkloadGetter) error {
 
 // Capacity implementation
 func (v ZMachine) Capacity() (gridtypes.Capacity, error) {
-	var sru uint64
-	if v.Size > 250*gridtypes.Megabyte {
-		sru += uint64(v.Size) - 250*uint64(gridtypes.Megabyte)
-	}
 	return gridtypes.Capacity{
 		CRU: uint64(v.ComputeCapacity.CPU),
 		MRU: v.ComputeCapacity.Memory,
-		SRU: gridtypes.Unit(sru),
+		SRU: v.RootSize(),
 	}, nil
 }
 
diff --git a/pkg/primitives/vm.go b/pkg/primitives/vm.go
index 041e4d4b..bacd923c 100644
--- a/pkg/primitives/vm.go
+++ b/pkg/primitives/vm.go
@@ -222,10 +222,7 @@ func (p *Primitives) virtualMachineProvisionImpl(ctx context.Context, wl *gridty
 		if err := flist.Unmount(ctx, wl.ID.String()); err != nil {
 			return result, errors.Wrapf(err, "failed to unmount flist: %s", wl.ID.String())
 		}
-		rootfsSize := config.Size
-		if rootfsSize < 250*gridtypes.Megabyte {
-			rootfsSize = 250 * gridtypes.Megabyte
-		}
+		rootfsSize := config.RootSize()
 		// create a persisted volume for the vm. we don't do it automatically
 		// via the flist, so we have control over when to decomission this volume.
 		// remounting in RW mode
diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go
index 03e967a3..6209cce9 100644
--- a/pkg/storage/storage.go
+++ b/pkg/storage/storage.go
@@ -23,6 +23,8 @@ import (
 )
 
 const (
+	// SSDOverProvisionFactor over provision factor for SSDs
+	SSDOverProvisionFactor = 2
 	// CacheTarget is the path where the cache disk is mounted
 	CacheTarget = "/var/cache"
 	// cacheLabel is the name of the cache
@@ -645,7 +647,7 @@ func (s *Module) checkForCandidates(size gridtypes.Unit, mounted bool) ([]candid
 			Uint64("new size", reserved+uint64(size)).
 			Msgf("usage of pool %s", pool.Name())
 		// Make sure adding this filesystem would not bring us over the disk limit
-		if reserved+uint64(size) > usage.Size {
+		if reserved+uint64(size) > usage.Size*SSDOverProvisionFactor {
 			log.Info().Msgf("Disk does not have enough space left to hold filesystem")
 
 			if !poolIsMounted && !mounted {


Test patch:
diff --git a/pkg/gridtypes/zos/zmachine_test.go b/pkg/gridtypes/zos/zmachine_test.go
new file mode 100644
index 00000000..dfce5e19
--- /dev/null
+++ b/pkg/gridtypes/zos/zmachine_test.go
@@ -0,0 +1,96 @@
+package zos
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/threefoldtech/zos/pkg/gridtypes"
+)
+
+func TestZMachineSRU(t *testing.T) {
+
+	type Case struct {
+		Expected gridtypes.Unit
+		VM       ZMachine
+	}
+	cases := []Case{
+		{
+			Expected: 50 * gridtypes.Gigabyte,
+			VM: ZMachine{
+
+				ComputeCapacity: MachineCapacity{
+					CPU:    1,
+					Memory: 8 * gridtypes.Gigabyte,
+				},
+			},
+		},
+		{
+			Expected: 25 * gridtypes.Gigabyte,
+			VM: ZMachine{
+
+				ComputeCapacity: MachineCapacity{
+					CPU:    1,
+					Memory: 4 * gridtypes.Gigabyte,
+				},
+			},
+		},
+		{
+			Expected: 50 * gridtypes.Gigabyte,
+			VM: ZMachine{
+
+				ComputeCapacity: MachineCapacity{
+					CPU:    2,
+					Memory: 4 * gridtypes.Gigabyte,
+				},
+			},
+		},
+		{
+			Expected: 75 * gridtypes.Gigabyte,
+			VM: ZMachine{
+
+				ComputeCapacity: MachineCapacity{
+					CPU:    3,
+					Memory: 4 * gridtypes.Gigabyte,
+				},
+			},
+		},
+		{
+			Expected: 125 * gridtypes.Gigabyte,
+			VM: ZMachine{
+
+				ComputeCapacity: MachineCapacity{
+					CPU:    4,
+					Memory: 5 * gridtypes.Gigabyte,
+				},
+			},
+		},
+		{
+			Expected: 6400 * gridtypes.Megabyte,
+			VM: ZMachine{
+
+				ComputeCapacity: MachineCapacity{
+					CPU:    1,
+					Memory: 1 * gridtypes.Gigabyte,
+				},
+			},
+		},
+		{
+			Expected: 1600000 * gridtypes.Kilobyte, // ~= 1562.5 Megabytes
+			VM: ZMachine{
+
+				ComputeCapacity: MachineCapacity{
+					CPU:    1,
+					Memory: 250 * gridtypes.Megabyte,
+				},
+			},
+		},
+	}
+
+	for _, test := range cases {
+		expected := test.Expected
+		vm := test.VM
+		t.Run(vm.ComputeCapacity.String(), func(t *testing.T) {
+			require.Equal(t, expected, vm.RootSize())
+		})
+	}
+}
diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go
index ce63322e..5d2cf8e3 100644
--- a/pkg/storage/storage_test.go
+++ b/pkg/storage/storage_test.go
@@ -337,7 +337,7 @@ func TestVDiskFindCandidatesNoSpace(t *testing.T) {
 		name:     "pool-1",
 		reserved: 2000,
 		usage: filesystem.Usage{
-			Size: 10000,
+			Size: 5000,
 			Used: 100,
 		},
 		ptype: zos.SSDDevice,
@@ -347,7 +347,7 @@ func TestVDiskFindCandidatesNoSpace(t *testing.T) {
 		name:     "pool-2",
 		reserved: 1000,
 		usage: filesystem.Usage{
-			Size: 10000,
+			Size: 5000,
 			Used: 100,
 		},
 		ptype: zos.SSDDevice,
@@ -386,3 +386,36 @@ func TestVDiskFindCandidatesNoSpace(t *testing.T) {
 		t.Fail()
 	}
 }
+
+func TestVDiskFindCandidatesOverProvision(t *testing.T) {
+	require := require.New(t)
+
+	pool1 := &testPool{
+		name:     "pool-1",
+		reserved: 2000,
+		usage: filesystem.Usage{
+			Size: 5000,
+			Used: 100,
+		},
+		ptype: zos.SSDDevice,
+	}
+
+	mod := Module{
+		ssds: []filesystem.Pool{
+			pool1,
+		},
+	}
+
+	sub := &testVolume{
+		name: vdiskVolumeName,
+	}
+
+	pool1.On("Volumes").Return([]filesystem.Volume{sub}, nil)
+
+	_, err := mod.diskFindCandidate(8000)
+	require.NoError(err)
+
+	_, err = mod.diskFindCandidate(10000)
+	require.Error(err)
+
+}