Repository: threefoldtech/zos
Pull request #1908 (2023-02-21 13:34:39)

validate backend
## Related issues

- https://github.com/threefoldtech/zos/issues/1886
- https://github.com/threefoldtech/zos/issues/1906


Patch:
diff --git a/client/node.go b/client/node.go
index 11a17e3c..4745d0d5 100644
--- a/client/node.go
+++ b/client/node.go
@@ -186,6 +186,13 @@ func (n *NodeClient) Counters(ctx context.Context) (total gridtypes.Capacity, us
 	return result.Total, result.Used, result.System, nil
 }
 
+// Pools returns statistics of separate pools
+func (n *NodeClient) Pools(ctx context.Context) (pools []pkg.PoolMetrics, err error) {
+	const cmd = "zos.storage.pools"
+	err = n.bus.Call(ctx, n.nodeTwin, cmd, nil, &pools)
+	return
+}
+
 // NetworkListWGPorts return a list of all "taken" ports on the node. A new deployment
 // should be careful to use a free port for its network setup.
 func (n *NodeClient) NetworkListWGPorts(ctx context.Context) ([]uint16, error) {
diff --git a/cmds/modules/provisiond/main.go b/cmds/modules/provisiond/main.go
index e5ab4ff5..1bcadf69 100644
--- a/cmds/modules/provisiond/main.go
+++ b/cmds/modules/provisiond/main.go
@@ -395,6 +395,8 @@ func action(cli *cli.Context) error {
 		return errors.Wrap(err, "failed to create statistics api")
 	}
 
+	setupStorageRmb(zosRouter, cl)
+
 	_ = mbus.NewDeploymentMessageBus(zosRouter, engine)
 
 	log.Info().Msg("running rmb handler")
@@ -427,3 +429,11 @@ func getNodeReserved(cl zbus.Client, available gridtypes.Capacity) (counter grid
 
 	return
 }
+
+func setupStorageRmb(router rmb.Router, cl zbus.Client) {
+	storage := router.Subroute("storage")
+	storage.WithHandler("pools", func(ctx context.Context, payload []byte) (interface{}, error) {
+		stub := stubs.NewStorageModuleStub(cl)
+		return stub.Metrics(ctx)
+	})
+}
diff --git a/docs/manual/api.md b/docs/manual/api.md
index 04613db0..6d784d41 100644
--- a/docs/manual/api.md
+++ b/docs/manual/api.md
@@ -65,6 +65,23 @@ Capacity {
 
 > Note that, `used` capacity equal the full workload reserved capacity PLUS the system reserved capacity
 so `used = user_used + system`, while `system` is only the amount of resourced reserved by `zos` itself
+
+# Storage
+## List separate pools with capacity
+| command |body| return|
+|---|---|---|
+| `zos.storage.pools` | - |`[]Pool`|
+
+List all node pools with their types, size and used space
+where
+```json
+Pool {
+    "name": "pool-id",
+    "type": "(ssd|hdd)",
+    "size": <size in bytes>,
+    "used": <used in bytes>
+}
+```
 # Network
 ## List Wireguard Ports
 | command |body| return|
diff --git a/go.mod b/go.mod
index 12493644..ab9cfe1c 100644
--- a/go.mod
+++ b/go.mod
@@ -70,6 +70,7 @@ require (
 	github.com/coreos/go-iptables v0.6.0 // indirect
 	github.com/cosmos/go-bip39 v1.0.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
+	github.com/dave/jennifer v1.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/crypto/blake256 v1.0.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v3 v3.0.0 // indirect
diff --git a/go.sum b/go.sum
index ffba039a..4bca489a 100644
--- a/go.sum
+++ b/go.sum
@@ -182,6 +182,7 @@ github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW
 github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
 github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I=
 github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
+github.com/dave/jennifer v1.3.0 h1:p3tl41zjjCZTNBytMwrUuiAnherNUZktlhPTKoF/sEk=
 github.com/dave/jennifer v1.3.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go
index e6afb0b6..cf2a8e6b 100644
--- a/pkg/gateway/gateway.go
+++ b/pkg/gateway/gateway.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"net"
-	"net/url"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -565,22 +564,13 @@ func (g *gatewayModule) setupRouting(wlID string, fqdn string, backends []string
 	}
 	servers := make([]Server, len(backends))
 	for idx, backend := range backends {
+		if err := zos.Backend(backend).Valid(TLSPassthrough); err != nil {
+			return errors.Wrapf(err, "failed to validate backend '%s'", backend)
+		}
 		if TLSPassthrough {
-			if err := zos.Backend(backend).Valid(true); err != nil {
-				return errors.Wrap(err, "couldn't validate backend host")
-			}
-			u, err := url.Parse(backend)
-			log.Debug().Str("hostname", u.Host).Str("backend", backend).Msg("tls passthrough")
-			if err != nil {
-				return errors.Wrap(err, "couldn't parse backend host")
-			}
-			servers[idx] = Server{
-				Address: u.Host,
-			}
+			servers[idx] = Server{Address: backend}
 		} else {
-			servers[idx] = Server{
-				Url: backend,
-			}
+			servers[idx] = Server{Url: backend}
 		}
 	}
 	route := fmt.Sprintf("%s-route", wlID)
diff --git a/pkg/gridtypes/workload.go b/pkg/gridtypes/workload.go
index 27e77da2..008d7a9c 100644
--- a/pkg/gridtypes/workload.go
+++ b/pkg/gridtypes/workload.go
@@ -194,6 +194,10 @@ func (w *Workload) Challenge(i io.Writer) error {
 		return err
 	}
 
+	if _, err := fmt.Fprintf(i, "%s", w.Name); err != nil {
+		return err
+	}
+
 	if _, err := fmt.Fprintf(i, "%s", w.Type.String()); err != nil {
 		return err
 	}
diff --git a/pkg/gridtypes/zos/gw_fqdn.go b/pkg/gridtypes/zos/gw_fqdn.go
index 56fbdc0f..080cee26 100644
--- a/pkg/gridtypes/zos/gw_fqdn.go
+++ b/pkg/gridtypes/zos/gw_fqdn.go
@@ -3,9 +3,11 @@ package zos
 import (
 	"fmt"
 	"io"
+	"math"
 	"net"
 	"net/url"
 	"regexp"
+	"strconv"
 
 	"github.com/pkg/errors"
 	"github.com/threefoldtech/zos/pkg/gridtypes"
@@ -17,26 +19,43 @@ var (
 
 type Backend string
 
-// check if valid http://ip:port, http://ip or ip:port
-func (b Backend) Valid(tls bool) error {
-	u, err := url.Parse(string(b))
-	if err != nil {
-		return errors.Wrap(err, "failed to parse backend")
-	}
-	if tls {
-		if u.Scheme != "" {
-			return fmt.Errorf("scheme expected to be empty")
+// Parse accepts http://ip:port, http://ip or ip:port
+// checks if backend string is a valid string based on the tlsPassthrough parameter
+// ip:port is only valid in case of tlsPassthrough is true
+// http://ip:port or http://ip is valid in case of tlsPassthrough is false
+func (b Backend) Valid(tlsPassthrough bool) error {
+	var hostName string
+	if tlsPassthrough {
+		host, port, err := net.SplitHostPort(string(b))
+		if err != nil {
+			return fmt.Errorf("failed to parse backend %s with error: %w", b, err)
+		}
+
+		parsedPort, err := strconv.ParseUint(port, 10, 64)
+		if err != nil {
+			return fmt.Errorf("invalid port in backend: %s", port)
+		}
+
+		if parsedPort > math.MaxUint16 {
+			return fmt.Errorf("port '%s' must be <= 65535", port)
+		}
+
+		hostName = host
+	} else {
+		u, err := url.Parse(string(b))
+		if err != nil {
+			return fmt.Errorf("failed to parse backend with error: %w", err)
 		}
-		if u.Port() == "" {
-			return fmt.Errorf("missing port in backend address")
+
+		if u.Scheme != "http" {
+			return fmt.Errorf("scheme expected to be http")
 		}
-	} else if u.Scheme != "http" {
-		return fmt.Errorf("scheme expected to be http")
+		hostName = u.Hostname()
 	}
 
-	ip := net.ParseIP(u.Hostname())
+	ip := net.ParseIP(hostName)
 	if len(ip) == 0 || ip.IsLoopback() {
-		return fmt.Errorf("invalid ip address in backend: %s", u.Hostname())
+		return fmt.Errorf("invalid ip address in backend: %s", hostName)
 	}
 	return nil
 }
diff --git a/pkg/primitives/zdb/zdb.go b/pkg/primitives/zdb/zdb.go
index 8a09b6ef..bd81d1a1 100644
--- a/pkg/primitives/zdb/zdb.go
+++ b/pkg/primitives/zdb/zdb.go
@@ -172,20 +172,13 @@ func (p *Manager) zdbProvisionImpl(ctx context.Context, wl *gridtypes.WorkloadWi
 			}, nil
 		}
 
-		// we did not find the namespace, but is this container
-		// a possible candidate to hold the new namespace?
-		reserved, err := index.Reserved()
-		if err != nil {
-			return zos.ZDBResult{}, errors.Wrap(err, "failed to check total reserved size")
-		}
-
 		device, err := storage.DeviceLookup(ctx, container.Name)
 		if err != nil {
 			log.Error().Err(err).Str("container", string(id)).Msg("failed to inspect zdb device")
 			continue
 		}
 
-		if reserved+uint64(config.Size) <= uint64(device.Usage.Size) {
+		if uint64(device.Usage.Used)+uint64(config.Size) <= uint64(device.Usage.Size) {
 			candidates = append(candidates, container)
 		}
 	}
diff --git a/pkg/storage.go b/pkg/storage.go
index c4e769cf..934ae9d4 100644
--- a/pkg/storage.go
+++ b/pkg/storage.go
@@ -153,6 +153,16 @@ type StorageModule interface {
 
 	// DeviceLookup inspects a previously allocated device
 	DeviceLookup(name string) (Device, error)
+
+	// Capacity
+	Metrics() ([]PoolMetrics, error)
+}
+
+type PoolMetrics struct {
+	Name string         `json:"name"`
+	Type DeviceType     `json:"type"`
+	Size gridtypes.Unit `json:"size"`
+	Used gridtypes.Unit `json:"used"`
 }
 
 // VDisk info returned by a call to inspect
diff --git a/pkg/storage/filesystem/btrfs.go b/pkg/storage/filesystem/btrfs.go
index e2862c03..1b503806 100644
--- a/pkg/storage/filesystem/btrfs.go
+++ b/pkg/storage/filesystem/btrfs.go
@@ -367,7 +367,8 @@ func (v *btrfsVolume) Usage() (usage Usage, err error) {
 	if used == 0 {
 		// in case no limit is set on the subvolume, we assume
 		// it's size is the size of the files on that volumes
-		used, err = FilesUsage(v.Path())
+		// or a special case when the volume is a zdb volume
+		used, err = volumeUsage(v.Path())
 		if err != nil {
 			return usage, errors.Wrap(err, "failed to get subvolume usage")
 		}
diff --git a/pkg/storage/filesystem/utils.go b/pkg/storage/filesystem/utils.go
index 9f6573d6..32df76ea 100644
--- a/pkg/storage/filesystem/utils.go
+++ b/pkg/storage/filesystem/utils.go
@@ -12,6 +12,12 @@ import (
 	"syscall"
 
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+	"github.com/threefoldtech/zos/pkg/zdb"
+)
+
+const (
+	ZdbVolume = "zdb"
 )
 
 func getMountTarget(f io.Reader, device string) (string, bool) {
@@ -54,6 +60,25 @@ func FilesUsage(path string) (uint64, error) {
 	return total, err
 }
 
+func volumeUsage(path string) (uint64, error) {
+	name := filepath.Base(path)
+	if name != ZdbVolume {
+		return FilesUsage(path)
+	}
+
+	// if this is zdb volume we try something else
+	index := zdb.NewIndex(path)
+	usage, err := index.Reserved()
+	if err != nil {
+		log.Error().Err(err).Msg("failed to open zdb index")
+		// if the used size by zdb was not
+		// possible to calculate
+		return FilesUsage(path)
+	}
+
+	return usage, nil
+}
+
 // GetMountTarget returns the mount target of a device or false if the
 // device is not mounted.
 // panic, it panics if it can't read /proc/mounts
diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go
index e34e3205..4cdd12f8 100644
--- a/pkg/storage/storage.go
+++ b/pkg/storage/storage.go
@@ -296,6 +296,54 @@ func (s *Module) initialize() error {
 	return nil
 }
 
+func (s *Module) poolUsage(pool filesystem.Pool) (uint64, error) {
+	_, err := pool.Mounted()
+	if errors.Is(err, filesystem.ErrDeviceNotMounted) {
+		return 0, nil
+	} else if err != nil {
+		return 0, errors.Wrap(err, "failed to check pool mount status")
+	}
+
+	usage, err := pool.Usage()
+	if err != nil {
+		return 0, errors.Wrap(err, "failed to check pool usage")
+	}
+
+	return usage.Used, nil
+}
+
+func (s *Module) Metrics() ([]pkg.PoolMetrics, error) {
+	var metrics []pkg.PoolMetrics
+
+	for i, pools := range [][]filesystem.Pool{s.ssds, s.hdds} {
+		// this is just to avoid writing the same loop twice
+		typ := zos.SSDDevice
+		if i == 1 {
+			typ = zos.HDDDevice
+		}
+
+		for _, pool := range pools {
+			size := pool.Device().Size
+			used, err := s.poolUsage(pool)
+
+			if err != nil {
+				log.Error().Err(err).Msg("failed to check pool usage")
+				continue
+			}
+
+			metrics = append(metrics, pkg.PoolMetrics{
+				Type: typ,
+				Name: pool.Name(),
+				Size: gridtypes.Unit(size),
+				Used: gridtypes.Unit(used),
+			})
+		}
+
+	}
+
+	return metrics, nil
+}
+
 func (s *Module) shutdownUnusedPools(vm bool) error {
 	log.Debug().Msg("shutting down unused disks")
 	for _, sets := range [][]filesystem.Pool{s.ssds, s.hdds} {
diff --git a/pkg/stubs/storage_stub.go b/pkg/stubs/storage_stub.go
index 2ec0a4f0..f46f2e74 100644
--- a/pkg/stubs/storage_stub.go
+++ b/pkg/stubs/storage_stub.go
@@ -258,6 +258,23 @@ func (s *StorageModuleStub) DiskWrite(ctx context.Context, arg0 string, arg1 str
 	return
 }
 
+func (s *StorageModuleStub) Metrics(ctx context.Context) (ret0 []pkg.PoolMetrics, ret1 error) {
+	args := []interface{}{}
+	result, err := s.client.RequestContext(ctx, s.module, s.object, "Metrics", args...)
+	if err != nil {
+		panic(err)
+	}
+	result.PanicOnError()
+	ret1 = result.CallError()
+	loader := zbus.Loader{
+		&ret0,
+	}
+	if err := result.Unmarshal(&loader); err != nil {
+		panic(err)
+	}
+	return
+}
+
 func (s *StorageModuleStub) Monitor(ctx context.Context) (<-chan pkg.PoolsStats, error) {
 	ch := make(chan pkg.PoolsStats, 1)
 	recv, err := s.client.Stream(ctx, s.module, s.object, "Monitor")


Test patch:
diff --git a/pkg/gridtypes/zos/gw_fqdn_test.go b/pkg/gridtypes/zos/gw_fqdn_test.go
new file mode 100644
index 00000000..1a57220a
--- /dev/null
+++ b/pkg/gridtypes/zos/gw_fqdn_test.go
@@ -0,0 +1,115 @@
+package zos
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestValidBackend(t *testing.T) {
+	require := require.New(t)
+
+	t.Run("test ip:port cases", func(t *testing.T) {
+		backend := Backend("1.1.1.1:10")
+		err := backend.Valid(true)
+		require.NoError(err)
+
+		backend = Backend("1.1.1.1:10")
+		err = backend.Valid(false)
+		require.Error(err)
+
+		backend = Backend("1.1.1.1")
+		err = backend.Valid(true)
+		require.Error(err)
+
+		backend = Backend("1.1.1.1:port")
+		err = backend.Valid(true)
+		require.Error(err)
+
+		backend = Backend("ip:10")
+		err = backend.Valid(true)
+		require.Error(err)
+
+		backend = Backend("1.1.1.1:1000000")
+		err = backend.Valid(true)
+		require.Error(err)
+	})
+
+	t.Run("test http://ip:port cases", func(t *testing.T) {
+		backend := Backend("http://1.1.1.1:10")
+		err := backend.Valid(false)
+		require.NoError(err)
+
+		backend = Backend("http://1.1.1.1:10")
+		err = backend.Valid(true)
+		require.Error(err)
+
+		backend = Backend("http://1.1.1.1:port")
+		err = backend.Valid(false)
+		require.Error(err)
+
+		backend = Backend("http://ip:10")
+		err = backend.Valid(false)
+		require.Error(err)
+	})
+
+	t.Run("test http://ip cases", func(t *testing.T) {
+		backend := Backend("http://1.1.1.1")
+		err := backend.Valid(false)
+		require.NoError(err)
+
+		backend = Backend("http://1.1.1.1")
+		err = backend.Valid(true)
+		require.Error(err)
+
+		backend = Backend("http://ip")
+		err = backend.Valid(false)
+		require.Error(err)
+	})
+}
+
+func TestValidBackendIP6(t *testing.T) {
+	require := require.New(t)
+
+	t.Run("test ip:port cases", func(t *testing.T) {
+		backend := Backend("[2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]:10")
+		err := backend.Valid(true)
+		require.NoError(err)
+
+		backend = Backend("[2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]:10")
+		err = backend.Valid(false)
+		require.Error(err)
+
+		backend = Backend("[2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]:port")
+		err = backend.Valid(true)
+		require.Error(err)
+
+		backend = Backend("[2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]:1000000")
+		err = backend.Valid(true)
+		require.Error(err)
+	})
+
+	t.Run("test http://ip:port cases", func(t *testing.T) {
+		backend := Backend("http://[2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]:10")
+		err := backend.Valid(false)
+		require.NoError(err)
+
+		backend = Backend("http://[2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]:10")
+		err = backend.Valid(true)
+		require.Error(err)
+
+		backend = Backend("http://[2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]:port")
+		err = backend.Valid(false)
+		require.Error(err)
+	})
+
+	t.Run("test http://ip cases", func(t *testing.T) {
+		backend := Backend("http://[2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]")
+		err := backend.Valid(false)
+		require.NoError(err)
+
+		backend = Backend("http://[2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]")
+		err = backend.Valid(true)
+		require.Error(err)
+	})
+}
diff --git a/pkg/upgrade/hub_test.go b/pkg/upgrade/hub_test.go
index 99169409..bfe47d2a 100644
--- a/pkg/upgrade/hub_test.go
+++ b/pkg/upgrade/hub_test.go
@@ -42,7 +42,7 @@ func TestInfo(t *testing.T) {
 }
 
 func TestInfoGet(t *testing.T) {
-	const flist = "tf-zos/zos:0.1.0-rc1.flist"
+	const flist = "thabet/redis.flist"
 
 	var hub HubClient
 	info, err := hub.Info(flist)