Repository: threefoldtech/zos
Pull request #1756 (2022-06-27 09:30:52)

Br pub exit issue
This PR includes
- Fix public exit nic detection, errors where not handled correctly which caused some nics to be selected when they shouldn't
- Show the current node setup on UI (single, dual) and what nic is selected
- Add authorized (only the farmer can make) calls to the node rmb interface:
  - List all physical devices on the node. 
  - Choose to switch which interface is used by br-pub as an exit interface. The choices can be either `zos` to fall back to single nic setup, or any other free physical nic.
  - 


Patch:
diff --git a/client/node.go b/client/node.go
index 4b61114f..025a8bb8 100644
--- a/client/node.go
+++ b/client/node.go
@@ -85,6 +85,23 @@ type Version struct {
 	ZInit string `json:"zinit"`
 }
 
+type Interface struct {
+	IPs []string `json:"ips"`
+	Mac string   `json:"mac"`
+}
+
+type ExitDevice struct {
+	// IsSingle is set to true if br-pub
+	// is connected to zos bridge
+	IsSingle bool `json:"is_single"`
+	// IsDual is set to true if br-pub is
+	// connected to a physical nic
+	IsDual bool `json:"is_dual"`
+	// AsDualInterface is set to the physical
+	// interface name if IsDual is true
+	AsDualInterface string `json:"dual_interface"`
+}
+
 type args map[string]interface{}
 
 // NewNodeClient creates a new node RMB client. This client then can be used to
@@ -183,19 +200,42 @@ func (n *NodeClient) HasPublicIPv6(ctx context.Context) (bool, error) {
 	return result, nil
 }
 
-func (n *NodeClient) NetworkListInterfaces(ctx context.Context) (map[string][]net.IP, error) {
+func (n *NodeClient) NetworkListInterfaces(ctx context.Context) (result map[string][]net.IP, err error) {
 	const cmd = "zos.network.interfaces"
-	var result map[string][]net.IP
 
-	if err := n.bus.Call(ctx, n.nodeTwin, cmd, nil, &result); err != nil {
-		return nil, err
-	}
+	err = n.bus.Call(ctx, n.nodeTwin, cmd, nil, &result)
 
-	return result, nil
+	return
 }
 
-// NetworkListIPs list taken public IPs on the node
-func (n *NodeClient) NetworkListIPs(ctx context.Context) ([]string, error) {
+// NetworkListAllInterfaces return all physical devices on a node
+func (n *NodeClient) NetworkListAllInterfaces(ctx context.Context) (result map[string]Interface, err error) {
+	const cmd = "zos.network.admin.interfaces"
+
+	err = n.bus.Call(ctx, n.nodeTwin, cmd, nil, &result)
+
+	return
+
+}
+
+// NetworkSetPublicExitDevice select which physical interface to use as an exit device
+// setting `iface` to `zos` will then make node run in a single nic setup.
+func (n *NodeClient) NetworkSetPublicExitDevice(ctx context.Context, iface string) error {
+	const cmd = "zos.network.admin.set_public_nic"
+
+	return n.bus.Call(ctx, n.nodeTwin, cmd, iface, nil)
+}
+
+// NetworkGetPublicExitDevice gets the current dual nic setup of the node.
+func (n *NodeClient) NetworkGetPublicExitDevice(ctx context.Context) (exit ExitDevice, err error) {
+	const cmd = "zos.network.admin.get_public_nic"
+
+	err = n.bus.Call(ctx, n.nodeTwin, cmd, nil, &exit)
+	return
+}
+
+// NetworkListPublicIPs list taken public IPs on the node
+func (n *NodeClient) NetworkListPublicIPs(ctx context.Context) ([]string, error) {
 	const cmd = "zos.network.list_public_ips"
 	var result []string
 
@@ -211,10 +251,7 @@ func (n *NodeClient) NetworkListIPs(ctx context.Context) ([]string, error) {
 func (n *NodeClient) NetworkGetPublicConfig(ctx context.Context) (cfg pkg.PublicConfig, err error) {
 	const cmd = "zos.network.public_config_get"
 
-	if err = n.bus.Call(ctx, n.nodeTwin, cmd, nil, &cfg); err != nil {
-		return
-	}
-
+	err = n.bus.Call(ctx, n.nodeTwin, cmd, nil, &cfg)
 	return
 }
 
diff --git a/cmds/internet/main.go b/cmds/internet/main.go
index 71c7dc35..586b29ba 100644
--- a/cmds/internet/main.go
+++ b/cmds/internet/main.go
@@ -76,7 +76,7 @@ func configureZOS() error {
 
 		log.Info().Msg("Start network bootstrap")
 
-		ifaceConfigs, err := bootstrap.AnalyseLinks(
+		ifaceConfigs, err := bootstrap.AnalyzeLinks(
 			bootstrap.RequiresIPv4,
 			bootstrap.PhysicalFilter,
 			bootstrap.PluggedFilter)
diff --git a/cmds/modules/networkd/main.go b/cmds/modules/networkd/main.go
index 83cf18b8..bb04b235 100644
--- a/cmds/modules/networkd/main.go
+++ b/cmds/modules/networkd/main.go
@@ -5,11 +5,11 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
-	"path/filepath"
 	"time"
 
 	"github.com/pkg/errors"
 	"github.com/threefoldtech/zos/pkg/network/public"
+	"github.com/threefoldtech/zos/pkg/rmb"
 	"github.com/urfave/cli/v2"
 
 	"github.com/cenkalti/backoff/v3"
@@ -25,9 +25,8 @@ import (
 )
 
 const (
-	redisSocket      = "unix:///var/run/redis.sock"
-	module           = "network"
-	publicConfigFile = "public-config.json"
+	redisSocket = "unix:///var/run/redis.sock"
+	module      = "network"
 )
 
 // Module is entry point for module
@@ -55,6 +54,10 @@ func action(cli *cli.Context) error {
 		broker string = cli.String("broker")
 	)
 
+	if err := os.MkdirAll(root, 0755); err != nil {
+		return errors.Wrap(err, "fail to create module root")
+	}
+
 	waitYggdrasilBin()
 
 	if err := bootstrap.DefaultBridgeValid(); err != nil {
@@ -81,8 +84,8 @@ func action(cli *cli.Context) error {
 		return errors.Wrap(err, "failed to host firewall rules")
 	}
 
-	publicCfgPath := filepath.Join(root, publicConfigFile)
-	public.SetPersistence(publicCfgPath)
+	public.SetPersistence(root)
+
 	pub, err := public.LoadPublicConfig()
 	log.Debug().Err(err).Msgf("public interface configred: %+v", pub)
 	if err != nil && err != public.ErrNoPublicConfig {
@@ -139,24 +142,54 @@ func action(cli *cli.Context) error {
 		}
 	}
 
-	log.Info().Msg("start zbus server")
-	if err := os.MkdirAll(root, 0750); err != nil {
-		return errors.Wrap(err, "fail to create module root")
-	}
-
 	networker, err := network.NewNetworker(identity, dmz, ygg)
 	if err != nil {
 		return errors.Wrap(err, "error creating network manager")
 	}
 
-	if err := startServer(ctx, broker, networker); err != nil {
+	mBus, err := rmb.New(broker)
+	if err != nil {
+		return errors.Wrap(err, "failed to initialize message bus")
+	}
+
+	zosRouter := mBus.Subroute("zos")
+	zosRouter.Use(rmb.LoggerMiddleware)
+
+	if _, err := network.NewNetworkMessageBus(zosRouter, networker); err != nil {
+		return errors.Wrap(err, "failed to initialize rmb api")
+	}
+
+	// we need to start both rmb server and zbus server.
+	go func(ctx context.Context) {
+		if err := startRmbServer(ctx, mBus); err != nil {
+			log.Fatal().Err(err).Msg("rmb exited unexpectedly")
+		}
+	}(ctx)
+
+	log.Info().Msg("start zbus server")
+	if err := startZBusServer(ctx, broker, networker); err != nil {
 		return errors.Wrap(err, "unexpected error")
 	}
 
 	return nil
 }
 
-func startServer(ctx context.Context, broker string, networker pkg.Networker) error {
+func startRmbServer(ctx context.Context, bus *rmb.MessageBus) error {
+	log.Info().
+		Msg("starting networkd rmb module")
+
+	for _, handler := range bus.Handlers() {
+		log.Debug().Msgf("registered handler: %s", handler)
+	}
+
+	if err := bus.Run(ctx); err != nil && err != context.Canceled {
+		return err
+	}
+
+	return nil
+}
+
+func startZBusServer(ctx context.Context, broker string, networker pkg.Networker) error {
 
 	server, err := zbus.NewRedisServer(module, broker, 1)
 	if err != nil {
diff --git a/cmds/modules/noded/public.go b/cmds/modules/noded/public.go
index 82f73c28..36fe9a46 100644
--- a/cmds/modules/noded/public.go
+++ b/cmds/modules/noded/public.go
@@ -2,6 +2,7 @@ package noded
 
 import (
 	"context"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
@@ -60,17 +61,25 @@ reapply:
 			}
 		}
 
-		for event := range events {
-			if event.Kind == pkg.EventSubscribed {
-				// the events has re-subscribed, so possible
-				// loss of events.
-				// then we-reapply
-				continue reapply
-			}
+		for {
+			select {
+			case event := <-events:
+				if event.Kind == pkg.EventSubscribed {
+					// the events has re-subscribed, so possible
+					// loss of events.
+					// then we-reapply
+					continue reapply
+				}
 
-			log.Info().Msgf("got a public config update: %+v", event.PublicConfig)
-			if err := setPublicConfig(ctx, cl, event.PublicConfig); err != nil {
-				return errors.Wrap(err, "failed to set public config")
+				log.Info().Msgf("got a public config update: %+v", event.PublicConfig)
+				if err := setPublicConfig(ctx, cl, event.PublicConfig); err != nil {
+					return errors.Wrap(err, "failed to set public config")
+				}
+			case <-time.After(2 * time.Hour):
+				// last resort, if none of the events
+				// was received, it will be a good idea to just
+				// check every 2 hours for changes.
+				continue reapply
 			}
 		}
 	}
diff --git a/cmds/modules/provisiond/main.go b/cmds/modules/provisiond/main.go
index 3e5150ef..43e2f1a2 100644
--- a/cmds/modules/provisiond/main.go
+++ b/cmds/modules/provisiond/main.go
@@ -148,14 +148,6 @@ func action(cli *cli.Context) error {
 	)
 	prom.Instrument()
 
-	mBus, err := rmb.New(msgBrokerCon)
-	if err != nil {
-		return errors.Wrap(err, "Failed to initialize message bus")
-	}
-
-	zosRouter := mBus.Subroute("zos")
-	zosRouter.Use(rmb.LoggerMiddleware)
-
 	// the v1 endpoint will be used by all components to register endpoints
 	// that are specific for that component
 	//v1 := router.PathPrefix("/api/v1").Subrouter()
@@ -223,10 +215,6 @@ func action(cli *cli.Context) error {
 		provisioners,
 	)
 
-	if err := primitives.NewStatisticsMessageBus(zosRouter, statistics); err != nil {
-		return errors.Wrap(err, "failed to create statistics api")
-	}
-
 	mgr, err := environment.GetSubstrate()
 	if err != nil {
 		return err
@@ -383,10 +371,21 @@ func action(cli *cli.Context) error {
 		log.Info().Msg("zbus server stopped")
 	}()
 
-	// register message bug api
-	setupMessageBusses(zosRouter, cl, engine)
+	mBus, err := rmb.New(msgBrokerCon)
+	if err != nil {
+		return errors.Wrap(err, "Failed to initialize message bus")
+	}
+
+	zosRouter := mBus.Subroute("zos")
+	zosRouter.Use(rmb.LoggerMiddleware)
+	// attach statistics module to rmb
+	if err := primitives.NewStatisticsMessageBus(zosRouter, statistics); err != nil {
+		return errors.Wrap(err, "failed to create statistics api")
+	}
 
-	log.Info().Msg("running messagebus")
+	_ = mbus.NewDeploymentMessageBus(zosRouter, engine)
+
+	log.Info().Msg("running rmb handler")
 
 	for _, handler := range mBus.Handlers() {
 		log.Debug().Msgf("registered handler: %s", handler)
@@ -421,15 +420,6 @@ func getNodeReserved(cl zbus.Client, available gridtypes.Capacity) (counter prim
 	return
 }
 
-func setupMessageBusses(router rmb.Router, cl zbus.Client, engine provision.Engine) error {
-
-	_ = mbus.NewDeploymentMessageBus(router, engine)
-
-	_ = mbus.NewNetworkMessagebus(router, engine, cl)
-
-	return nil
-}
-
 func storageMigration(db *storage.BoltStorage, fs *fsStorage.Fs) error {
 	log.Info().Msg("starting storage migration")
 	twins, err := fs.Twins()
diff --git a/cmds/modules/zui/main.go b/cmds/modules/zui/main.go
index b6b236da..61aae0e5 100644
--- a/cmds/modules/zui/main.go
+++ b/cmds/modules/zui/main.go
@@ -71,11 +71,11 @@ func action(ctx *cli.Context) error {
 
 	netgrid := ui.NewGrid()
 	netgrid.Title = "Network"
-	netgrid.SetRect(0, 7, width, 13)
+	netgrid.SetRect(0, 7, width, 14)
 
 	provision := ui.NewGrid()
 	provision.Title = "Provision"
-	provision.SetRect(0, 13, width, 19)
+	provision.SetRect(0, 14, width, 20)
 	provision.Border = false
 
 	var flag signalFlag
diff --git a/cmds/modules/zui/net.go b/cmds/modules/zui/net.go
index a60b200f..e61ede12 100644
--- a/cmds/modules/zui/net.go
+++ b/cmds/modules/zui/net.go
@@ -2,7 +2,7 @@ package zui
 
 import (
 	"context"
-	_ "fmt"
+	"fmt"
 	"net"
 	"strings"
 
@@ -24,6 +24,7 @@ func addressRender(ctx context.Context, table *widgets.Table, client zbus.Client
 		{"DMZ", "Not configured"},
 		{"YGG", "Not configured"},
 		{"PUB", "Not configured"},
+		{"DUL", "Not configured"},
 	}
 
 	stub := stubs.NewNetworkerStub(client)
@@ -82,6 +83,14 @@ func addressRender(ctx context.Context, table *widgets.Table, client zbus.Client
 				table.Rows[3][1] = str
 			}
 
+			exit, err := stub.GetPublicExitDevice(ctx)
+			dual := exit.String()
+			if err != nil {
+				dual = fmt.Sprintf("error: %s", err)
+			}
+
+			table.Rows[4][1] = dual
+
 			render.Signal()
 		}
 	}()
diff --git a/pkg/network.go b/pkg/network.go
index e90799d7..a628d5e0 100644
--- a/pkg/network.go
+++ b/pkg/network.go
@@ -2,6 +2,7 @@ package pkg
 
 import (
 	"context"
+	"fmt"
 	"net"
 	"reflect"
 
@@ -36,6 +37,34 @@ type YggdrasilTap struct {
 	Gateway net.IPNet
 }
 
+type Interface struct {
+	Name string
+	IPs  []net.IPNet
+	Mac  string
+}
+
+type ExitDevice struct {
+	// IsSingle is set to true if br-pub
+	// is connected to zos bridge
+	IsSingle bool `json:"is_single"`
+	// IsDual is set to true if br-pub is
+	// connected to a physical nic
+	IsDual bool `json:"is_dual"`
+	// AsDualInterface is set to the physical
+	// interface name if IsDual is true
+	AsDualInterface string `json:"dual_interface"`
+}
+
+func (e *ExitDevice) String() string {
+	if e.IsSingle {
+		return "single"
+	} else if e.IsDual {
+		return fmt.Sprintf("dual(%s)", e.AsDualInterface)
+	}
+
+	return "unknown"
+}
+
 //Networker is the interface for the network module
 type Networker interface {
 	// Ready return nil is networkd is ready to operate
@@ -146,6 +175,13 @@ type Networker interface {
 
 	// Addrs return the IP addresses of interface
 	// if the interface is in a network namespace netns needs to be not empty
+	// if iface is empty, return ALL interfaces in the given namespace
+	// if they are physical
+	Interfaces(iface string, netns string) (map[string]Interface, error)
+
+	// Addrs return the IP addresses of interface
+	// if the interface is in a network namespace netns needs to be not empty
+	// [obsolete] please use Interfaces instead
 	Addrs(iface string, netns string) (ips []net.IP, mac string, err error)
 
 	WireguardPorts() ([]uint, error)
@@ -158,6 +194,11 @@ type Networker interface {
 	// Get node public namespace config
 	GetPublicConfig() (PublicConfig, error)
 
+	// GetPublicExitDevice either return "singe" or "dual(<nic>)"
+	GetPublicExitDevice() (ExitDevice, error)
+
+	SetPublicExitDevice(iface string) error
+
 	// Monitoring methods
 
 	// ZOSAddresses monitoring streams for ZOS bridge IPs
diff --git a/pkg/network/bootstrap/bootstrap.go b/pkg/network/bootstrap/bootstrap.go
index 31e0192d..90ff25d6 100644
--- a/pkg/network/bootstrap/bootstrap.go
+++ b/pkg/network/bootstrap/bootstrap.go
@@ -5,7 +5,10 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"reflect"
+	"runtime"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 
@@ -55,20 +58,35 @@ func (a byAddr) Len() int           { return len(a) }
 func (a byAddr) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
 func (a byAddr) Less(i, j int) bool { return bytes.Compare(a[i].IP, a[j].IP) < 0 }
 
-// AnalyseLinks is used to gather the IP that each interfaces would be from DHCP and SLAAC
+func filterName(filter Filter) string {
+	fn := runtime.FuncForPC(reflect.ValueOf(filter).Pointer())
+	if fn == nil {
+		return fmt.Sprintf("filter(%+v)", filter)
+	}
+	name := fn.Name()
+	parts := strings.Split(name, ".")
+	idx := len(parts) - 1
+	if idx < 0 {
+		return name
+	}
+
+	return parts[idx]
+}
+
+// AnalyzeLinks is used to gather the IP that each interfaces would be from DHCP and SLAAC
 // it returns the IPs and routes for each interfaces for both IPv4 and IPv6
 //
 // It will list all the physical interfaces that have a cable plugged in it
 // create a network namespace per interfaces
 // start a DHCP probe on each interfaces and gather the IPs and routes received
-func AnalyseLinks(requires Requires, filters ...Filter) ([]IfaceConfig, error) {
+func AnalyzeLinks(requires Requires, filters ...Filter) ([]IfaceConfig, error) {
 	links, err := netlink.LinkList()
 	if err != nil {
 		log.Error().Err(err).Msgf("failed to list interfaces")
 		return nil, err
 	}
 
-	filterred := links[:0]
+	filtered := links[:0]
 filter:
 	for _, link := range links {
 		log := log.With().Str("interface", link.Attrs().Name).Str("type", link.Type()).Logger()
@@ -85,23 +103,22 @@ filter:
 			}
 
 			if !ok {
-				log.Info().Msg("link didn't match filter crateria, skip testing")
+				log.Info().Msgf("link didn't match filter criteria (%v), skip testing", filterName(filter))
 				continue filter
 			}
 		}
 		log.Info().Msg("link valid. testing link for connectivity")
-		filterred = append(filterred, link)
+		filtered = append(filtered, link)
 	}
 
 	wg := sync.WaitGroup{}
 
-	ch := make(chan IfaceConfig)
+	ch := make(chan analyzeResult)
 	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
 	defer cancel()
-
-	for _, link := range filterred {
+	for _, link := range filtered {
 		wg.Add(1)
-		analyseLink(ctx, &wg, ch, requires, link)
+		analyzeLink(ctx, &wg, ch, requires, link)
 	}
 
 	go func() {
@@ -110,16 +127,19 @@ filter:
 	}()
 
 	configs := make([]IfaceConfig, 0, len(links))
-	for cfg := range ch {
-		log.Info().Str("link", cfg.Name).Str("info", fmt.Sprintf("%+v", cfg)).Msg("info")
-		configs = append(configs, cfg)
+	for res := range ch {
+		if res.err != nil {
+			log.Error().Err(res.err).Str("interface", res.cfg.Name).Msg("error analyzing interface")
+			continue
+		}
+		configs = append(configs, res.cfg)
 	}
 
 	return configs, nil
 }
 
-// AnalyseLink gets information about link
-func AnalyseLink(ctx context.Context, requires Requires, link netlink.Link) (cfg IfaceConfig, err error) {
+// AnalyzeLink gets information about link
+func AnalyzeLink(ctx context.Context, requires Requires, link netlink.Link) (cfg IfaceConfig, err error) {
 	cfg.Name = link.Attrs().Name
 	if link.Attrs().MasterIndex != 0 {
 		// this is to avoid breaking setups if a link is
@@ -225,14 +245,16 @@ func AnalyseLink(ctx context.Context, requires Requires, link netlink.Link) (cfg
 	return
 }
 
-func analyseLink(ctx context.Context, wg *sync.WaitGroup, out chan<- IfaceConfig, requires Requires, link netlink.Link) {
+type analyzeResult struct {
+	cfg IfaceConfig
+	err error
+}
+
+func analyzeLink(ctx context.Context, wg *sync.WaitGroup, out chan<- analyzeResult, requires Requires, link netlink.Link) {
 	go func() {
 		defer wg.Done()
-		cfg, err := AnalyseLink(ctx, requires, link)
-		if err != nil {
-			log.Error().Err(err).Str("interface", link.Attrs().Name).Msg("error analysing interface")
-		}
-		out <- cfg
+		cfg, err := AnalyzeLink(ctx, requires, link)
+		out <- analyzeResult{cfg, err}
 	}()
 }
 
@@ -291,6 +313,10 @@ func PhysicalFilter(link netlink.Link) (bool, error) {
 	return link.Type() == "device", nil
 }
 
+func VEthFilter(link netlink.Link) (bool, error) {
+	return link.Type() == "veth", nil
+}
+
 // PluggedFilter returns true if link is plugged in
 func PluggedFilter(link netlink.Link) (bool, error) {
 	if err := netlink.LinkSetUp(link); err != nil {
@@ -306,6 +332,15 @@ func NotAttachedFilter(link netlink.Link) (bool, error) {
 	return link.Attrs().MasterIndex == 0, nil
 }
 
+func NotIpsAssignedFilter(link netlink.Link) (bool, error) {
+	adds, err := netlink.AddrList(link, netlink.FAMILY_ALL)
+	if err != nil {
+		return false, err
+	}
+
+	return len(adds) == 0, nil
+}
+
 func isPrivateIP(ip net.IP) bool {
 	privateIPBlocks := []*net.IPNet{}
 	for _, cidr := range []string{
diff --git a/pkg/network/mbus.go b/pkg/network/mbus.go
new file mode 100644
index 00000000..74d78a6c
--- /dev/null
+++ b/pkg/network/mbus.go
@@ -0,0 +1,146 @@
+package network
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+
+	"github.com/pkg/errors"
+	"github.com/threefoldtech/zos/pkg"
+	"github.com/threefoldtech/zos/pkg/environment"
+	"github.com/threefoldtech/zos/pkg/rmb"
+)
+
+// Network message bus api
+type Network struct {
+	mgr pkg.Networker
+}
+
+// NewNetworkMessageBus creates a new messagebus instance
+func NewNetworkMessageBus(router rmb.Router, mgr pkg.Networker) (*Network, error) {
+	api := &Network{
+		mgr: mgr,
+	}
+
+	if err := api.setup(router); err != nil {
+		return nil, err
+	}
+
+	return api, nil
+}
+
+func (n *Network) hasPublicIPv6Handler(ctx context.Context, payload []byte) (interface{}, error) {
+	return n.hasPublicIPv6(ctx), nil
+}
+
+func (n *Network) setup(router rmb.Router) error {
+
+	// network handlers
+	sub := router.Subroute("network")
+	sub.WithHandler("list_wg_ports", n.listPorts)
+	sub.WithHandler("public_config_get", n.getPublicConfig)
+	sub.WithHandler("interfaces", n.listInterfaces)
+	sub.WithHandler("has_ipv6", n.hasPublicIPv6Handler)
+
+	admin := sub.Subroute("admin")
+	env, err := environment.Get()
+	if err != nil {
+		return errors.Wrap(err, "failed to get environment")
+	}
+	mgr, err := environment.GetSubstrate()
+	if err != nil {
+		return errors.Wrap(err, "failed to get substrate")
+	}
+	mw, err := rmb.Authorized(mgr, uint32(env.FarmerID))
+	if err != nil {
+		return errors.Wrap(err, "failed to initialized admin mw")
+	}
+	admin.Use(mw)
+	admin.WithHandler("interfaces", n.listAllInterfaces)
+	admin.WithHandler("set_public_nic", n.setPublicNic)
+	admin.WithHandler("get_public_nic", n.getPublicNic)
+
+	return nil
+}
+
+func (n *Network) listAllInterfaces(ctx context.Context, _ []byte) (interface{}, error) {
+	// list all interfaces on node
+	type Interface struct {
+		IPs []string `json:"ips"`
+		Mac string   `json:"mac"`
+	}
+
+	interfaces, err := n.mgr.Interfaces("", "")
+	if err != nil {
+		return nil, err
+	}
+	output := make(map[string]Interface)
+	for name, inf := range interfaces {
+		output[name] = Interface{
+			Mac: inf.Mac,
+			IPs: func() []string {
+				var ips []string
+				for _, ip := range inf.IPs {
+					ips = append(ips, ip.String())
+				}
+				return ips
+			}(),
+		}
+	}
+
+	return output, nil
+}
+
+func (n *Network) setPublicNic(ctx context.Context, data []byte) (interface{}, error) {
+	var iface string
+	if err := json.Unmarshal(data, &iface); err != nil {
+		return nil, fmt.Errorf("failed to decode input, expecting string")
+	}
+
+	return nil, n.mgr.SetPublicExitDevice(iface)
+}
+
+func (n *Network) getPublicNic(ctx context.Context, _ []byte) (interface{}, error) {
+	return n.mgr.GetPublicExitDevice()
+}
+
+func (n *Network) listPorts(ctx context.Context, _ []byte) (interface{}, error) {
+	return n.mgr.WireguardPorts()
+}
+
+func (n *Network) hasPublicIPv6(ctx context.Context) interface{} {
+	ipData, err := n.mgr.GetPublicIPv6Subnet()
+	return ipData.IP != nil && err == nil
+}
+func (n *Network) listInterfaces(ctx context.Context, _ []byte) (interface{}, error) {
+
+	results := make(map[string][]net.IP)
+	type q struct {
+		inf    string
+		ns     string
+		rename string
+	}
+	for _, i := range []q{{"zos", "", "zos"}, {"nygg6", "ndmz", "ygg"}} {
+		ips, _, err := n.mgr.Addrs(i.inf, i.ns)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to get ips for '%s' interface", i)
+		}
+
+		results[i.rename] = func() []net.IP {
+			list := make([]net.IP, 0, len(ips))
+			for _, item := range ips {
+				ip := net.IP(item)
+				list = append(list, ip)
+			}
+
+			return list
+		}()
+	}
+
+	return results, nil
+}
+
+func (n *Network) getPublicConfig(ctx context.Context, _ []byte) (interface{}, error) {
+	return n.mgr.GetPublicConfig()
+}
diff --git a/pkg/network/networker.go b/pkg/network/networker.go
index 66886b1f..8502d21c 100644
--- a/pkg/network/networker.go
+++ b/pkg/network/networker.go
@@ -18,6 +18,7 @@ import (
 
 	"github.com/threefoldtech/zos/pkg/cache"
 	"github.com/threefoldtech/zos/pkg/gridtypes/zos"
+	"github.com/threefoldtech/zos/pkg/network/bootstrap"
 	"github.com/threefoldtech/zos/pkg/network/ndmz"
 	"github.com/threefoldtech/zos/pkg/network/public"
 	"github.com/threefoldtech/zos/pkg/network/tuntap"
@@ -131,7 +132,7 @@ func (n *networker) WireguardPorts() ([]uint, error) {
 	return n.portSet.List()
 }
 
-func (n networker) attachYgg(id string, netNs ns.NetNS) (net.IPNet, error) {
+func (n *networker) attachYgg(id string, netNs ns.NetNS) (net.IPNet, error) {
 	// new hardware address for the ygg interface
 	hw := ifaceutil.HardwareAddrFromInputBytes([]byte("ygg:" + id))
 
@@ -167,7 +168,7 @@ func (n networker) attachYgg(id string, netNs ns.NetNS) (net.IPNet, error) {
 	return ip, nil
 }
 
-func (n networker) detachYgg(id string, netNs ns.NetNS) error {
+func (n *networker) detachYgg(id string, netNs ns.NetNS) error {
 	return netNs.Do(func(_ ns.NetNS) error {
 		link, err := netlink.LinkByName(ZDBYggIface)
 		if err != nil {
@@ -182,7 +183,7 @@ func (n networker) detachYgg(id string, netNs ns.NetNS) error {
 
 // prepare creates a unique namespace (based on id) with "prefix"
 // and make sure it's wired to the bridge on host namespace
-func (n networker) prepare(id, prefix, bridge string) (string, error) {
+func (n *networker) prepare(id, prefix, bridge string) (string, error) {
 	hw := ifaceutil.HardwareAddrFromInputBytes([]byte("pub:" + id))
 
 	netNSName := prefix + strings.Replace(hw.String(), ":", "", -1)
@@ -204,7 +205,7 @@ func (n networker) prepare(id, prefix, bridge string) (string, error) {
 	return netNSName, err
 }
 
-func (n networker) destroy(ns string) error {
+func (n *networker) destroy(ns string) error {
 	if !namespace.Exists(ns) {
 		return nil
 	}
@@ -219,16 +220,16 @@ func (n networker) destroy(ns string) error {
 	return namespace.Delete(nSpace)
 }
 
-//func (n networker) NSPrepare(id string, )
+//func (n *networker) NSPrepare(id string, )
 // ZDBPrepare sends a macvlan interface into the
 // network namespace of a ZDB container
-func (n networker) ZDBPrepare(id string) (string, error) {
+func (n *networker) ZDBPrepare(id string) (string, error) {
 	return n.prepare(id, zdbNamespacePrefix, types.PublicBridge)
 }
 
 // ZDBDestroy is the opposite of ZDPrepare, it makes sure network setup done
 // for zdb is rewind. ns param is the namespace return by the ZDBPrepare
-func (n networker) ZDBDestroy(ns string) error {
+func (n *networker) ZDBDestroy(ns string) error {
 	panic("not implemented")
 	// if !strings.HasPrefix(ns, zdbNamespacePrefix) {
 	// 	return fmt.Errorf("invalid zdb namespace name '%s'", ns)
@@ -237,7 +238,7 @@ func (n networker) ZDBDestroy(ns string) error {
 	// return n.destroy(ns)
 }
 
-func (n networker) createMacVlan(iface string, master string, hw net.HardwareAddr, ips []*net.IPNet, routes []*netlink.Route, netNs ns.NetNS) error {
+func (n *networker) createMacVlan(iface string, master string, hw net.HardwareAddr, ips []*net.IPNet, routes []*netlink.Route, netNs ns.NetNS) error {
 	var macVlan *netlink.Macvlan
 	err := netNs.Do(func(_ ns.NetNS) error {
 		var err error
@@ -587,7 +588,7 @@ func (n *networker) GetPublicIPv6Subnet() (net.IPNet, error) {
 
 // GetSubnet of a local network resource identified by the network ID, ipv4 and ipv6
 // subnet respectively
-func (n networker) GetSubnet(networkID pkg.NetID) (net.IPNet, error) {
+func (n *networker) GetSubnet(networkID pkg.NetID) (net.IPNet, error) {
 	localNR, err := n.networkOf(string(networkID))
 	if err != nil {
 		return net.IPNet{}, errors.Wrapf(err, "couldn't load network with id (%s)", networkID)
@@ -597,7 +598,7 @@ func (n networker) GetSubnet(networkID pkg.NetID) (net.IPNet, error) {
 }
 
 // GetNet of a network identified by the network ID
-func (n networker) GetNet(networkID pkg.NetID) (net.IPNet, error) {
+func (n *networker) GetNet(networkID pkg.NetID) (net.IPNet, error) {
 	localNR, err := n.networkOf(string(networkID))
 	if err != nil {
 		return net.IPNet{}, errors.Wrapf(err, "couldn't load network with id (%s)", networkID)
@@ -609,7 +610,7 @@ func (n networker) GetNet(networkID pkg.NetID) (net.IPNet, error) {
 // GetDefaultGwIP returns the IPs of the default gateways inside the network
 // resource identified by the network ID on the local node, for IPv4 and IPv6
 // respectively
-func (n networker) GetDefaultGwIP(networkID pkg.NetID) (net.IP, net.IP, error) {
+func (n *networker) GetDefaultGwIP(networkID pkg.NetID) (net.IP, net.IP, error) {
 	localNR, err := n.networkOf(string(networkID))
 	if err != nil {
 		return nil, nil, errors.Wrapf(err, "couldn't load network with id (%s)", networkID)
@@ -630,39 +631,84 @@ func (n networker) GetDefaultGwIP(networkID pkg.NetID) (net.IP, net.IP, error) {
 }
 
 // GetIPv6From4 generates an IPv6 address from a given IPv4 address in a NR
-func (n networker) GetIPv6From4(networkID pkg.NetID, ip net.IP) (net.IPNet, error) {
+func (n *networker) GetIPv6From4(networkID pkg.NetID, ip net.IP) (net.IPNet, error) {
 	if ip.To4() == nil {
 		return net.IPNet{}, errors.New("invalid IPv4 address")
 	}
 	return net.IPNet{IP: nr.Convert4to6(string(networkID), ip), Mask: net.CIDRMask(64, 128)}, nil
 }
 
-// Addrs return the IP addresses of interface
-func (n networker) Addrs(iface string, netns string) (ips []net.IP, mac string, err error) {
-	f := func(_ ns.NetNS) error {
-		link, err := netlink.LinkByName(iface)
+func (n *networker) SetPublicExitDevice(iface string) error {
+	link, err := netlink.LinkByName(iface)
+	if err != nil {
+		return err
+	}
+
+	return public.SetPublicExitLink(link)
+}
+
+func (n *networker) Interfaces(iface string, netns string) (map[string]pkg.Interface, error) {
+	getter := func(iface string) ([]netlink.Link, error) {
+		if iface != "" {
+			l, err := netlink.LinkByName(iface)
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to get interface %s", iface)
+			}
+			return []netlink.Link{l}, nil
+		}
+
+		all, err := netlink.LinkList()
 		if err != nil {
-			return errors.Wrapf(err, "failed to get interface %s", iface)
+			return nil, err
+		}
+		filtered := all[:0]
+		for _, l := range all {
+			name := l.Attrs().Name
+
+			if name == "lo" ||
+				(l.Type() != "device" && name != types.DefaultBridge) {
+
+				continue
+			}
+
+			filtered = append(filtered, l)
 		}
 
-		mac = link.Attrs().HardwareAddr.String()
+		return filtered, nil
+	}
 
-		addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL)
+	interfaces := make(map[string]pkg.Interface)
+	f := func(_ ns.NetNS) error {
+		links, err := getter(iface)
 		if err != nil {
-			return errors.Wrapf(err, "failed to list addresses of interfaces %s", iface)
+			return errors.Wrapf(err, "failed to get interfaces (query: '%s')", iface)
 		}
-		ips = make([]net.IP, 0, len(addrs))
-		for _, addr := range addrs {
-			ip := addr.IP
-			if ip6 := ip.To16(); ip6 != nil {
-				// ipv6
-				if !ip6.IsGlobalUnicast() || ifaceutil.IsULA(ip6) {
-					// skip if not global or is ula address
-					continue
+
+		for _, link := range links {
+
+			addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL)
+			if err != nil {
+				return errors.Wrapf(err, "failed to list addresses of interfaces %s", iface)
+			}
+			ips := make([]net.IPNet, 0, len(addrs))
+			for _, addr := range addrs {
+				ip := addr.IP
+				if ip6 := ip.To16(); ip6 != nil {
+					// ipv6
+					if !ip6.IsGlobalUnicast() || ifaceutil.IsULA(ip6) {
+						// skip if not global or is ula address
+						continue
+					}
 				}
+
+				ips = append(ips, *addr.IPNet)
 			}
 
-			ips = append(ips, addr.IP)
+			interfaces[link.Attrs().Name] = pkg.Interface{
+				Name: link.Attrs().Name,
+				Mac:  link.Attrs().HardwareAddr.String(),
+				IPs:  ips,
+			}
 		}
 
 		return nil
@@ -671,20 +717,38 @@ func (n networker) Addrs(iface string, netns string) (ips []net.IP, mac string,
 	if netns != "" {
 		netNS, err := namespace.GetByName(netns)
 		if err != nil {
-			return nil, mac, errors.Wrapf(err, "failed to get network namespace %s", netns)
+			return nil, errors.Wrapf(err, "failed to get network namespace %s", netns)
 		}
 		defer netNS.Close()
 
 		if err := netNS.Do(f); err != nil {
-			return nil, mac, err
+			return nil, err
 		}
 	} else {
 		if err := f(nil); err != nil {
-			return nil, mac, err
+			return nil, err
 		}
 	}
 
-	return ips, mac, nil
+	return interfaces, nil
+}
+
+// [obsolete] use Interfaces instead Addrs return the IP addresses of interface
+func (n *networker) Addrs(iface string, netns string) (ips []net.IP, mac string, err error) {
+	if iface == "" {
+		return ips, mac, fmt.Errorf("iface cannot be empty")
+	}
+	interfaces, err := n.Interfaces(iface, netns)
+	if err != nil {
+		return nil, "", err
+	}
+
+	inf := interfaces[iface]
+	mac = inf.Mac
+	for _, ip := range inf.IPs {
+		ips = append(ips, ip.IP)
+	}
+	return
 }
 
 // CreateNR implements pkg.Networker interface
@@ -900,6 +964,21 @@ func (n *networker) SetPublicConfig(cfg pkg.PublicConfig) error {
 	return nil
 }
 
+func (n *networker) GetPublicExitDevice() (pkg.ExitDevice, error) {
+	exit, err := public.GetPublicExitLink()
+	if err != nil {
+		return pkg.ExitDevice{}, err
+	}
+
+	// if exit is over veth then we going over zos bridge
+	// hence it's a single nic setup
+	if ok, _ := bootstrap.VEthFilter(exit); ok {
+		return pkg.ExitDevice{IsSingle: true}, nil
+	}
+
+	return pkg.ExitDevice{IsDual: true, AsDualInterface: exit.Attrs().Name}, nil
+}
+
 // Get node public namespace config
 func (n *networker) GetPublicConfig() (pkg.PublicConfig, error) {
 	// TODO: instea of loading, this actually must get
diff --git a/pkg/network/public/persist.go b/pkg/network/public/persist.go
index 34a1678e..982215ca 100644
--- a/pkg/network/public/persist.go
+++ b/pkg/network/public/persist.go
@@ -3,25 +3,36 @@ package public
 import (
 	"encoding/json"
 	"os"
+	"path/filepath"
 
 	"github.com/pkg/errors"
 	"github.com/threefoldtech/zos/pkg"
 )
 
+const (
+	publicConfigFile = "public-config.json"
+	publicExitFile   = "public-exit"
+)
+
 var (
 	// persistencePath is path to config file.
 	persistencePath = ""
 )
 
 func SetPersistence(path string) {
+	stat, err := os.Stat(path)
+	if err != nil || !stat.IsDir() {
+		panic("invalid persistence path must be an existing directory")
+	}
+
 	persistencePath = path
 }
 
-func getPersistencePath() string {
+func getPersistencePath(file string) string {
 	if persistencePath == "" {
 		panic("public config persistence path is not set")
 	}
-	return persistencePath
+	return filepath.Join(persistencePath, file)
 }
 
 // ErrNoPublicConfig is the error returns by ReadPubIface when no public
@@ -30,8 +41,7 @@ var ErrNoPublicConfig = errors.New("no public configuration")
 
 // LoadPublicConfig loads public config from file
 func LoadPublicConfig() (*pkg.PublicConfig, error) {
-
-	file, err := os.Open(getPersistencePath())
+	file, err := os.Open(getPersistencePath(publicConfigFile))
 	if os.IsNotExist(err) {
 		// it's not an error to not have config
 		// but we return a nil config
@@ -51,7 +61,7 @@ func LoadPublicConfig() (*pkg.PublicConfig, error) {
 
 // SavePublicConfig stores public config in a file
 func SavePublicConfig(cfg pkg.PublicConfig) error {
-	file, err := os.Create(getPersistencePath())
+	file, err := os.Create(getPersistencePath(publicConfigFile))
 	if err != nil {
 		return errors.Wrap(err, "failed to create configuration file")
 	}
diff --git a/pkg/network/public/public.go b/pkg/network/public/public.go
index dbef993c..37a986e6 100644
--- a/pkg/network/public/public.go
+++ b/pkg/network/public/public.go
@@ -2,7 +2,9 @@ package public
 
 import (
 	"fmt"
+	"io/ioutil"
 	"net"
+	"os"
 
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/pkg/errors"
@@ -26,6 +28,8 @@ const (
 	// PublicBridge public bridge name, exists only after a call to EnsurePublicSetup
 	PublicBridge    = types.PublicBridge
 	PublicNamespace = types.PublicNamespace
+
+	// TODO: This pass need to come from
 )
 
 // EnsurePublicBridge makes sure that the public bridge exists
@@ -87,9 +91,7 @@ func IPs() ([]net.IPNet, error) {
 }
 
 func setupPublicBridge(br *netlink.Bridge) error {
-	// find possible exit interface.
-	// or fall back to zos.
-	exit, err := findPossibleExit()
+	exit, err := getExitNic()
 	if err != nil {
 		return errors.Wrap(err, "failed to find possible exit")
 	}
@@ -100,15 +102,144 @@ func setupPublicBridge(br *netlink.Bridge) error {
 		return errors.Wrapf(err, "failed to get link '%s' by name", exit)
 	}
 
-	if err := netlink.LinkSetUp(exitLink); err != nil {
-		return errors.Wrapf(err, "failed to set link '%s' up", exitLink.Attrs().Name)
+	return attachPublicToExit(br, exitLink)
+}
+
+func attachPublicToExit(br *netlink.Bridge, exit netlink.Link) error {
+	if err := netlink.LinkSetUp(exit); err != nil {
+		return errors.Wrapf(err, "failed to set link '%s' up", exit.Attrs().Name)
 	}
 
-	if err := bridge.Attach(exitLink, br, toZosVeth); err != nil {
+	if err := bridge.Attach(exit, br, toZosVeth); err != nil {
 		return errors.Wrap(err, "failed to attach exit nic to public bridge 'br-pub'")
 	}
 
-	return nil
+	// persist this value for next boot
+	return ioutil.WriteFile(
+		getPersistencePath(publicExitFile),
+		[]byte(exit.Attrs().Name),
+		0644,
+	)
+}
+
+// if no persisted exit link file found, we need
+// to make sure to persist the current exit nic
+// in case of some nodes has their public nic
+// wired correctly manually (on some dev envs)
+// so to make sure node uses the same nic on reboot
+// we need to query the current active exit, and store
+// it
+func persistExitNicIfNotFound(exit netlink.Link) error {
+	log.Debug().Msg("persisting current public bridge uplink")
+	path := getPersistencePath(publicExitFile)
+	if _, err := os.Stat(path); err == nil || !os.IsNotExist(err) {
+		return err
+	}
+
+	name := types.DefaultBridge
+	if ok, _ := bootstrap.PhysicalFilter(exit); ok {
+		name = exit.Attrs().Name
+	}
+
+	// persist this value for next boot
+	return ioutil.WriteFile(
+		path,
+		[]byte(name),
+		0644,
+	)
+}
+
+func GetPublicExitLink() (netlink.Link, error) {
+	// return the upstream (exit) link for br-pub
+	br, err := bridge.Get(PublicBridge)
+	if err != nil {
+		return nil, errors.Wrap(err, "no public bridge found")
+	}
+
+	all, err := netlink.LinkList()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to list node nics")
+	}
+
+	// public bridge can be wired to either
+	matches := []bootstrap.Filter{
+		// a nic
+		bootstrap.PhysicalFilter,
+		// a veth pair to another bridge (zos always)
+		bootstrap.VEthFilter,
+	}
+
+	for _, link := range all {
+		for _, match := range matches {
+			if ok, _ := match(link); !ok {
+				continue
+			}
+
+			if link.Attrs().MasterIndex == br.Index {
+				return link, nil
+			}
+		}
+	}
+
+	return nil, os.ErrNotExist
+}
+
+// SetPublicExitLink rewires the br-pub to a different exit (upstream) device.
+// this upstream device can either be a physical free device, or zos bridge.
+// the method is idempotent.
+func SetPublicExitLink(link netlink.Link) error {
+	// we can only attach to either physical nic, or zos bridge
+	log.Debug().
+		Str("type", link.Type()).
+		Str("name", link.Attrs().Name).
+		Int("master", link.Attrs().MasterIndex).
+		Msg("trying to set public exit interface")
+
+	if link.Type() != "device" && link.Attrs().Name != types.DefaultBridge {
+		return fmt.Errorf("invalid exit bridge must be a physical nic or the default bridge")
+	}
+
+	br, err := bridge.Get(PublicBridge)
+	if err != nil {
+		return err
+	}
+
+	if link.Type() == "device" {
+		// already attached
+		if link.Attrs().MasterIndex == br.Index {
+			return nil
+		} else if link.Attrs().MasterIndex != 0 {
+			return fmt.Errorf("device is '%s' already used", link.Attrs().Name)
+		}
+	}
+
+	current, err := GetPublicExitLink()
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	if current != nil {
+		log.Debug().Str("type", current.Type()).Str("name", current.Attrs().Name).Msg("current attached exit is")
+
+		// disconnect br pub based on the type of the current uplink
+
+		if veth, _ := bootstrap.VEthFilter(current); veth {
+			// br pub is already connected to zos
+			if link.Attrs().Name == "zos" {
+				return nil
+			}
+
+			// otherwise we remove this link
+			if err := netlink.LinkDel(current); err != nil {
+				return errors.Wrap(err, "failed to unhook public bridge from zos bridge")
+			}
+		} else if err := netlink.LinkSetMasterByIndex(current, 0); err != nil {
+			// otherwise we try to remove the nic from br-pub
+			return errors.Wrap(err, "failed to unhook public bridge from physical nic")
+		}
+	}
+
+	return attachPublicToExit(br, link)
 }
 
 func HasPublicSetup() bool {
@@ -186,7 +317,17 @@ func GetPublicSetup() (pkg.PublicConfig, error) {
 	return cfg, err
 }
 
-// EnsurePublicSetup create the public setup, it's okay to have inf == nil
+// EnsurePublicSetup create the public setup, it's okay to have inf == nil.
+// this method need to be called at least once in the life of the node. to make bridges are created
+// and wired correctly, and initialize public name space if `inf` is found.
+// Public bridge wiring (to exit nic) is remembered from last boot. If no exit nic is set
+// the node tries to detect the exit br-pub nic based on the following criteria
+// - physical nic
+// - wired and has a signal
+// - can get public slaac IPv6
+//
+// if no nic is found zos is selected.
+// changes to the br-pub exit nic can then be done later with SetPublicExitLink
 func EnsurePublicSetup(nodeID pkg.Identifier, inf *pkg.PublicConfig) (*netlink.Bridge, error) {
 	log.Debug().Msg("ensure public setup")
 	br, err := ensurePublicBridge()
@@ -194,24 +335,19 @@ func EnsurePublicSetup(nodeID pkg.Identifier, inf *pkg.PublicConfig) (*netlink.B
 		return nil, err
 	}
 
-	// find if we have anything attached to this bridge (so initialization)
-	// has been done before.
-	all, err := netlink.LinkList()
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to list host links")
-	}
-
-	filtered := all[:0]
-	for _, link := range all {
-		if link.Attrs().MasterIndex == br.Index {
-			filtered = append(filtered, link)
-		}
-	}
+	exit, err := GetPublicExitLink()
 
-	if len(filtered) == 0 {
+	if os.IsNotExist(err) {
+		// bridge is not initialized, wire it.
+		log.Debug().Msg("no public bridge uplink found, setting up...")
 		if err := setupPublicBridge(br); err != nil {
 			return nil, err
 		}
+	} else if err != nil {
+		return nil, errors.Wrap(err, "failed to get current public bridge uplink")
+	} else if err := persistExitNicIfNotFound(exit); err != nil {
+		log.Error().Err(err).
+			Msg("failed to persist current public bridge uplink")
 	}
 
 	if inf != nil {
@@ -223,10 +359,45 @@ func EnsurePublicSetup(nodeID pkg.Identifier, inf *pkg.PublicConfig) (*netlink.B
 	return br, netlink.LinkSetUp(br)
 }
 
-func findPossibleExit() (string, error) {
-	log.Debug().Msg("find possible ipv6 exit interface")
+func getPersistedExitNic() (string, error) {
+	path := getPersistencePath(publicExitFile)
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+
+	name := string(data)
+	// we need to check if an actual device exits with that name or not
+	if _, err := netlink.LinkByName(name); err != nil {
+		return "", os.ErrNotExist
+	}
 
-	links, err := bootstrap.AnalyseLinks(
+	return name, nil
+}
+
+// getExitNic gets the possible "device" that need to used by br-pub
+// as a traffic upstream device. this can be set by the "farmer" for this
+// node, or can
+func getExitNic() (string, error) {
+	// if previous exit bridge was selected and persisted we need to use this
+	if dev, err := getPersistedExitNic(); err != nil && !os.IsNotExist(err) {
+		return "", errors.Wrap(err, "failed to load exit nic setup")
+	} else if err == nil {
+		return dev, nil
+	}
+
+	dev, err := detectExitNic()
+	if err != nil {
+		return "", err
+	}
+
+	return dev, nil
+}
+
+func detectExitNic() (string, error) {
+	log.Debug().Msg("find possible ipv6 exit interface")
+	// otherwise we try to find the right one
+	links, err := bootstrap.AnalyzeLinks(
 		bootstrap.RequiresIPv6,
 		bootstrap.PhysicalFilter,
 		bootstrap.NotAttachedFilter,
@@ -235,7 +406,7 @@ func findPossibleExit() (string, error) {
 
 	log.Debug().Int("found", len(links)).Msg("found possible links")
 	if err != nil {
-		return "", errors.Wrap(err, "failed to analyse links")
+		return "", errors.Wrap(err, "failed to analyze links")
 	}
 
 	for _, link := range links {
diff --git a/pkg/provision/mbus/api.go b/pkg/provision/mbus/api.go
index 14d2add2..e3a89763 100644
--- a/pkg/provision/mbus/api.go
+++ b/pkg/provision/mbus/api.go
@@ -4,6 +4,9 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/pkg/errors"
+	"github.com/threefoldtech/zos/pkg/gridtypes"
+	"github.com/threefoldtech/zos/pkg/gridtypes/zos"
 	"github.com/threefoldtech/zos/pkg/provision"
 	"github.com/threefoldtech/zos/pkg/rmb"
 )
@@ -33,6 +36,53 @@ func (d *Deployments) setup(router rmb.Router) {
 	sub.WithHandler("delete", d.deleteHandler)
 	sub.WithHandler("get", d.getHandler)
 	sub.WithHandler("changes", d.changesHandler)
+
+	net := router.Subroute("network")
+	net.WithHandler("list_public_ips", d.listPublicIps)
+}
+
+func (n *Deployments) listPublicIps(ctx context.Context, _ []byte) (interface{}, error) {
+	storage := n.engine.Storage()
+	// for efficiency this method should just find out configured public Ips.
+	// but currently the only way to do this is by scanning the nft rules
+	// a nother less efficient but good for now solution is to scan all
+	// reservations and find the ones with public IPs.
+
+	twins, err := storage.Twins()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to list twins")
+	}
+	ips := make([]string, 0)
+	for _, twin := range twins {
+		deploymentsIDs, err := storage.ByTwin(twin)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to list twin deployment")
+		}
+		for _, id := range deploymentsIDs {
+			deployment, err := storage.Get(twin, id)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to load deployment")
+			}
+			workloads := deployment.ByType(zos.PublicIPv4Type, zos.PublicIPType)
+
+			for _, workload := range workloads {
+				if workload.Result.State != gridtypes.StateOk {
+					continue
+				}
+
+				var result zos.PublicIPResult
+				if err := workload.Result.Unmarshal(&result); err != nil {
+					return nil, err
+				}
+
+				if result.IP.IP != nil {
+					ips = append(ips, result.IP.String())
+				}
+			}
+		}
+	}
+
+	return ips, nil
 }
 
 func (d *Deployments) updateHandler(ctx context.Context, payload []byte) (interface{}, error) {
diff --git a/pkg/provision/mbus/network.go b/pkg/provision/mbus/network.go
deleted file mode 100644
index 74d27a9b..00000000
--- a/pkg/provision/mbus/network.go
+++ /dev/null
@@ -1,172 +0,0 @@
-package mbus
-
-import (
-	"context"
-	"net"
-
-	"github.com/pkg/errors"
-	"github.com/threefoldtech/zbus"
-	"github.com/threefoldtech/zos/pkg/gridtypes"
-	"github.com/threefoldtech/zos/pkg/gridtypes/zos"
-	"github.com/threefoldtech/zos/pkg/provision"
-	"github.com/threefoldtech/zos/pkg/provision/mw"
-	"github.com/threefoldtech/zos/pkg/rmb"
-	"github.com/threefoldtech/zos/pkg/stubs"
-)
-
-// Network message bus api
-type Network struct {
-	engine provision.Engine
-	cl     zbus.Client
-}
-
-// NewNetworkMessagebus creates a new messagebus instance
-func NewNetworkMessagebus(router rmb.Router, engine provision.Engine, cl zbus.Client) *Network {
-
-	api := &Network{
-		engine: engine,
-		cl:     cl,
-	}
-	api.setup(router)
-	return api
-}
-
-func (n *Network) listPortsHandler(ctx context.Context, payload []byte) (interface{}, error) {
-	data, err := n.listPorts(ctx)
-	if err != nil {
-		return nil, err.Err()
-	}
-	return data, nil
-}
-
-func (n *Network) hasPublicIPv6Handler(ctx context.Context, payload []byte) (interface{}, error) {
-	return n.hasPublicIPv6(ctx), nil
-}
-func (n *Network) interfacesHandler(ctx context.Context, payload []byte) (interface{}, error) {
-	data, err := n.listInterfaces(ctx)
-	if err != nil {
-		return nil, err.Err()
-	}
-	return data, nil
-}
-
-func (n *Network) listPublicIPsHandler(ctx context.Context, payload []byte) (interface{}, error) {
-	data, err := n.listPublicIps()
-	if err != nil {
-		return nil, err.Err()
-	}
-	return data, nil
-}
-
-func (n *Network) getPublicConfigHandler(ctx context.Context, payload []byte) (interface{}, error) {
-	data, err := n.getPublicConfig(ctx)
-	if err != nil {
-		return nil, err.Err()
-	}
-	return data, nil
-}
-
-func (n *Network) setup(router rmb.Router) {
-
-	// network handlers
-	sub := router.Subroute("network")
-	sub.WithHandler("list_wg_ports", n.listPortsHandler)
-	sub.WithHandler("list_public_ips", n.listPublicIPsHandler)
-	sub.WithHandler("public_config_get", n.getPublicConfigHandler)
-	sub.WithHandler("interfaces", n.interfacesHandler)
-	sub.WithHandler("has_ipv6", n.hasPublicIPv6Handler)
-}
-
-func (n *Network) listPorts(ctx context.Context) (interface{}, mw.Response) {
-	ports, err := stubs.NewNetworkerStub(n.cl).WireguardPorts(ctx)
-	if err != nil {
-		return nil, mw.Error(err)
-	}
-
-	return ports, nil
-}
-
-func (n *Network) hasPublicIPv6(ctx context.Context) interface{} {
-	ipData, err := stubs.NewNetworkerStub(n.cl).GetPublicIPv6Subnet(ctx)
-	return ipData.IP != nil && err == nil
-}
-func (n *Network) listInterfaces(ctx context.Context) (interface{}, mw.Response) {
-	mgr := stubs.NewNetworkerStub(n.cl)
-	results := make(map[string][]net.IP)
-	type q struct {
-		inf    string
-		ns     string
-		rename string
-	}
-	for _, i := range []q{{"zos", "", "zos"}, {"nygg6", "ndmz", "ygg"}} {
-		ips, _, err := mgr.Addrs(ctx, i.inf, i.ns)
-		if err != nil {
-			return nil, mw.Error(errors.Wrapf(err, "failed to get ips for '%s' interface", i))
-		}
-
-		results[i.rename] = func() []net.IP {
-			list := make([]net.IP, 0, len(ips))
-			for _, item := range ips {
-				ip := net.IP(item)
-				list = append(list, ip)
-			}
-
-			return list
-		}()
-	}
-
-	return results, nil
-}
-
-func (n *Network) listPublicIps() (interface{}, mw.Response) {
-	storage := n.engine.Storage()
-	// for efficiency this method should just find out configured public Ips.
-	// but currently the only way to do this is by scanning the nft rules
-	// a nother less efficient but good for now solution is to scan all
-	// reservations and find the ones with public IPs.
-
-	twins, err := storage.Twins()
-	if err != nil {
-		return nil, mw.Error(errors.Wrap(err, "failed to list twins"))
-	}
-	ips := make([]string, 0)
-	for _, twin := range twins {
-		deploymentsIDs, err := storage.ByTwin(twin)
-		if err != nil {
-			return nil, mw.Error(errors.Wrap(err, "failed to list twin deployment"))
-		}
-		for _, id := range deploymentsIDs {
-			deployment, err := storage.Get(twin, id)
-			if err != nil {
-				return nil, mw.Error(errors.Wrap(err, "failed to load deployment"))
-			}
-			workloads := deployment.ByType(zos.PublicIPv4Type, zos.PublicIPType)
-
-			for _, workload := range workloads {
-				if workload.Result.State != gridtypes.StateOk {
-					continue
-				}
-
-				var result zos.PublicIPResult
-				if err := workload.Result.Unmarshal(&result); err != nil {
-					return nil, mw.Error(err)
-				}
-
-				if result.IP.IP != nil {
-					ips = append(ips, result.IP.String())
-				}
-			}
-		}
-	}
-
-	return ips, nil
-}
-
-func (n *Network) getPublicConfig(ctx context.Context) (interface{}, mw.Response) {
-	cfg, err := stubs.NewNetworkerStub(n.cl).GetPublicConfig(ctx)
-	if err != nil {
-		return nil, mw.NotFound(err)
-	}
-
-	return cfg, nil
-}
diff --git a/pkg/provision/mbus/reservation.go b/pkg/provision/mbus/reservation.go
index fc0ff6c6..2a8026cc 100644
--- a/pkg/provision/mbus/reservation.go
+++ b/pkg/provision/mbus/reservation.go
@@ -63,30 +63,6 @@ func (d *Deployments) createOrUpdate(ctx context.Context, payload []byte, update
 	return nil, mw.Accepted()
 }
 
-// func (d *Deployments) delete(ctx context.Context, payload []byte) (interface{}, mw.Response) {
-// 	var args idArgs
-// 	err := json.Unmarshal(payload, &args)
-// 	if err != nil {
-// 		return nil, mw.Error(err)
-// 	}
-
-// 	ctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
-// 	defer cancel()
-
-// 	err = d.engine.Deprovision(ctx, rmb.GetTwinID(ctx), args.ContractID,
-// 		fmt.Sprintf("requested by twin %d over rmb api", rmb.GetTwinID(ctx)),
-// 	)
-// 	if err == context.DeadlineExceeded {
-// 		return nil, mw.Unavailable(ctx.Err())
-// 	} else if errors.Is(err, provision.ErrDeploymentNotExists) {
-// 		return nil, mw.NotFound(err)
-// 	} else if err != nil {
-// 		return nil, mw.Error(err)
-// 	}
-
-// 	return nil, mw.Accepted()
-// }
-
 func (d *Deployments) get(ctx context.Context, payload []byte) (interface{}, mw.Response) {
 	var args idArgs
 	err := json.Unmarshal(payload, &args)
diff --git a/pkg/rmb/mw.go b/pkg/rmb/mw.go
index ed684219..fac0036f 100644
--- a/pkg/rmb/mw.go
+++ b/pkg/rmb/mw.go
@@ -2,8 +2,11 @@ package rmb
 
 import (
 	"context"
+	"fmt"
 
+	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
+	"github.com/threefoldtech/substrate-client"
 )
 
 // LoggerMiddleware simple logger middleware.
@@ -19,3 +22,30 @@ func LoggerMiddleware(ctx context.Context, payload []byte) (context.Context, err
 var (
 	_ Middleware = LoggerMiddleware
 )
+
+// Authorized middleware allows only admins to make these calls
+func Authorized(mgr substrate.Manager, farmID uint32) (Middleware, error) {
+	sub, err := mgr.Substrate()
+	if err != nil {
+		return nil, err
+	}
+	defer sub.Close()
+	farm, err := sub.GetFarm(farmID)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get farm")
+	}
+
+	farmer, err := sub.GetTwin(uint32(farm.TwinID))
+	if err != nil {
+		return nil, err
+	}
+
+	return func(ctx context.Context, payload []byte) (context.Context, error) {
+		user := GetTwinID(ctx)
+		if user != uint32(farmer.ID) {
+			return nil, fmt.Errorf("unauthorized")
+		}
+
+		return ctx, nil
+	}, nil
+}
diff --git a/pkg/storage.go b/pkg/storage.go
index 1081459e..c4e769cf 100644
--- a/pkg/storage.go
+++ b/pkg/storage.go
@@ -21,7 +21,7 @@ type ErrNotEnoughSpace struct {
 }
 
 func (e ErrNotEnoughSpace) Error() string {
-	return fmt.Sprintf("Not enough space left in pools of this type %s", e.DeviceType)
+	return fmt.Sprintf("not enough space left in pools of this type %s", e.DeviceType)
 }
 
 // ErrInvalidDeviceType raised when trying to allocate space on unsupported device type
diff --git a/pkg/stubs/network_stub.go b/pkg/stubs/network_stub.go
index 4d6cb0fa..b1bb6d48 100644
--- a/pkg/stubs/network_stub.go
+++ b/pkg/stubs/network_stub.go
@@ -183,6 +183,23 @@ func (s *NetworkerStub) GetPublicConfig(ctx context.Context) (ret0 pkg.PublicCon
 	return
 }
 
+func (s *NetworkerStub) GetPublicExitDevice(ctx context.Context) (ret0 pkg.ExitDevice, ret1 error) {
+	args := []interface{}{}
+	result, err := s.client.RequestContext(ctx, s.module, s.object, "GetPublicExitDevice", args...)
+	if err != nil {
+		panic(err)
+	}
+	result.PanicOnError()
+	ret1 = result.CallError()
+	loader := zbus.Loader{
+		&ret0,
+	}
+	if err := result.Unmarshal(&loader); err != nil {
+		panic(err)
+	}
+	return
+}
+
 func (s *NetworkerStub) GetPublicIPv6Subnet(ctx context.Context) (ret0 net.IPNet, ret1 error) {
 	args := []interface{}{}
 	result, err := s.client.RequestContext(ctx, s.module, s.object, "GetPublicIPv6Subnet", args...)
@@ -217,6 +234,23 @@ func (s *NetworkerStub) GetSubnet(ctx context.Context, arg0 zos.NetID) (ret0 net
 	return
 }
 
+func (s *NetworkerStub) Interfaces(ctx context.Context, arg0 string, arg1 string) (ret0 interface{}, ret1 error) {
+	args := []interface{}{arg0, arg1}
+	result, err := s.client.RequestContext(ctx, s.module, s.object, "Interfaces", args...)
+	if err != nil {
+		panic(err)
+	}
+	result.PanicOnError()
+	ret1 = result.CallError()
+	loader := zbus.Loader{
+		&ret0,
+	}
+	if err := result.Unmarshal(&loader); err != nil {
+		panic(err)
+	}
+	return
+}
+
 func (s *NetworkerStub) Namespace(ctx context.Context, arg0 zos.NetID) (ret0 string) {
 	args := []interface{}{arg0}
 	result, err := s.client.RequestContext(ctx, s.module, s.object, "Namespace", args...)
@@ -447,6 +481,21 @@ func (s *NetworkerStub) SetPublicConfig(ctx context.Context, arg0 pkg.PublicConf
 	return
 }
 
+func (s *NetworkerStub) SetPublicExitDevice(ctx context.Context, arg0 string) (ret0 error) {
+	args := []interface{}{arg0}
+	result, err := s.client.RequestContext(ctx, s.module, s.object, "SetPublicExitDevice", args...)
+	if err != nil {
+		panic(err)
+	}
+	result.PanicOnError()
+	ret0 = result.CallError()
+	loader := zbus.Loader{}
+	if err := result.Unmarshal(&loader); err != nil {
+		panic(err)
+	}
+	return
+}
+
 func (s *NetworkerStub) SetupPrivTap(ctx context.Context, arg0 zos.NetID, arg1 string) (ret0 string, ret1 error) {
 	args := []interface{}{arg0, arg1}
 	result, err := s.client.RequestContext(ctx, s.module, s.object, "SetupPrivTap", args...)
diff --git a/tools/certify/main.go b/tools/certify/main.go
index 338f4316..4dc13dcf 100644
--- a/tools/certify/main.go
+++ b/tools/certify/main.go
@@ -58,12 +58,12 @@ func run(opt options) error {
 			Uint32("node-id", uint32(node.ID)).
 			Uint32("farm-id", uint32(node.FarmID)).
 			Bool("secure-boot", node.SecureBoot).
-			Bool("certified", node.CertificationType.IsCertified).
+			Bool("certified", node.Certification.IsCertified).
 			Logger()
 
 		log.Debug().Msg("node found")
 
-		if !node.SecureBoot || node.CertificationType.IsCertified {
+		if !node.SecureBoot || node.Certification.IsCertified {
 			// notthing to do anyway
 			continue
 		}
@@ -74,7 +74,7 @@ func run(opt options) error {
 			continue
 		}
 
-		if err := cl.SetNodeCertificate(sudo, uint32(node.ID), substrate.CertificationType{IsCertified: true}); err != nil {
+		if err := cl.SetNodeCertificate(sudo, uint32(node.ID), substrate.NodeCertification{IsCertified: true}); err != nil {
 			log.Error().Err(err).Msg("failed to mark node as certified")
 			continue
 		}


Test patch:
diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go
index 18ff9a0b..cfb17530 100644
--- a/pkg/storage/storage_test.go
+++ b/pkg/storage/storage_test.go
@@ -259,7 +259,7 @@ func TestCreateSubvolNoSpaceLeft(t *testing.T) {
 
 	_, err := mod.createSubvolWithQuota(20000, "sub")
 
-	require.EqualError(err, "Not enough space left in pools of this type ssd")
+	require.EqualError(err, "not enough space left in pools of this type ssd")
 }
 
 func TestVDiskFindCandidatesHasEnoughSpace(t *testing.T) {