Repository: threefoldtech/zos
Pull request #1320 (2021-06-23 12:04:31)

ygg in zmachines
Support yggdrasil IPs inside zmachine.


Patch:
diff --git a/pkg/container/watch.go b/pkg/container/watch.go
index 6be6887e..8cb42c94 100644
--- a/pkg/container/watch.go
+++ b/pkg/container/watch.go
@@ -122,5 +122,6 @@ func (c *Module) Watch(ctx context.Context) {
 		}
 
 		log.Err(err).Msg("error while watching events from containerd")
+		<-time.After(3 * time.Second)
 	}
 }
diff --git a/pkg/flist/flist.go b/pkg/flist/flist.go
index de0b5117..19d4eb62 100644
--- a/pkg/flist/flist.go
+++ b/pkg/flist/flist.go
@@ -516,7 +516,7 @@ func (f *flistModule) Unmount(name string) error {
 	}
 
 	if f.valid(mountpoint) == ErrAlreadyMounted {
-		if err := f.system.Unmount(mountpoint, syscall.MNT_DETACH); err != nil {
+		if err := f.system.Unmount(mountpoint, 0); err != nil {
 			log.Error().Err(err).Str("path", mountpoint).Msg("fail to umount flist")
 		}
 	}
diff --git a/pkg/gridtypes/zos/zmachine.go b/pkg/gridtypes/zos/zmachine.go
index 0465e01f..364539ed 100644
--- a/pkg/gridtypes/zos/zmachine.go
+++ b/pkg/gridtypes/zos/zmachine.go
@@ -21,6 +21,7 @@ type MachineInterface struct {
 type MachineNetwork struct {
 	PublicIP   gridtypes.Name     `json:"public_ip"`
 	Interfaces []MachineInterface `json:"interfaces"`
+	Planetary  bool               `json:"planetary"`
 }
 
 // Challenge builder
@@ -29,6 +30,10 @@ func (n *MachineNetwork) Challenge(w io.Writer) error {
 		return err
 	}
 
+	if _, err := fmt.Fprintf(w, "%t", n.Planetary); err != nil {
+		return err
+	}
+
 	for _, inf := range n.Interfaces {
 		if _, err := fmt.Fprintf(w, "%s", inf.Network); err != nil {
 			return err
diff --git a/pkg/network.go b/pkg/network.go
index 21335b71..1a24af4c 100644
--- a/pkg/network.go
+++ b/pkg/network.go
@@ -26,6 +26,14 @@ type ContainerNetworkConfig struct {
 	YggdrasilIP bool
 }
 
+// YggdrasilTap structure
+type YggdrasilTap struct {
+	Name    string
+	HW      net.HardwareAddr
+	IP      net.IPNet
+	Gateway net.IPNet
+}
+
 //Networker is the interface for the network module
 type Networker interface {
 	// Ready return nil is networkd is ready to operate
@@ -37,15 +45,17 @@ type Networker interface {
 	// Delete a network resource
 	DeleteNR(Network) error
 
-	// Join a network (with network id) will create a new isolated namespace
-	// that is hooked to the network bridge with a veth pair, and assign it a
-	// new IP from the network resource range. The method return the new namespace
-	// name.
-	// The member name specifies the name of the member, and must be unique
-	// The NetID is the network id to join
-	Join(networkdID NetID, containerID string, cfg ContainerNetworkConfig) (join Member, err error)
-	// Leave delete a container nameapce created by Join
-	Leave(networkdID NetID, containerID string) (err error)
+	// deprecated all uses taps now
+
+	// // Join a network (with network id) will create a new isolated namespace
+	// // that is hooked to the network bridge with a veth pair, and assign it a
+	// // new IP from the network resource range. The method return the new namespace
+	// // name.
+	// // The member name specifies the name of the member, and must be unique
+	// // The NetID is the network id to join
+	// Join(networkdID NetID, containerID string, cfg ContainerNetworkConfig) (join Member, err error)
+	// // Leave delete a container nameapce created by Join
+	// Leave(networkdID NetID, containerID string) (err error)
 
 	// ZDBPrepare creates a network namespace with a macvlan interface into it
 	// to allow the 0-db container to be publicly accessible
@@ -58,9 +68,12 @@ type Networker interface {
 	// for zdb is rewind. ns param is the namespace return by the ZDBPrepare
 	ZDBDestroy(ns string) error
 
-	// SetupTap sets up a tap device in the network namespace for the networkID. It is hooked
+	// SetupPrivTap sets up a tap device in the network namespace for the networkID. It is hooked
 	// to the network bridge. The name of the tap interface is returned
-	SetupTap(networkID NetID, name string) (string, error)
+	SetupPrivTap(networkID NetID, name string) (string, error)
+
+	// SetupYggTap sets up a tap device in the host namespace for the yggdrasil ip
+	SetupYggTap(name string) (YggdrasilTap, error)
 
 	// TapExists checks if the tap device with the given name exists already
 	TapExists(name string) (bool, error)
@@ -117,12 +130,16 @@ type Networker interface {
 
 	WireguardPorts() ([]uint, error)
 
+	// Public Config
+
 	// Set node public namespace config
 	SetPublicConfig(cfg PublicConfig) error
 
 	// Get node public namespace config
 	GetPublicConfig() (PublicConfig, error)
 
+	// Monitoring methods
+
 	// ZOSAddresses monitoring streams for ZOS bridge IPs
 	ZOSAddresses(ctx context.Context) <-chan NetlinkAddresses
 
diff --git a/pkg/network/networker.go b/pkg/network/networker.go
index 67836650..91d50e4c 100644
--- a/pkg/network/networker.go
+++ b/pkg/network/networker.go
@@ -128,123 +128,6 @@ func (n *networker) WireguardPorts() ([]uint, error) {
 	return n.portSet.List()
 }
 
-func (n *networker) Join(networkdID pkg.NetID, containerID string, cfg pkg.ContainerNetworkConfig) (join pkg.Member, err error) {
-	// TODO:
-	// 1- Make sure this network id is actually deployed
-	// 2- Check if the requested network config is doable
-	// 3- Create a new namespace, then create a veth pair inside this namespace
-	// 4- Hook one end to the NR bridge
-	// 5- Assign IP to the veth endpoint inside the namespace.
-	// 6- return the namespace name
-
-	log.Info().Str("network-id", string(networkdID)).Msg("joining network")
-
-	localNR, err := n.networkOf(string(networkdID))
-	if err != nil {
-		return join, errors.Wrapf(err, "couldn't load network with id (%s)", networkdID)
-	}
-
-	ipv4Only, err := n.ndmz.IsIPv4Only()
-	if err != nil {
-		return join, errors.Wrap(err, "failed to check ipv6 support")
-	}
-	if cfg.PublicIP6 && ipv4Only {
-		return join, errors.Errorf("this node runs in IPv4 only mode and you asked for a public IPv6. Impossible to fulfill the request")
-	}
-
-	netRes, err := nr.New(localNR)
-	if err != nil {
-		return join, errors.Wrap(err, "failed to load network resource")
-	}
-
-	ips := make([]net.IP, len(cfg.IPs))
-	for i, addr := range cfg.IPs {
-		ips[i] = net.ParseIP(addr)
-	}
-
-	join, err = netRes.Join(nr.ContainerConfig{
-		ContainerID: containerID,
-		IPs:         ips,
-		PublicIP6:   cfg.PublicIP6,
-		IPv4Only:    ipv4Only,
-	})
-	if err != nil {
-		return join, errors.Wrap(err, "failed to load network resource")
-	}
-
-	netNs, err := namespace.GetByName(join.Namespace)
-	if err != nil {
-		return join, errors.Wrap(err, "failed to found a valid network interface to use as parent for 0-db container")
-	}
-	defer netNs.Close()
-
-	if cfg.PublicIP6 {
-		hw := ifaceutil.HardwareAddrFromInputBytes([]byte(containerID))
-		if err = n.createMacVlan("pub", public.PublicBridge, hw, nil, nil, netNs); err != nil {
-			return join, errors.Wrap(err, "failed to create public macvlan interface")
-		}
-	}
-
-	if cfg.YggdrasilIP {
-		var (
-			ips    []*net.IPNet
-			routes []*netlink.Route
-		)
-
-		hw := ifaceutil.HardwareAddrFromInputBytes([]byte("ygg:" + containerID))
-		ip, err := n.ygg.SubnetFor(hw)
-		if err != nil {
-			return join, err
-		}
-
-		ips = []*net.IPNet{
-			{
-				IP:   ip,
-				Mask: net.CIDRMask(64, 128),
-			},
-		}
-		join.YggdrasilIP = ip
-
-		gw, err := n.ygg.Gateway()
-		if err != nil {
-			return join, err
-		}
-
-		routes = []*netlink.Route{
-			{
-				Dst: &net.IPNet{
-					IP:   net.ParseIP("200::"),
-					Mask: net.CIDRMask(7, 128),
-				},
-				Gw: gw.IP,
-				// LinkIndex:... this is set by macvlan.Install
-			},
-		}
-
-		if err := n.createMacVlan("ygg", types.YggBridge, hw, ips, routes, netNs); err != nil {
-			return join, errors.Wrap(err, "failed to create yggdrasil macvlan interface")
-		}
-	}
-
-	return join, nil
-}
-
-func (n *networker) Leave(networkdID pkg.NetID, containerID string) error {
-	log.Info().Str("network-id", string(networkdID)).Msg("leaving network")
-
-	localNR, err := n.networkOf(string(networkdID))
-	if err != nil {
-		return errors.Wrapf(err, "couldn't load network with id (%s)", networkdID)
-	}
-
-	netRes, err := nr.New(localNR)
-	if err != nil {
-		return errors.Wrap(err, "failed to load network resource")
-	}
-
-	return netRes.Leave(containerID)
-}
-
 // ZDBPrepare sends a macvlan interface into the
 // network namespace of a ZDB container
 func (n networker) ZDBPrepare(id string) (string, error) {
@@ -355,7 +238,7 @@ func (n networker) createMacVlan(iface string, master string, hw net.HardwareAdd
 
 // SetupTap interface in the network resource. We only allow 1 tap interface to be
 // set up per NR currently
-func (n *networker) SetupTap(networkID pkg.NetID, name string) (string, error) {
+func (n *networker) SetupPrivTap(networkID pkg.NetID, name string) (string, error) {
 	log.Info().Str("network-id", string(networkID)).Msg("Setting up tap interface")
 
 	localNR, err := n.networkOf(string(networkID))
@@ -378,6 +261,10 @@ func (n *networker) SetupTap(networkID pkg.NetID, name string) (string, error) {
 		return "", errors.Wrap(err, "could not get network namespace tap device name")
 	}
 
+	if ifaceutil.Exists(tapIface, nil) {
+		return tapIface, nil
+	}
+
 	_, err = tuntap.CreateTap(tapIface, bridgeName)
 
 	return tapIface, err
@@ -431,6 +318,43 @@ func (n *networker) SetupPubTap(pubIPReservationID string) (string, error) {
 	return tapIface, err
 }
 
+// SetupYggTap sets up a tap device in the host namespace for the yggdrasil ip
+func (n *networker) SetupYggTap(name string) (tap pkg.YggdrasilTap, err error) {
+	log.Info().Str("pubip-res-id", string(name)).Msg("Setting up public tap interface")
+
+	tapIface, err := tapName(name)
+	if err != nil {
+		return tap, errors.Wrap(err, "could not get network namespace tap device name")
+	}
+
+	tap.Name = tapIface
+
+	hw := ifaceutil.HardwareAddrFromInputBytes([]byte("ygg:" + name))
+	tap.HW = hw
+	ip, err := n.ygg.SubnetFor(hw)
+	if err != nil {
+		return tap, err
+	}
+
+	tap.IP = net.IPNet{
+		IP:   ip,
+		Mask: net.CIDRMask(64, 128),
+	}
+
+	gw, err := n.ygg.Gateway()
+	if err != nil {
+		return tap, err
+	}
+
+	tap.Gateway = gw
+	if ifaceutil.Exists(tapIface, nil) {
+		return tap, nil
+	}
+
+	_, err = tuntap.CreateTap(tapIface, types.YggBridge)
+	return tap, err
+}
+
 // PubTapExists checks if the tap device for the public network exists already
 func (n *networker) PubTapExists(pubIPReservationID string) (bool, error) {
 	log.Info().Str("pubip-res-id", string(pubIPReservationID)).Msg("Checking if public tap interface exists")
diff --git a/pkg/primitives/vm.go b/pkg/primitives/vm.go
index 04f2094e..9669ae20 100644
--- a/pkg/primitives/vm.go
+++ b/pkg/primitives/vm.go
@@ -1,11 +1,11 @@
 package primitives
 
 import (
-	"bytes"
 	"context"
 	"crypto/md5"
 	"encoding/json"
 	"fmt"
+	"net"
 	"path/filepath"
 
 	"github.com/jbenet/go-base58"
@@ -74,7 +74,7 @@ func (p *Primitives) mountsToDisks(ctx context.Context, deployment gridtypes.Dep
 
 	return results, nil
 }
-func (p *Primitives) virtualMachineProvisionImpl(ctx context.Context, wl *gridtypes.WorkloadWithID) (result KubernetesResult, err error) {
+func (p *Primitives) virtualMachineProvisionImpl(ctx context.Context, wl *gridtypes.WorkloadWithID) (result zos.ZMachineResult, err error) {
 	var (
 		storage = stubs.NewVDiskModuleStub(p.zbus)
 		network = stubs.NewNetworkerStub(p.zbus)
@@ -93,24 +93,12 @@ func (p *Primitives) virtualMachineProvisionImpl(ctx context.Context, wl *gridty
 	}
 	// Should config.Vaid() be called here?
 
-	deployment := provision.GetDeployment(ctx)
-
 	// the config is validated by the engine. we now only support only one
 	// private network
-	netConfig := config.Network.Interfaces[0]
-	netID := zos.NetworkID(deployment.TwinID, netConfig.Network)
-
-	// hash to avoid tapName > 16 errors
-	tapName := hashDeployment(wl.ID)
-
-	exists, err := network.TapExists(ctx, tapName)
-	if err != nil {
-		return result, errors.Wrap(err, "could not check if tap device exists")
-	}
-
-	if exists {
-		return result, errors.New("found a tap device with the same name. shouldn't happen")
+	if len(config.Network.Interfaces) != 1 {
+		return result, fmt.Errorf("only one private network is support")
 	}
+	netConfig := config.Network.Interfaces[0]
 
 	// check if public ipv4 is supported, should this be requested
 	if !config.Network.PublicIP.IsEmpty() && !network.PublicIPv4Support(ctx) {
@@ -120,18 +108,55 @@ func (p *Primitives) virtualMachineProvisionImpl(ctx context.Context, wl *gridty
 	result.ID = wl.ID.String()
 	result.IP = netConfig.IP.String()
 
+	deployment := provision.GetDeployment(ctx)
+
+	networkInfo := pkg.VMNetworkInfo{
+		Nameservers: []net.IP{net.ParseIP("8.8.8.8"), net.ParseIP("1.1.1.1"), net.ParseIP("2001:4860:4860::8888")},
+	}
+
+	defer func() {
+		if err != nil {
+			for _, nic := range networkInfo.Ifaces {
+				if nic.Public {
+					network.DisconnectPubTap(ctx, nic.Tap)
+				} else {
+					network.RemoveTap(ctx, nic.Tap)
+				}
+			}
+		}
+	}()
+
+	for _, nic := range config.Network.Interfaces {
+		inf, err := p.newPrivNetworkInterface(ctx, deployment, wl, nic)
+		if err != nil {
+			return result, err
+		}
+		networkInfo.Ifaces = append(networkInfo.Ifaces, inf)
+	}
+
+	if !config.Network.PublicIP.IsEmpty() {
+		inf, err := p.newPubNetworkInterface(ctx, deployment, config)
+		if err != nil {
+			return result, err
+		}
+		networkInfo.Ifaces = append(networkInfo.Ifaces, inf)
+	}
+
+	if config.Network.Planetary {
+		inf, err := p.newYggNetworkInterface(ctx, wl)
+		if err != nil {
+			return result, err
+		}
+
+		log.Debug().Msgf("Planetary: %+v", inf)
+		networkInfo.Ifaces = append(networkInfo.Ifaces, inf)
+	}
 	// - mount flist RO
 	mnt, err := flist.Mount(ctx, wl.ID.String(), config.FList, pkg.ReadOnlyMountOptions)
 	if err != nil {
 		return result, errors.Wrapf(err, "failed to mount flist: %s", wl.ID.String())
 	}
 
-	// defer func() {
-	// 	if err != nil {
-	// 		flist.Unmount(ctx, wl.ID.String())
-	// 	}
-	// }()
-
 	var imageInfo FListInfo
 	// - detect type (container or VM)
 	imageInfo, err = getFlistInfo(mnt)
@@ -231,44 +256,7 @@ func (p *Primitives) virtualMachineProvisionImpl(ctx context.Context, wl *gridty
 	// - Attach mounts
 	// - boot
 
-	var iface string
-	iface, err = network.SetupTap(ctx, netID, tapName)
-	if err != nil {
-		return result, errors.Wrap(err, "could not set up tap device")
-	}
-
-	defer func() {
-		if err != nil {
-			_ = network.RemoveTap(ctx, tapName)
-		}
-	}()
-
-	var pubIface string
-	if len(config.Network.PublicIP) > -0 {
-		ipWl, err := deployment.Get(config.Network.PublicIP)
-		if err != nil {
-			return zos.KubernetesResult{}, err
-		}
-		name := ipWl.ID.String()
-		pubIface, err = network.SetupPubTap(ctx, name)
-		if err != nil {
-			return result, errors.Wrap(err, "could not set up tap device for public network")
-		}
-
-		defer func() {
-			if err != nil {
-				_ = network.RemovePubTap(ctx, name)
-			}
-		}()
-	}
-
-	var netInfo pkg.VMNetworkInfo
-	netInfo, err = p.buildNetworkInfo(ctx, deployment, iface, pubIface, config)
-	if err != nil {
-		return result, errors.Wrap(err, "could not generate network info")
-	}
-
-	err = p.vmRun(ctx, wl.ID.String(), &config, boot, disks, imageInfo, cmd, netInfo)
+	err = p.vmRun(ctx, wl.ID.String(), &config, boot, disks, imageInfo, cmd, networkInfo)
 	if err != nil {
 		// attempt to delete the vm, should the process still be lingering
 		vm.Delete(ctx, wl.ID.String())
@@ -300,10 +288,12 @@ func (p *Primitives) vmDecomission(ctx context.Context, wl *gridtypes.WorkloadWi
 		log.Error().Err(err).Msg("failed to unmount machine flist")
 	}
 
-	tapName := hashDeployment(wl.ID)
+	for _, inf := range cfg.Network.Interfaces {
+		tapName := tapNameFromName(wl.ID, string(inf.Network))
 
-	if err := network.RemoveTap(ctx, tapName); err != nil {
-		return errors.Wrap(err, "could not clean up tap device")
+		if err := network.RemoveTap(ctx, tapName); err != nil {
+			return errors.Wrap(err, "could not clean up tap device")
+		}
 	}
 
 	if len(cfg.Network.PublicIP) > 0 {
@@ -351,10 +341,12 @@ func (p *Primitives) vmRun(
 	return vm.Run(ctx, kubevm)
 }
 
-func hashDeployment(wid gridtypes.WorkloadID) string {
-	buf := bytes.Buffer{}
-	buf.WriteString(fmt.Sprint(wid))
-	h := md5.Sum(buf.Bytes())
+func tapNameFromName(id gridtypes.WorkloadID, network string) string {
+	m := md5.New()
+
+	fmt.Fprintf(m, "%s:%s", id.String(), network)
+
+	h := m.Sum(nil)
 	b := base58.Encode(h[:])
 	if len(b) > 13 {
 		b = b[:13]
diff --git a/pkg/primitives/vm_utils.go b/pkg/primitives/vm_utils.go
index 49335468..0a63b390 100644
--- a/pkg/primitives/vm_utils.go
+++ b/pkg/primitives/vm_utils.go
@@ -16,91 +16,126 @@ import (
 	"github.com/threefoldtech/zos/pkg/stubs"
 )
 
-// Kubernetes type
-type Kubernetes = zos.Kubernetes
+func (p *Primitives) newYggNetworkInterface(ctx context.Context, wl *gridtypes.WorkloadWithID) (pkg.VMIface, error) {
+	network := stubs.NewNetworkerStub(p.zbus)
+
+	//TODO: if we use `ygg` as a network name. this will conflict
+	//if the user has a network that is called `ygg`.
+	tapName := tapNameFromName(wl.ID, "ygg")
+	iface, err := network.SetupYggTap(ctx, tapName)
+	if err != nil {
+		return pkg.VMIface{}, errors.Wrap(err, "could not set up tap device")
+	}
 
-// KubernetesResult type
-type KubernetesResult = zos.KubernetesResult
+	out := pkg.VMIface{
+		Tap: iface.Name,
+		MAC: iface.HW.String(),
+		IPs: []net.IPNet{
+			iface.IP,
+		},
+		Routes: []pkg.Route{
+			{
+				Net: net.IPNet{
+					IP:   net.ParseIP("200::"),
+					Mask: net.CIDRMask(7, 128),
+				},
+				Gateway: iface.Gateway.IP,
+			},
+		},
+		Public: false,
+	}
+
+	return out, nil
+}
 
-func (p *Primitives) buildNetworkInfo(ctx context.Context, deployment gridtypes.Deployment, iface string, pubIface string, cfg ZMachine) (pkg.VMNetworkInfo, error) {
+func (p *Primitives) newPrivNetworkInterface(ctx context.Context, dl gridtypes.Deployment, wl *gridtypes.WorkloadWithID, inf zos.MachineInterface) (pkg.VMIface, error) {
 	network := stubs.NewNetworkerStub(p.zbus)
-	netConfig := cfg.Network.Interfaces[0]
-	netID := zos.NetworkID(deployment.TwinID, netConfig.Network)
+	netID := zos.NetworkID(dl.TwinID, inf.Network)
+
 	subnet, err := network.GetSubnet(ctx, netID)
 	if err != nil {
-		return pkg.VMNetworkInfo{}, errors.Wrapf(err, "could not get network resource subnet")
+		return pkg.VMIface{}, errors.Wrapf(err, "could not get network resource subnet")
 	}
 
-	if !subnet.Contains(netConfig.IP) {
-		return pkg.VMNetworkInfo{}, fmt.Errorf("IP %s is not part of local nr subnet %s", netConfig.IP.String(), subnet.String())
+	if !subnet.Contains(inf.IP) {
+		return pkg.VMIface{}, fmt.Errorf("IP %s is not part of local nr subnet %s", inf.IP.String(), subnet.String())
 	}
 
 	privNet, err := network.GetNet(ctx, netID)
 	if err != nil {
-		return pkg.VMNetworkInfo{}, errors.Wrapf(err, "could not get network range")
+		return pkg.VMIface{}, errors.Wrapf(err, "could not get network range")
 	}
 
 	addrCIDR := net.IPNet{
-		IP:   netConfig.IP,
+		IP:   inf.IP,
 		Mask: subnet.Mask,
 	}
 
 	gw4, gw6, err := network.GetDefaultGwIP(ctx, netID)
 	if err != nil {
-		return pkg.VMNetworkInfo{}, errors.Wrap(err, "could not get network resource default gateway")
+		return pkg.VMIface{}, errors.Wrap(err, "could not get network resource default gateway")
+	}
+
+	privIP6, err := network.GetIPv6From4(ctx, netID, inf.IP)
+	if err != nil {
+		return pkg.VMIface{}, errors.Wrap(err, "could not convert private ipv4 to ipv6")
 	}
 
-	privIP6, err := network.GetIPv6From4(ctx, netID, netConfig.IP)
+	tapName := tapNameFromName(wl.ID, string(inf.Network))
+	iface, err := network.SetupPrivTap(ctx, netID, tapName)
 	if err != nil {
-		return pkg.VMNetworkInfo{}, errors.Wrap(err, "could not convert private ipv4 to ipv6")
-	}
-
-	networkInfo := pkg.VMNetworkInfo{
-		Ifaces: []pkg.VMIface{{
-			Tap:            iface,
-			MAC:            "", // rely on static IP configuration so we don't care here
-			IP4AddressCIDR: addrCIDR,
-			IP4GatewayIP:   net.IP(gw4),
-			IP4Net:         privNet,
-			IP6AddressCIDR: privIP6,
-			IP6GatewayIP:   gw6,
-			Public:         false,
-		}},
-		Nameservers: []net.IP{net.ParseIP("8.8.8.8"), net.ParseIP("1.1.1.1"), net.ParseIP("2001:4860:4860::8888")},
-	}
-
-	pubIP := cfg.Network.PublicIP
-	if len(pubIP) > 0 {
-		// A public ip is set, load the reservation, extract the ip and make a config
-		// for it
-		ipWl, err := deployment.Get(pubIP)
-		if err != nil {
-			return pkg.VMNetworkInfo{}, err
-		}
-
-		pubIP, pubGw, err := p.getPubIPConfig(ipWl)
-		if err != nil {
-			return pkg.VMNetworkInfo{}, errors.Wrap(err, "could not get public ip config")
-		}
-
-		// the mac address uses the global workload id
-		// this needs to be the same as how we get it in the actual IP reservation
-		mac := ifaceutil.HardwareAddrFromInputBytes([]byte(ipWl.ID.String()))
-
-		iface := pkg.VMIface{
-			Tap:            pubIface,
-			MAC:            mac.String(), // mac so we always get the same IPv6 from slaac
-			IP4AddressCIDR: pubIP,
-			IP4GatewayIP:   pubGw,
-			// for now we get ipv6 from slaac, so leave ipv6 stuffs this empty
-			//
-			Public: true,
-		}
-
-		networkInfo.Ifaces = append(networkInfo.Ifaces, iface)
-	}
-
-	return networkInfo, nil
+		return pkg.VMIface{}, errors.Wrap(err, "could not set up tap device")
+	}
+
+	out := pkg.VMIface{
+		Tap: iface,
+		MAC: "", // rely on static IP configuration so we don't care here
+		IPs: []net.IPNet{
+			addrCIDR, privIP6,
+		},
+		Routes: []pkg.Route{
+			{Net: privNet, Gateway: gw4},
+		},
+		IP4DefaultGateway: net.IP(gw4),
+		IP6DefaultGateway: gw6,
+		Public:            false,
+	}
+
+	return out, nil
+}
+
+func (p *Primitives) newPubNetworkInterface(ctx context.Context, deployment gridtypes.Deployment, cfg ZMachine) (pkg.VMIface, error) {
+	network := stubs.NewNetworkerStub(p.zbus)
+	ipWl, err := deployment.Get(cfg.Network.PublicIP)
+	if err != nil {
+		return pkg.VMIface{}, err
+	}
+	name := ipWl.ID.String()
+
+	pubIP, pubGw, err := p.getPubIPConfig(ipWl)
+	if err != nil {
+		return pkg.VMIface{}, errors.Wrap(err, "could not get public ip config")
+	}
+
+	pubIface, err := network.SetupPubTap(ctx, name)
+	if err != nil {
+		return pkg.VMIface{}, errors.Wrap(err, "could not set up tap device for public network")
+	}
+
+	// the mac address uses the global workload id
+	// this needs to be the same as how we get it in the actual IP reservation
+	mac := ifaceutil.HardwareAddrFromInputBytes([]byte(ipWl.ID.String()))
+
+	return pkg.VMIface{
+		Tap: pubIface,
+		MAC: mac.String(), // mac so we always get the same IPv6 from slaac
+		IPs: []net.IPNet{
+			pubIP,
+		},
+		IP4DefaultGateway: pubGw,
+		// for now we get ipv6 from slaac, so leave ipv6 stuffs this empty
+		Public: true,
+	}, nil
 }
 
 // Get the public ip, and the gateway from the reservation ID
diff --git a/pkg/provision/mbus/reservation.go b/pkg/provision/mbus/reservation.go
index bffb5801..d9ff671d 100644
--- a/pkg/provision/mbus/reservation.go
+++ b/pkg/provision/mbus/reservation.go
@@ -36,7 +36,7 @@ func (d *Deployments) createOrUpdate(ctx context.Context, payload []byte, update
 	defer cancel()
 
 	action := d.engine.Provision
-	if !update {
+	if update {
 		action = d.engine.Update
 	}
 
diff --git a/pkg/stubs/network_stub.go b/pkg/stubs/network_stub.go
index ba7fe71d..e8a0779e 100644
--- a/pkg/stubs/network_stub.go
+++ b/pkg/stubs/network_stub.go
@@ -201,35 +201,6 @@ func (s *NetworkerStub) GetSubnet(ctx context.Context, arg0 zos.NetID) (ret0 net
 	return
 }
 
-func (s *NetworkerStub) Join(ctx context.Context, arg0 zos.NetID, arg1 string, arg2 pkg.ContainerNetworkConfig) (ret0 pkg.Member, ret1 error) {
-	args := []interface{}{arg0, arg1, arg2}
-	result, err := s.client.RequestContext(ctx, s.module, s.object, "Join", args...)
-	if err != nil {
-		panic(err)
-	}
-	if err := result.Unmarshal(0, &ret0); err != nil {
-		panic(err)
-	}
-	ret1 = new(zbus.RemoteError)
-	if err := result.Unmarshal(1, &ret1); err != nil {
-		panic(err)
-	}
-	return
-}
-
-func (s *NetworkerStub) Leave(ctx context.Context, arg0 zos.NetID, arg1 string) (ret0 error) {
-	args := []interface{}{arg0, arg1}
-	result, err := s.client.RequestContext(ctx, s.module, s.object, "Leave", args...)
-	if err != nil {
-		panic(err)
-	}
-	ret0 = new(zbus.RemoteError)
-	if err := result.Unmarshal(0, &ret0); err != nil {
-		panic(err)
-	}
-	return
-}
-
 func (s *NetworkerStub) PubIPFilterExists(ctx context.Context, arg0 string) (ret0 bool) {
 	args := []interface{}{arg0}
 	result, err := s.client.RequestContext(ctx, s.module, s.object, "PubIPFilterExists", args...)
@@ -354,6 +325,22 @@ func (s *NetworkerStub) SetPublicConfig(ctx context.Context, arg0 pkg.PublicConf
 	return
 }
 
+func (s *NetworkerStub) SetupPrivTap(ctx context.Context, arg0 zos.NetID, arg1 string) (ret0 string, ret1 error) {
+	args := []interface{}{arg0, arg1}
+	result, err := s.client.RequestContext(ctx, s.module, s.object, "SetupPrivTap", args...)
+	if err != nil {
+		panic(err)
+	}
+	if err := result.Unmarshal(0, &ret0); err != nil {
+		panic(err)
+	}
+	ret1 = new(zbus.RemoteError)
+	if err := result.Unmarshal(1, &ret1); err != nil {
+		panic(err)
+	}
+	return
+}
+
 func (s *NetworkerStub) SetupPubIPFilter(ctx context.Context, arg0 string, arg1 string, arg2 string, arg3 string, arg4 string) (ret0 error) {
 	args := []interface{}{arg0, arg1, arg2, arg3, arg4}
 	result, err := s.client.RequestContext(ctx, s.module, s.object, "SetupPubIPFilter", args...)
@@ -383,9 +370,9 @@ func (s *NetworkerStub) SetupPubTap(ctx context.Context, arg0 string) (ret0 stri
 	return
 }
 
-func (s *NetworkerStub) SetupTap(ctx context.Context, arg0 zos.NetID, arg1 string) (ret0 string, ret1 error) {
-	args := []interface{}{arg0, arg1}
-	result, err := s.client.RequestContext(ctx, s.module, s.object, "SetupTap", args...)
+func (s *NetworkerStub) SetupYggTap(ctx context.Context, arg0 string) (ret0 pkg.YggdrasilTap, ret1 error) {
+	args := []interface{}{arg0}
+	result, err := s.client.RequestContext(ctx, s.module, s.object, "SetupYggTap", args...)
 	if err != nil {
 		panic(err)
 	}
diff --git a/pkg/vm.go b/pkg/vm.go
index 4ab59101..a7a27328 100644
--- a/pkg/vm.go
+++ b/pkg/vm.go
@@ -11,24 +11,28 @@ import (
 
 //go:generate zbusc -module vmd -version 0.0.1 -name manager -package stubs github.com/threefoldtech/zos/pkg+VMModule stubs/vmd_stub.go
 
+// Route structure
+type Route struct {
+	Net net.IPNet
+	// Gateway can be nil, in that
+	// case the device is used as a dev instead
+	Gateway net.IP
+}
+
 // VMIface structure
 type VMIface struct {
 	// Tap device name
 	Tap string
 	// Mac address of the device
 	MAC string
-	// Address of the device in the form of cidr for ipv4
-	IP4AddressCIDR net.IPNet
-	// Gateway address for ipv4
-	IP4GatewayIP net.IP
-	// Full subnet for the IP4 resource. This allows configuration of networking for
-	// non local subnets (i.e. NR on other nodes).
-	// Does not need to be set for public ifaces
-	IP4Net net.IPNet
-	// Address of the device in the form of cidr for ipv6
-	IP6AddressCIDR net.IPNet
-	// Gateway address for ipv6
-	IP6GatewayIP net.IP
+	// ips assigned to this interface
+	IPs []net.IPNet
+	// extra routes on this interface
+	Routes []Route
+	// IP4DefaultGateway address for ipv4
+	IP4DefaultGateway net.IP
+	// IP6DefaultGateway address for ipv6
+	IP6DefaultGateway net.IP
 	// Private or public network
 	Public bool
 }
@@ -70,6 +74,7 @@ type Boot struct {
 // KernelArgs are arguments passed to the kernel
 type KernelArgs map[string]string
 
+// String builds commandline string
 func (s KernelArgs) String() string {
 	var buf bytes.Buffer
 	for k, v := range s {
@@ -105,6 +110,13 @@ func (s KernelArgs) String() string {
 	return buf.String()
 }
 
+// Extend the arguments with set of extra arguments
+func (s KernelArgs) Extend(k KernelArgs) {
+	for a, b := range k {
+		s[a] = b
+	}
+}
+
 // VM config structure
 type VM struct {
 	// virtual machine name, or ID
diff --git a/pkg/vm/manager.go b/pkg/vm/manager.go
index fcbabe35..5f1bb07b 100644
--- a/pkg/vm/manager.go
+++ b/pkg/vm/manager.go
@@ -1,6 +1,7 @@
 package vm
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io/ioutil"
@@ -26,10 +27,17 @@ const (
 	socketDir = "/var/run/cloud-hypervisor"
 	configDir = "config"
 	logsDir   = "logs"
-
-	defaultKernelArgs = "ro console=ttyS0 noapic reboot=k panic=1 pci=off nomodules"
 )
 
+var (
+	//defaultKernelArgs if no args are set
+	defaultKernelArgs = pkg.KernelArgs{
+		"rw":      "",
+		"console": "ttyS0",
+		"reboot":  "k",
+		"panic":   "1",
+	}
+)
 var (
 	protectedKernelEnv = map[string]struct{}{
 		"init":       {},
@@ -126,7 +134,25 @@ func (m *Module) Exists(id string) bool {
 	return err == nil
 }
 
-func (m *Module) makeNetwork(vm *pkg.VM) ([]Interface, string, error) {
+func (m *Module) buildRouteParam(defaultGw net.IP, table map[string]string) string {
+	var buf bytes.Buffer
+	if defaultGw != nil {
+		buf.WriteString(fmt.Sprintf("default,%s", defaultGw.String()))
+	}
+
+	for k, v := range table {
+		if buf.Len() > 0 {
+			buf.WriteRune(';')
+		}
+		buf.WriteString(k)
+		buf.WriteRune(',')
+		buf.WriteString(v)
+	}
+
+	return buf.String()
+}
+
+func (m *Module) makeNetwork(vm *pkg.VM) ([]Interface, pkg.KernelArgs, error) {
 	// assume there is always at least 1 iface present
 
 	// we do 2 things here:
@@ -138,53 +164,65 @@ func (m *Module) makeNetwork(vm *pkg.VM) ([]Interface, string, error) {
 	// method uses a custom script inside the image to set proper IP. The config
 	// is also passed through the command line.
 
+	args := pkg.KernelArgs{}
+	v4Routes := make(map[string]string)
+	v6Routes := make(map[string]string)
+	var defaultGw4 net.IP
+	var defaultGw6 net.IP
+
 	nics := make([]Interface, 0, len(vm.Network.Ifaces))
 	for i, ifcfg := range vm.Network.Ifaces {
-		nics = append(nics, Interface{
+		nic := Interface{
 			ID:  fmt.Sprintf("eth%d", i),
 			Tap: ifcfg.Tap,
 			Mac: ifcfg.MAC,
-		})
-	}
-
-	cmdLineSections := make([]string, 0, len(vm.Network.Ifaces)+1)
-	for i, ifcfg := range vm.Network.Ifaces {
-		cmdLineSections = append(cmdLineSections, m.makeNetCmdLine(i, ifcfg))
-	}
-	dnsSection := make([]string, 0, len(vm.Network.Nameservers))
-	for _, ns := range vm.Network.Nameservers {
-		dnsSection = append(dnsSection, ns.String())
-	}
-	cmdLineSections = append(cmdLineSections, fmt.Sprintf("net_dns=%s", strings.Join(dnsSection, ",")))
+		}
+		nics = append(nics, nic)
 
-	cmdline := strings.Join(cmdLineSections, " ")
+		var ips []string
+		for _, ip := range ifcfg.IPs {
+			ips = append(ips, ip.String())
+		}
+		// configure nic ips
+		args[fmt.Sprintf("net_%s", nic.ID)] = strings.Join(ips, ";")
+		// configure nic routes
+		if defaultGw4 == nil && ifcfg.IP4DefaultGateway != nil {
+			defaultGw4 = ifcfg.IP4DefaultGateway
+		}
 
-	return nics, cmdline, nil
-}
+		if defaultGw6 == nil && ifcfg.IP6DefaultGateway != nil {
+			defaultGw6 = ifcfg.IP6DefaultGateway
+		}
+		// one extra check to always use public nic as default
+		// gw
+		if ifcfg.Public && ifcfg.IP4DefaultGateway != nil {
+			defaultGw4 = ifcfg.IP4DefaultGateway
+		}
 
-func (m *Module) makeNetCmdLine(idx int, ifcfg pkg.VMIface) string {
-	// net_%ifacename=%ip4_cidr,$ip4_gw[,$ip4_route],$ipv6_cidr,$ipv6_gw,public|priv
-	ip4Elems := make([]string, 0, 3)
-	ip4Elems = append(ip4Elems, ifcfg.IP4AddressCIDR.String())
-	ip4Elems = append(ip4Elems, ifcfg.IP4GatewayIP.String())
-	if len(ifcfg.IP4Net.IP) > 0 {
-		ip4Elems = append(ip4Elems, ifcfg.IP4Net.String())
+		// inserting extra routes in right places
+		for _, route := range ifcfg.Routes {
+			table := v4Routes
+			if route.Net.IP.To4() == nil {
+				table = v6Routes
+			}
+			gw := nic.ID
+			if route.Gateway != nil {
+				gw = route.Gateway.String()
+			}
+			table[route.Net.String()] = gw
+		}
 	}
 
-	ip6Elems := make([]string, 0, 3)
-	if ifcfg.IP6AddressCIDR.IP.To16() != nil {
-		ip6Elems = append(ip6Elems, ifcfg.IP6AddressCIDR.String())
-		ip6Elems = append(ip6Elems, ifcfg.IP6GatewayIP.String())
-	} else {
-		ip6Elems = append(ip6Elems, "slaac")
-	}
+	args["net_r4"] = m.buildRouteParam(defaultGw4, v4Routes)
+	args["net_r6"] = m.buildRouteParam(defaultGw6, v6Routes)
 
-	privPub := "priv"
-	if ifcfg.Public {
-		privPub = "public"
+	dnsSection := make([]string, 0, len(vm.Network.Nameservers))
+	for _, ns := range vm.Network.Nameservers {
+		dnsSection = append(dnsSection, ns.String())
 	}
+	args["net_dns"] = strings.Join(dnsSection, ";")
 
-	return fmt.Sprintf("net_eth%d=%s,%s,%s", idx, strings.Join(ip4Elems, ","), strings.Join(ip6Elems, ","), privPub)
+	return nics, args, nil
 }
 
 func (m *Module) tail(path string) (string, error) {
@@ -286,6 +324,7 @@ func (m *Module) Run(vm pkg.VM) error {
 	cmdline := vm.KernelArgs
 	if cmdline == nil {
 		cmdline = pkg.KernelArgs{}
+		cmdline.Extend(defaultKernelArgs)
 	}
 
 	var fs []VirtioFS
@@ -330,25 +369,14 @@ func (m *Module) Run(vm pkg.VM) error {
 		return err
 	}
 
-	var kargs strings.Builder
-	kargs.WriteString(args)
-
-	if kargs.Len() != 0 {
-		kargs.WriteRune(' ')
-	}
-
-	if len(cmdline) > 0 {
-		kargs.WriteString(cmdline.String())
-	} else {
-		kargs.WriteString(defaultKernelArgs)
-	}
+	cmdline.Extend(args)
 
 	machine := Machine{
 		ID: vm.Name,
 		Boot: Boot{
 			Kernel: vm.KernelImage,
 			Initrd: vm.InitrdImage,
-			Args:   kargs.String(),
+			Args:   cmdline.String(),
 		},
 		Config: Config{
 			CPU:       CPU(vm.CPU),


Test patch:
diff --git a/pkg/flist/flist_test.go b/pkg/flist/flist_test.go
index f44ca754..0f7e7152 100644
--- a/pkg/flist/flist_test.go
+++ b/pkg/flist/flist_test.go
@@ -169,7 +169,7 @@ func TestMountUnmount(t *testing.T) {
 	os.Remove(cmder.m["pid"])
 	strg.On("ReleaseFilesystem", mock.Anything, filepath.Base(mnt)).Return(nil)
 
-	sys.On("Unmount", mnt, syscall.MNT_DETACH).Return(nil)
+	sys.On("Unmount", mnt, 0).Return(nil)
 
 	err = flister.Unmount(name)
 	require.NoError(t, err)
@@ -203,7 +203,7 @@ func TestMountUnmountRO(t *testing.T) {
 	os.Remove(cmder.m["pid"])
 	strg.On("ReleaseFilesystem", mock.Anything, filepath.Base(mnt)).Return(nil)
 
-	sys.On("Unmount", mnt, syscall.MNT_DETACH).Return(nil)
+	sys.On("Unmount", mnt, 0).Return(nil)
 
 	err = flister.Unmount(name)
 	require.NoError(t, err)