Repository: threefoldtech/zos (go)

Issue:
used wireguard port are not properly reported on testnet
```python
for n in nodes: 
   2     print(n.node_id, n.wg_ports)                                                                                                                                                                     
2fi9ZZiBGW4G9pnrN656bMfW6x55RSoHDeMrd9pgSA8T []
72CP8QPhMSpF7MbSvNR1TYZFbTnbRiuyvq5xwcoRNAib []
D1sY7tgHX6fVvjrpCotr4SmezVyRShLTufqhEpBQxK1v []
6m9gFPTYLRiCH7FD8yU5VWwXZdfegKUvXZgrJoS1EJfQ []
2m3nHPSAMyZFSeg5HPozic2NGBMtrXrBkhtNcVmd5Ss6 []
3dAnxcykEDgKVQdTRKmktggL2MZbm3CPSdS9Tdoy4HAF []
3BwDVNRigau8Zk6BQkJt5PLxrHQnJ5NTfNvAtuubC1eS []
A7FmQZ72h7FzjkJMGXmzLDFyfyxzitDZYuernGG97nv7 []
7fHSAHEvUGtUcYSqLtpGq8ANssPikTyyHC52FddDYF4Y []
FjwyHVvfATkVb4Puh4x6jCMS79TVVgSYagAuZTxWrsbj []
HT9yNvHRQs65dyRJeztAdEnXKweCX2bhgvD9i1WFPUNs - 1,000

9211BFV7MFwktD2b8jHE9Ub3fHRtaYQyBBfwT9kEKA7q []
HugtVL51BFNLbZbbxnWu2GEe8hV97YVPac19zy5wwNpT []
9KAbX21NGbZYupBJ6EeeWx3ZTKDx7ADevr8qtmEa5WkC []
9kcLeTuseybGHGWw2YXvdu4kk2jZzyZCaCHV9t6Axqqx []
3h4TKp11bNWjb2UemgrVwayuPnYcs2M1bccXvi3jPR2Y []
FUq4Sz7CdafZYV2qJmTe3Rs4U4fxtJFcnV6mPNgGbmRg []
5Pb5NMBQWLTWhXK2cCM8nS6JZrnP2HaTP452TfMMYT9p []
DUF2knurdMuX2eJVp9o7tXq4eNBy2fbxBoWhrMXWPEtF []
8zdqjFD7GLsSSfsTgFYcGusw91gQ3tdx7jbUhJep2a5X []
6chi1iSczxfF4U2iyCcJwkwWnwzcDgQHzCRExK9r4V1j []
AT4hKXnFWPNsnE38sUoERDVzxL9mNysqDWQSdHkum7YD - 8,192

J1Xb2piba1vZM2pSmWP24CPgREnoTce7EuUJysttCyz6 - 8,193
- 8,194
- 1,000

9161hj9VnM41HVvTr7nHQKcgFBx8Yf6AjWduMCrBXQip - 8,193
- 8,194
- 1,000

33zKcb6YBSuYDVrTsm6AjBuTpujP4na7kL6ZrShcVKbF - 8,194
- 1,000
- 8,193

FLeByELHK22evRYqJokHsCMCF9wQpLtuyehUp1LgR5eM []
49RnnxyRJ7r2So6UMBV2UDyLtoyRjmAf8NEnDnzpMtmY []
ByTfXHH7RogHNLpTpBWbn34ofC2tfCBVhFKJxYnMBoNi []
A34YUGenHKyhjDMAUKZe4cVDtJM2wQ4n4XRkfGUUEYdy - 1,000
- 8,192

HYwvrxCy5z1QPALC5gTAiAMYWLH7orQ8xWEDbEyLiJV7 - 8,192
- 1,000

CBDY1Fu4CuxGpdU3zLL9QT5DGaRkxjpuJmzV6V5CBWg4 - 8,192
- 1,000

Hb6oVe2B5v9UBzDcDeQfZGn5bwFeM2R3rJh6U93AWfiN - 8,192
- 1,000

54S1qFXxWgnjmvEFVvqUbR7dHhvCshLbQrp2UpmE7GhZ - 8,192
- 1,000

9WhwSbM2xBNb9E3ws3PNJfyeajnKXWDZAMBLZMXCA9jf - 8,192
- 1,000

2hgRioV9ZKe8Apnm84TZn8Bn5XczyPU2nkPmozUY4rYw - 8,192
- 1,000

6gBWuYT8MTCJiRJp3mLiPLSXDJpDCGrLyi4HpVissb5j - 8,192
- 1,000

59R1GU6qh4RiWQRvUFm4GzwBbTM2JStyTFmgpzruq9Gt []
292NEoCJ98FfhB62qZXXKqBQyLUij54roeH5VwBuXsSp []
HARGRBPyxF315bXDBCdaTnAD9cqMpwxvQnnzkZjhpVCA []
Ddstj2hPbGip8Ci26TUaEzY52DHZs3JphLFXy9UgWSRS []
FZZxePXsMfWpoepiPtdmc7EjRs3i8cmFF8xXJsFHKFB4 []
Aiux6s3V8wg398FHzWWy3LC81C8vuP4JLkck7uR9T3pD []
D9pVi21QMozECuZ6inVKQZP5UFrtusKUhW5mfW5fRxbK []
35A85gq6FRBFjMfdHDE6d7j5BYr786h4afy1wNcmdzre []
4TksThCbMYo8THcsADEckaCwurHSbn6EvDapfXvga97r []
JE3GQ7LhStnoTR5mSfJZE1gcLhTr2oMiURVH1C1ipsgo []
7Qa7fxRPtMc5R72cnNms4XWD9PJkeQJ6iD8iTU9T8p2o []
9if6GSLuz1awA9EAywgYWDdgyH7Aq79X68kv9adJNAFL []
```
This is an output of all the wireguard port marked as used for all the nodes from testnet.
While when I check the amount of network resource deployed on some nodes there should be more wireguard port marked as used.

This also make new provision of network fails cause user doesn't know which port is used or not. Which make there reservation failed with 
```
[+] provisiond: 2020-03-02T14:10:15Z error failed to apply provision error="failed to create network resource for network U7Xfof9T9CHuL: failed to configure network resource: failed to configure wireguard interface: failed to bring wireguard interface w-U7Xfof9T9CHuL up: address already in use" id=1159-1```
cause the port picked but the user is actually already used

Pull request description:
networkd: make reserved wireguard port persistent between network restart
fixes #556

Before this commit networkd would loose all the reserved port after
a restart and thus publish a wrong list of reserved ports on BCDB.

We now keep this list of reserved port on cache disk so it survive
a restart.

We still do clean up after a reboot since all the network resources are
also going to be re-created so the list of reserved port will be rebuild
from scratch

Patch:
diff --git a/cmds/networkd/main.go b/cmds/networkd/main.go
index fe49d60b..bfdf147c 100644
--- a/cmds/networkd/main.go
+++ b/cmds/networkd/main.go
@@ -55,17 +55,12 @@ func main() {
 		log.Fatal().Err(err).Msg("failed to connect to zbus broker")
 	}
 
-	if err := os.MkdirAll(root, 0750); err != nil {
-		log.Fatal().Err(err).Msgf("fail to create module root")
-	}
-
 	db, err := bcdbClient()
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to connect to BCDB")
 	}
 
 	identity := stubs.NewIdentityManagerStub(client)
-	networker := network.NewNetworker(identity, db, root)
 	nodeID := identity.NodeID()
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -133,6 +128,15 @@ func main() {
 	go startAddrWatch(ctx, nodeID, db, ifaces)
 
 	log.Info().Msg("start zbus server")
+	if err := os.MkdirAll(root, 0750); err != nil {
+		log.Fatal().Err(err).Msgf("fail to create module root")
+	}
+
+	networker, err := network.NewNetworker(identity, db, root)
+	if err != nil {
+		log.Fatal().Err(err).Msg("error creating network manager")
+	}
+
 	if err := startServer(ctx, broker, networker); err != nil {
 		log.Fatal().Err(err).Msg("unexpected error")
 	}
diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go
new file mode 100644
index 00000000..97e40ed3
--- /dev/null
+++ b/pkg/cache/cache.go
@@ -0,0 +1,36 @@
+package cache
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/threefoldtech/zos/pkg/storage"
+)
+
+// ForeverDir creates a new cache directory that is stored on the persistent cache.
+// This means data stored in this directory will survive reboot
+// It is the caller's responsibility to remove the directory when no longer needed.
+func ForeverDir(name string) (string, error) {
+	name = filepath.Join(storage.CacheTarget, "cache", name)
+	return name, os.MkdirAll(name, 0700)
+}
+
+// VolatileDir creates a new cache directory that is stored on a tmpfs.
+// This means data stored in this directory will NOT survive a reboot.
+// Use this when you need to store data that needs to survice deamon reboot but not between reboots
+// It is the caller's responsibility to remove the directory when no longer needed.
+// If the directory already exist, the the function does nothing and return successfully
+func VolatileDir(name string, size uint64) (string, error) {
+	const volatileBaseDir = "/var/run/cache"
+	name = filepath.Join(volatileBaseDir, name)
+	if err := os.MkdirAll(volatileBaseDir, 0700); err != nil {
+		return name, err
+	}
+
+	if err := os.Mkdir(name, 0700); err != nil {
+		return "", err
+	}
+	return name, syscall.Mount("", name, "tmpfs", 0, fmt.Sprintf("size=%d", size))
+}
diff --git a/pkg/gedis/converter.go b/pkg/gedis/converter.go
index d276aa06..85d793d4 100644
--- a/pkg/gedis/converter.go
+++ b/pkg/gedis/converter.go
@@ -103,7 +103,7 @@ func networkReservation(i interface{}) types.TfgridReservationNetwork1 {
 
 		for y, peer := range nr.Peers {
 			network.NetworkResources[i].Peers[y] = types.WireguardPeer1{
-				IPRange:    nr.Subnet.ToSchema(),
+				IPRange:    peer.Subnet.ToSchema(),
 				Endpoint:   peer.Endpoint,
 				PublicKey:  peer.WGPublicKey,
 				AllowedIPs: make([]string, len(peer.AllowedIPs)),
diff --git a/pkg/network/ndmz/ipam.go b/pkg/network/ndmz/ipam.go
index cce933a4..4f8692a1 100644
--- a/pkg/network/ndmz/ipam.go
+++ b/pkg/network/ndmz/ipam.go
@@ -10,9 +10,8 @@ import (
 
 // allocateIPv4 allocates a unique IPv4 for the entity defines by the given id (for example container id, or a vm).
 // in the network with netID, and NetResource.
-func allocateIPv4(networkID string) (*net.IPNet, error) {
-	// FIXME: path to the cache disk shouldn't be hardcoded here
-	store, err := disk.New("ndmz", ipamPath)
+func allocateIPv4(networkID, leaseDir string) (*net.IPNet, error) {
+	store, err := disk.New("ndmz", leaseDir)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/network/ndmz/ndmz.go b/pkg/network/ndmz/ndmz.go
index 31b9b93a..c9bbdef3 100644
--- a/pkg/network/ndmz/ndmz.go
+++ b/pkg/network/ndmz/ndmz.go
@@ -5,13 +5,10 @@ import (
 	"fmt"
 	"net"
 	"os"
-	"path/filepath"
 	"time"
 
 	"github.com/cenkalti/backoff/v3"
-	"github.com/termie/go-shutil"
 	"github.com/threefoldtech/zos/pkg"
-	"github.com/threefoldtech/zos/pkg/app"
 	"github.com/threefoldtech/zos/pkg/network/ifaceutil"
 	"github.com/threefoldtech/zos/pkg/network/types"
 
@@ -51,47 +48,8 @@ const (
 	nrPubIface = "public"
 )
 
-var ipamPath = "/var/cache/modules/networkd/lease"
-
 //Create create the NDMZ network namespace and configure its default routes and addresses
 func Create(nodeID pkg.Identifier) error {
-	path := filepath.Join(ipamPath, "ndmz")
-
-	if app.IsFirstBoot("networkd-dmz") {
-		log.Info().Msg("first boot, empty reservation cache")
-
-		if err := os.RemoveAll(path); err != nil {
-			return err
-		}
-
-		// TODO @zaibon: remove once all the network has applies this fix
-
-		// This check ensure we do not delete current lease from the node
-		// running a previous version of this code.
-		// if the old leases directory exists, we copy the content
-		// into the new location and delete the old directory
-		var err error
-		oldFolder := filepath.Join(ipamPath, "dmz")
-		if _, err = os.Stat(oldFolder); err == nil {
-			if err := shutil.CopyTree(oldFolder, path, nil); err != nil {
-				return err
-			}
-			err = os.RemoveAll(oldFolder)
-		} else {
-			err = os.MkdirAll(path, 0770)
-		}
-		if err != nil {
-			return err
-		}
-
-		if err := app.MarkBooted("networkd-dmz"); err != nil {
-			return errors.Wrap(err, "fail to mark provisiond as booted")
-		}
-
-	} else {
-		log.Info().Msg("restart detected, keep IPAM lease cache intact")
-	}
-
 	netNS, err := namespace.GetByName(NetNSNDMZ)
 	if err != nil {
 		netNS, err = namespace.Create(NetNSNDMZ)
@@ -374,7 +332,7 @@ func applyFirewall() error {
 }
 
 // AttachNR links a network resource to the NDMZ
-func AttachNR(networkID string, nr *nr.NetResource) error {
+func AttachNR(networkID string, nr *nr.NetResource, ipamLeaseDir string) error {
 	nrNSName, err := nr.Namespace()
 	if err != nil {
 		return err
@@ -392,7 +350,7 @@ func AttachNR(networkID string, nr *nr.NetResource) error {
 	}
 
 	return nrNS.Do(func(_ ns.NetNS) error {
-		addr, err := allocateIPv4(networkID)
+		addr, err := allocateIPv4(networkID, ipamLeaseDir)
 		if err != nil {
 			return errors.Wrap(err, "ip allocation for network resource")
 		}
diff --git a/pkg/network/networker.go b/pkg/network/networker.go
index 29e9a7b5..f33ec23b 100644
--- a/pkg/network/networker.go
+++ b/pkg/network/networker.go
@@ -5,11 +5,15 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"os"
 	"path/filepath"
 	"time"
 
+	"github.com/termie/go-shutil"
+
+	"github.com/threefoldtech/zos/pkg/cache"
 	"github.com/threefoldtech/zos/pkg/network/ndmz"
 	"github.com/threefoldtech/zos/pkg/network/tuntap"
 
@@ -35,26 +39,102 @@ import (
 
 const (
 	// ZDBIface is the name of the interface used in the 0-db network namespace
-	ZDBIface = "zdb0"
+	ZDBIface     = "zdb0"
+	wgPortDir    = "wireguard_ports"
+	networkDir   = "networks"
+	ipamLeaseDir = "ndmz-lease"
+	ipamPath     = "/var/cache/modules/networkd/lease"
+)
+
+const (
+	mib = 1024 * 1024
 )
 
 type networker struct {
-	identity   pkg.IdentityManager
-	storageDir string
-	tnodb      TNoDB
-	portSet    *set.UintSet
+	identity     pkg.IdentityManager
+	networkDir   string
+	ipamLeaseDir string
+	tnodb        TNoDB
+	portSet      *set.UintSet
 }
 
 // NewNetworker create a new pkg.Networker that can be used over zbus
-func NewNetworker(identity pkg.IdentityManager, tnodb TNoDB, storageDir string) pkg.Networker {
+func NewNetworker(identity pkg.IdentityManager, tnodb TNoDB, storageDir string) (pkg.Networker, error) {
+
+	vd, err := cache.VolatileDir("networkd", 50*mib)
+	if err != nil && !os.IsExist(err) {
+		return nil, fmt.Errorf("failed to create networkd cache directory: %w", err)
+	}
+
+	wgDir := filepath.Join(vd, wgPortDir)
+	if err := os.MkdirAll(wgDir, 0700); err != nil {
+		return nil, fmt.Errorf("failed to create wireguard port cache directory: %w", err)
+	}
+
+	nwDir := filepath.Join(vd, networkDir)
+	ipamLease := filepath.Join(vd, ipamLeaseDir)
+
+	oldPath := filepath.Join(ipamPath, "ndmz")
+	newPath := filepath.Join(ipamLease, "ndmz")
+	if _, err := os.Stat(oldPath); err == nil {
+		if err := shutil.CopyTree(oldPath, newPath, nil); err != nil {
+			return nil, err
+		}
+		_ = os.RemoveAll(ipamPath)
+	}
+
+	//TODO: remove once all the node have move the network into the volatile directory
+	if _, err = os.Stat(storageDir); err == nil {
+		if err := copyNetworksToVolatile(storageDir, nwDir); err != nil {
+			return nil, fmt.Errorf("failed to copy old networks directory: %w", err)
+		}
+	}
+
 	nw := &networker{
-		identity:   identity,
-		storageDir: storageDir,
-		tnodb:      tnodb,
-		portSet:    set.NewUint(),
+		identity:     identity,
+		tnodb:        tnodb,
+		networkDir:   nwDir,
+		ipamLeaseDir: ipamLease,
+		portSet:      set.NewUint(wgDir),
 	}
 
-	return nw
+	return nw, nil
+}
+
+func copyNetworksToVolatile(src, dst string) error {
+	log.Info().Msg("move network cached file to volatile directory")
+	if err := os.MkdirAll(dst, 0700); err != nil {
+		return err
+	}
+
+	infos, err := ioutil.ReadDir(src)
+	if err != nil {
+		return err
+	}
+
+	copy := func(src string, dst string) error {
+		log.Info().Str("source", src).Str("dst", dst).Msg("copy file")
+		// Read all content of src to data
+		data, err := ioutil.ReadFile(src)
+		if err != nil {
+			return err
+		}
+		// Write data to dst
+		return ioutil.WriteFile(dst, data, 0644)
+	}
+
+	for _, info := range infos {
+		if info.IsDir() {
+			continue
+		}
+
+		s := filepath.Join(src, info.Name())
+		d := filepath.Join(dst, info.Name())
+		if err := copy(s, d); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 var _ pkg.Networker = (*networker)(nil)
@@ -387,6 +467,12 @@ func (n networker) Addrs(iface string, netns string) ([]net.IP, error) {
 
 // CreateNR implements pkg.Networker interface
 func (n *networker) CreateNR(network pkg.Network) (string, error) {
+	defer func() {
+		if err := n.publishWGPorts(); err != nil {
+			log.Warn().Err(err).Msg("failed to publish wireguard port to BCDB")
+		}
+	}()
+
 	var err error
 	var nodeID = n.identity.NodeID().Identity()
 
@@ -416,7 +502,6 @@ func (n *networker) CreateNR(network pkg.Network) (string, error) {
 	// check if there is a reserved wireguard port for this NR already
 	// or if we need to update it
 	storedNet, err := n.networkOf(string(network.NetID))
-
 	if err != nil && !os.IsNotExist(err) {
 		return "", err
 	}
@@ -426,17 +511,15 @@ func (n *networker) CreateNR(network pkg.Network) (string, error) {
 		if err != nil {
 			return "", err
 		}
-		if netNR.WGListenPort != storedNR.WGListenPort {
-			if err := n.releasePort(storedNR.WGListenPort); err != nil {
-				return "", err
-			}
-		}
-	} else if os.IsNotExist(err) {
-		if err := n.reservePort(netNR.WGListenPort); err != nil {
+		if err := n.releasePort(storedNR.WGListenPort); err != nil {
 			return "", err
 		}
 	}
 
+	if err := n.reservePort(netNR.WGListenPort); err != nil {
+		return "", err
+	}
+
 	netr, err := nr.New(network.NetID, netNR, &network.IPRange.IPNet)
 	if err != nil {
 		return "", err
@@ -461,7 +544,7 @@ func (n *networker) CreateNR(network pkg.Network) (string, error) {
 		return "", errors.Wrap(err, "failed to create network resource")
 	}
 
-	if err := ndmz.AttachNR(string(network.NetID), netr); err != nil {
+	if err := ndmz.AttachNR(string(network.NetID), netr, n.ipamLeaseDir); err != nil {
 		return "", errors.Wrapf(err, "failed to attach network resource to DMZ bridge")
 	}
 
@@ -471,7 +554,7 @@ func (n *networker) CreateNR(network pkg.Network) (string, error) {
 	}
 
 	// map the network ID to the network namespace
-	path := filepath.Join(n.storageDir, string(network.NetID))
+	path := filepath.Join(n.networkDir, string(network.NetID))
 	file, err := os.Create(path)
 	if err != nil {
 		cleanup()
@@ -494,7 +577,7 @@ func (n *networker) CreateNR(network pkg.Network) (string, error) {
 }
 
 func (n *networker) networkOf(id string) (*pkg.Network, error) {
-	path := filepath.Join(n.storageDir, string(id))
+	path := filepath.Join(n.networkDir, string(id))
 	file, err := os.Open(path)
 	if err != nil {
 		return nil, err
@@ -530,6 +613,12 @@ func (n *networker) networkOf(id string) (*pkg.Network, error) {
 
 // DeleteNR implements pkg.Networker interface
 func (n *networker) DeleteNR(network pkg.Network) error {
+	defer func() {
+		if err := n.publishWGPorts(); err != nil {
+			log.Warn().Msg("failed to publish wireguard port to BCDB")
+		}
+	}()
+
 	netNR, err := ResourceByNodeID(n.identity.NodeID().Identity(), network.NetResources)
 	if err != nil {
 		return err
@@ -550,7 +639,7 @@ func (n *networker) DeleteNR(network pkg.Network) error {
 	}
 
 	// map the network ID to the network namespace
-	path := filepath.Join(n.storageDir, string(network.NetID))
+	path := filepath.Join(n.networkDir, string(network.NetID))
 	if err := os.Remove(path); err != nil {
 		log.Error().Err(err).Msg("failed to remove file mapping between network ID and namespace")
 	}
@@ -575,28 +664,32 @@ func (n *networker) extractPrivateKey(hexKey string) (string, error) {
 }
 
 func (n *networker) reservePort(port uint16) error {
+	log.Debug().Uint16("port", port).Msg("reserve wireguard port")
 	err := n.portSet.Add(uint(port))
 	if err != nil {
 		return errors.Wrap(err, "wireguard listen port already in use, pick another one")
 	}
 
-	if err := n.tnodb.PublishWGPort(n.identity.NodeID(), n.portSet.List()); err != nil {
-		n.portSet.Remove(uint(port))
-		return errors.Wrap(err, "fail to publish wireguard port to bcdb")
-	}
-
 	return nil
 }
 
 func (n *networker) releasePort(port uint16) error {
+	log.Debug().Uint16("port", port).Msg("release wireguard port")
 	n.portSet.Remove(uint(port))
+	return nil
+}
 
-	if err := n.tnodb.PublishWGPort(n.identity.NodeID(), n.portSet.List()); err != nil {
+func (n *networker) publishWGPorts() error {
+	ports, err := n.portSet.List()
+	if err != nil {
+		return err
+	}
+
+	if err := n.tnodb.PublishWGPort(n.identity.NodeID(), ports); err != nil {
 		// maybe retry a couple of times ?
 		// having bdb and the node out of sync is pretty bad
 		return errors.Wrap(err, "fail to publish wireguard port to bcdb")
 	}
-
 	return nil
 }
 
diff --git a/pkg/set/uint.go b/pkg/set/uint.go
index 872752e3..4aa6943a 100644
--- a/pkg/set/uint.go
+++ b/pkg/set/uint.go
@@ -1,8 +1,15 @@
 package set
 
 import (
+	"errors"
 	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
 	"sync"
+
+	"github.com/rs/zerolog/log"
 )
 
 // ErrConflict is return when trying to add a port
@@ -18,27 +25,37 @@ func (e ErrConflict) Error() string {
 // UintSet is a set containing uint
 type UintSet struct {
 	sync.RWMutex
-	m map[uint]struct{}
+	root string
 }
 
 // NewUint creates a new set for uint
-func NewUint() *UintSet {
+// path if the directory where to store the set on disk
+// the directory pointed by path must exists already
+func NewUint(path string) *UintSet {
 	return &UintSet{
-		m: make(map[uint]struct{}),
+		root: path,
 	}
 }
 
+func (p *UintSet) path(i uint) string {
+	return filepath.Join(p.root, fmt.Sprintf("%d", i))
+}
+
 // Add tries to add port to the set. If port is already
 // present errPortConflict is return otherwise nil is returned
 func (p *UintSet) Add(i uint) error {
 	p.Lock()
 	defer p.Unlock()
 
-	_, exists := p.m[i]
-	if exists {
-		return ErrConflict{i}
+	f, err := os.OpenFile(p.path(i), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0440)
+	if err != nil {
+		if errors.Is(err, os.ErrExist) {
+			return ErrConflict{Port: i}
+		}
+		return err
 	}
-	p.m[i] = struct{}{}
+
+	f.Close()
 	return nil
 }
 
@@ -49,20 +66,28 @@ func (p *UintSet) Remove(i uint) {
 	p.Lock()
 	defer p.Unlock()
 
-	_, exists := p.m[i]
-	if exists {
-		delete(p.m, i)
-	}
+	_ = os.RemoveAll(p.path(i))
 }
 
 // List returns a list of uint present in the set
-func (p *UintSet) List() []uint {
+func (p *UintSet) List() ([]uint, error) {
 	p.RLock()
 	defer p.RUnlock()
 
-	l := make([]uint, 0, len(p.m))
-	for i := range p.m {
-		l = append(l, i)
+	infos, err := ioutil.ReadDir(p.root)
+	if err != nil {
+		return nil, err
+	}
+	l := make([]uint, 0, len(infos))
+
+	for _, info := range infos {
+		s := filepath.Base(info.Name())
+		i, err := strconv.Atoi(s)
+		if err != nil {
+			log.Warn().Err(err).Msg("file with wrong formatted name found in reserved wireguard port cache")
+			continue
+		}
+		l = append(l, uint(i))
 	}
-	return l
+	return l, nil
 }
diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go
index 930eab3c..c3722d1d 100644
--- a/pkg/storage/storage.go
+++ b/pkg/storage/storage.go
@@ -18,7 +18,8 @@ import (
 )
 
 const (
-	cacheTarget = "/var/cache"
+	// CacheTarget is the path where the cache disk is mounted
+	CacheTarget = "/var/cache"
 	cacheLabel  = "zos-cache"
 	cacheSize   = 20 * 1024 * 1024 * 1024 // 20GB
 )
@@ -396,11 +397,11 @@ func (s *storageModule) ensureCache() error {
 		cacheFs = fs
 	}
 
-	if !filesystem.IsMountPoint(cacheTarget) {
-		log.Debug().Msgf("Mounting cache partition in %s", cacheTarget)
-		return filesystem.BindMount(cacheFs, cacheTarget)
+	if !filesystem.IsMountPoint(CacheTarget) {
+		log.Debug().Msgf("Mounting cache partition in %s", CacheTarget)
+		return filesystem.BindMount(cacheFs, CacheTarget)
 	}
-	log.Debug().Msgf("Cache partition already mounted in %s", cacheTarget)
+	log.Debug().Msgf("Cache partition already mounted in %s", CacheTarget)
 	return nil
 }
 


Test patch:
diff --git a/pkg/network/ndmz/ndmz_test.go b/pkg/network/ndmz/ndmz_test.go
index 376f5e20..77d89f1b 100644
--- a/pkg/network/ndmz/ndmz_test.go
+++ b/pkg/network/ndmz/ndmz_test.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net"
+	"os"
 	"sync"
 	"testing"
 
@@ -38,37 +39,31 @@ func TestIpv6(t *testing.T) {
 }
 
 func TestIPv4Allocate(t *testing.T) {
-	var (
-		err    error
-		origin = ipamPath
-	)
-
-	ipamPath, err = ioutil.TempDir("", "")
+	ipamPath, err := ioutil.TempDir("", "")
 	require.NoError(t, err)
-	defer func() { ipamPath = origin }()
+	t.Cleanup(func() {
+		_ = os.RemoveAll(ipamPath)
+	})
 
-	addr, err := allocateIPv4("network1")
+	addr, err := allocateIPv4("network1", ipamPath)
 	require.NoError(t, err)
 
-	addr2, err := allocateIPv4("network1")
+	addr2, err := allocateIPv4("network1", ipamPath)
 	require.NoError(t, err)
 
 	assert.Equal(t, addr.String(), addr2.String())
 
-	addr3, err := allocateIPv4("network2")
+	addr3, err := allocateIPv4("network2", ipamPath)
 	require.NoError(t, err)
 	assert.NotEqual(t, addr.String(), addr3.String())
 }
 
 func TestIPv4AllocateConcurent(t *testing.T) {
-	var (
-		err    error
-		origin = ipamPath
-	)
-
-	ipamPath, err = ioutil.TempDir("", "")
+	ipamPath, err := ioutil.TempDir("", "")
 	require.NoError(t, err)
-	defer func() { ipamPath = origin }()
+	t.Cleanup(func() {
+		_ = os.RemoveAll(ipamPath)
+	})
 
 	wg := sync.WaitGroup{}
 	wg.Add(10)
@@ -80,7 +75,7 @@ func TestIPv4AllocateConcurent(t *testing.T) {
 			defer wg.Done()
 			for y := 0; y < 10; y++ {
 				nw := fmt.Sprintf("network%d%d", i, y)
-				addr, err := allocateIPv4(nw)
+				addr, err := allocateIPv4(nw, ipamPath)
 				require.NoError(t, err)
 				c <- addr
 			}
diff --git a/pkg/set/uint_test.go b/pkg/set/uint_test.go
index dda0f0fa..e1cb4fbd 100644
--- a/pkg/set/uint_test.go
+++ b/pkg/set/uint_test.go
@@ -2,36 +2,57 @@ package set
 
 import (
 	"errors"
+	"io/ioutil"
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
+func testDir(t *testing.T) string {
+	td, err := ioutil.TempDir("", "")
+	require.NoError(t, err)
+
+	path := filepath.Join(td, "wireguard_ports")
+	err = os.MkdirAll(path, 0770)
+	require.NoError(t, err)
+
+	return path
+}
+
 func TestNew(t *testing.T) {
-	s := NewUint()
-	assert.NotNil(t, s.m)
+	td := testDir(t)
+
+	s := NewUint(td)
+	assert.Equal(t, td, s.root)
 }
 
 func TestAdd(t *testing.T) {
-	s := NewUint()
+	td := testDir(t)
+
+	s := NewUint(td)
 
 	for i := 0; i < 10; i++ {
 		err := s.Add(uint(i))
 		assert.NoError(t, err)
 	}
 
-	for i := 0; i < 10; i++ {
-		_, ok := s.m[uint(i)]
-		assert.True(t, ok, "%d should be present in the map", i)
-	}
+	infos, err := ioutil.ReadDir(s.root)
+	require.NoError(t, err)
 
-	err := s.Add(0)
+	assert.EqualValues(t, 10, len(infos))
+
+	err = s.Add(0)
 	assert.Error(t, err, "adding already present number should return an error")
 	assert.True(t, errors.Is(err, ErrConflict{}))
 }
 
 func TestRemove(t *testing.T) {
-	s := NewUint()
+	td := testDir(t)
+
+	s := NewUint(td)
 
 	for i := 0; i < 10; i++ {
 		err := s.Add(uint(i))
@@ -42,13 +63,18 @@ func TestRemove(t *testing.T) {
 		s.Remove(uint(i))
 	}
 
-	assert.EqualValues(t, 0, len(s.m))
+	infos, err := ioutil.ReadDir(s.root)
+	require.NoError(t, err)
+
+	assert.EqualValues(t, 0, len(infos))
 
 	s.Remove(99999) //ensure remove never panics
 }
 
 func TestList(t *testing.T) {
-	s := NewUint()
+	td := testDir(t)
+
+	s := NewUint(td)
 
 	err := s.Add(1)
 	assert.NoError(t, err)
@@ -57,12 +83,10 @@ func TestList(t *testing.T) {
 	err = s.Add(3)
 	assert.NoError(t, err)
 
-	l := s.List()
+	l, err := s.List()
+	require.NoError(t, err)
+
 	expected := []uint{1, 2, 3}
 	assert.Equal(t, len(expected), len(l))
 	assert.Subset(t, expected, l)
 }
-
-func TestConcurent(t *testing.T) {
-	// TODO
-}