Repository: threefoldtech/zos
Pull request #584 (2020-03-18 09:02:39)

Fixes empty reservation chache bug
issue: https://github.com/threefoldtech/zos/issues/581

codecov[bot]: # [Codecov](https://codecov.io/gh/threefoldtech/zos/pull/584?src=pr&el=h1) Report
> Merging [#584](https://codecov.io/gh/threefoldtech/zos/pull/584?src=pr&el=desc) into [master](https://codecov.io/gh/threefoldtech/zos/commit/c47cd8bb1dc08e7f164cb937fb116a8d38de4e06&el=desc) will **decrease** coverage by `0.02%`.
> The diff coverage is `0.00%`.

[![Impacted file tree graph](https://codecov.io/gh/threefoldtech/zos/pull/584/graphs/tree.svg?width=650&height=150&src=pr&token=f7Vmq1OAEm)](https://codecov.io/gh/threefoldtech/zos/pull/584?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master     #584      +/-   ##
==========================================
- Coverage   36.00%   35.98%   -0.03%     
==========================================
  Files          71       71              
  Lines        5255     5258       +3     
==========================================
  Hits         1892     1892              
- Misses       3093     3096       +3     
  Partials      270      270              
```


| [Impacted Files](https://codecov.io/gh/threefoldtech/zos/pull/584?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [pkg/provision/local\_store.go](https://codecov.io/gh/threefoldtech/zos/pull/584/diff?src=pr&el=tree#diff-cGtnL3Byb3Zpc2lvbi9sb2NhbF9zdG9yZS5nbw==) | `28.64% <0.00%> (-0.46%)` | :arrow_down: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/threefoldtech/zos/pull/584?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/threefoldtech/zos/pull/584?src=pr&el=footer). Last update [c47cd8b...35af773](https://codecov.io/gh/threefoldtech/zos/pull/584?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/bootstrap/bootstrap/src/zfs.rs b/bootstrap/bootstrap/src/zfs.rs
index 5aab9233..a382ca46 100644
--- a/bootstrap/bootstrap/src/zfs.rs
+++ b/bootstrap/bootstrap/src/zfs.rs
@@ -75,7 +75,7 @@ impl Zfs {
                 debug!("installing file {:?}", dst);
                 std::fs::copy(&src, &dst)?;
             } else {
-                error!("unsupported file type: ({:?}): {:?}", src, typ)
+                debug!("skipping: ({:?}): {:?}", src, typ)
             }
         }
 
diff --git a/cmds/capacityd/main.go b/cmds/capacityd/main.go
index 48c25c74..a1dfbd3b 100644
--- a/cmds/capacityd/main.go
+++ b/cmds/capacityd/main.go
@@ -11,7 +11,6 @@ import (
 	"github.com/threefoldtech/zos/pkg/app"
 	"github.com/threefoldtech/zos/pkg/capacity"
 	"github.com/threefoldtech/zos/pkg/environment"
-	"github.com/threefoldtech/zos/pkg/gedis"
 	"github.com/threefoldtech/zos/pkg/monitord"
 	"github.com/threefoldtech/zos/pkg/stubs"
 	"github.com/threefoldtech/zos/pkg/utils"
@@ -161,16 +160,6 @@ func bcdbClient() (capacity.Store, error) {
 		return nil, errors.Wrap(err, "failed to parse node environment")
 	}
 
-	// use the bcdb mock for dev and test
-	if env.RunningMode == environment.RunningDev {
-		return capacity.NewHTTPStore(env.BcdbURL), nil
-	}
-
-	// use gedis for production bcdb
-	store, err := gedis.New(env.BcdbURL, env.BcdbPassword)
-	if err != nil {
-		return nil, errors.Wrap(err, "fail to connect to BCDB")
-	}
+	return capacity.NewHTTPStore(env.BcdbURL), nil
 
-	return capacity.NewBCDBStore(store), nil
 }
diff --git a/cmds/identityd/main.go b/cmds/identityd/main.go
index 79b7b478..5387786e 100644
--- a/cmds/identityd/main.go
+++ b/cmds/identityd/main.go
@@ -12,7 +12,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/threefoldtech/zos/pkg/app"
 	"github.com/threefoldtech/zos/pkg/flist"
-	"github.com/threefoldtech/zos/pkg/gedis"
 	"github.com/threefoldtech/zos/pkg/geoip"
 	"github.com/threefoldtech/zos/pkg/stubs"
 	"github.com/threefoldtech/zos/pkg/upgrade"
@@ -443,17 +442,7 @@ func bcdbClient() (identity.IDStore, error) {
 		return nil, errors.Wrap(err, "failed to parse node environment")
 	}
 
-	// use the bcdb mock for dev and test
-	if env.RunningMode == environment.RunningDev {
-		return identity.NewHTTPIDStore(env.BcdbURL), nil
-	}
-
-	// use gedis for production bcdb
-	store, err := gedis.New(env.BcdbURL, env.BcdbPassword)
-	if err != nil {
-		return nil, errors.Wrap(err, "fail to connect to BCDB")
-	}
-	return store, nil
+	return identity.NewHTTPIDStore(env.BcdbURL), nil
 }
 
 func registerNode(nodeID pkg.Identifier, farmID pkg.FarmID, version string, store identity.IDStore, loc geoip.Location) error {
diff --git a/cmds/networkd/main.go b/cmds/networkd/main.go
index fe49d60b..bc395b70 100644
--- a/cmds/networkd/main.go
+++ b/cmds/networkd/main.go
@@ -14,7 +14,6 @@ import (
 	"github.com/threefoldtech/zos/pkg"
 	"github.com/threefoldtech/zos/pkg/app"
 	"github.com/threefoldtech/zos/pkg/environment"
-	"github.com/threefoldtech/zos/pkg/gedis"
 	"github.com/threefoldtech/zos/pkg/network"
 	"github.com/threefoldtech/zos/pkg/network/bootstrap"
 	"github.com/threefoldtech/zos/pkg/network/ndmz"
@@ -55,17 +54,12 @@ func main() {
 		log.Fatal().Err(err).Msg("failed to connect to zbus broker")
 	}
 
-	if err := os.MkdirAll(root, 0750); err != nil {
-		log.Fatal().Err(err).Msgf("fail to create module root")
-	}
-
 	db, err := bcdbClient()
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to connect to BCDB")
 	}
 
 	identity := stubs.NewIdentityManagerStub(client)
-	networker := network.NewNetworker(identity, db, root)
 	nodeID := identity.NodeID()
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -133,6 +127,15 @@ func main() {
 	go startAddrWatch(ctx, nodeID, db, ifaces)
 
 	log.Info().Msg("start zbus server")
+	if err := os.MkdirAll(root, 0750); err != nil {
+		log.Fatal().Err(err).Msgf("fail to create module root")
+	}
+
+	networker, err := network.NewNetworker(identity, db, root)
+	if err != nil {
+		log.Fatal().Err(err).Msg("error creating network manager")
+	}
+
 	if err := startServer(ctx, broker, networker); err != nil {
 		log.Fatal().Err(err).Msg("unexpected error")
 	}
@@ -189,15 +192,5 @@ func bcdbClient() (network.TNoDB, error) {
 		return nil, errors.Wrap(err, "failed to parse node environment")
 	}
 
-	// use the bcdb mock for dev and test
-	if env.RunningMode == environment.RunningDev {
-		return tnodb.NewHTTPTNoDB(env.BcdbURL), nil
-	}
-
-	// use gedis for production bcdb
-	store, err := gedis.New(env.BcdbURL, env.BcdbPassword)
-	if err != nil {
-		return nil, errors.Wrap(err, "fail to connect to BCDB")
-	}
-	return store, nil
+	return tnodb.NewHTTPTNoDB(env.BcdbURL), nil
 }
diff --git a/cmds/provisiond/main.go b/cmds/provisiond/main.go
index a3596fe3..4a1488b9 100644
--- a/cmds/provisiond/main.go
+++ b/cmds/provisiond/main.go
@@ -5,12 +5,12 @@ import (
 	"flag"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/threefoldtech/zos/pkg"
 	"github.com/threefoldtech/zos/pkg/app"
 	"github.com/threefoldtech/zos/pkg/environment"
-	"github.com/threefoldtech/zos/pkg/gedis"
 
 	"github.com/threefoldtech/zos/pkg/stubs"
 	"github.com/threefoldtech/zos/pkg/utils"
@@ -53,6 +53,14 @@ func main() {
 		zerolog.SetGlobalLevel(zerolog.DebugLevel)
 	}
 
+	// keep checking if limited-cache flag is set
+	if app.CheckFlag(app.LimitedCache) {
+		log.Error().Msg("failed cache reservation! Retrying every 30 seconds...")
+		for app.CheckFlag(app.LimitedCache) {
+			time.Sleep(time.Second * 30)
+		}
+	}
+
 	flag.Parse()
 
 	if err := os.MkdirAll(storageDir, 0770); err != nil {
@@ -108,6 +116,7 @@ func main() {
 	)
 
 	engine := provision.New(nodeID.Identity(), source, localStore, remoteStore)
+
 	server.Register(zbus.ObjectID{Name: module, Version: "0.0.1"}, pkg.ProvisionMonitor(engine))
 
 	log.Info().
@@ -145,15 +154,5 @@ func bcdbClient() (store, error) {
 		return nil, errors.Wrap(err, "failed to parse node environment")
 	}
 
-	// use the bcdb mock for dev and test
-	if env.RunningMode == environment.RunningDev {
-		return provision.NewHTTPStore(env.BcdbURL), nil
-	}
-
-	// use gedis for production bcdb
-	store, err := gedis.New(env.BcdbURL, env.BcdbPassword)
-	if err != nil {
-		return nil, errors.Wrap(err, "fail to connect to BCDB")
-	}
-	return store, nil
+	return provision.NewHTTPStore(env.BcdbURL), nil
 }
diff --git a/cmds/storaged/main.go b/cmds/storaged/main.go
index b9f6f731..082bf922 100644
--- a/cmds/storaged/main.go
+++ b/cmds/storaged/main.go
@@ -38,7 +38,7 @@ func main() {
 
 	storageModule, err := storage.New()
 	if err != nil {
-		log.Fatal().Err(err).Msg("Error initializing storage module")
+		log.Fatal().Err(err).Msg("failed to initialize storage module")
 	}
 
 	server, err := zbus.NewRedisServer(module, msgBrokerCon, workerNr)
@@ -46,14 +46,15 @@ func main() {
 		log.Fatal().Err(err).Msg("fail to connect to message broker server")
 	}
 
+	server.Register(zbus.ObjectID{Name: "storage", Version: "0.0.1"}, storageModule)
+
 	vdiskModule, err := storage.NewVDiskModule(storageModule)
 	if err != nil {
-		log.Fatal().Err(err)
+		log.Error().Err(err).Bool("limited-cache", app.CheckFlag(app.LimitedCache)).Msg("failed to initialize virtual disk module")
+	} else {
+		server.Register(zbus.ObjectID{Name: "vdisk", Version: "0.0.1"}, vdiskModule)
 	}
 
-	server.Register(zbus.ObjectID{Name: "storage", Version: "0.0.1"}, storageModule)
-	server.Register(zbus.ObjectID{Name: "vdisk", Version: "0.0.1"}, vdiskModule)
-
 	log.Info().
 		Str("broker", msgBrokerCon).
 		Uint("worker nr", workerNr).
diff --git a/docs/kubernetes/README.md b/docs/kubernetes/README.md
index 5996e10c..c9494c88 100644
--- a/docs/kubernetes/README.md
+++ b/docs/kubernetes/README.md
@@ -17,6 +17,10 @@ Prerequisite:
 - [kubectl](https://kubernetes.io/fr/docs/tasks/tools/install-kubectl/)
 - [wireguard](https://www.wireguard.com/install/)
 
+Optionally (if you want to deploy the charts);
+
+- helm (v3) ([install instructions](https://helm.sh/docs/intro/install))
+
 the script arguments are `startup.sh {GITHUB_ACCOUNT_OR_SSH_PUB_KEY} {CIDR/16} {DURATION} {NUMBER_OF_NODES} {VM_SIZE}`
 
 ```
@@ -78,21 +82,21 @@ If you want an API to explore the nodes on the grid you can go to : [https://exp
 ### Networking
 
 We now need to create a network between the nodes. Hopefully tfuser can help us generate the configuration file.
-Let's first create a network named `kubetest` with a large cidr `173.30.0.0/16` and write that configuration inside a file `network.json`
+Let's first create a network named `kubetest` with a large cidr `172.30.0.0/16` and write that configuration inside a file `network.json`
 
 ```
-$ ./tfuser generate --schema network.json network create --name kubetest --cidr 173.30.0.0/16
+$ ./tfuser generate --schema network.json network create --name kubetest --cidr 172.30.0.0/16
 ```
 
 now let's add the nodes that we have selected inside our network by specifying the path to our `network.json` network configuration file. We also specify a port for wireguard to use and a subnet to use on the nodes.
 
 ```
 $ ./tfuser generate --schema network.json network add-node --node qzuTJJVd5boi6Uyoco1WWnSgzTb7q8uN79AjBT9x9N3  \
-				      --subnet 173.30.1.0/24
+				      --subnet 172.30.1.0/24
 $ ./tfuser generate --schema network.json network add-node --node 3NAkUYqm5iPRmNAnmLfjwdreqDssvsebj4uPUt9BxFPm \
-				      --subnet 173.30.2.0/24
+				      --subnet 172.30.2.0/24
 $ ./tfuser generate --schema network.json network add-node --node FTothsg9ZuJubAEzZByEgQQUmkWM637x93YH1QSJM242 \
-				      --subnet 173.30.3.0/24
+				      --subnet 172.30.3.0/24
 ```
 
 there are two levels here :
@@ -106,7 +110,7 @@ We will need an external access to the network to be able to ssh into our VMs.
 
 ```
 $ ./tfuser generate --schema network.json network add-access --node qzuTJJVd5boi6Uyoco1WWnSgzTb7q8uN79AjBT9x9N3 \
-				       --subnet 173.30.4.0/24 --ip4 > wg.conf
+				       --subnet 172.30.4.0/24 --ip4 > wg.conf
 ```
 
 We generated the wireguard, a secure and fast VPN, configuration file that we will use here after in this guide.
@@ -116,7 +120,7 @@ here is an example
 
 ```
 $ ./tfuser generate --schema network.json network add-access --node BpTAry1Na2s1J8RAHNyDsbvaBSM3FjR4gMXEKga3UPbs \
-				      --subnet 173.30.4.0/24 > wg.conf
+				      --subnet 172.30.4.0/24 > wg.conf
 ```
 
 Let's take a look at a graphical representation of our network
@@ -168,20 +172,20 @@ Let's now provision some Kubernetes VM on those nodes
 ### Provision
 
 Let's provision a kubernetes VM on our first node and assign an IP allowed in the network `kubetest` defined here above.
-Replace `SSH_PUB_KEY` by your ssh public key and ssh key algorithm e.g. `"ssh-rsa AAAAB3NzaC1yc2EAAA.."`. We choose an IP for the VM that is part of the subnet we previously defined for the node. We can't choose 173.30.3.1 as it is the IP of the network ressource itself.
+Replace `SSH_PUB_KEY` by your ssh public key and ssh key algorithm e.g. `"ssh-rsa AAAAB3NzaC1yc2EAAA.."`. We choose an IP for the VM that is part of the subnet we previously defined for the node. We can't choose 172.30.3.1 as it is the IP of the network ressource itself.
 
 ```
-./tfuser generate --schema kube1.json kubernetes --size 1 --network-id kubetest --ip 173.30.1.2 --secret token --node qzuTJJVd5boi6Uyoco1WWnSgzTb7q8uN79AjBT9x9N3  --ssh-keys "{SSH_PUB_KEY}" &&  \
+./tfuser generate --schema kube1.json kubernetes --size 1 --network-id kubetest --ip 172.30.1.2 --secret token --node qzuTJJVd5boi6Uyoco1WWnSgzTb7q8uN79AjBT9x9N3  --ssh-keys "{SSH_PUB_KEY}" &&  \
  ./tfuser -d provision --schema kube1.json --duration 2 --seed user.seed --node qzuTJJVd5boi6Uyoco1WWnSgzTb7q8uN79AjBT9x9N3
 ```
 
 ```
-./tfuser generate --schema kube2.json kubernetes --size 1 --network-id kubetest --ip 173.30.2.2 --master-ips 173.30.1.2 --secret token --node 3NAkUYqm5iPRmNAnmLfjwdreqDssvsebj4uPUt9BxFPm  --ssh-keys "{SSH_PUB_KEY}" &&  \
+./tfuser generate --schema kube2.json kubernetes --size 1 --network-id kubetest --ip 172.30.2.2 --master-ips 172.30.1.2 --secret token --node 3NAkUYqm5iPRmNAnmLfjwdreqDssvsebj4uPUt9BxFPm  --ssh-keys "{SSH_PUB_KEY}" &&  \
  ./tfuser -d provision --schema kube2.json --duration 2 --seed user.seed --node 3NAkUYqm5iPRmNAnmLfjwdreqDssvsebj4uPUt9BxFPm
 ```
 
 ```
-./tfuser generate --schema kube3.json kubernetes --size 1 --network-id kubetest --ip 173.30.3.2 --master-ips 173.30.1.2 --secret token --node FTothsg9ZuJubAEzZByEgQQUmkWM637x93YH1QSJM242  --ssh-keys "{SSH_PUB_KEY}" &&  \
+./tfuser generate --schema kube3.json kubernetes --size 1 --network-id kubetest --ip 172.30.3.2 --master-ips 172.30.1.2 --secret token --node FTothsg9ZuJubAEzZByEgQQUmkWM637x93YH1QSJM242  --ssh-keys "{SSH_PUB_KEY}" &&  \
  ./tfuser -d provision --schema kube3.json --duration 2 --seed user.seed --node FTothsg9ZuJubAEzZByEgQQUmkWM637x93YH1QSJM242
 ```
 
@@ -200,7 +204,7 @@ interface: wg
 
 peer: BQE9qUNPKEH59Fy6B2xyMz0KrRfBDIdDm4Bd23ro8DM=
   endpoint: 185.69.166.246:3561
-  allowed ips: 173.30.5.0/24, 100.64.30.5/32
+  allowed ips: 172.30.5.0/24, 100.64.30.5/32
   latest handshake: 2 minutes, 43 seconds ago
   transfer: 14.26 KiB received, 19.14 KiB sent
   persistent keepalive: every 20 seconds
@@ -210,12 +214,12 @@ peer: BQE9qUNPKEH59Fy6B2xyMz0KrRfBDIdDm4Bd23ro8DM=
 log into your VM
 
 ```
-$ ping 173.30.1.2
-$ ssh rancher@173.30.1.2
-The authenticity of host '173.30.1.2 (173.30.1.2)' can't be established.
+$ ping 172.30.1.2
+$ ssh rancher@172.30.1.2
+The authenticity of host '172.30.1.2 (172.30.1.2)' can't be established.
 ECDSA key fingerprint is SHA256:Q4kQ94B8QaSbo1EsyI8dQrgBkZyk/USda72c8nwVwIE.
 Are you sure you want to continue connecting (yes/no)? yes
-Warning: Permanently added '173.30.1.2' (ECDSA) to the list of known hosts.
+Warning: Permanently added '172.30.1.2' (ECDSA) to the list of known hosts.
 Welcome to k3OS!
 
 Refer to https://github.com/rancher/k3os for README and issues
@@ -242,7 +246,11 @@ Copy the config so that we can use kubectl from our local machine. By default it
 Execute this command on your local machine not in a remote shell
 
 ```
-$ scp rancher@173.30.1.2:/etc/rancher/k3s/k3s.yaml ./k3s.yaml
+$ scp rancher@172.30.1.2:/etc/rancher/k3s/k3s.yaml ./k3s.yaml
+
+> IMPORTANT: Do not forget to update the server IP in the copied yaml file.
+> By default, it will be pointing to localhost, this must be changed into the
+> IP you used to connect to the VM.
 ```
 
 If you already have a kube config file usually located in `~/.kube/config`
@@ -255,7 +263,7 @@ apiVersion: v1
 clusters:
 - cluster:
     certificate-authority-data: LS0tLS1CRUdLKjhdDGhjDHKHKhDBJWakNCL3FBREFnRUNBZ0VBTUFvR0NDcUdTTTQ5QkFNQ01DTXhJVEFmQmdOVkJBTU1HR3N6Y3kxelpYSjIKWlhJdFkyRkFNVFU0TURjME9EQXhOakFlRncweU1EQXlNRE14TmpRd01UWmFGdzB6TURBeE16RXhOalF3TVRaYQpNQ014SVRBZkJnTlZCQU1NR0dzemN5MXpaWEoyWlhJdFkyRkFNVFU0TURjME9EQXhOakPPOIHjkDHDJHGkFnRUddaW9tdVR1MXQ1aVRlZDhHaVFrQ2FrdnRWL2xpRGJ3MUlxSS94dEkKWmUya2Y3Tm1mL0txR3IrMzN5SVZ5Q0tkaEdlelBCbEsvanNUSkZVSWpzdWpJekFoTUE0R0ExVWREd0DezdzedzenTlZIUk1CQWY4RUJUQURBUUgvTUFvR0NDcUdTTTQ5QkFNQ0EwY0FNRVFDSUJFNTYzcUttY2xiClVQWHc2UXJCbWxQUmlrbWdCVnY0VHlkMVZ0TWNXY3JYQWlCVlJPY3RjMTF1TXFrOGJWVHJOVFNiN0lFS3ZkRjAKelluMzhwME41MdLUVORCBDRVJUSUZJQ0FURS0D=
-    server: https://170.30.1.2:6443
+    server: https://172.30.1.2:6443
   name: k3s
 - context:
     cluster: k3s
@@ -343,7 +351,7 @@ $ kubectl get -o jsonpath="{.spec.ports[0].nodePort}" services wordpress
 31004
 ```
 
-We can browse any nodes url on port 31004 to find the wordpress website e.g. [http://173.30.3.2:31004/](http://173.30.3.2:31004/)
+We can browse any nodes url on port 31004 to find the wordpress website e.g. [http://172.30.3.2:31004/](http://172.30.3.2:31004/)
 and after some setup screens you will access your articles
 
 ![wordpress first article](ressources/wordpress/wordpress.png)
@@ -354,8 +362,8 @@ K3s comes with helm support built-in. Let's try to deploy prometheus and grafana
 
 ```
 $ kubectl create namespace mon
-$ helm install --namespace mon --skip-crds  my-release stable/prometheus-operator
-$ helm list
+$ helm install --namespace mon my-release stable/prometheus-operator
+$ helm -n mon list
 NAME            NAMESPACE       REVISION        UPDATED                                 STATUS          CHART                      APP VERSION
 my-release      mon             1               2020-02-04 16:08:41.70990744 +0100 CET  deployed        prometheus-operator-8.5.11 0.34.0
 $ kubectl config set-context --current --namespace=mon
diff --git a/docs/kubernetes/advanced_k8s.md b/docs/kubernetes/advanced_k8s.md
index 21dedb9b..f8a07b54 100644
--- a/docs/kubernetes/advanced_k8s.md
+++ b/docs/kubernetes/advanced_k8s.md
@@ -31,105 +31,6 @@ volume-test                  1/1     Running   0          2m
 
 ```
 
-### Rook
-
-Rook is an open source cloud-native storage orchestrator, providing the platform, framework, and support for a diverse set of storage solutions to natively integrate with cloud-native environments.
-
-Rook turns storage software into self-managing, self-scaling, and self-healing storage services. It does this by automating deployment, bootstrapping, configuration, provisioning, scaling, upgrading, migration, disaster recovery, monitoring, and resource management. Rook uses the facilities provided by the underlying cloud-native container management, scheduling and orchestration platform to perform its duties.
-
-installing [rook](https://rook.io/docs/rook/v1.2/)
-
-```
-git clone --single-branch --branch release-1.2 https://github.com/rook/rook.git
-cd cluster/examples/kubernetes/ceph
-kubectl create -f common.yaml
-kubectl create -f operator.yaml
-kubectl create -f cluster-test.yaml
-```
-
-### Installing [rook NFS](https://rook.io/docs/rook/v1.2/nfs.html)
-
-NFS allows remote hosts to mount filesystems over a network and interact with those filesystems as though they are mounted locally. This enables system administrators to consolidate resources onto centralized servers on the network.
-
-#### First deploy the Rook NFS operator using the following commands:
-
-```
-$ cd resources/storage/rook-NFS
-$ kubectl create -f 1-operator.yaml
-```
-
-We will create a NFS server instance that exports storage that is backed by the default StorageClass. In k3s environments storageClass "local-path": Only support ReadWriteOnce access mode so for the PVC taht must be created before creating NFS CRD instance.
-
-```
-$ kubectl create -f 2-nfs.yaml
-$ kubectl get pvc -n rook-nfs
-NAME                STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
-nfs-default-claim   Bound    pvc-9804c4f1-80e1-45ec-bf05-3dbdd012564e   1Gi        RWO            local-path     3m49s
-$ kubectl get po -n rook-nfs
-NAME         READY   STATUS    RESTARTS   AGE
-rook-nfs-0   1/1     Running   0          16s
-```
-
-#### Accessing the Export through dynamic NFS provisioning
-
-Once the NFS Operator and an instance of NFSServer is deployed. A storageclass has to be created to dynamically provisioning volumes.
-The StorageClass need to have the following 3 parameters passed.
-
-- exportName: It tells the provisioner which export to use for provisioning the volumes.
-- nfsServerName: It is the name of the NFSServer instance.
-- nfsServerNamespace: It namespace where the NFSServer instance is running.
-
-```
-$ kubectl create -f 3-sc.yaml
-```
-
-Once the above storageclass has been created create a PV claim referencing the storageclass as shown in the example given below.
-
-```
-$ kubectl create -f 4-pvc.yaml
-```
-
-#### Consuming the Export
-
-Now we can consume the PV that we just created by creating an example web server app that uses the above `PersistentVolumeClaim` to claim the exported volume. There are 2 pods that comprise this example:
-
-- A web server pod that will read and display the contents of the NFS share
-- A writer pod that will write random data to the NFS share so the website will continually update
-  Start both the busybox pod (writer) and the web server from the ressources/storage/rook-NFS folder:
-
-```
-kubectl create -f busybox-rc.yaml
-kubectl create -f web-rc.yaml
-```
-
-CANT CREATE POD SUCK IN CONTAINER CREATING
-
-```
-  Warning  FailedMount       15m                   kubelet, zv2k8s-04  Unable to attach or mount volumes: unmounted volumes=[rook-nfs-vol], unattached volumes=[default-token-f6sx2 rook-nfs-vol]: timed out waiting for the condition
-  Warning  FailedMount       2m31s (x10 over 16m)  kubelet, zv2k8s-04  MountVolume.SetUp failed for volume "pvc-903d405d-0c9c-4af7-bc2d-356fc03905fb" : mount failed: exit status 255
-Mounting command: mount
-Mounting arguments: -t nfs 10.43.102.171:/nfs-default-claim /var/lib/kubelet/pods/91d477ac-7e3f-4957-b427-0a3f2a68847b/volumes/kubernetes.io~nfs/pvc-903d405d-0c9c-4af7-bc2d-356fc03905fb
-```
-
-probably need nfs-common
-
-### Installing [rook CEPH](https://rook.io/docs/rook/v1.2/ceph.html)
-
-RBD
-
-Rook Ceph requires a Linux kernel built with the RBD module. Many distributions of Linux have this module but some don’t, e.g. the GKE Container-Optimised OS (COS) does not have RBD. You can test your Kubernetes nodes by running modprobe rbd. If it says ‘not found’, you may have to rebuild your kernel or choose a different Linux distribution.
-
-CephFS
-
-If you will be creating volumes from a Ceph shared file system (CephFS), the recommended minimum kernel version is 4.17. If you have a kernel version less than 4.17, the requested PVC sizes will not be enforced. Storage quotas will only be enforced on newer kernels.
-
-No RDB module and kernel < 4.17
-
-```
-usr/lib/modules/4.14.82-Zero-OS/kernel/drivers # find /usr/lib/modules | grep rdb
-/usr/lib/modules/4.14.82-Zero-OS/kernel/drivers/media/rc/keymaps/rc-avermedia-cardbus.ko
-```
-
 ## Load balancing and external IP
 
 #### Klipper
diff --git a/docs/kubernetes/startup.sh b/docs/kubernetes/startup.sh
index fca8435d..2a4e5485 100755
--- a/docs/kubernetes/startup.sh
+++ b/docs/kubernetes/startup.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 #set -Eeuo pipefail
 set -Eeo pipefail
-DEFAULT_NETWORK_RANGE="173.30"
+DEFAULT_NETWORK_RANGE="172.30"
 CIDR_16="${DEFAULT_NETWORK_RANGE}.0.0/16"
 CUR_DIR=$PWD
 echo "CUR_DIR:" $CUR_DIR
diff --git a/docs/network/Network-V2.md b/docs/network/Network-V2.md
new file mode 100644
index 00000000..59e16aee
--- /dev/null
+++ b/docs/network/Network-V2.md
@@ -0,0 +1,315 @@
+# 0-OS v2 and it's network
+
+## Introduction
+
+0-OS nodes participating in the Threefold grid, need connectivity of course. They need to be able to communicate over 
+the Internet with each-other in order to do various things:
+
+- download it's OS modules
+- perform OS module upgrades
+- register itself to the grid, and send regular updates about it's status
+- query the grid for tasks to execute
+- build and run the Overlay Network
+- download flists and the effective files to cache
+
+The nodes themselves can have connectivity in a few different ways:
+
+- Only have RFC1918 private addresses, connected to the Internet through NAT, NO IPv6
+  Mostly, these are single-NIC (Network card) machines that can host some workloads through the Overlay Network, but 
+  cant't expose services directly. These are HIDDEN nodes, and are mostly booted with an USB stick from 
+  bootstrap.grid.tf .
+- Dual-stacked: having RFC1918 private IPv4 and public IPv6 , where the IPv6 addresses are received from a home router, 
+but firewalled for outgoing traffic only. These nodes are effectively also HIDDEN
+- Nodes with 2 NICs, one that has effectively a NIC connected to a segment that has real public 
+addresses (IPv4 and/or IPv6) and one NIC that is used for booting and local 
+management. (OOB) (like in the drawing for farmer setup)
+
+For Farmers, we need to have Nodes to be reachable over IPv6, so that the nodes can:
+
+- expose services to be proxied into containers/vms
+- act as aggregating nodes for Overlay Networks for HIDDEN Nodes
+
+Some Nodes in Farms should also have a publicly reachable IPv4, to make sure that clients that only have IPv4 can 
+effectively reach exposed services.
+
+But we need to stress the importance of IPv6 availability when you're running a multi-node farm in a datacentre: as the 
+grid is boldly claiming to be a new Internet, we should make sure we adhere to the new protocols that are future-proof. 
+Hence: IPv6 is the base, and IPv4 is just there to accomodate the transition.
+
+Nowadays, RIPE can't even hand out consecutive /22 IPv4 blocks any more for new LIRs, so you'll be bound to market to 
+get IPv4, mostly at rates of 10-15 Euro per IP. Things tend to get costly that way.
+
+So anyway, IPv6 is not an afterthought in 0-OS, we're starting with it.
+
+## Physical setup for farmers
+
+```text
+                      XXXXX  XXX
+                     XX   XXX  XXXXX XXX
+                    X           X      XXX
+                   X                     X
+                   X      INTERNET       X
+                   XXX        X          X
+                     XXXXX   XX    XX XXXX
+                      +X XXXX  XX XXXXX
+                      |
+                      |
+                      |
+                      |
+                      |
+               +------+--------+
+               | FIREWALL/     |
+               | ROUTER        |
+               +--+----------+-+
+                  |          |
+      +-----------+----+   +-+--------------+
+      |  switch/       |   |  switch/       |
+      |  vlan segment  |   |  vlan segment  |
+      +-+---------+----+   +---+------------+
+        |         |            |
++-------+-------+ |OOB         | PUBLIC
+| PXE / dhcp    | |            |
+| Ser^er        | |            |
++---------------+ |            |
+                  |            |
+            +-----+------------+----------+
+            |                             |
+            |                             +--+
+            |                             |  |
+            |  NODES                      |  +--+
+            +--+--------------------------+  |  |
+               |                             |  |
+               +--+--------------------------+  |
+                  |                             |
+                  +-----------------------------+
+```
+
+The PXE/dhcp can also be done by the firewall, your mileage may vary.
+
+## Switch and firewall configs
+
+Single switch, multiple switch, it all boils down to the same:
+
+- one port is an access port on an OOB vlan/segment
+- one port is connected to a public vlan/segment
+
+The farmer makes sure that every node receives properly an IPv4 address in the OOB segment through means of dhcp, so 
+that with a PXE config or USB, a node can effectively start it's boot process:
+
+- Download kernel and initrd
+- Download and mount the system flists so that the 0-OS daemons can start
+- Register itself on the grid
+- Query the grid for tasks to execute
+
+For the PUBLIC side of the Nodes, there are a few things to consider:
+
+- It's the farmer's job to inform the grid what node gets an IP address, be it IPv4 or IPv4.
+- Nodes that don't receive and IPv4 address will connect to the IPv4 net through the NATed OOB network
+- A farmer is responsible to provide and IPv6 prefix on at least one segment, and have a Router Advertisement daemon 
+runnig to provide for SLAAC addressin on that segment.
+- That IPv6 Prefix on the public segment should not be firewalled, as it's impossible to know in your firewall what 
+ports will get exposed for the proxies.
+
+The Nodes themselves have nothing listening that points into the host OS itself, and are by themselves also firewalled. 
+In dev mode, there is an ssh server with a key-only login, accessible by a select few ;-)
+
+## DHCP/Radvd/RA/DHCP6
+
+For home networks, there is not much to do, a Node will get an IPv4 Private(rfc1918) address , and most probaly and 
+ipv6 address in a /64 prefix, but is not reachable over ipv6, unless the firewall is disabled for IPv6. As we can't 
+rely on the fact that that is possible, we assume these nodes to be HIDDEN.
+
+A normal self-respecting Firewall or IP-capable switch can hand out IP[46] addresses, some can 
+even bootp/tftp to get nodes booted over the network.
+We are (full of hope) assuming that you would have such a beast to configure and splice your network 
+in multiple segments.  
+A segment is a physical network separation. That can be port-based vlans, or even separate switches, whatver rocks your 
+boat, the keyword is here **separate**.
+
+On both segments you will need a way to hand out IPv4 addresses based on MAC addresses of the nodes. Yes, there is some 
+administration to do, but it's a one-off, and really necessary, because you really need to know whic physical machine 
+has which IP. For lights-out management and location of machines that is a must.
+
+So you'll need a list of mac addresses to add to your dhcp server for IPv4, to make sure you know which machine has 
+received what IPv4 Address.
+That is necessary for 2 things:
+
+- locate the node if something is amiss, like be able to pinpoint a node's disk in case it broke (which it will)
+- have the node be reachable all the time, without the need to update the grid and network configs every time the node 
+boots.
+
+## What happens under the hood (farmer)
+
+While we did our uttermost best to keep IPv4 address needs to a strict minimum, at least one Node will need an IPv4 address for handling everything that is Overlay Networks.  
+For Containers to reach the Internet, any type of connectivity will do, be it NAT or though an Internal DMZ that has a 
+routable IPv4 address.
+
+Internally, a lot of things are being set-up to have a node properly participate in the grid, as well to be prepared to partake in the User's Overlay Networks.
+
+A node connects itself to 'the Internet' depending on a few states.
+
+1. It lives in a fully private network (like it would be connected directly to a port on a home router)
+
+```
+      XX  XXX
+    XXX     XXXXXX
+   X  Internet   X
+   XXXXXXX   XXXXX
+       XX  XXX
+      XX   X
+         X+X
+          |
+          |
+ +--------+-----------+
+ | HOME /             |
+ | SOHO router        |
+ |                    |
+ +--------+-----------+
+          |
+          |  Private space IPv4
+          |  (192.168.1.0/24)
+          |
++---------+------------+
+|                      |
+|  NODE                |
+|                      |
+|                      |
+|                      |
+|                      |
+|                      |
++----------------------+
+```
+
+1. It lives in a fully public network (like it is connected directly to an uplink and has a public ipv4 address)
+
+```
+      XX  XXX
+    XXX     XXXXXX
+   X  Internet   X
+   XXXXXXX   XXXXX
+       XX  XXX
+      XX   X
+         X+X
+          |
+          | fully public space ipv4/6
+          | 185.69.166.0/24
+          | 2a02:1802:5e:0:1000::abcd/64
+          |
++---------+------------+
+|                      |
+|  NODE                |
+|                      |
++----------------------+
+
+```
+The node is fully reachable
+
+1. It lives in a datacentre, where a farmer manages the Network.
+
+A little Drawing :
+
+```text
++----------------------------------------------------+
+|  switch                                            |
+|                                                    |
+|                                                    |
++----------+-------------------------------------+---+
+           |                                     |
+  access   |                                     |
+  mgmt     |                     +---------------+
+  vlan     |                     | access
+           |                     | public
+           |                     | vlan
+           |                     |
+   +-------+---------------------+------+
+   |                                    |
+   |     nic1                  nic2     |
+   |                                    |
+   |                                    |
+   |                                    |
+   |   NODE                             |
+   |                                    |
+   |                                    |
+   |                                    |
+   +------------------------------------+
+
+```
+
+Or the more elaborate drawing on top that should be sufficient for a sysadmin to comprehend.
+
+Although:
+
+- we don't (yet) support nic bonding (next release)
+- we don't (yet) support vlans, so your ports on switch/router need to be access ports to vlans to your router/firewall
+
+
+## yeayea, but really ... what now ?
+
+Ok, what are the constraints?
+
+A little foreword:
+ZosV2 uses IPv6 as it's base for networking, where the oldie IPv4 is merely an afterthought. So for it to work properly in it's actual incantation (we are working to get it to do IPv4-only too), for now, we need the node to live in a space that provides IPv6 __too__ .  
+IPV4 and IPv6 are very different beasts, so any machine connected to the Internet wil do both on the same network. So basically your computer talks 2 different languages, when it comes to communicating. That is the same for ZOS, where right now, it's mother tongue is IPv6.
+
+So your zos for V2 can start in different settings
+1) you are a farmer, your ISP can provide you with IPv6
+Ok, you're all set, aside from a public IPv4 DHCP, you need to run a Stateless-Only SLAAC Router Advertiser (ZOS does NOT do DHCP6).
+
+1) you are a farmer, your ISP asks you what the hell IPv6 is
+That is problematic right now, wait for the next release of ZosV2
+
+1) you are a farmer, with only one node , at home, and on your PC https://ipv6.net tells you you have IPv6 on your PC.
+That means your home router received an IPV6 allocation from the ISP, 
+Your'e all set, your node will boot, and register to the grid. If you know what you're doing, you can configure your router to allow all ipv6 traffic in forwarding mode to the specifice mac address of your node. (we'll explain later)
+1) you are a farmer, with a few nodes somewhere that are registered on the grid in V1, but you have no clue if IPv6 is supported where these nodes live 
+1) you have a ThreefoldToken node at home, and still do not have a clue
+
+Basically it boils down also in a few other cases
+
+1) the physical network where a node lives has: IPv6 and Private space IPv4 
+1) the physical network where a node lives has: IPv6 and Public IPv4
+1) the physical network where a node lives has: only IPv4
+
+But it bloils down to : call your ISP, ask for IPv6. It's the future, for yout ISP, it's time. There is no way to circumvent it. No way.
+
+
+OK, then, now what.
+
+1) you're a farmer with a bunch of nodes somewhere in a DC
+
+  - your nodes are connected once (with one NIC) to a switch/router  
+  Then your router will have :
+    - a segment that carries IPv4 __and__ IPv6:
+
+    - for IPv4, there are 2 possibilities:
+      - it's RFC1918 (Private space) -> you NAT that subnet (e.g. 192.168.1.0/24) towards the Public Internet 
+      
+        - you __will__ have difficulty to designate a IPv4 public entrypoint into your farm
+        - your workloads will be only reachable through the overlay
+        - your storage will not be reachable
+        
+      - you received a (small, because of the scarceness of IPv4 addresses, your ISP will give you only limited and pricy IPv4 adresses) IPv4 range you can utilise
+
+        - things are better, the nodes can live in public ipv4 space, where they can be used as entrypoint
+        - standard configuration that works
+
+    - for IPv6, your router is a Routing advertiser that provides SLAAC (Stateless, unmanaged) for that segment, working witha /64 prefix
+
+      - the nodes will reachable over IPv6
+      - storage backend will be available for the full grid
+      - everything will just work
+      
+      Best solution for single NIC: 
+        - an ipv6 prefx
+        - an ipv4 subnet (however small)
+
+  - your nodes have 2 connections, and you wnat to differ management from user traffic
+  
+    - same applies as above, where the best outcome will be obtained with a real IPv6 prefix allocation and a small public subnet that is routable.
+      - the second NIC (typically 10GBit) will then carry everything public, and the first nic will just be there for managent, living in Private space for IPv4, mostly without IPv6 
+      - your switch needs to be configured to provide port-based vlans, so the segments are properly separated, and your router needs to reflect that vlan config so that separation is handeled by the firewall in the router (iptables, pf, acl, ...)
+
+
+  
+
+
diff --git a/pkg/app/flag.go b/pkg/app/flag.go
new file mode 100644
index 00000000..957e78b4
--- /dev/null
+++ b/pkg/app/flag.go
@@ -0,0 +1,53 @@
+package app
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	flagsDir = "/tmp/flags"
+	// LimitedCache represent the flag cache couldn't mount on ssd or hdd
+	LimitedCache = "limited-cache"
+)
+
+// SetFlag is used when the /var/cache cannot be mounted on a SSD or HDD,
+// it will mount the cache disk on a temporary file system in memory.
+func SetFlag(key string) error {
+	// creating a file that will be used as a flag
+	// the flag can 'warn' other deamons that the /var/cache is not on HDD or SDD
+	if err := os.MkdirAll(flagsDir, os.ModePerm); err != nil {
+		return errors.Wrap(err, "failed to create flags directory")
+	}
+
+	f, err := os.Create(filepath.Join(flagsDir, key))
+	if err != nil {
+		return errors.Wrap(err, "failed to create the flag file")
+	}
+
+	return f.Close()
+}
+
+// CheckFlag checks the status of a flag based on a key
+func CheckFlag(key string) bool {
+	_, err := os.Stat(filepath.Join(flagsDir, key))
+	return !os.IsNotExist(err)
+}
+
+// DeleteFlag deletes (unsets) a given flag based on a key
+func DeleteFlag(key string) error {
+	// to avoid "path injection"
+	path := filepath.Join(flagsDir, key)
+	if filepath.Dir(path) != flagsDir {
+		return fmt.Errorf("trying to delete a directory outside of the flags boundaries")
+	}
+
+	if err := os.RemoveAll(filepath.Join(flagsDir, key)); err != nil {
+		errors.Wrap(err, "failed to remove the flag file")
+		return err
+	}
+	return nil
+}
diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go
new file mode 100644
index 00000000..97e40ed3
--- /dev/null
+++ b/pkg/cache/cache.go
@@ -0,0 +1,36 @@
+package cache
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/threefoldtech/zos/pkg/storage"
+)
+
+// ForeverDir creates a new cache directory that is stored on the persistent cache.
+// This means data stored in this directory will survive reboot
+// It is the caller's responsibility to remove the directory when no longer needed.
+func ForeverDir(name string) (string, error) {
+	name = filepath.Join(storage.CacheTarget, "cache", name)
+	return name, os.MkdirAll(name, 0700)
+}
+
+// VolatileDir creates a new cache directory that is stored on a tmpfs.
+// This means data stored in this directory will NOT survive a reboot.
+// Use this when you need to store data that needs to survice deamon reboot but not between reboots
+// It is the caller's responsibility to remove the directory when no longer needed.
+// If the directory already exist, the the function does nothing and return successfully
+func VolatileDir(name string, size uint64) (string, error) {
+	const volatileBaseDir = "/var/run/cache"
+	name = filepath.Join(volatileBaseDir, name)
+	if err := os.MkdirAll(volatileBaseDir, 0700); err != nil {
+		return name, err
+	}
+
+	if err := os.Mkdir(name, 0700); err != nil {
+		return "", err
+	}
+	return name, syscall.Mount("", name, "tmpfs", 0, fmt.Sprintf("size=%d", size))
+}
diff --git a/pkg/capacity/store.go b/pkg/capacity/store.go
index a0f9e7d4..f56b9648 100644
--- a/pkg/capacity/store.go
+++ b/pkg/capacity/store.go
@@ -76,6 +76,8 @@ func (s *HTTPStore) Register(nodeID pkg.Identifier, c Capacity, d dmi.DMI, disks
 	if err != nil {
 		return err
 	}
+	defer resp.Body.Close()
+
 	if resp.StatusCode != http.StatusOK {
 		return fmt.Errorf("wrong response status code received: %v", resp.Status)
 	}
@@ -100,6 +102,8 @@ func (s *HTTPStore) Ping(nodeID pkg.Identifier, uptime uint64) error {
 	if err != nil {
 		return err
 	}
+	defer resp.Body.Close()
+
 	if resp.StatusCode != http.StatusOK {
 		return fmt.Errorf("wrong response status code received: %v", resp.Status)
 	}
diff --git a/pkg/container/startup.go b/pkg/container/startup.go
index fa85f45b..ddff1b96 100644
--- a/pkg/container/startup.go
+++ b/pkg/container/startup.go
@@ -17,14 +17,26 @@ type entry struct {
 type args struct {
 	Name string
 	Dir  string
+	Args []string
 	Env  map[string]string
 }
 
 func (e entry) Entrypoint() string {
 	if e.Name == "core.system" ||
 		e.Name == "core.base" && e.Args.Name != "" {
-		return e.Args.Name
+		var buf strings.Builder
+		buf.WriteString(e.Args.Name)
+		for _, arg := range e.Args.Args {
+			buf.WriteRune(' ')
+			arg = strings.Replace(arg, "\"", "\\\"", -1)
+			buf.WriteRune('"')
+			buf.WriteString(arg)
+			buf.WriteRune('"')
+		}
+
+		return buf.String()
 	}
+
 	return ""
 }
 
diff --git a/pkg/environment/environment.go b/pkg/environment/environment.go
index 8c540c21..4c530a85 100644
--- a/pkg/environment/environment.go
+++ b/pkg/environment/environment.go
@@ -63,7 +63,7 @@ var (
 
 	envTest = Environment{
 		RunningMode: RunningTest,
-		BcdbURL:     "tcp://explorer.testnet.grid.tf:8901",
+		BcdbURL:     "https://explorer.testnet.grid.tf/explorer",
 		// ProvisionTimeout:  120,
 		// ProvisionInterval: 10,
 	}
@@ -71,7 +71,7 @@ var (
 	// same as testnet for now. will be updated the day of the launch of production network
 	envProd = Environment{
 		RunningMode: RunningMain,
-		BcdbURL:     "tcp://explorer.grid.tf:8901",
+		BcdbURL:     "https://explorer.grid.tf/explorer",
 		// ProvisionTimeout:  240,
 		// ProvisionInterval: 20,
 	}
diff --git a/pkg/flist/flist.go b/pkg/flist/flist.go
index 94f4f591..d785b91a 100644
--- a/pkg/flist/flist.go
+++ b/pkg/flist/flist.go
@@ -349,6 +349,7 @@ func (f *flistModule) downloadFlist(url string) (string, error) {
 	if err != nil {
 		return "", err
 	}
+	defer resp.Body.Close()
 
 	if resp.StatusCode != 200 {
 		return "", fmt.Errorf("fail to download flist: %v", resp.Status)
diff --git a/pkg/gedis/converter.go b/pkg/gedis/converter.go
index d276aa06..fc3afa9f 100644
--- a/pkg/gedis/converter.go
+++ b/pkg/gedis/converter.go
@@ -103,7 +103,7 @@ func networkReservation(i interface{}) types.TfgridReservationNetwork1 {
 
 		for y, peer := range nr.Peers {
 			network.NetworkResources[i].Peers[y] = types.WireguardPeer1{
-				IPRange:    nr.Subnet.ToSchema(),
+				IPRange:    peer.Subnet.ToSchema(),
 				Endpoint:   peer.Endpoint,
 				PublicKey:  peer.WGPublicKey,
 				AllowedIPs: make([]string, len(peer.AllowedIPs)),
@@ -204,8 +204,8 @@ func k8sReservation(i interface{}, nodeID string) types.TfgridWorkloadsReservati
 		NetworkID:     string(k.NetworkID),
 		Ipaddress:     k.IP,
 		ClusterSecret: k.ClusterSecret,
-		MasterIps:     make([]net.IP, 0, len(k.MasterIPs)),
-		SSHKeys:       make([]string, 0, len(k.SSHKeys)),
+		MasterIps:     make([]net.IP, len(k.MasterIPs)),
+		SSHKeys:       make([]string, len(k.SSHKeys)),
 	}
 
 	copy(k8s.MasterIps, k.MasterIPs)
diff --git a/pkg/gedis/gedis.go b/pkg/gedis/gedis.go
index fa9a9ce1..a9dfda20 100644
--- a/pkg/gedis/gedis.go
+++ b/pkg/gedis/gedis.go
@@ -122,7 +122,8 @@ func newRedisPool(address string) (*redis.Pool, error) {
 
 			return nil
 		},
-		MaxActive:   10,
+		MaxActive:   5,
+		MaxIdle:     5,
 		IdleTimeout: 1 * time.Minute,
 		Wait:        true,
 	}, nil
diff --git a/pkg/gedis/types/provision/tfgrid_reservation_1.go b/pkg/gedis/types/provision/tfgrid_reservation_1.go
index 30d7556d..78bf3a38 100644
--- a/pkg/gedis/types/provision/tfgrid_reservation_1.go
+++ b/pkg/gedis/types/provision/tfgrid_reservation_1.go
@@ -1,15 +1,13 @@
 package provision
 
 import (
-	"encoding/json"
-
 	schema "github.com/threefoldtech/zos/pkg/schema"
 )
 
 // TfgridReservation1 jsx schema
 type TfgridReservation1 struct {
 	ID                  uint                                 `json:"id"`
-	JSON                json.RawMessage                      `json:"json"`
+	JSON                string                               `json:"json"`
 	DataReservation     TfgridReservationData1               `json:"data_reservation"`
 	CustomerTid         int64                                `json:"customer_tid"`
 	CustomerSignature   string                               `json:"customer_signature"`
diff --git a/pkg/identity/store.go b/pkg/identity/store.go
index c3b06309..4775078f 100644
--- a/pkg/identity/store.go
+++ b/pkg/identity/store.go
@@ -2,11 +2,14 @@ package identity
 
 import (
 	"bytes"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 
+	"github.com/jbenet/go-base58"
+
 	"github.com/threefoldtech/zos/pkg/gedis/types/directory"
 
 	"github.com/threefoldtech/zos/pkg/geoip"
@@ -33,6 +36,9 @@ func NewHTTPIDStore(url string) IDStore {
 // RegisterNode implements the IDStore interface
 func (s *httpIDStore) RegisterNode(node pkg.Identifier, farm pkg.FarmID, version string, loc geoip.Location) (string, error) {
 	buf := bytes.Buffer{}
+
+	pk := base58.Decode(node.Identity())
+
 	err := json.NewEncoder(&buf).Encode(directory.TfgridNode2{
 		NodeID:    node.Identity(),
 		FarmID:    uint64(farm),
@@ -44,6 +50,7 @@ func (s *httpIDStore) RegisterNode(node pkg.Identifier, farm pkg.FarmID, version
 			Latitude:  loc.Latitude,
 			Longitude: loc.Longitute,
 		},
+		PublicKeyHex: hex.EncodeToString(pk),
 	})
 	if err != nil {
 		return "", err
@@ -53,6 +60,8 @@ func (s *httpIDStore) RegisterNode(node pkg.Identifier, farm pkg.FarmID, version
 	if err != nil {
 		return "", err
 	}
+	defer resp.Body.Close()
+
 	if resp.StatusCode != http.StatusCreated {
 		return "", fmt.Errorf("wrong response status code received: %v", resp.Status)
 	}
diff --git a/pkg/network/ndmz/ipam.go b/pkg/network/ndmz/ipam.go
index cce933a4..4f8692a1 100644
--- a/pkg/network/ndmz/ipam.go
+++ b/pkg/network/ndmz/ipam.go
@@ -10,9 +10,8 @@ import (
 
 // allocateIPv4 allocates a unique IPv4 for the entity defines by the given id (for example container id, or a vm).
 // in the network with netID, and NetResource.
-func allocateIPv4(networkID string) (*net.IPNet, error) {
-	// FIXME: path to the cache disk shouldn't be hardcoded here
-	store, err := disk.New("ndmz", ipamPath)
+func allocateIPv4(networkID, leaseDir string) (*net.IPNet, error) {
+	store, err := disk.New("ndmz", leaseDir)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/network/ndmz/ndmz.go b/pkg/network/ndmz/ndmz.go
index 31b9b93a..c9bbdef3 100644
--- a/pkg/network/ndmz/ndmz.go
+++ b/pkg/network/ndmz/ndmz.go
@@ -5,13 +5,10 @@ import (
 	"fmt"
 	"net"
 	"os"
-	"path/filepath"
 	"time"
 
 	"github.com/cenkalti/backoff/v3"
-	"github.com/termie/go-shutil"
 	"github.com/threefoldtech/zos/pkg"
-	"github.com/threefoldtech/zos/pkg/app"
 	"github.com/threefoldtech/zos/pkg/network/ifaceutil"
 	"github.com/threefoldtech/zos/pkg/network/types"
 
@@ -51,47 +48,8 @@ const (
 	nrPubIface = "public"
 )
 
-var ipamPath = "/var/cache/modules/networkd/lease"
-
 //Create create the NDMZ network namespace and configure its default routes and addresses
 func Create(nodeID pkg.Identifier) error {
-	path := filepath.Join(ipamPath, "ndmz")
-
-	if app.IsFirstBoot("networkd-dmz") {
-		log.Info().Msg("first boot, empty reservation cache")
-
-		if err := os.RemoveAll(path); err != nil {
-			return err
-		}
-
-		// TODO @zaibon: remove once all the network has applies this fix
-
-		// This check ensure we do not delete current lease from the node
-		// running a previous version of this code.
-		// if the old leases directory exists, we copy the content
-		// into the new location and delete the old directory
-		var err error
-		oldFolder := filepath.Join(ipamPath, "dmz")
-		if _, err = os.Stat(oldFolder); err == nil {
-			if err := shutil.CopyTree(oldFolder, path, nil); err != nil {
-				return err
-			}
-			err = os.RemoveAll(oldFolder)
-		} else {
-			err = os.MkdirAll(path, 0770)
-		}
-		if err != nil {
-			return err
-		}
-
-		if err := app.MarkBooted("networkd-dmz"); err != nil {
-			return errors.Wrap(err, "fail to mark provisiond as booted")
-		}
-
-	} else {
-		log.Info().Msg("restart detected, keep IPAM lease cache intact")
-	}
-
 	netNS, err := namespace.GetByName(NetNSNDMZ)
 	if err != nil {
 		netNS, err = namespace.Create(NetNSNDMZ)
@@ -374,7 +332,7 @@ func applyFirewall() error {
 }
 
 // AttachNR links a network resource to the NDMZ
-func AttachNR(networkID string, nr *nr.NetResource) error {
+func AttachNR(networkID string, nr *nr.NetResource, ipamLeaseDir string) error {
 	nrNSName, err := nr.Namespace()
 	if err != nil {
 		return err
@@ -392,7 +350,7 @@ func AttachNR(networkID string, nr *nr.NetResource) error {
 	}
 
 	return nrNS.Do(func(_ ns.NetNS) error {
-		addr, err := allocateIPv4(networkID)
+		addr, err := allocateIPv4(networkID, ipamLeaseDir)
 		if err != nil {
 			return errors.Wrap(err, "ip allocation for network resource")
 		}
diff --git a/pkg/network/networker.go b/pkg/network/networker.go
index 29e9a7b5..f33ec23b 100644
--- a/pkg/network/networker.go
+++ b/pkg/network/networker.go
@@ -5,11 +5,15 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"os"
 	"path/filepath"
 	"time"
 
+	"github.com/termie/go-shutil"
+
+	"github.com/threefoldtech/zos/pkg/cache"
 	"github.com/threefoldtech/zos/pkg/network/ndmz"
 	"github.com/threefoldtech/zos/pkg/network/tuntap"
 
@@ -35,26 +39,102 @@ import (
 
 const (
 	// ZDBIface is the name of the interface used in the 0-db network namespace
-	ZDBIface = "zdb0"
+	ZDBIface     = "zdb0"
+	wgPortDir    = "wireguard_ports"
+	networkDir   = "networks"
+	ipamLeaseDir = "ndmz-lease"
+	ipamPath     = "/var/cache/modules/networkd/lease"
+)
+
+const (
+	mib = 1024 * 1024
 )
 
 type networker struct {
-	identity   pkg.IdentityManager
-	storageDir string
-	tnodb      TNoDB
-	portSet    *set.UintSet
+	identity     pkg.IdentityManager
+	networkDir   string
+	ipamLeaseDir string
+	tnodb        TNoDB
+	portSet      *set.UintSet
 }
 
 // NewNetworker create a new pkg.Networker that can be used over zbus
-func NewNetworker(identity pkg.IdentityManager, tnodb TNoDB, storageDir string) pkg.Networker {
+func NewNetworker(identity pkg.IdentityManager, tnodb TNoDB, storageDir string) (pkg.Networker, error) {
+
+	vd, err := cache.VolatileDir("networkd", 50*mib)
+	if err != nil && !os.IsExist(err) {
+		return nil, fmt.Errorf("failed to create networkd cache directory: %w", err)
+	}
+
+	wgDir := filepath.Join(vd, wgPortDir)
+	if err := os.MkdirAll(wgDir, 0700); err != nil {
+		return nil, fmt.Errorf("failed to create wireguard port cache directory: %w", err)
+	}
+
+	nwDir := filepath.Join(vd, networkDir)
+	ipamLease := filepath.Join(vd, ipamLeaseDir)
+
+	oldPath := filepath.Join(ipamPath, "ndmz")
+	newPath := filepath.Join(ipamLease, "ndmz")
+	if _, err := os.Stat(oldPath); err == nil {
+		if err := shutil.CopyTree(oldPath, newPath, nil); err != nil {
+			return nil, err
+		}
+		_ = os.RemoveAll(ipamPath)
+	}
+
+	//TODO: remove once all the node have move the network into the volatile directory
+	if _, err = os.Stat(storageDir); err == nil {
+		if err := copyNetworksToVolatile(storageDir, nwDir); err != nil {
+			return nil, fmt.Errorf("failed to copy old networks directory: %w", err)
+		}
+	}
+
 	nw := &networker{
-		identity:   identity,
-		storageDir: storageDir,
-		tnodb:      tnodb,
-		portSet:    set.NewUint(),
+		identity:     identity,
+		tnodb:        tnodb,
+		networkDir:   nwDir,
+		ipamLeaseDir: ipamLease,
+		portSet:      set.NewUint(wgDir),
 	}
 
-	return nw
+	return nw, nil
+}
+
+func copyNetworksToVolatile(src, dst string) error {
+	log.Info().Msg("move network cached file to volatile directory")
+	if err := os.MkdirAll(dst, 0700); err != nil {
+		return err
+	}
+
+	infos, err := ioutil.ReadDir(src)
+	if err != nil {
+		return err
+	}
+
+	copy := func(src string, dst string) error {
+		log.Info().Str("source", src).Str("dst", dst).Msg("copy file")
+		// Read all content of src to data
+		data, err := ioutil.ReadFile(src)
+		if err != nil {
+			return err
+		}
+		// Write data to dst
+		return ioutil.WriteFile(dst, data, 0644)
+	}
+
+	for _, info := range infos {
+		if info.IsDir() {
+			continue
+		}
+
+		s := filepath.Join(src, info.Name())
+		d := filepath.Join(dst, info.Name())
+		if err := copy(s, d); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 var _ pkg.Networker = (*networker)(nil)
@@ -387,6 +467,12 @@ func (n networker) Addrs(iface string, netns string) ([]net.IP, error) {
 
 // CreateNR implements pkg.Networker interface
 func (n *networker) CreateNR(network pkg.Network) (string, error) {
+	defer func() {
+		if err := n.publishWGPorts(); err != nil {
+			log.Warn().Err(err).Msg("failed to publish wireguard port to BCDB")
+		}
+	}()
+
 	var err error
 	var nodeID = n.identity.NodeID().Identity()
 
@@ -416,7 +502,6 @@ func (n *networker) CreateNR(network pkg.Network) (string, error) {
 	// check if there is a reserved wireguard port for this NR already
 	// or if we need to update it
 	storedNet, err := n.networkOf(string(network.NetID))
-
 	if err != nil && !os.IsNotExist(err) {
 		return "", err
 	}
@@ -426,17 +511,15 @@ func (n *networker) CreateNR(network pkg.Network) (string, error) {
 		if err != nil {
 			return "", err
 		}
-		if netNR.WGListenPort != storedNR.WGListenPort {
-			if err := n.releasePort(storedNR.WGListenPort); err != nil {
-				return "", err
-			}
-		}
-	} else if os.IsNotExist(err) {
-		if err := n.reservePort(netNR.WGListenPort); err != nil {
+		if err := n.releasePort(storedNR.WGListenPort); err != nil {
 			return "", err
 		}
 	}
 
+	if err := n.reservePort(netNR.WGListenPort); err != nil {
+		return "", err
+	}
+
 	netr, err := nr.New(network.NetID, netNR, &network.IPRange.IPNet)
 	if err != nil {
 		return "", err
@@ -461,7 +544,7 @@ func (n *networker) CreateNR(network pkg.Network) (string, error) {
 		return "", errors.Wrap(err, "failed to create network resource")
 	}
 
-	if err := ndmz.AttachNR(string(network.NetID), netr); err != nil {
+	if err := ndmz.AttachNR(string(network.NetID), netr, n.ipamLeaseDir); err != nil {
 		return "", errors.Wrapf(err, "failed to attach network resource to DMZ bridge")
 	}
 
@@ -471,7 +554,7 @@ func (n *networker) CreateNR(network pkg.Network) (string, error) {
 	}
 
 	// map the network ID to the network namespace
-	path := filepath.Join(n.storageDir, string(network.NetID))
+	path := filepath.Join(n.networkDir, string(network.NetID))
 	file, err := os.Create(path)
 	if err != nil {
 		cleanup()
@@ -494,7 +577,7 @@ func (n *networker) CreateNR(network pkg.Network) (string, error) {
 }
 
 func (n *networker) networkOf(id string) (*pkg.Network, error) {
-	path := filepath.Join(n.storageDir, string(id))
+	path := filepath.Join(n.networkDir, string(id))
 	file, err := os.Open(path)
 	if err != nil {
 		return nil, err
@@ -530,6 +613,12 @@ func (n *networker) networkOf(id string) (*pkg.Network, error) {
 
 // DeleteNR implements pkg.Networker interface
 func (n *networker) DeleteNR(network pkg.Network) error {
+	defer func() {
+		if err := n.publishWGPorts(); err != nil {
+			log.Warn().Msg("failed to publish wireguard port to BCDB")
+		}
+	}()
+
 	netNR, err := ResourceByNodeID(n.identity.NodeID().Identity(), network.NetResources)
 	if err != nil {
 		return err
@@ -550,7 +639,7 @@ func (n *networker) DeleteNR(network pkg.Network) error {
 	}
 
 	// map the network ID to the network namespace
-	path := filepath.Join(n.storageDir, string(network.NetID))
+	path := filepath.Join(n.networkDir, string(network.NetID))
 	if err := os.Remove(path); err != nil {
 		log.Error().Err(err).Msg("failed to remove file mapping between network ID and namespace")
 	}
@@ -575,28 +664,32 @@ func (n *networker) extractPrivateKey(hexKey string) (string, error) {
 }
 
 func (n *networker) reservePort(port uint16) error {
+	log.Debug().Uint16("port", port).Msg("reserve wireguard port")
 	err := n.portSet.Add(uint(port))
 	if err != nil {
 		return errors.Wrap(err, "wireguard listen port already in use, pick another one")
 	}
 
-	if err := n.tnodb.PublishWGPort(n.identity.NodeID(), n.portSet.List()); err != nil {
-		n.portSet.Remove(uint(port))
-		return errors.Wrap(err, "fail to publish wireguard port to bcdb")
-	}
-
 	return nil
 }
 
 func (n *networker) releasePort(port uint16) error {
+	log.Debug().Uint16("port", port).Msg("release wireguard port")
 	n.portSet.Remove(uint(port))
+	return nil
+}
 
-	if err := n.tnodb.PublishWGPort(n.identity.NodeID(), n.portSet.List()); err != nil {
+func (n *networker) publishWGPorts() error {
+	ports, err := n.portSet.List()
+	if err != nil {
+		return err
+	}
+
+	if err := n.tnodb.PublishWGPort(n.identity.NodeID(), ports); err != nil {
 		// maybe retry a couple of times ?
 		// having bdb and the node out of sync is pretty bad
 		return errors.Wrap(err, "fail to publish wireguard port to bcdb")
 	}
-
 	return nil
 }
 
diff --git a/pkg/provision/local_store.go b/pkg/provision/local_store.go
index 889617e5..c1a951e8 100644
--- a/pkg/provision/local_store.go
+++ b/pkg/provision/local_store.go
@@ -109,7 +109,15 @@ func NewFSStore(root string) (*FSStore, error) {
 }
 
 func (s *FSStore) removeAllButPersistent(rootPath string) error {
-	err := filepath.Walk(rootPath, func(path string, info os.FileInfo, r error) error {
+	// if rootPath is not present on the filesystem, return
+	_, err := os.Stat(rootPath)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	err = filepath.Walk(rootPath, func(path string, info os.FileInfo, r error) error {
 		if r != nil {
 			return r
 		}
diff --git a/pkg/provision/provision.go b/pkg/provision/provision.go
index 87d37dd2..f9236e71 100644
--- a/pkg/provision/provision.go
+++ b/pkg/provision/provision.go
@@ -68,6 +68,9 @@ func NewErrTemporary(err error) error {
 
 // Error implements the errors.Error interface
 func (e ErrTemporary) Error() string {
+	if e.err == nil {
+		return ""
+	}
 	return e.err.Error()
 }
 
diff --git a/pkg/provision/remote_store.go b/pkg/provision/remote_store.go
index cd72e5e3..5f0af0e3 100644
--- a/pkg/provision/remote_store.go
+++ b/pkg/provision/remote_store.go
@@ -12,6 +12,7 @@ import (
 	"github.com/rs/zerolog/log"
 	"github.com/threefoldtech/zos/pkg"
 	generated "github.com/threefoldtech/zos/pkg/gedis/types/provision"
+	"github.com/threefoldtech/zos/pkg/schema"
 )
 
 // HTTPStore is a reservation store
@@ -130,9 +131,31 @@ func (s *HTTPStore) Get(id string) (*Reservation, error) {
 func (s *HTTPStore) Feedback(nodeID string, r *Result) error {
 	url := fmt.Sprintf("%s/reservations/workloads/%s/%s", s.baseURL, r.ID, nodeID)
 
+	var rType generated.TfgridReservationResult1CategoryEnum
+	switch r.Type {
+	case VolumeReservation:
+		rType = generated.TfgridReservationResult1CategoryVolume
+	case ContainerReservation:
+		rType = generated.TfgridReservationResult1CategoryContainer
+	case ZDBReservation:
+		rType = generated.TfgridReservationResult1CategoryZdb
+	case NetworkReservation:
+		rType = generated.TfgridReservationResult1CategoryNetwork
+	}
+
+	result := generated.TfgridReservationResult1{
+		Category:   rType,
+		WorkloadID: r.ID,
+		DataJSON:   r.Data,
+		Signature:  r.Signature,
+		State:      generated.TfgridReservationResult1StateEnum(r.State),
+		Message:    r.Error,
+		Epoch:      schema.Date{Time: r.Created},
+	}
+
 	buf := &bytes.Buffer{}
 
-	if err := json.NewEncoder(buf).Encode(r); err != nil {
+	if err := json.NewEncoder(buf).Encode(result); err != nil {
 		return err
 	}
 
diff --git a/pkg/schema/types.go b/pkg/schema/types.go
index e738bf02..0744823a 100644
--- a/pkg/schema/types.go
+++ b/pkg/schema/types.go
@@ -222,7 +222,6 @@ func (e *Email) UnmarshalText(text []byte) error {
 		return fmt.Errorf("invalid email address: %s", v)
 	}
 
-	fmt.Println("setting email to:", v)
 	*e = Email(v)
 	return nil
 }
diff --git a/pkg/set/uint.go b/pkg/set/uint.go
index 872752e3..4aa6943a 100644
--- a/pkg/set/uint.go
+++ b/pkg/set/uint.go
@@ -1,8 +1,15 @@
 package set
 
 import (
+	"errors"
 	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
 	"sync"
+
+	"github.com/rs/zerolog/log"
 )
 
 // ErrConflict is return when trying to add a port
@@ -18,27 +25,37 @@ func (e ErrConflict) Error() string {
 // UintSet is a set containing uint
 type UintSet struct {
 	sync.RWMutex
-	m map[uint]struct{}
+	root string
 }
 
 // NewUint creates a new set for uint
-func NewUint() *UintSet {
+// path if the directory where to store the set on disk
+// the directory pointed by path must exists already
+func NewUint(path string) *UintSet {
 	return &UintSet{
-		m: make(map[uint]struct{}),
+		root: path,
 	}
 }
 
+func (p *UintSet) path(i uint) string {
+	return filepath.Join(p.root, fmt.Sprintf("%d", i))
+}
+
 // Add tries to add port to the set. If port is already
 // present errPortConflict is return otherwise nil is returned
 func (p *UintSet) Add(i uint) error {
 	p.Lock()
 	defer p.Unlock()
 
-	_, exists := p.m[i]
-	if exists {
-		return ErrConflict{i}
+	f, err := os.OpenFile(p.path(i), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0440)
+	if err != nil {
+		if errors.Is(err, os.ErrExist) {
+			return ErrConflict{Port: i}
+		}
+		return err
 	}
-	p.m[i] = struct{}{}
+
+	f.Close()
 	return nil
 }
 
@@ -49,20 +66,28 @@ func (p *UintSet) Remove(i uint) {
 	p.Lock()
 	defer p.Unlock()
 
-	_, exists := p.m[i]
-	if exists {
-		delete(p.m, i)
-	}
+	_ = os.RemoveAll(p.path(i))
 }
 
 // List returns a list of uint present in the set
-func (p *UintSet) List() []uint {
+func (p *UintSet) List() ([]uint, error) {
 	p.RLock()
 	defer p.RUnlock()
 
-	l := make([]uint, 0, len(p.m))
-	for i := range p.m {
-		l = append(l, i)
+	infos, err := ioutil.ReadDir(p.root)
+	if err != nil {
+		return nil, err
+	}
+	l := make([]uint, 0, len(infos))
+
+	for _, info := range infos {
+		s := filepath.Base(info.Name())
+		i, err := strconv.Atoi(s)
+		if err != nil {
+			log.Warn().Err(err).Msg("file with wrong formatted name found in reserved wireguard port cache")
+			continue
+		}
+		l = append(l, uint(i))
 	}
-	return l
+	return l, nil
 }
diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go
index 930eab3c..3413b6d1 100644
--- a/pkg/storage/storage.go
+++ b/pkg/storage/storage.go
@@ -7,6 +7,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/google/uuid"
@@ -14,11 +15,13 @@ import (
 	log "github.com/rs/zerolog/log"
 	"github.com/shirou/gopsutil/disk"
 	"github.com/threefoldtech/zos/pkg"
+	"github.com/threefoldtech/zos/pkg/app"
 	"github.com/threefoldtech/zos/pkg/storage/filesystem"
 )
 
 const (
-	cacheTarget = "/var/cache"
+	// CacheTarget is the path where the cache disk is mounted
+	CacheTarget = "/var/cache"
 	cacheLabel  = "zos-cache"
 	cacheSize   = 20 * 1024 * 1024 * 1024 // 20GB
 )
@@ -364,7 +367,12 @@ func (s *storageModule) ensureCache() error {
 
 	var cacheFs filesystem.Volume
 
-	// check if we already have a cache
+	if filesystem.IsMountPoint(CacheTarget) {
+		log.Debug().Msgf("Cache partition already mounted in %s", CacheTarget)
+		return nil
+	}
+
+	// check if cache volume available
 	for idx := range s.volumes {
 		filesystems, err := s.volumes[idx].Volumes()
 		if err != nil {
@@ -385,23 +393,41 @@ func (s *storageModule) ensureCache() error {
 	if cacheFs == nil {
 		log.Debug().Msgf("No cache found, try to create new cache")
 
+		log.Debug().Msgf("Trying to create new cache on SSD")
 		fs, err := s.createSubvol(cacheSize, cacheLabel, pkg.SSDDevice)
-		if errors.Is(err, pkg.ErrNotEnoughSpace{}) {
-			// No space on SSD (probably no SSD in the node at all), try HDD
-			fs, err = s.createSubvol(cacheSize, cacheLabel, pkg.HDDDevice)
+
+		if err != nil {
+			log.Warn().Err(err).Msg("failed to create new cache on SSD")
+		} else {
+			cacheFs = fs
 		}
+	}
+
+	if cacheFs == nil {
+		log.Debug().Msgf("Trying to create new cache on HDD")
+		fs, err := s.createSubvol(cacheSize, cacheLabel, pkg.HDDDevice)
+
 		if err != nil {
-			return err
+			log.Warn().Err(err).Msg("failed to create new cache on HDD")
+		} else {
+			cacheFs = fs
 		}
-		cacheFs = fs
 	}
 
-	if !filesystem.IsMountPoint(cacheTarget) {
-		log.Debug().Msgf("Mounting cache partition in %s", cacheTarget)
-		return filesystem.BindMount(cacheFs, cacheTarget)
+	if cacheFs == nil {
+		log.Warn().Msg("failed to create persisted cache disk. Running on limited cache")
+
+		// set limited cache flag
+		if err := app.SetFlag("limited-cache"); err != nil {
+			return err
+		}
+
+		// when everything failed, mount the Tmpfs
+		return syscall.Mount("", "/var/cache", "tmpfs", 0, "size=500M")
 	}
-	log.Debug().Msgf("Cache partition already mounted in %s", cacheTarget)
-	return nil
+
+	log.Debug().Msgf("Mounting cache partition in %s", CacheTarget)
+	return filesystem.BindMount(cacheFs, CacheTarget)
 }
 
 // createSubvol creates a subvolume with the given name and limits it to the given size
diff --git a/pkg/upgrade/hub.go b/pkg/upgrade/hub.go
index 1e795395..1d127a63 100644
--- a/pkg/upgrade/hub.go
+++ b/pkg/upgrade/hub.go
@@ -61,6 +61,7 @@ func (h *hubClient) Info(flist string) (info flistInfo, err error) {
 	if response.StatusCode != http.StatusOK {
 		return info, fmt.Errorf("failed to get flist info: %s", response.Status)
 	}
+	defer response.Body.Close()
 
 	dec := json.NewDecoder(response.Body)
 
diff --git a/tools/bcdb_mock/main.go b/tools/bcdb_mock/main.go
index 23d6826f..54598edd 100644
--- a/tools/bcdb_mock/main.go
+++ b/tools/bcdb_mock/main.go
@@ -61,7 +61,7 @@ func main() {
 
 	log.Printf("start on %s\n", listen)
 	r := handlers.LoggingHandler(os.Stderr, router)
-	r = handlers.CORS()(r)
+	r = handlers.CORS(handlers.AllowedOrigins([]string{"*"}))(r)
 
 	s := &http.Server{
 		Addr:    listen,
diff --git a/tools/bcdb_mock/migration/convert.py b/tools/bcdb_mock/migration/convert.py
new file mode 100644
index 00000000..9389063f
--- /dev/null
+++ b/tools/bcdb_mock/migration/convert.py
@@ -0,0 +1,28 @@
+from yaml import load, CLoader
+import json
+import sys
+import os
+
+
+def main(args):
+    if len(args) == 0:
+        raise Exception("required file name")
+    name = args[0]
+    input = open(name)
+
+    data = load(input, CLoader)
+    input.close()
+
+    output_name = "%s.json" % name
+    output = open(output_name, 'w')
+    try:
+        json.dump(data, output)
+    except Exception as e:
+        os.remove(output_name)
+        raise e
+    finally:
+        output.close()
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])
diff --git a/tools/bcdb_mock/migration/migration.go b/tools/bcdb_mock/migration/migration.go
new file mode 100644
index 00000000..4a823858
--- /dev/null
+++ b/tools/bcdb_mock/migration/migration.go
@@ -0,0 +1,171 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+	"github.com/threefoldtech/zos/pkg/app"
+	"github.com/threefoldtech/zos/tools/bcdb_mock/mw"
+
+	"go.mongodb.org/mongo-driver/mongo"
+
+	directory "github.com/threefoldtech/zos/tools/bcdb_mock/pkg/directory/types"
+	phonebook "github.com/threefoldtech/zos/tools/bcdb_mock/pkg/phonebook/types"
+	workloads "github.com/threefoldtech/zos/tools/bcdb_mock/pkg/workloads/types"
+)
+
+func foreach(root string, f func(p string, r io.Reader) error) error {
+	files, err := ioutil.ReadDir(root)
+	if err != nil {
+		return err
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			continue
+		}
+
+		if filepath.Ext(file.Name()) != ".json" {
+			continue
+		}
+
+		p := filepath.Join(root, file.Name())
+		fd, err := os.Open(p)
+		if err != nil {
+			return err
+		}
+
+		if err := f(p, fd); err != nil {
+			fd.Close()
+			return err
+		}
+
+		fd.Close()
+	}
+
+	return nil
+}
+
+// Migrator callback
+type Migrator func(root string, db *mongo.Database) error
+
+func migrateFarms(root string, db *mongo.Database) error {
+	col := db.Collection(directory.FarmCollection)
+	return foreach(root, func(p string, r io.Reader) error {
+		var farm directory.Farm
+		if err := json.NewDecoder(r).Decode(&farm); err != nil {
+			return errors.Wrapf(err, "failed to load file '%s'", p)
+		}
+
+		_, err := col.InsertOne(context.TODO(), farm)
+		if err != nil {
+			log.Error().Err(err).Msgf("failed to insert farm '%s'", p)
+		}
+
+		return nil
+	})
+}
+
+func migrateNodes(root string, db *mongo.Database) error {
+	col := db.Collection(directory.NodeCollection)
+	return foreach(root, func(p string, r io.Reader) error {
+		var node directory.Node
+		if err := json.NewDecoder(r).Decode(&node); err != nil {
+			return errors.Wrapf(err, "failed to load file '%s'", p)
+		}
+
+		if err := node.Validate(); err != nil {
+			return errors.Wrapf(err, "file '%s'", p)
+		}
+
+		_, err := col.InsertOne(context.TODO(), node)
+		if err != nil {
+			log.Error().Err(err).Msgf("failed to insert node '%s'", p)
+		}
+
+		return nil
+	})
+}
+
+func migrateUsers(root string, db *mongo.Database) error {
+	col := db.Collection(phonebook.UserCollection)
+	return foreach(root, func(p string, r io.Reader) error {
+		var user phonebook.User
+		if err := json.NewDecoder(r).Decode(&user); err != nil {
+			return errors.Wrapf(err, "failed to load file '%s'", p)
+		}
+
+		_, err := col.InsertOne(context.TODO(), user)
+		if err != nil {
+			log.Error().Err(err).Msgf("failed to insert user '%s'", p)
+		}
+
+		return nil
+	})
+}
+
+func migrateReservations(root string, db *mongo.Database) error {
+	col := db.Collection(workloads.ReservationCollection)
+	return foreach(root, func(p string, r io.Reader) error {
+		var reservation workloads.Reservation
+		if err := json.NewDecoder(r).Decode(&reservation); err != nil {
+			return errors.Wrapf(err, "failed to load file '%s'", p)
+		}
+
+		_, err := col.InsertOne(context.TODO(), reservation)
+		if err != nil {
+			log.Error().Err(err).Msgf("failed to insert reservation '%s'", p)
+		}
+
+		return nil
+	})
+}
+
+func main() {
+	app.Initialize()
+
+	var (
+		root   string
+		dbConf string
+		name   string
+	)
+
+	flag.StringVar(&dbConf, "mongo", "mongodb://localhost:27017", "connection string to mongo database")
+	flag.StringVar(&name, "name", "explorer", "database name")
+	flag.StringVar(&root, "root", "", "root directory of the bcdb exported data")
+
+	flag.Parse()
+	if strings.EqualFold(root, "") {
+		log.Fatal().Msg("root option is required")
+	}
+
+	db, err := mw.NewDatabaseMiddleware(name, dbConf)
+	if err != nil {
+		log.Fatal().Err(err).Msg("failed to connect to database")
+	}
+
+	if err := directory.Setup(context.TODO(), db.Database()); err != nil {
+		log.Fatal().Err(err).Msg("failed to setup directory indexes")
+	}
+
+	types := map[string]Migrator{
+		"tfgrid_directory/tfgrid.directory.farm.1/yaml":        migrateFarms,
+		"tfgrid_directory/tfgrid.directory.node.2/yaml":        migrateNodes,
+		"phonebook/tfgrid.phonebook.user.1/yaml":               migrateUsers,
+		"tfgrid_workloads/tfgrid.workloads.reservation.1/yaml": migrateReservations,
+	}
+
+	for typ, migrator := range types {
+		if err := migrator(filepath.Join(root, typ), db.Database()); err != nil {
+			log.Error().Err(err).Str("root", typ).Msg("migration failed")
+		}
+	}
+}
diff --git a/tools/bcdb_mock/migration/readme.me b/tools/bcdb_mock/migration/readme.me
new file mode 100644
index 00000000..f0a6dc52
--- /dev/null
+++ b/tools/bcdb_mock/migration/readme.me
@@ -0,0 +1,15 @@
+## Migration
+this migration tool accepts the path to a bcdb exported data directory and iterate over all structures that we need
+to import in the database.
+
+> *NOTE:* You need to convert all the data from yaml to json data. We don't support importing the yaml data. to make this easy
+please use the `convert.py` util as follows
+
+```bash
+find ~/bcdb_exports/18_Mar_2020_11_48_38/ -name '*.yaml'  -print -exec python convert.py {} \;
+```
+
+Then run migration like
+```bash
+./migration -name explorer -root ~/tmp/bcdb_exports/18_Mar_2020_11_48_38
+```
diff --git a/tools/bcdb_mock/models/generated/workloads/workloads.go b/tools/bcdb_mock/models/generated/workloads/workloads.go
index 83457705..7691b209 100644
--- a/tools/bcdb_mock/models/generated/workloads/workloads.go
+++ b/tools/bcdb_mock/models/generated/workloads/workloads.go
@@ -9,7 +9,7 @@ import (
 
 type TfgridWorkloadsReservation1 struct {
 	ID                  schema.ID                                     `bson:"_id" json:"id"`
-	Json                json.RawMessage                               `bson:"json" json:"json"`
+	Json                string                                        `bson:"json" json:"json"`
 	DataReservation     TfgridWorkloadsReservationData1               `bson:"data_reservation" json:"data_reservation"`
 	CustomerTid         int64                                         `bson:"customer_tid" json:"customer_tid"`
 	CustomerSignature   string                                        `bson:"customer_signature" json:"customer_signature"`
diff --git a/tools/bcdb_mock/models/utils.go b/tools/bcdb_mock/models/utils.go
index b670468a..35146c53 100644
--- a/tools/bcdb_mock/models/utils.go
+++ b/tools/bcdb_mock/models/utils.go
@@ -4,6 +4,7 @@ import (
 	"net/http"
 	"strconv"
 
+	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
 
@@ -22,7 +23,7 @@ func Page(p int64, size ...int64) Pager {
 		ps = size[0]
 	}
 	skip := p * ps
-	return options.Find().SetLimit(ps).SetSkip(skip)
+	return options.Find().SetLimit(ps).SetSkip(skip).SetSort(bson.D{{Key: "_id", Value: -1}})
 }
 
 // PageFromRequest return page information from the page & size url params
@@ -37,6 +38,11 @@ func PageFromRequest(r *http.Request) Pager {
 
 	if len(p) != 0 {
 		page, _ = strconv.ParseInt(p, 10, 64)
+		// user facing pages are start from 1
+		page--
+		if page < 0 {
+			page = 0
+		}
 	}
 	if len(s) != 0 {
 		size, _ = strconv.ParseInt(s, 10, 64)
diff --git a/tools/bcdb_mock/pkg/directory/farms_handlers.go b/tools/bcdb_mock/pkg/directory/farms_handlers.go
index 6379b6a7..98db84ae 100644
--- a/tools/bcdb_mock/pkg/directory/farms_handlers.go
+++ b/tools/bcdb_mock/pkg/directory/farms_handlers.go
@@ -2,11 +2,14 @@ package directory
 
 import (
 	"encoding/json"
-	"log"
+	"fmt"
 	"net/http"
 	"strconv"
 
+	"github.com/rs/zerolog/log"
+
 	"github.com/threefoldtech/zos/pkg/schema"
+	"github.com/threefoldtech/zos/tools/bcdb_mock/models"
 	"github.com/threefoldtech/zos/tools/bcdb_mock/mw"
 	directory "github.com/threefoldtech/zos/tools/bcdb_mock/pkg/directory/types"
 
@@ -14,7 +17,7 @@ import (
 )
 
 func (s *FarmAPI) registerFarm(r *http.Request) (interface{}, mw.Response) {
-	log.Println("farm register request received")
+	log.Info().Msg("farm register request received")
 
 	db := mw.Database(r)
 	defer r.Body.Close()
@@ -42,7 +45,13 @@ func (s *FarmAPI) registerFarm(r *http.Request) (interface{}, mw.Response) {
 
 func (s *FarmAPI) listFarm(r *http.Request) (interface{}, mw.Response) {
 	db := mw.Database(r)
-	farms, err := s.List(r.Context(), db)
+
+	tid, err := parseOwnerID(r)
+	if err != nil {
+		return nil, mw.Error(err, http.StatusBadRequest)
+	}
+
+	farms, err := s.List(r.Context(), db, tid, models.PageFromRequest(r))
 	if err != nil {
 		return nil, mw.Error(err)
 	}
@@ -50,10 +59,22 @@ func (s *FarmAPI) listFarm(r *http.Request) (interface{}, mw.Response) {
 	return farms, nil
 }
 
+func parseOwnerID(r *http.Request) (tid int64, err error) {
+	stid := r.URL.Query().Get("owner")
+	if stid != "" {
+		tid, err = strconv.ParseInt(stid, 10, 64)
+		if err != nil {
+			return tid, fmt.Errorf("owner should be a integer")
+		}
+	}
+	log.Info().Msgf("owner id %d", tid)
+	return tid, err
+}
+
 func (s *FarmAPI) cockpitListFarm(r *http.Request) (interface{}, mw.Response) {
 	// TODO: do we need this ?
 	db := mw.Database(r)
-	farms, err := s.List(r.Context(), db)
+	farms, err := s.List(r.Context(), db, 0)
 	if err != nil {
 		return nil, mw.Error(err)
 	}
diff --git a/tools/bcdb_mock/pkg/directory/farms_store.go b/tools/bcdb_mock/pkg/directory/farms_store.go
index 5916cc3a..fec8c294 100644
--- a/tools/bcdb_mock/pkg/directory/farms_store.go
+++ b/tools/bcdb_mock/pkg/directory/farms_store.go
@@ -4,10 +4,11 @@ import (
 	"context"
 
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
 	"github.com/threefoldtech/zos/pkg/schema"
-	"github.com/threefoldtech/zos/tools/bcdb_mock/models"
 	directory "github.com/threefoldtech/zos/tools/bcdb_mock/pkg/directory/types"
 	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
 )
 
 // FarmAPI holds farm releated handlers
@@ -15,15 +16,20 @@ type FarmAPI struct{}
 
 // List farms
 // TODO: add paging arguments
-func (s *FarmAPI) List(ctx context.Context, db *mongo.Database) ([]directory.Farm, error) {
+func (s *FarmAPI) List(ctx context.Context, db *mongo.Database, tid int64, opts ...*options.FindOptions) ([]directory.Farm, error) {
 	var filter directory.FarmFilter
-	//options.Find().
-	cur, err := filter.Find(ctx, db, models.Page(0))
+
+	if tid != 0 {
+		log.Info().Msgf("with owner id %d", tid)
+		filter = filter.WithOwner(tid)
+	}
+
+	cur, err := filter.Find(ctx, db, opts...)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to list farms")
 	}
 	defer cur.Close(ctx)
-	var out []directory.Farm
+	out := []directory.Farm{}
 	if err := cur.All(ctx, &out); err != nil {
 		return nil, errors.Wrap(err, "failed to load farm list")
 	}
diff --git a/tools/bcdb_mock/pkg/directory/node_handlers.go b/tools/bcdb_mock/pkg/directory/node_handlers.go
index c6a2de83..6309a5ea 100644
--- a/tools/bcdb_mock/pkg/directory/node_handlers.go
+++ b/tools/bcdb_mock/pkg/directory/node_handlers.go
@@ -3,14 +3,12 @@ package directory
 import (
 	"encoding/json"
 	"net/http"
-	"strconv"
 
-	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 
 	"github.com/threefoldtech/zos/pkg/capacity"
 	"github.com/threefoldtech/zos/pkg/network/types"
-	"github.com/threefoldtech/zos/pkg/schema"
+	"github.com/threefoldtech/zos/tools/bcdb_mock/models"
 	generated "github.com/threefoldtech/zos/tools/bcdb_mock/models/generated/directory"
 	"github.com/threefoldtech/zos/tools/bcdb_mock/mw"
 	directory "github.com/threefoldtech/zos/tools/bcdb_mock/pkg/directory/types"
@@ -43,9 +41,13 @@ func (s *NodeAPI) registerNode(r *http.Request) (interface{}, mw.Response) {
 
 func (s *NodeAPI) nodeDetail(r *http.Request) (interface{}, mw.Response) {
 	nodeID := mux.Vars(r)["node_id"]
+	q := nodeQuery{}
+	if err := q.Parse(r); err != nil {
+		return nil, err
+	}
 	db := mw.Database(r)
 
-	node, err := s.Get(r.Context(), db, nodeID)
+	node, err := s.Get(r.Context(), db, nodeID, q.Proofs)
 	if err != nil {
 		return nil, mw.NotFound(err)
 	}
@@ -54,23 +56,13 @@ func (s *NodeAPI) nodeDetail(r *http.Request) (interface{}, mw.Response) {
 }
 
 func (s *NodeAPI) listNodes(r *http.Request) (interface{}, mw.Response) {
-	sFarm := r.URL.Query().Get("farm")
-
-	var (
-		farm uint64
-		err  error
-	)
-
-	if sFarm != "" {
-		farm, err = strconv.ParseUint(sFarm, 10, 64)
-		if err != nil {
-			return nil, mw.BadRequest(errors.Wrap(err, "invalid farm id"))
-		}
+	q := nodeQuery{}
+	if err := q.Parse(r); err != nil {
+		return nil, err
 	}
 
 	db := mw.Database(r)
-
-	nodes, err := s.List(r.Context(), db, schema.ID(farm))
+	nodes, err := s.List(r.Context(), db, q, models.PageFromRequest(r))
 	if err != nil {
 		return nil, mw.Error(err)
 	}
@@ -79,23 +71,14 @@ func (s *NodeAPI) listNodes(r *http.Request) (interface{}, mw.Response) {
 }
 
 func (s *NodeAPI) cockpitListNodes(r *http.Request) (interface{}, mw.Response) {
-	sFarm := r.URL.Query().Get("farm")
-
-	var (
-		farm uint64
-		err  error
-	)
-
-	if sFarm != "" {
-		farm, err = strconv.ParseUint(sFarm, 10, 64)
-		if err != nil {
-			return nil, mw.BadRequest(errors.Wrap(err, "invalid farm id"))
-		}
+	q := nodeQuery{}
+	if err := q.Parse(r); err != nil {
+		return nil, err
 	}
 
 	db := mw.Database(r)
 
-	nodes, err := s.List(r.Context(), db, schema.ID(farm))
+	nodes, err := s.List(r.Context(), db, q, models.PageFromRequest(r))
 	if err != nil {
 		return nil, mw.Error(err)
 	}
diff --git a/tools/bcdb_mock/pkg/directory/nodes_store.go b/tools/bcdb_mock/pkg/directory/nodes_store.go
index d7f82203..ba912119 100644
--- a/tools/bcdb_mock/pkg/directory/nodes_store.go
+++ b/tools/bcdb_mock/pkg/directory/nodes_store.go
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"net/http"
 	"sort"
+	"strconv"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -14,30 +15,88 @@ import (
 	"github.com/threefoldtech/zos/pkg/capacity"
 	"github.com/threefoldtech/zos/pkg/capacity/dmi"
 	"github.com/threefoldtech/zos/pkg/schema"
-	"github.com/threefoldtech/zos/tools/bcdb_mock/models"
 	generated "github.com/threefoldtech/zos/tools/bcdb_mock/models/generated/directory"
 	"github.com/threefoldtech/zos/tools/bcdb_mock/mw"
 	directory "github.com/threefoldtech/zos/tools/bcdb_mock/pkg/directory/types"
+	"go.mongodb.org/mongo-driver/bson"
 	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
 )
 
 // NodeAPI holds api for nodes
 type NodeAPI struct{}
 
+type nodeQuery struct {
+	FarmID  int64
+	Country string
+	City    string
+	CRU     int64
+	MRU     int64
+	SRU     int64
+	HRU     int64
+	Proofs  bool
+}
+
+func (n *nodeQuery) Parse(r *http.Request) mw.Response {
+	var err error
+	n.FarmID, err = queryInt(r, "farm")
+	if err != nil {
+		return mw.BadRequest(errors.Wrap(err, "invalid farm id"))
+	}
+	n.Country = r.URL.Query().Get("country")
+	n.City = r.URL.Query().Get("city")
+	n.CRU, err = queryInt(r, "cru")
+	if err != nil {
+		return mw.BadRequest(errors.Wrap(err, "invalid cru"))
+	}
+	n.MRU, err = queryInt(r, "mru")
+	if err != nil {
+		return mw.BadRequest(errors.Wrap(err, "invalid mru"))
+	}
+	n.SRU, err = queryInt(r, "sru")
+	if err != nil {
+		return mw.BadRequest(errors.Wrap(err, "invalid sru"))
+	}
+	n.HRU, err = queryInt(r, "hru")
+	if err != nil {
+		return mw.BadRequest(errors.Wrap(err, "invalid hru"))
+	}
+	n.Proofs = r.URL.Query().Get("proofs") == "true"
+
+	return nil
+}
+
+func queryInt(r *http.Request, q string) (int64, error) {
+	s := r.URL.Query().Get(q)
+	if s != "" {
+		return strconv.ParseInt(s, 10, 64)
+	}
+	return 0, nil
+}
+
 // List farms
 // TODO: add paging arguments
-func (s *NodeAPI) List(ctx context.Context, db *mongo.Database, farm schema.ID) ([]directory.Node, error) {
+func (s *NodeAPI) List(ctx context.Context, db *mongo.Database, q nodeQuery, opts ...*options.FindOptions) ([]directory.Node, error) {
 	var filter directory.NodeFilter
-	if farm > 0 {
-		filter = filter.WithFarmID(farm)
+	if q.FarmID > 0 {
+		filter = filter.WithFarmID(schema.ID(q.FarmID))
+	}
+	filter = filter.WithTotalCap(q.CRU, q.MRU, q.HRU, q.SRU)
+	filter = filter.WithLocation(q.Country, q.City)
+
+	if !q.Proofs {
+		projection := bson.D{
+			{Key: "proofs", Value: 0},
+		}
+		opts = append(opts, options.Find().SetProjection(projection))
 	}
 
-	cur, err := filter.Find(ctx, db, models.Page(0))
+	cur, err := filter.Find(ctx, db, opts...)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to list nodes")
 	}
 	defer cur.Close(ctx)
-	var out []directory.Node
+	out := []directory.Node{}
 	if err := cur.All(ctx, &out); err != nil {
 		return nil, errors.Wrap(err, "failed to load node list")
 	}
@@ -46,10 +105,10 @@ func (s *NodeAPI) List(ctx context.Context, db *mongo.Database, farm schema.ID)
 }
 
 // Get a single node
-func (s *NodeAPI) Get(ctx context.Context, db *mongo.Database, nodeID string) (directory.Node, error) {
+func (s *NodeAPI) Get(ctx context.Context, db *mongo.Database, nodeID string, includeProofs bool) (directory.Node, error) {
 	var filter directory.NodeFilter
 	filter = filter.WithNodeID(nodeID)
-	return filter.Get(ctx, db)
+	return filter.Get(ctx, db, includeProofs)
 }
 
 // Exists tests if node exists
@@ -120,7 +179,7 @@ func (s *NodeAPI) SetInterfaces(ctx context.Context, db *mongo.Database, nodeID
 
 // SetPublicConfig sets node public config
 func (s *NodeAPI) SetPublicConfig(ctx context.Context, db *mongo.Database, nodeID string, cfg generated.TfgridDirectoryNodePublicIface1) error {
-	node, err := s.Get(ctx, db, nodeID)
+	node, err := s.Get(ctx, db, nodeID, false)
 	if err != nil {
 		return err
 	}
diff --git a/tools/bcdb_mock/pkg/directory/types/farm.go b/tools/bcdb_mock/pkg/directory/types/farm.go
index 0fc0bffb..61eba1d9 100644
--- a/tools/bcdb_mock/pkg/directory/types/farm.go
+++ b/tools/bcdb_mock/pkg/directory/types/farm.go
@@ -18,7 +18,8 @@ var (
 )
 
 const (
-	farmCollection = "farm"
+	// FarmCollection db collection name
+	FarmCollection = "farm"
 )
 
 //Farm mongo db wrapper for generated TfgridDirectoryFarm
@@ -54,9 +55,14 @@ func (f FarmFilter) WithName(name string) FarmFilter {
 	return append(f, bson.E{Key: "name", Value: name})
 }
 
+// WithOwner filter farm by owner ID
+func (f FarmFilter) WithOwner(tid int64) FarmFilter {
+	return append(f, bson.E{Key: "threebot_id", Value: tid})
+}
+
 // Find run the filter and return a cursor result
 func (f FarmFilter) Find(ctx context.Context, db *mongo.Database, opts ...*options.FindOptions) (*mongo.Cursor, error) {
-	col := db.Collection(farmCollection)
+	col := db.Collection(FarmCollection)
 	if f == nil {
 		f = FarmFilter{}
 	}
@@ -68,7 +74,7 @@ func (f FarmFilter) Get(ctx context.Context, db *mongo.Database) (farm Farm, err
 	if f == nil {
 		f = FarmFilter{}
 	}
-	col := db.Collection(farmCollection)
+	col := db.Collection(FarmCollection)
 	result := col.FindOne(ctx, f, options.FindOne())
 
 	err = result.Err()
@@ -86,8 +92,8 @@ func FarmCreate(ctx context.Context, db *mongo.Database, farm Farm) (schema.ID,
 		return 0, err
 	}
 
-	col := db.Collection(farmCollection)
-	id, err := models.NextID(ctx, db, farmCollection)
+	col := db.Collection(FarmCollection)
+	id, err := models.NextID(ctx, db, FarmCollection)
 	if err != nil {
 		return id, err
 	}
diff --git a/tools/bcdb_mock/pkg/directory/types/node.go b/tools/bcdb_mock/pkg/directory/types/node.go
index 3eaf463a..bea0d796 100644
--- a/tools/bcdb_mock/pkg/directory/types/node.go
+++ b/tools/bcdb_mock/pkg/directory/types/node.go
@@ -15,7 +15,8 @@ import (
 )
 
 const (
-	nodeCollection = "node"
+	// NodeCollection db collection name
+	NodeCollection = "node"
 )
 
 // Node model
@@ -66,18 +67,34 @@ func (f NodeFilter) WithFarmID(id schema.ID) NodeFilter {
 // WithTotalCap filter with total cap only units that > 0 are used
 // in the query
 func (f NodeFilter) WithTotalCap(cru, mru, hru, sru int64) NodeFilter {
-	for k, v := range map[string]int64{"cru": cru, "mru": mru, "hru": hru, "sru": sru} {
+	for k, v := range map[string]int64{
+		"total_resources.cru": cru,
+		"total_resources.mru": mru,
+		"total_resources.hru": hru,
+		"total_resources.sru": sru} {
 		if v > 0 {
-			f = append(f, bson.E{Key: "$gt", Value: bson.M{k: v}})
+			f = append(f, bson.E{Key: k, Value: bson.M{"$gte": v}})
 		}
 	}
 
 	return f
 }
 
+// WithLocation search the nodes that are located in country and or city
+func (f NodeFilter) WithLocation(country, city string) NodeFilter {
+	if country != "" {
+		f = append(f, bson.E{Key: "location.country", Value: country})
+	}
+	if city != "" {
+		f = append(f, bson.E{Key: "location.city", Value: city})
+	}
+
+	return f
+}
+
 // Find run the filter and return a cursor result
 func (f NodeFilter) Find(ctx context.Context, db *mongo.Database, opts ...*options.FindOptions) (*mongo.Cursor, error) {
-	col := db.Collection(nodeCollection)
+	col := db.Collection(NodeCollection)
 	if f == nil {
 		f = NodeFilter{}
 	}
@@ -86,13 +103,22 @@ func (f NodeFilter) Find(ctx context.Context, db *mongo.Database, opts ...*optio
 }
 
 // Get one farm that matches the filter
-func (f NodeFilter) Get(ctx context.Context, db *mongo.Database) (node Node, err error) {
+func (f NodeFilter) Get(ctx context.Context, db *mongo.Database, includeproofs bool) (node Node, err error) {
 	if f == nil {
 		f = NodeFilter{}
 	}
 
-	col := db.Collection(nodeCollection)
-	result := col.FindOne(ctx, f, options.FindOne())
+	col := db.Collection(NodeCollection)
+
+	var projection bson.D
+	if !includeproofs {
+		projection = bson.D{
+			{Key: "proofs", Value: 0},
+		}
+	} else {
+		projection = bson.D{}
+	}
+	result := col.FindOne(ctx, f, options.FindOne().SetProjection(projection))
 
 	err = result.Err()
 	if err != nil {
@@ -105,7 +131,7 @@ func (f NodeFilter) Get(ctx context.Context, db *mongo.Database) (node Node, err
 
 // Count number of documents matching
 func (f NodeFilter) Count(ctx context.Context, db *mongo.Database) (int64, error) {
-	col := db.Collection(nodeCollection)
+	col := db.Collection(NodeCollection)
 	if f == nil {
 		f = NodeFilter{}
 	}
@@ -129,10 +155,10 @@ func NodeCreate(ctx context.Context, db *mongo.Database, node Node) (schema.ID,
 	var filter NodeFilter
 	filter = filter.WithNodeID(node.NodeId)
 	var id schema.ID
-	current, err := filter.Get(ctx, db)
+	current, err := filter.Get(ctx, db, false)
 	if err != nil {
 		//TODO: check that this is a NOT FOUND error
-		id, err = models.NextID(ctx, db, nodeCollection)
+		id, err = models.NextID(ctx, db, NodeCollection)
 		if err != nil {
 			return id, err
 		}
@@ -147,7 +173,7 @@ func NodeCreate(ctx context.Context, db *mongo.Database, node Node) (schema.ID,
 	}
 
 	node.Updated = schema.Date{Time: time.Now()}
-	col := db.Collection(nodeCollection)
+	col := db.Collection(NodeCollection)
 	result := col.FindOneAndUpdate(ctx, filter, bson.M{"$set": node}, options.FindOneAndUpdate().SetUpsert(true))
 	return id, result.Err()
 }
@@ -157,7 +183,7 @@ func nodeUpdate(ctx context.Context, db *mongo.Database, nodeID string, value in
 		return fmt.Errorf("invalid node id")
 	}
 
-	col := db.Collection(nodeCollection)
+	col := db.Collection(NodeCollection)
 	var filter NodeFilter
 	filter = filter.WithNodeID(nodeID)
 	_, err := col.UpdateOne(ctx, filter, bson.M{
@@ -217,7 +243,7 @@ func NodePushProof(ctx context.Context, db *mongo.Database, nodeID string, proof
 		return fmt.Errorf("invalid node id")
 	}
 
-	col := db.Collection(nodeCollection)
+	col := db.Collection(NodeCollection)
 	var filter NodeFilter
 	filter = filter.WithNodeID(nodeID)
 	_, err := col.UpdateOne(ctx, filter, bson.M{
diff --git a/tools/bcdb_mock/pkg/directory/types/setup.go b/tools/bcdb_mock/pkg/directory/types/setup.go
index 3202908b..7e13042a 100644
--- a/tools/bcdb_mock/pkg/directory/types/setup.go
+++ b/tools/bcdb_mock/pkg/directory/types/setup.go
@@ -13,7 +13,7 @@ import (
 // Setup sets up indexes for types, must be called at least
 // Onetime during the life time of the object
 func Setup(ctx context.Context, db *mongo.Database) error {
-	farm := db.Collection(farmCollection)
+	farm := db.Collection(FarmCollection)
 	_, err := farm.Indexes().CreateMany(ctx, []mongo.IndexModel{
 		{
 			Keys:    bson.M{"name": 1},
@@ -24,7 +24,7 @@ func Setup(ctx context.Context, db *mongo.Database) error {
 		log.Error().Err(err).Msg("failed to initialize farm index")
 	}
 
-	node := db.Collection(nodeCollection)
+	node := db.Collection(NodeCollection)
 
 	nodeIdexes := []mongo.IndexModel{
 		{
diff --git a/tools/bcdb_mock/pkg/phonebook/types/setup.go b/tools/bcdb_mock/pkg/phonebook/types/setup.go
index 093fb310..321efe08 100644
--- a/tools/bcdb_mock/pkg/phonebook/types/setup.go
+++ b/tools/bcdb_mock/pkg/phonebook/types/setup.go
@@ -12,7 +12,7 @@ import (
 // Setup sets up indexes for types, must be called at least
 // Onetime during the life time of the object
 func Setup(ctx context.Context, db *mongo.Database) error {
-	farm := db.Collection(userCollection)
+	farm := db.Collection(UserCollection)
 	_, err := farm.Indexes().CreateMany(ctx, []mongo.IndexModel{
 		{
 			Keys:    bson.M{"name": 1},
@@ -24,7 +24,7 @@ func Setup(ctx context.Context, db *mongo.Database) error {
 		},
 	})
 	if err != nil {
-		log.Error().Err(err).Msg("failed to initialize farm index")
+		log.Error().Err(err).Msg("failed to initialize user index")
 	}
 
 	return err
diff --git a/tools/bcdb_mock/pkg/phonebook/types/user.go b/tools/bcdb_mock/pkg/phonebook/types/user.go
index 5baf401e..67251c0e 100644
--- a/tools/bcdb_mock/pkg/phonebook/types/user.go
+++ b/tools/bcdb_mock/pkg/phonebook/types/user.go
@@ -18,12 +18,13 @@ import (
 )
 
 const (
-	userCollection = "user"
+	// UserCollection db collection name
+	UserCollection = "user"
 )
 
 var (
 	// ErrUserExists returned if user with same name exists
-	ErrUserExists = errors.New("user with same name exists")
+	ErrUserExists = errors.New("user with same name or email exists")
 	// ErrUserNotFound is returned if user is not found
 	ErrUserNotFound = errors.New("user not found")
 	// ErrAuthorization returned if user is not allowed to do an operation
@@ -57,16 +58,25 @@ type UserFilter bson.D
 
 // WithID filters user with ID
 func (f UserFilter) WithID(id schema.ID) UserFilter {
+	if id == 0 {
+		return f
+	}
 	return append(f, bson.E{Key: "_id", Value: id})
 }
 
 // WithName filters user with name
 func (f UserFilter) WithName(name string) UserFilter {
+	if name == "" {
+		return f
+	}
 	return append(f, bson.E{Key: "name", Value: name})
 }
 
 // WithEmail filters user with email
 func (f UserFilter) WithEmail(email string) UserFilter {
+	if email == "" {
+		return f
+	}
 	return append(f, bson.E{Key: "email", Value: email})
 }
 
@@ -75,7 +85,7 @@ func (f UserFilter) Find(ctx context.Context, db *mongo.Database, opts ...*optio
 	if f == nil {
 		f = UserFilter{}
 	}
-	return db.Collection(userCollection).Find(ctx, f, opts...)
+	return db.Collection(UserCollection).Find(ctx, f, opts...)
 }
 
 // Get single user
@@ -84,7 +94,7 @@ func (f UserFilter) Get(ctx context.Context, db *mongo.Database) (user User, err
 		f = UserFilter{}
 	}
 
-	result := db.Collection(userCollection).FindOne(ctx, f, options.FindOne())
+	result := db.Collection(UserCollection).FindOne(ctx, f, options.FindOne())
 	if err = result.Err(); err != nil {
 		return
 	}
@@ -114,7 +124,7 @@ func UserCreate(ctx context.Context, db *mongo.Database, name, email, pubkey str
 	}
 	// else ErrNoDocuments
 
-	id := models.MustID(ctx, db, userCollection)
+	id := models.MustID(ctx, db, UserCollection)
 	user = User{
 		ID:     id,
 		Name:   name,
@@ -122,8 +132,17 @@ func UserCreate(ctx context.Context, db *mongo.Database, name, email, pubkey str
 		Pubkey: pubkey,
 	}
 
-	col := db.Collection(userCollection)
+	col := db.Collection(UserCollection)
 	_, err = col.InsertOne(ctx, user)
+	if err != nil {
+		if merr, ok := err.(mongo.WriteException); ok {
+			errCode := merr.WriteErrors[0].Code
+			if errCode == 11000 {
+				return user, ErrUserExists
+			}
+		}
+		return user, err
+	}
 	return
 }
 
@@ -186,7 +205,7 @@ func UserUpdate(ctx context.Context, db *mongo.Database, id schema.ID, signature
 	}
 
 	// actually update the user with final data
-	if _, err := db.Collection(userCollection).UpdateOne(ctx, filter, bson.M{"$set": current}); err != nil {
+	if _, err := db.Collection(UserCollection).UpdateOne(ctx, filter, bson.M{"$set": current}); err != nil {
 		return err
 	}
 
diff --git a/tools/bcdb_mock/pkg/phonebook/user.go b/tools/bcdb_mock/pkg/phonebook/user.go
index e73a0c0c..00eaa748 100644
--- a/tools/bcdb_mock/pkg/phonebook/user.go
+++ b/tools/bcdb_mock/pkg/phonebook/user.go
@@ -84,18 +84,9 @@ func (u *UserAPI) register(r *http.Request) (interface{}, mw.Response) {
 }
 
 func (u *UserAPI) list(r *http.Request) (interface{}, mw.Response) {
-	var (
-		name  = r.FormValue("name")
-		email = r.FormValue("email")
-	)
-
 	var filter types.UserFilter
-	if len(name) != 0 {
-		filter = filter.WithName(name)
-	}
-	if len(email) != 0 {
-		filter = filter.WithEmail(email)
-	}
+	filter = filter.WithName(r.FormValue("name"))
+	filter = filter.WithEmail(r.FormValue("email"))
 
 	db := mw.Database(r)
 	cur, err := filter.Find(r.Context(), db, models.PageFromRequest(r))
@@ -126,7 +117,6 @@ func (u *UserAPI) get(r *http.Request) (interface{}, mw.Response) {
 	if err != nil {
 		return nil, mw.BadRequest(err)
 	}
-
 	var filter types.UserFilter
 	filter = filter.WithID(userID)
 
diff --git a/tools/bcdb_mock/pkg/workloads/types/pipeline.go b/tools/bcdb_mock/pkg/workloads/types/pipeline.go
index d6ee4ab9..f6344c33 100644
--- a/tools/bcdb_mock/pkg/workloads/types/pipeline.go
+++ b/tools/bcdb_mock/pkg/workloads/types/pipeline.go
@@ -4,6 +4,7 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
 	generated "github.com/threefoldtech/zos/tools/bcdb_mock/models/generated/workloads"
 )
 
@@ -31,6 +32,7 @@ func (p *Pipeline) checkProvisionSignatures() bool {
 	// required quorum has been reached
 
 	request := p.r.DataReservation.SigningRequestProvision
+	log.Debug().Msgf("%+v", request)
 	if request.QuorumMin == 0 {
 		return true
 	}
@@ -101,6 +103,7 @@ func (p *Pipeline) Next() (Reservation, bool) {
 	if p.r.Expired() || p.checkDeleteSignatures() {
 		// reservation has expired
 		// set its status (next action) to delete
+		log.Debug().Int64("id", int64(p.r.ID)).Msg("expired or to be deleted")
 		p.r.NextAction = generated.TfgridWorkloadsReservation1NextActionDelete
 		return p.r, true
 	}
@@ -115,22 +118,27 @@ func (p *Pipeline) Next() (Reservation, bool) {
 			if time.Until(p.r.DataReservation.ExpirationProvisioning.Time) <= 0 {
 				// exceeded
 				// TODO: I think this should be set to "delete" not "invalid"
+				log.Debug().Int64("id", int64(p.r.ID)).Msg("expired")
 				p.r.NextAction = generated.TfgridWorkloadsReservation1NextActionInvalid
 			} else {
+				log.Debug().Int64("id", int64(p.r.ID)).Msg("ready to sign")
 				p.r.NextAction = generated.TfgridWorkloadsReservation1NextActionSign
 			}
 		case generated.TfgridWorkloadsReservation1NextActionSign:
 			// this stage will not change unless all
 			if p.checkProvisionSignatures() {
+				log.Debug().Int64("id", int64(p.r.ID)).Msg("ready to pay")
 				p.r.NextAction = generated.TfgridWorkloadsReservation1NextActionPay
 			}
 		case generated.TfgridWorkloadsReservation1NextActionPay:
 			// TODO: here we should actually start the payment process
 			// but this is not implemented yet, so now we just need to move
 			// to deploy
+			log.Debug().Int64("id", int64(p.r.ID)).Msg("ready to deploy")
 			p.r.NextAction = generated.TfgridWorkloadsReservation1NextActionDeploy
 		case generated.TfgridWorkloadsReservation1NextActionDeploy:
 			//nothing to do
+			log.Debug().Int64("id", int64(p.r.ID)).Msg("let's deploy")
 		}
 
 		if current == p.r.NextAction {
diff --git a/tools/bcdb_mock/pkg/workloads/types/reservation.go b/tools/bcdb_mock/pkg/workloads/types/reservation.go
index d775cac0..360f20a6 100644
--- a/tools/bcdb_mock/pkg/workloads/types/reservation.go
+++ b/tools/bcdb_mock/pkg/workloads/types/reservation.go
@@ -21,7 +21,8 @@ import (
 )
 
 const (
-	reservationCollection = "reservation"
+	// ReservationCollection db collection name
+	ReservationCollection = "reservation"
 	queueCollection       = "workqueue"
 )
 
@@ -97,7 +98,7 @@ func (f ReservationFilter) Get(ctx context.Context, db *mongo.Database) (reserva
 		f = ReservationFilter{}
 	}
 
-	result := db.Collection(reservationCollection).FindOne(ctx, f)
+	result := db.Collection(ReservationCollection).FindOne(ctx, f)
 	if err = result.Err(); err != nil {
 		return
 	}
@@ -111,7 +112,7 @@ func (f ReservationFilter) Find(ctx context.Context, db *mongo.Database, opts ..
 	if f == nil {
 		f = ReservationFilter{}
 	}
-	return db.Collection(reservationCollection).Find(ctx, f, opts...)
+	return db.Collection(ReservationCollection).Find(ctx, f, opts...)
 }
 
 // Reservation is a wrapper around generated type
@@ -203,7 +204,7 @@ func (r *Reservation) SignatureVerify(pk string, sig []byte) error {
 		return errors.Wrap(err, "failed to write id to buffer")
 	}
 
-	if _, err := buf.Write(r.Json); err != nil {
+	if _, err := buf.WriteString(r.Json); err != nil {
 		return errors.Wrap(err, "failed to write json to buffer")
 	}
 
@@ -372,10 +373,10 @@ func (r *Reservation) Workloads(nodeID string) []Workload {
 // NOTE: use reservations only that are returned from calling Pipeline.Next()
 // no validation is done here, this is just a CRUD operation
 func ReservationCreate(ctx context.Context, db *mongo.Database, r Reservation) (schema.ID, error) {
-	id := models.MustID(ctx, db, reservationCollection)
+	id := models.MustID(ctx, db, ReservationCollection)
 	r.ID = id
 
-	_, err := db.Collection(reservationCollection).InsertOne(ctx, r)
+	_, err := db.Collection(ReservationCollection).InsertOne(ctx, r)
 	if err != nil {
 		return 0, err
 	}
@@ -388,7 +389,7 @@ func ReservationSetNextAction(ctx context.Context, db *mongo.Database, id schema
 	var filter ReservationFilter
 	filter = filter.WithID(id)
 
-	col := db.Collection(reservationCollection)
+	col := db.Collection(ReservationCollection)
 	_, err := col.UpdateOne(ctx, filter, bson.M{
 		"$set": bson.M{
 			"next_action": action,
@@ -417,7 +418,7 @@ func ReservationPushSignature(ctx context.Context, db *mongo.Database, id schema
 
 	var filter ReservationFilter
 	filter = filter.WithID(id)
-	col := db.Collection(reservationCollection)
+	col := db.Collection(ReservationCollection)
 	// NOTE: this should be a transaction not a bulk write
 	// but i had so many issues with transaction, and i couldn't
 	// get it to work. so I used bulk write in place instead
@@ -524,7 +525,7 @@ func (r *Result) Verify(pk string) error {
 // ResultPush pushes result to a reservation result array.
 // NOTE: this is just a crud operation, no validation is done here
 func ResultPush(ctx context.Context, db *mongo.Database, id schema.ID, result Result) error {
-	col := db.Collection(reservationCollection)
+	col := db.Collection(ReservationCollection)
 	var filter ReservationFilter
 	filter = filter.WithID(id)
 
diff --git a/tools/bcdb_mock/pkg/workloads/types/setup.go b/tools/bcdb_mock/pkg/workloads/types/setup.go
index d2763c91..41c919e4 100644
--- a/tools/bcdb_mock/pkg/workloads/types/setup.go
+++ b/tools/bcdb_mock/pkg/workloads/types/setup.go
@@ -11,7 +11,7 @@ import (
 // Setup sets up indexes for types, must be called at least
 // Onetime during the life time of the object
 func Setup(ctx context.Context, db *mongo.Database) error {
-	col := db.Collection(reservationCollection)
+	col := db.Collection(ReservationCollection)
 	indexes := []mongo.IndexModel{
 		{
 			Keys: bson.M{"data_reservation.networks.network_resources.node_id": 1},
diff --git a/tools/tfuser/cmds_provision.go b/tools/tfuser/cmds_provision.go
index ab11e685..cd94194c 100644
--- a/tools/tfuser/cmds_provision.go
+++ b/tools/tfuser/cmds_provision.go
@@ -158,7 +158,7 @@ func cmdsProvision(c *cli.Context) error {
 		return err
 	}
 
-	jsx.JSON = bytes
+	jsx.JSON = string(bytes)
 	signature, err := crypto.Sign(keypair.PrivateKey, []byte(jsx.JSON))
 	if err != nil {
 		return errors.Wrap(err, "failed to sign the reservation")
@@ -218,7 +218,7 @@ func cmdsDeleteReservation(c *cli.Context) error {
 	if _, err := buf.WriteString(fmt.Sprint(resID)); err != nil {
 		return err
 	}
-	if _, err := buf.Write(reservation.JSON); err != nil {
+	if _, err := buf.Write([]byte(reservation.JSON)); err != nil {
 		return err
 	}
 


Test patch:
diff --git a/pkg/app/flag_test.go b/pkg/app/flag_test.go
new file mode 100644
index 00000000..bce547f7
--- /dev/null
+++ b/pkg/app/flag_test.go
@@ -0,0 +1,19 @@
+package app
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDeleteFlag(t *testing.T) {
+	key := LimitedCache
+	err := DeleteFlag(key)
+	require.NoError(t, err)
+}
+
+func TestDeleteBadFlag(t *testing.T) {
+	key := "/"
+	err := DeleteFlag(key)
+	require.Error(t, err)
+}
diff --git a/pkg/container/startup_test.go b/pkg/container/startup_test.go
index 3313c358..6ebc6768 100644
--- a/pkg/container/startup_test.go
+++ b/pkg/container/startup_test.go
@@ -5,6 +5,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/google/shlex"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/BurntSushi/toml"
@@ -19,6 +20,11 @@ name = "core.system"
 [startup.entry.args]
 name = "/start"
 dir = "/data"
+args = [
+	"extra",
+	"argument with spaces",
+	"with \"skip\"",
+]
 
 [startup.entry.args.env]
 DIFFICULTY = "easy"
@@ -37,6 +43,8 @@ func TestParseStartup(t *testing.T) {
 	assert.Equal(t, "core.system", entry.Name)
 	assert.Equal(t, "/start", entry.Args.Name)
 	assert.Equal(t, "/data", entry.Args.Dir)
+	assert.Equal(t, []string{"extra", "argument with spaces", "with \"skip\""}, entry.Args.Args)
+
 	assert.Equal(t, map[string]string{
 		"DIFFICULTY":  "easy",
 		"LEVEL":       "world",
@@ -52,7 +60,12 @@ func TestStartupEntrypoint(t *testing.T) {
 
 	entry, ok := e.Entries["entry"]
 	require.True(t, ok)
-	assert.Equal(t, entry.Entrypoint(), "/start")
+	assert.Equal(t, entry.Entrypoint(), `/start "extra" "argument with spaces" "with \"skip\""`)
+
+	parts, err := shlex.Split(entry.Entrypoint())
+	require.NoError(t, err)
+
+	assert.Equal(t, []string{"/start", "extra", "argument with spaces", "with \"skip\""}, parts)
 }
 
 func TestStartupEnvs(t *testing.T) {
diff --git a/pkg/network/ndmz/ndmz_test.go b/pkg/network/ndmz/ndmz_test.go
index 376f5e20..77d89f1b 100644
--- a/pkg/network/ndmz/ndmz_test.go
+++ b/pkg/network/ndmz/ndmz_test.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net"
+	"os"
 	"sync"
 	"testing"
 
@@ -38,37 +39,31 @@ func TestIpv6(t *testing.T) {
 }
 
 func TestIPv4Allocate(t *testing.T) {
-	var (
-		err    error
-		origin = ipamPath
-	)
-
-	ipamPath, err = ioutil.TempDir("", "")
+	ipamPath, err := ioutil.TempDir("", "")
 	require.NoError(t, err)
-	defer func() { ipamPath = origin }()
+	t.Cleanup(func() {
+		_ = os.RemoveAll(ipamPath)
+	})
 
-	addr, err := allocateIPv4("network1")
+	addr, err := allocateIPv4("network1", ipamPath)
 	require.NoError(t, err)
 
-	addr2, err := allocateIPv4("network1")
+	addr2, err := allocateIPv4("network1", ipamPath)
 	require.NoError(t, err)
 
 	assert.Equal(t, addr.String(), addr2.String())
 
-	addr3, err := allocateIPv4("network2")
+	addr3, err := allocateIPv4("network2", ipamPath)
 	require.NoError(t, err)
 	assert.NotEqual(t, addr.String(), addr3.String())
 }
 
 func TestIPv4AllocateConcurent(t *testing.T) {
-	var (
-		err    error
-		origin = ipamPath
-	)
-
-	ipamPath, err = ioutil.TempDir("", "")
+	ipamPath, err := ioutil.TempDir("", "")
 	require.NoError(t, err)
-	defer func() { ipamPath = origin }()
+	t.Cleanup(func() {
+		_ = os.RemoveAll(ipamPath)
+	})
 
 	wg := sync.WaitGroup{}
 	wg.Add(10)
@@ -80,7 +75,7 @@ func TestIPv4AllocateConcurent(t *testing.T) {
 			defer wg.Done()
 			for y := 0; y < 10; y++ {
 				nw := fmt.Sprintf("network%d%d", i, y)
-				addr, err := allocateIPv4(nw)
+				addr, err := allocateIPv4(nw, ipamPath)
 				require.NoError(t, err)
 				c <- addr
 			}
diff --git a/pkg/provision/provision_test.go b/pkg/provision/provision_test.go
index e2085fc9..368cfe5d 100644
--- a/pkg/provision/provision_test.go
+++ b/pkg/provision/provision_test.go
@@ -77,3 +77,9 @@ func TestErrTemporary(t *testing.T) {
 	assert.True(t, errors.Is(wrap2, ErrTemporary{}))
 	assert.Equal(t, err.Error(), wrap1.Error())
 }
+
+func TestErrTemporaryNil(t *testing.T) {
+	wrap1 := NewErrTemporary(nil)
+	assert.Equal(t, wrap1.Error(), "")
+	assert.True(t, errors.Is(wrap1, ErrTemporary{}))
+}
diff --git a/pkg/set/uint_test.go b/pkg/set/uint_test.go
index dda0f0fa..e1cb4fbd 100644
--- a/pkg/set/uint_test.go
+++ b/pkg/set/uint_test.go
@@ -2,36 +2,57 @@ package set
 
 import (
 	"errors"
+	"io/ioutil"
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
+func testDir(t *testing.T) string {
+	td, err := ioutil.TempDir("", "")
+	require.NoError(t, err)
+
+	path := filepath.Join(td, "wireguard_ports")
+	err = os.MkdirAll(path, 0770)
+	require.NoError(t, err)
+
+	return path
+}
+
 func TestNew(t *testing.T) {
-	s := NewUint()
-	assert.NotNil(t, s.m)
+	td := testDir(t)
+
+	s := NewUint(td)
+	assert.Equal(t, td, s.root)
 }
 
 func TestAdd(t *testing.T) {
-	s := NewUint()
+	td := testDir(t)
+
+	s := NewUint(td)
 
 	for i := 0; i < 10; i++ {
 		err := s.Add(uint(i))
 		assert.NoError(t, err)
 	}
 
-	for i := 0; i < 10; i++ {
-		_, ok := s.m[uint(i)]
-		assert.True(t, ok, "%d should be present in the map", i)
-	}
+	infos, err := ioutil.ReadDir(s.root)
+	require.NoError(t, err)
 
-	err := s.Add(0)
+	assert.EqualValues(t, 10, len(infos))
+
+	err = s.Add(0)
 	assert.Error(t, err, "adding already present number should return an error")
 	assert.True(t, errors.Is(err, ErrConflict{}))
 }
 
 func TestRemove(t *testing.T) {
-	s := NewUint()
+	td := testDir(t)
+
+	s := NewUint(td)
 
 	for i := 0; i < 10; i++ {
 		err := s.Add(uint(i))
@@ -42,13 +63,18 @@ func TestRemove(t *testing.T) {
 		s.Remove(uint(i))
 	}
 
-	assert.EqualValues(t, 0, len(s.m))
+	infos, err := ioutil.ReadDir(s.root)
+	require.NoError(t, err)
+
+	assert.EqualValues(t, 0, len(infos))
 
 	s.Remove(99999) //ensure remove never panics
 }
 
 func TestList(t *testing.T) {
-	s := NewUint()
+	td := testDir(t)
+
+	s := NewUint(td)
 
 	err := s.Add(1)
 	assert.NoError(t, err)
@@ -57,12 +83,10 @@ func TestList(t *testing.T) {
 	err = s.Add(3)
 	assert.NoError(t, err)
 
-	l := s.List()
+	l, err := s.List()
+	require.NoError(t, err)
+
 	expected := []uint{1, 2, 3}
 	assert.Equal(t, len(expected), len(l))
 	assert.Subset(t, expected, l)
 }
-
-func TestConcurent(t *testing.T) {
-	// TODO
-}