Repository: threefoldtech/zos
Pull request #429 (2019-12-03 09:25:00)

zdb: correct the way storage allocation is done for 0-db sub volumes


Patch:
diff --git a/bootstrap/Makefile b/bootstrap/Makefile
index 3840443e..aa44d11c 100644
--- a/bootstrap/Makefile
+++ b/bootstrap/Makefile
@@ -6,12 +6,10 @@ install: build
 	mkdir -p ${ROOT}/etc/zinit/
 	mkdir -p ${ROOT}/bin
 	mkdir -p ${ROOT}/sbin
-	
-	# install interent 
-	cp ../etc/zinit/internet.yaml ${ROOT}/etc/zinit/
+
+	# install interent
 	cp ../bin/internet ${ROOT}/bin
-	
+
 	# install bootstrap
 	cp -a etc ${ROOT}
 	cp bootstrap/target/x86_64-unknown-linux-musl/release/bootstrap ${ROOT}/sbin/
-	
diff --git a/bootstrap/bootstrap/src/bootstrap.rs b/bootstrap/bootstrap/src/bootstrap.rs
index 51cb8959..3ea1f9f3 100644
--- a/bootstrap/bootstrap/src/bootstrap.rs
+++ b/bootstrap/bootstrap/src/bootstrap.rs
@@ -1,6 +1,5 @@
 use super::config;
 use super::hub;
-use super::kparams;
 use super::workdir::WorkDir;
 use super::zfs::Zfs;
 use super::zinit;
@@ -9,51 +8,18 @@ use failure::Error;
 use retry;
 
 const FLIST_REPO: &str = "tf-zos";
+const BIN_REPO: &str = "tf-zos-bins";
 const FLIST_INFO_FILE: &str = "/tmp/flist.info";
 const FLIST_NAME_FILE: &str = "/tmp/flist.name";
 const WORKDIR: &str = "/tmp/bootstrap";
 
 type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug)]
-enum RunMode {
-    Prod,
-    Test,
-    Dev,
-}
-
-fn runmode() -> Result<RunMode> {
-    let params = kparams::params()?;
-    let mode = match params.get("runmode") {
-        Some(mode) => match mode {
-            Some(mode) => match mode.as_ref() {
-                "prod" => RunMode::Prod,
-                "dev" => RunMode::Dev,
-                "test" => RunMode::Test,
-                m => {
-                    bail!("unknown runmode: {}", m);
-                }
-            },
-            None => {
-                //that's an error because runmode was
-                //provided as a kernel argumet but with no
-                //value
-                bail!("missing runmode value");
-            }
-        },
-        // runmode was not provided in cmdline
-        // so default is prod
-        None => RunMode::Prod,
-    };
-
-    Ok(mode)
-}
-
-fn boostrap_zos(mode: RunMode) -> Result<()> {
-    let flist = match mode {
-        RunMode::Prod => "zos:production:latest.flist",
-        RunMode::Dev => "zos:development:latest.flist",
-        RunMode::Test => "zos:testing:latest.flist",
+fn boostrap_zos(cfg: &config::Config) -> Result<()> {
+    let flist = match cfg.runmode {
+        config::RunMode::Prod => "zos:production:latest.flist",
+        config::RunMode::Dev => "zos:development:latest.flist",
+        config::RunMode::Test => "zos:testing:latest.flist",
     };
 
     debug!("using flist: {}/{}", FLIST_REPO, flist);
@@ -80,85 +46,14 @@ fn boostrap_zos(mode: RunMode) -> Result<()> {
     flist.write(FLIST_INFO_FILE)?;
     std::fs::write(FLIST_NAME_FILE, format!("{}/{}", FLIST_REPO, flist.name))?;
 
-    let result = retry::retry(retry::delay::Exponential::from_millis(200), || {
-        info!("download flist: {}", flist.name);
-
-        // the entire point of this match is the
-        // logging of the error.
-        match flist.download("machine.flist") {
-            Ok(ok) => Ok(ok),
-            Err(err) => {
-                error!("failed to download flist: {}", err);
-                bail!("failed to download flist: {}", err);
-            }
-        }
-    });
-
-    // I can't use the ? because error from retry
-    // is not compatible with failure::Error for
-    // some reason.
-    match result {
-        Err(err) => bail!("{:?}", err),
-        _ => (),
-    };
-
-    let fs = Zfs::mount("backend", "machine.flist", "root")?;
-    debug!("zfs started, now copying all files");
-    //copy everything to root
-    match fs.copy("/") {
-        Ok(_) => {}
-        Err(err) => bail!("failed to copy files to rootfs: {}", err),
-    };
-
-    // we need to find all yaml files under /etc/zinit to start monitoring them
-    let mut cfg = std::path::PathBuf::new();
-    cfg.push(&fs);
-    cfg.push("etc");
-    cfg.push("zinit");
-    let services = match std::fs::read_dir(&cfg) {
-        Ok(services) => services,
-        Err(err) => bail!("failed to read directory '{:?}': {}", cfg, err),
-    };
-    for service in services {
-        let service = service?;
-        let path = service.path();
-
-        if !path.is_file() {
-            continue;
-        }
-        let name = match path.file_name() {
-            Some(name) => match name.to_str() {
-                Some(name) => name,
-                None => {
-                    warn!("failed to process name: {:?}", path);
-                    continue;
-                }
-            },
-            None => continue,
-        };
-
-        match name.rfind(".yaml") {
-            None => continue,
-            Some(idx) => {
-                let service = &name[0..idx];
-                match zinit::monitor(service) {
-                    Ok(_) => {}
-                    Err(err) => {
-                        warn!("failed to monitor service '{}': {}", service, err);
-                    }
-                }
-            }
-        }
-    }
-    Ok(())
+    install_package(&flist)
 }
 
 /// bootstrap stage install and starts all zos daemons
-pub fn bootstrap(_: &config::Config) -> Result<()> {
-    let mode = runmode()?;
-    debug!("runmode: {:?}", mode);
+pub fn bootstrap(cfg: &config::Config) -> Result<()> {
+    debug!("runmode: {:?}", cfg.runmode);
     let result = WorkDir::run(WORKDIR, || -> Result<()> {
-        boostrap_zos(mode)?;
+        boostrap_zos(cfg)?;
         Ok(())
     })?;
 
@@ -207,16 +102,81 @@ fn update_bootstrap(debug: bool) -> Result<()> {
         Err(e) => bail!("failed to download flist: {:?}", e),
     };
 
+    // this trick here to allow overriding
+    // the current running bootstrap binary
+    let bin: Vec<String> = std::env::args().take(1).collect();
+    std::fs::rename(&bin[0], format!("{}.bak", &bin[0]))?;
+
+    install_package(&flist)
+}
+
+///install installs all binaries from the tf-zos-bins repo
+pub fn install(cfg: &config::Config) -> Result<()> {
+    let result = WorkDir::run(WORKDIR, || -> Result<()> {
+        install_packages(cfg)?;
+        Ok(())
+    })?;
+
+    result
+}
+
+fn install_packages(cfg: &config::Config) -> Result<()> {
+    let repo = match cfg.runmode {
+        config::RunMode::Prod => BIN_REPO.to_owned(),
+        config::RunMode::Dev => format!("{}.dev", BIN_REPO),
+        config::RunMode::Test => format!("{}.test", BIN_REPO),
+    };
+
+    let client = hub::Repo::new(&repo);
+    let packages = retry::retry(retry::delay::Exponential::from_millis(200), || {
+        info!("list packages in: {}", BIN_REPO);
+        //the full point of this match is the logging.
+        let packages = match client.list() {
+            Ok(info) => info,
+            Err(err) => {
+                error!("failed to list repo '{}': {}", BIN_REPO, err);
+                bail!("failed to list repo '{}': {}", BIN_REPO, err);
+            }
+        };
+
+        Ok(packages)
+    });
+
+    let packages = match packages {
+        Ok(packages) => packages,
+        Err(err) => bail!("failed to list '{}': {:?}", BIN_REPO, err),
+    };
+
+    let mut map = std::collections::HashMap::new();
+    for package in packages.iter() {
+        match install_package(package) {
+            Ok(_) => {}
+            Err(err) => warn!("failed to install package '{}': {}", package.url, err),
+        };
+
+        map.insert(format!("{}/{}", repo, package.name), package.clone());
+    }
+
+    let output = std::fs::OpenOptions::new()
+        .create(true)
+        .write(true)
+        .open("/tmp/bins.info")?;
+    serde_json::to_writer(&output, &map)?;
+
+    Ok(())
+}
+
+fn install_package(flist: &hub::Flist) -> Result<()> {
     let result = retry::retry(retry::delay::Exponential::from_millis(200), || {
         info!("download flist: {}", flist.name);
 
         // the entire point of this match is the
         // logging of the error.
-        match flist.download("bootstrap.flist") {
+        match flist.download(&flist.name) {
             Ok(ok) => Ok(ok),
             Err(err) => {
-                error!("failed to download flist: {}", err);
-                bail!("failed to download flist: {}", err);
+                error!("failed to download flist '{}': {}", flist.url, err);
+                bail!("failed to download flist '{}': {}", flist.url, err);
             }
         }
     });
@@ -229,13 +189,60 @@ fn update_bootstrap(debug: bool) -> Result<()> {
         _ => (),
     };
 
-    let fs = Zfs::mount("backend", "bootstrap.flist", "root")?;
+    let fs = Zfs::mount("backend", &flist.name, "root")?;
     debug!("zfs started, now copying all files");
 
-    // this trick here to allow overriding
-    // the current running bootstrap binary
-    let bin: Vec<String> = std::env::args().take(1).collect();
-    std::fs::rename(&bin[0], format!("{}.bak", &bin[0]))?;
+    fs.copy("/")?;
+
+    run_all(&fs)
+}
 
-    fs.copy("/")
+// run_all tries to run all services from an flist.
+// it will still try to run all other services defined
+// in the list of one or more failed. Returns error only
+// if failed to read the zinit directory
+fn run_all(fs: &Zfs) -> Result<()> {
+    let mut cfg = std::path::PathBuf::new();
+    cfg.push(fs);
+    cfg.push("etc");
+    cfg.push("zinit");
+    let services = match std::fs::read_dir(&cfg) {
+        Ok(services) => services,
+        Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(()),
+        Err(err) => bail!("failed to read directory '{:?}': {}", cfg, err),
+    };
+
+    for service in services {
+        let service = service?;
+        let path = service.path();
+
+        if !path.is_file() {
+            continue;
+        }
+        let name = match path.file_name() {
+            Some(name) => match name.to_str() {
+                Some(name) => name,
+                None => {
+                    warn!("failed to process name: {:?}", path);
+                    continue;
+                }
+            },
+            None => continue,
+        };
+
+        match name.rfind(".yaml") {
+            None => continue,
+            Some(idx) => {
+                let service = &name[0..idx];
+                match zinit::monitor(service) {
+                    Ok(_) => {}
+                    Err(err) => {
+                        warn!("failed to monitor service '{}': {}", service, err);
+                    }
+                }
+            }
+        }
+    }
+
+    Ok(())
 }
diff --git a/bootstrap/bootstrap/src/config.rs b/bootstrap/bootstrap/src/config.rs
index a6a5db63..63d04e87 100644
--- a/bootstrap/bootstrap/src/config.rs
+++ b/bootstrap/bootstrap/src/config.rs
@@ -1,11 +1,47 @@
+use super::kparams;
 use clap::{App, Arg};
 use failure::Error;
 
 type Result<T> = std::result::Result<T, Error>;
 
+#[derive(Debug)]
+pub enum RunMode {
+    Prod,
+    Test,
+    Dev,
+}
+
+fn runmode() -> Result<RunMode> {
+    let params = kparams::params()?;
+    let mode = match params.get("runmode") {
+        Some(mode) => match mode {
+            Some(mode) => match mode.as_ref() {
+                "prod" => RunMode::Prod,
+                "dev" => RunMode::Dev,
+                "test" => RunMode::Test,
+                m => {
+                    bail!("unknown runmode: {}", m);
+                }
+            },
+            None => {
+                //that's an error because runmode was
+                //provided as a kernel argumet but with no
+                //value
+                bail!("missing runmode value");
+            }
+        },
+        // runmode was not provided in cmdline
+        // so default is prod
+        None => RunMode::Prod,
+    };
+
+    Ok(mode)
+}
+
 pub struct Config {
     pub stage: u32,
     pub debug: bool,
+    pub runmode: RunMode,
 }
 
 impl Config {
@@ -44,6 +80,7 @@ impl Config {
         Ok(Config {
             stage: stage,
             debug: matches.occurrences_of("debug") > 0,
+            runmode: runmode()?,
         })
     }
 }
diff --git a/bootstrap/bootstrap/src/hub.rs b/bootstrap/bootstrap/src/hub.rs
index a5f38144..7849ddf8 100644
--- a/bootstrap/bootstrap/src/hub.rs
+++ b/bootstrap/bootstrap/src/hub.rs
@@ -15,19 +15,19 @@ pub struct Repo {
     name: String,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct Flist {
     #[serde(rename = "type")]
     pub kind: String,
     pub updated: u64,
-    pub size: u64,
+    #[serde(default)]
     pub md5: String,
     pub name: String,
     #[serde(default)]
     pub target: String,
 
     #[serde(skip)]
-    url: String,
+    pub url: String,
 }
 
 impl Repo {
@@ -41,6 +41,21 @@ impl Repo {
         }
     }
 
+    pub fn list(&self) -> Result<Vec<Flist>> {
+        let url = format!("{}/api/flist/{}", self.base, self.name,);
+
+        let mut response = get(&url)?;
+        let mut info: Vec<Flist> = match response.status() {
+            StatusCode::OK => response.json()?,
+            s => bail!("failed to get flist info: {}", s),
+        };
+        for flist in info.iter_mut() {
+            flist.url = format!("{}/{}/{}", self.base, self.name, flist.name);
+        }
+
+        Ok(info)
+    }
+
     pub fn get<T>(&self, flist: T) -> Result<Flist>
     where
         T: AsRef<str>,
@@ -121,4 +136,24 @@ mod tests {
         let _ = std::fs::remove_file(temp);
         Ok(())
     }
+
+    #[test]
+    fn test_list_repo() -> Result<()> {
+        let repo = Repo::new(String::from("azmy"));
+        let lists = repo.list()?;
+
+        let mut found: Option<&Flist> = None;
+        for flist in lists.iter() {
+            if flist.name == "test.flist" {
+                found = Some(flist);
+                break;
+            }
+        }
+
+        let found = found.unwrap();
+        assert_eq!(found.name, "test.flist");
+        assert_eq!(found.url, "https://hub.grid.tf/azmy/test.flist");
+
+        Ok(())
+    }
 }
diff --git a/bootstrap/bootstrap/src/main.rs b/bootstrap/bootstrap/src/main.rs
index 734ad35f..b053941d 100644
--- a/bootstrap/bootstrap/src/main.rs
+++ b/bootstrap/bootstrap/src/main.rs
@@ -29,7 +29,17 @@ fn app() -> Result<()> {
 
     // configure available stage
     let stages: Vec<fn(cfg: &Config) -> Result<()>> = vec![
-        |cfg| -> Result<()> { bootstrap::update(cfg) },
+        // self update
+        |cfg| -> Result<()> {
+            if cfg.debug {
+                // if debug is set, do not upgrade self.
+                return Ok(());
+            }
+            bootstrap::update(cfg)
+        },
+        // install all system binaries
+        |cfg| -> Result<()> { bootstrap::install(cfg) },
+        // install and start all services
         |cfg| -> Result<()> { bootstrap::bootstrap(cfg) },
     ];
 
diff --git a/etc/zinit/internet.yaml b/bootstrap/etc/zinit/internet.yaml
similarity index 100%
rename from etc/zinit/internet.yaml
rename to bootstrap/etc/zinit/internet.yaml
diff --git a/cmds/identityd/main.go b/cmds/identityd/main.go
index 37406f23..3f6f0d47 100644
--- a/cmds/identityd/main.go
+++ b/cmds/identityd/main.go
@@ -6,18 +6,20 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"syscall"
 	"time"
 
 	"github.com/pkg/errors"
 	"github.com/threefoldtech/zos/pkg/app"
+	"github.com/threefoldtech/zos/pkg/flist"
 	"github.com/threefoldtech/zos/pkg/gedis"
 	"github.com/threefoldtech/zos/pkg/geoip"
 	"github.com/threefoldtech/zos/pkg/stubs"
+	"github.com/threefoldtech/zos/pkg/upgrade"
 
 	"github.com/cenkalti/backoff/v3"
 	"github.com/threefoldtech/zos/pkg"
 	"github.com/threefoldtech/zos/pkg/environment"
-	"github.com/threefoldtech/zos/pkg/flist"
 	"github.com/threefoldtech/zos/pkg/identity"
 
 	"github.com/threefoldtech/zos/pkg/zinit"
@@ -26,7 +28,6 @@ import (
 
 	"github.com/rs/zerolog/log"
 	"github.com/threefoldtech/zbus"
-	"github.com/threefoldtech/zos/pkg/upgrade"
 	"github.com/threefoldtech/zos/pkg/utils"
 	"github.com/threefoldtech/zos/pkg/version"
 )
@@ -115,15 +116,6 @@ func main() {
 		log.Fatal().Err(err).Str("root", root).Msg("failed to create root directory")
 	}
 
-	zinit, err := zinit.New(zinitSocket)
-	if err != nil {
-		log.Fatal().Err(err).Msg("failed to connect to zinit")
-	}
-
-	// with volume allocation is set to nil this flister can be
-	// only used for RO mounts. Otherwise it will panic
-	flister := flist.New(root, nil)
-
 	boot := upgrade.Boot{}
 
 	bootMethod := boot.DetectBootMethod()
@@ -169,9 +161,9 @@ func main() {
 		return registerNode(nodeID, farmID, v, idStore, loc)
 	}
 
-	if err := backoff.Retry(func() error {
+	if err := backoff.RetryNotify(func() error {
 		return register(current)
-	}, backoff.NewExponentialBackOff()); err != nil {
+	}, backoff.NewExponentialBackOff(), retryNotify); err != nil {
 		log.Error().Err(err).Msg("failed to register node")
 	} else {
 		log.Info().Str("version", current).Msg("node registered successfully")
@@ -211,26 +203,101 @@ func main() {
 	// 4. Start watcher for new version
 	log.Info().Msg("start upgrade daemon")
 
+	upgradeLoop(ctx, &boot, root, debug, register)
+}
+
+func getBinsRepo() string {
+	env, _ := environment.Get()
+
+	switch env.RunningMode {
+	case environment.RunningDev:
+		return "tf-zos-bins.dev"
+	case environment.RunningTest:
+		return "tf-zos-bins.test"
+	default:
+		return "tf-zos-bins"
+	}
+}
+
+// allow reinstall if receive signal USR1
+// only allowed in debug mode
+func debugReinstall(boot *upgrade.Boot, up *upgrade.Upgrader) {
+	c := make(chan os.Signal)
+	signal.Notify(c, syscall.SIGUSR1)
+
+	go func() {
+		for range c {
+			current, err := boot.Current()
+			if err != nil {
+				log.Error().Err(err).Msg("couldn't get current flist info")
+				continue
+			}
+
+			up.Upgrade(current, current)
+		}
+	}()
+}
+func upgradeLoop(
+	ctx context.Context,
+	boot *upgrade.Boot,
+	root string,
+	debug bool,
+	register func(string) error) {
+
+	zinit, err := zinit.New(zinitSocket)
+	if err != nil {
+		log.Fatal().Err(err).Msg("failed to connect to zinit")
+	}
+
+	// with volume allocation is set to nil this flister can be
+	// only used for RO mounts. Otherwise it will panic
+	flister := flist.New(root, nil)
+
 	upgrader := upgrade.Upgrader{
 		FLister:      flister,
 		Zinit:        zinit,
 		NoSelfUpdate: debug,
 	}
 
-	watcher := upgrade.FListSemverWatcher{
+	if debug {
+		debugReinstall(boot, &upgrader)
+	}
+
+	flistWatcher := upgrade.FListSemverWatcher{
 		FList:    boot.Name(),
 		Current:  boot.MustVersion(), //if we are here version must be valid
 		Duration: 600 * time.Second,
 	}
 
-	events, err := watcher.Watch(ctx)
+	flistEvents, err := flistWatcher.Watch(ctx)
+	if err != nil {
+		log.Fatal().Err(err).Str("flist", flistWatcher.FList).Msg("failed to watch flist")
+	}
+
+	bins, err := boot.CurrentBins()
+	if err != nil {
+		log.Warn().Err(err).Msg("could not load current binaries list")
+	}
+
+	repoWatcher := upgrade.FListRepoWatcher{
+		Repo:    getBinsRepo(),
+		Current: bins,
+	}
+
+	repoEvents, err := repoWatcher.Watch(ctx)
 	if err != nil {
-		log.Fatal().Err(err).Str("flist", boot.Name()).Msg("failed to watch flist")
+		log.Fatal().Err(err).Str("repo", repoWatcher.Repo).Msg("failed to watch repo")
 	}
 
-	for event := range events {
-		switch event := event.(type) {
-		case *upgrade.FListEvent:
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case e := <-flistEvents:
+			if e == nil {
+				continue
+			}
+			event := e.(*upgrade.FListEvent)
 			// new flist available
 			version, err := event.Version()
 			if err != nil {
@@ -261,17 +328,43 @@ func main() {
 				log.Error().Err(err).Msg("failed to update boot information")
 			}
 
-			if err := backoff.Retry(func() error {
+			if err := backoff.RetryNotify(func() error {
 				return register(version.String())
-			}, backoff.NewExponentialBackOff()); err != nil {
+			}, backoff.NewExponentialBackOff(), retryNotify); err != nil {
 				log.Error().Err(err).Msg("failed to register node")
 			} else {
 				log.Info().Str("version", version.String()).Msg("node registered successfully")
 			}
+		case e := <-repoEvents:
+			if e == nil {
+				continue
+			}
+			event := e.(*upgrade.RepoEvent)
+			for _, bin := range event.ToDel {
+				if err := upgrader.UninstallBinary(bin); err != nil {
+					log.Error().Err(err).Str("flist", bin.Fqdn()).Msg("failed to uninstall flist")
+				}
+			}
+
+			for _, bin := range event.ToAdd {
+				if err := upgrader.InstallBinary(bin); err != nil {
+					log.Error().Err(err).Str("flist", bin.Fqdn()).Msg("failed to install flist")
+				}
+			}
+
+			if err := boot.SetBins(repoWatcher.Current); err != nil {
+				log.Error().Err(err).Msg("failed to update local db of installed binaries")
+			}
+
+			log.Debug().Msg("finish processing binary updates")
 		}
 	}
 }
 
+func retryNotify(err error, d time.Duration) {
+	log.Warn().Err(err).Str("sleep", d.String()).Msg("registration failed")
+}
+
 func identityMgr(root string) (pkg.IdentityManager, error) {
 	seedPath := filepath.Join(root, seedName)
 
diff --git a/etc/promtail/promtail.yaml b/etc/promtail/promtail.yaml
deleted file mode 100644
index ae933074..00000000
--- a/etc/promtail/promtail.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-server:
-  http_listen_port: 80
-  grpc_listen_port: 9095
-
-positions:
-  filename: /tmp/positions.yaml
-
-scrape_configs:
-- job_name: zos
-  static_configs:
-    - targets:
-        - localhost
-      labels:
-        job: zinit
-        node: "NODE-ID"
-        __path__: /var/cache/log/system.log
-  pipeline_stages:
-    - regex:
-        expression: '^\[.\]\s(?P<module>[^:]+):\s*(?P<body>.*)'
-    - regex:
-        # this might match other parts of the message.
-        expression: '(?P<level>info|error|debug|warn|fatal|panic)'
-        source: body
-    - regex:
-        expression: '(?P<time>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z(?:\d{2}\:\d{2})?)'
-        source: body
-    - timestamp:
-        source: time
-        format: RFC3339
-    - labels:
-        level:
-        module:
diff --git a/etc/zinit/containerd.yaml b/etc/zinit/containerd.yaml
deleted file mode 100644
index 029cacba..00000000
--- a/etc/zinit/containerd.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-exec: containerd
-test: ctr version
-after:
-  - node-ready
\ No newline at end of file
diff --git a/etc/zinit/promtail-config.yaml b/etc/zinit/promtail-config.yaml
deleted file mode 100644
index 81ab9366..00000000
--- a/etc/zinit/promtail-config.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-exec: |
-  sh -c 
-    'sed "s/NODE-ID/$(identityd -id)/g" -i /etc/promtail/promtail.yaml'
-oneshot: true
-after:
-  - boot
\ No newline at end of file
diff --git a/etc/zinit/promtail.yaml b/etc/zinit/promtail.yaml
deleted file mode 100644
index e964c52a..00000000
--- a/etc/zinit/promtail.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
-exec: |
-  promtail 
-  -config.file /etc/promtail/promtail.yaml
-  -client.url http://explorer.devnet.grid.tf:3100/loki/api/v1/push
-log: stdout
-after:
-  - promtail-config
-  - logger
diff --git a/go.mod b/go.mod
index 0b3cbab2..4b909b19 100644
--- a/go.mod
+++ b/go.mod
@@ -10,10 +10,10 @@ require (
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/cenkalti/backoff/v3 v3.0.0
 	github.com/containerd/cgroups v0.0.0-20190911145653-fc51bcbe4714 // indirect
-	github.com/containerd/containerd v1.2.3
+	github.com/containerd/containerd v1.3.2
 	github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6 // indirect
-	github.com/containerd/cri v1.11.1 // indirect
 	github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c // indirect
+	github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c // indirect
 	github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd // indirect
 	github.com/containernetworking/cni v0.7.1
 	github.com/containernetworking/plugins v0.8.2
@@ -21,7 +21,6 @@ require (
 	github.com/deckarep/golang-set v1.7.1
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/docker/distribution v2.7.1+incompatible // indirect
-	github.com/docker/docker v1.13.1 // indirect
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/go-redis/redis v6.15.5+incompatible
@@ -32,6 +31,7 @@ require (
 	github.com/gorilla/handlers v1.4.2
 	github.com/gorilla/mux v1.7.3
 	github.com/iancoleman/strcase v0.0.0-20190422225806-e506e3ef7365
+	github.com/imdario/mergo v0.3.8 // indirect
 	github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6
 	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
 	github.com/opencontainers/image-spec v1.0.1 // indirect
@@ -49,6 +49,7 @@ require (
 	github.com/vishvananda/netlink v1.0.0
 	github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
 	github.com/whs/nacl-sealed-box v0.0.0-20180930164530-92b9ba845d8d
+	go.etcd.io/bbolt v1.3.3 // indirect
 	golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472
 	golang.org/x/net v0.0.0-20191021144547-ec77196f6094 // indirect
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
@@ -59,3 +60,5 @@ require (
 	gopkg.in/yaml.v2 v2.2.4
 	gotest.tools v2.2.0+incompatible
 )
+
+replace github.com/docker/distribution v2.7.1+incompatible => github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible
diff --git a/go.sum b/go.sum
index 69dc37a5..1a05b448 100644
--- a/go.sum
+++ b/go.sum
@@ -20,14 +20,14 @@ github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4r
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/containerd/cgroups v0.0.0-20190911145653-fc51bcbe4714 h1:FglbJUAOM/NvNLkugiLF9L23nzmPi24NQgGOz4A198U=
 github.com/containerd/cgroups v0.0.0-20190911145653-fc51bcbe4714/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
-github.com/containerd/containerd v1.2.3 h1:6TvLXAk7vuAF72J3p6Fcw6c7Z8CDZFbZrHl+XIovyqk=
-github.com/containerd/containerd v1.2.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/containerd v1.3.2 h1:ForxmXkA6tPIvffbrDAcPUIB32QgXkt2XFj+F0UxetA=
+github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6 h1:NmTXa/uVnDyp0TY5MKi197+3HWcnYWfnHGyaFthlnGw=
 github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
-github.com/containerd/cri v1.11.1 h1:mR8+eNW4zEcbWGTGEpmDd7GzMmK7IMxMSVAZ2aIDKA4=
-github.com/containerd/cri v1.11.1/go.mod h1:DavH5Qa8+6jOmeOMO3dhWoqksucZDe06LfuhBz/xPZs=
 github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c h1:KFbqHhDeaHM7IfFtXHfUHMDaUStpM2YwBR+iJCIOsKk=
 github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
+github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c h1:+RqLdWzn0xFunb+sxXaEzHOg8NuEG/eaI+9C1xXX8Mw=
+github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8=
 github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd h1:bRLyitWw3PT/2YuVaCKTPg0cA5dOFKFwKtkfcP2dLsA=
 github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk=
 github.com/containernetworking/cni v0.7.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
@@ -55,10 +55,8 @@ github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9r
 github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
-github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo=
-github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible h1:dvc1KSkIYTVjZgHf/CTC2diTYC8PzhaA5sFISRfNVrE=
+github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8=
 github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
@@ -100,6 +98,8 @@ github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/iancoleman/strcase v0.0.0-20190422225806-e506e3ef7365 h1:ECW73yc9MY7935nNYXUkK7Dz17YuSUI9yqRqYS8aBww=
 github.com/iancoleman/strcase v0.0.0-20190422225806-e506e3ef7365/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE=
+github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ=
+github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
 github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6 h1:4zOlv2my+vf98jT1nQt4bT/yKWUImevYPJ2H344CloE=
 github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6/go.mod h1:r/8JmuR0qjuCiEhAolkfvdZgmPiHTnJaG0UXCSeR1Zo=
@@ -111,6 +111,8 @@ github.com/juju/testing v0.0.0-20190613124551-e81189438503/go.mod h1:63prj8cnj0t
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -144,6 +146,8 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1 h1:Lo6mRUjdS99f3zxYOUalftWHUoOGaDRqFk1+j0Q57/I=
+github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.14.3/go.mod h1:3WXPzbXEEliJ+a6UFE4vhIxV8qR1EML6ngzP9ug4eYg=
 github.com/rs/zerolog v1.15.0 h1:uPRuwkWF4J6fGsJ2R0Gn2jB1EQiav9k3S6CSdygQJXY=
@@ -160,8 +164,13 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5I
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.0.6 h1:hcP1GmhGigz/O7h1WVUM5KklBp1JoNS9FggWKdj/j3s=
 github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
@@ -186,14 +195,21 @@ github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6Ac
 github.com/whs/nacl-sealed-box v0.0.0-20180930164530-92b9ba845d8d h1:7UnhkLc0c3CSUXmDlQumHjDte0S2vW7TL+4mubdkiY4=
 github.com/whs/nacl-sealed-box v0.0.0-20180930164530-92b9ba845d8d/go.mod h1:ltQsZR7FRY+aC2OSr4UmsNI91hR831dJo2gZd50TSa4=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
+go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 h1:Gv7RPwsi3eZ2Fgewe3CBsuOebPwO27PoXzRpJPsvSSM=
 golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190509222800-a4d6f7feada5/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -203,16 +219,22 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20191021144547-ec77196f6094 h1:5O4U9trLjNpuhpynaDsqwCk+Tw6seqJz1EbqbnzHrc8=
 golang.org/x/net v0.0.0-20191021144547-ec77196f6094/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa h1:KIDDMLT1O0Nr7TSxp8xM5tJcdn8tgyAONntO829og1M=
@@ -224,6 +246,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
@@ -231,12 +255,17 @@ golang.zx2c4.com/wireguard/wgctrl v0.0.0-20190607034155-226bf4e412cd h1:p1UpgxMN
 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20190607034155-226bf4e412cd/go.mod h1:IZOMt4p2Xgw4FEU9w2Fq579uV0SgT7h5zxJnDyUHg60=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69 h1:4rNOqY4ULrKzS6twXa619uQgI7h9PaVd4ZhjFQ7C5zs=
+google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
 google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
 gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo=
@@ -253,4 +282,5 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/pkg/flist/flist.go b/pkg/flist/flist.go
index 3b5fc5a4..2c2939cf 100644
--- a/pkg/flist/flist.go
+++ b/pkg/flist/flist.go
@@ -235,6 +235,7 @@ func (f *flistModule) Umount(path string) error {
 	if err := syscall.Unmount(path, syscall.MNT_DETACH); err != nil {
 		log.Error().Err(err).Str("path", path).Msg("fail to umount flist")
 	}
+
 	if err := waitPidFile(time.Second*2, pidPath, false); err != nil {
 		log.Error().Err(err).Str("path", path).Msg("0-fs daemon did not stop properly")
 		return err
@@ -349,36 +350,31 @@ func random() (string, error) {
 // else it waits for the file to be deleted
 func waitPidFile(timeout time.Duration, path string, exists bool) error {
 	const delay = time.Millisecond * 100
-	cErr := make(chan error)
 
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 
-	go func() {
-		for {
-			select {
-			case <-ctx.Done():
-				cErr <- ctx.Err()
-			default:
-				_, err := os.Stat(path)
-				if exists {
-					if err != nil {
-						time.Sleep(delay)
-						continue
-					}
-					cErr <- nil
-				} else {
-					if err == nil {
-						time.Sleep(delay)
-						continue
-					}
-					cErr <- nil
-				}
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+			_, err := os.Stat(path)
+			// check exist but the file is not there yet,
+			// or check NOT exist but it is still there
+			// we try again
+			if (exists && os.IsNotExist(err)) || (!exists && err == nil) {
+				break //the select statement
+			} else if err != nil && !os.IsNotExist(err) {
+				//another error that is NOT IsNotExist.
+				return err
+			} else {
+				return nil
 			}
 		}
-	}()
 
-	return <-cErr
+		<-time.After(delay)
+	}
 }
 
 func waitMountedLog(timeout time.Duration, logfile string) error {
diff --git a/pkg/provision/container.go b/pkg/provision/container.go
index 0327b3da..320841ea 100644
--- a/pkg/provision/container.go
+++ b/pkg/provision/container.go
@@ -80,51 +80,25 @@ func containerProvisionImpl(ctx context.Context, reservation *Reservation) (Cont
 	tenantNS := fmt.Sprintf("ns%s", reservation.User)
 	containerID := reservation.ID
 
+	var config Container
+	if err := json.Unmarshal(reservation.Data, &config); err != nil {
+		return ContainerResult{}, err
+	}
+
 	// check if workload is already deployed
 	_, err := containerClient.Inspect(tenantNS, pkg.ContainerID(containerID))
 	if err == nil {
 		log.Info().Str("id", containerID).Msg("container already deployed")
 		return ContainerResult{
-			ID: containerID,
-			//IMPORTANT
-			//TODO: set IPv4 and IPv6 here
+			ID:   containerID,
+			IPv4: config.Network.IPs[0].String(),
 		}, nil
 	}
 
-	var config Container
-	if err := json.Unmarshal(reservation.Data, &config); err != nil {
-		return ContainerResult{}, err
-	}
-
 	if err := validateContainerConfig(config); err != nil {
 		return ContainerResult{}, errors.Wrap(err, "container provision schema not valid")
 	}
 
-	netID := networkID(reservation.User, string(config.Network.NetworkID))
-	log.Debug().
-		Str("network-id", string(netID)).
-		Str("config", fmt.Sprintf("%+v", config)).
-		Msg("deploying network")
-
-	networkMgr := stubs.NewNetworkerStub(GetZBus(ctx))
-
-	ips := make([]string, len(config.Network.IPs))
-	for i, ip := range config.Network.IPs {
-		ips[i] = ip.String()
-	}
-
-	join, err := networkMgr.Join(netID, containerID, ips)
-	if err != nil {
-		return ContainerResult{}, err
-	}
-
-	// TODO: Push IP back to bcdb
-	log.Info().
-		Str("ipv6", join.IPv6.String()).
-		Str("ipv4", join.IPv4.String()).
-		Str("container", reservation.ID).
-		Msg("assigned an IP")
-
 	log.Debug().Str("flist", config.FList).Msg("mounting flist")
 	mnt, err := flistClient.Mount(config.FList, config.FlistStorage, pkg.DefaultMountOptions)
 	if err != nil {
@@ -168,6 +142,31 @@ func containerProvisionImpl(ctx context.Context, reservation *Reservation) (Cont
 			},
 		)
 	}
+
+	netID := networkID(reservation.User, string(config.Network.NetworkID))
+	log.Debug().
+		Str("network-id", string(netID)).
+		Str("config", fmt.Sprintf("%+v", config)).
+		Msg("deploying network")
+
+	networkMgr := stubs.NewNetworkerStub(GetZBus(ctx))
+
+	ips := make([]string, len(config.Network.IPs))
+	for i, ip := range config.Network.IPs {
+		ips[i] = ip.String()
+	}
+
+	join, err := networkMgr.Join(netID, containerID, ips)
+	if err != nil {
+		return ContainerResult{}, err
+	}
+
+	log.Info().
+		Str("ipv6", join.IPv6.String()).
+		Str("ipv4", join.IPv4.String()).
+		Str("container", reservation.ID).
+		Msg("assigned an IP")
+
 	id, err := containerClient.Run(
 		tenantNS,
 		pkg.Container{
@@ -186,6 +185,10 @@ func containerProvisionImpl(ctx context.Context, reservation *Reservation) (Cont
 	)
 
 	if err != nil {
+		if err := networkMgr.Leave(netID, containerID); err != nil {
+			log.Error().Err(err).Msgf("failed leave containrt network namespace")
+		}
+
 		if err := flistClient.Umount(mnt); err != nil {
 			log.Error().Err(err).Str("path", mnt).Msgf("failed to unmount")
 		}
@@ -217,24 +220,25 @@ func containerDecommission(ctx context.Context, reservation *Reservation) error
 	}
 
 	info, err := container.Inspect(tenantNS, containerID)
-	if err != nil {
-		return errors.Wrapf(err, "failed to inspect container %s", containerID)
-	}
+	if err == nil {
+		if err := container.Delete(tenantNS, containerID); err != nil {
+			return errors.Wrapf(err, "failed to delete container %s", containerID)
+		}
 
-	if err := container.Delete(tenantNS, containerID); err != nil {
-		return errors.Wrapf(err, "failed to delete container %s", containerID)
-	}
+		rootFS := info.RootFS
+		if info.Interactive {
+			rootFS, err = findRootFS(info.Mounts)
+			if err != nil {
+				return err
+			}
+		}
 
-	rootFS := info.RootFS
-	if info.Interactive {
-		rootFS, err = findRootFS(info.Mounts)
-		if err != nil {
-			return err
+		if err := flist.Umount(rootFS); err != nil {
+			return errors.Wrapf(err, "failed to unmount flist at %s", rootFS)
 		}
-	}
 
-	if err := flist.Umount(rootFS); err != nil {
-		return errors.Wrapf(err, "failed to unmount flist at %s", rootFS)
+	} else {
+		log.Error().Err(err).Str("container", string(containerID)).Msg("failed to inspect container for decomission")
 	}
 
 	netID := networkID(reservation.User, string(config.Network.NetworkID))
diff --git a/pkg/storage/zdb.go b/pkg/storage/zdb.go
index baf1a436..e236a5e4 100644
--- a/pkg/storage/zdb.go
+++ b/pkg/storage/zdb.go
@@ -36,6 +36,22 @@ func (s *storageModule) Allocate(diskType pkg.DeviceType, size uint64, mode pkg.
 			continue
 		}
 
+		usage, err := pool.Usage()
+		if err != nil {
+			return "", "", errors.Wrapf(err, "failed to read usage of pool %s", pool.Name())
+		}
+
+		reserved, err := pool.Reserved()
+		if err != nil {
+			return "", "", errors.Wrapf(err, "failed to read reserved size of pool %s", pool.Name())
+		}
+
+		// Make sure adding this filesystem would not bring us over the disk limit
+		if reserved+size > usage.Size {
+			slog.Info().Msgf("Disk does not have enough space left to hold filesystem")
+			continue
+		}
+
 		volumes, err := pool.Volumes()
 		if err != nil {
 			return "", "", errors.Wrapf(err, "failed to list volume on pool %s", pool.Name())
@@ -48,27 +64,16 @@ func (s *storageModule) Allocate(diskType pkg.DeviceType, size uint64, mode pkg.
 				continue
 			}
 
-			usage, err := pool.Usage()
+			usage, err := volume.Usage()
 			if err != nil {
 				return "", "", errors.Wrapf(err, "failed to read usage of volume %s", volume.Name())
 			}
 
-			// skip pool with not enough space
-			reserved, err := pool.Reserved()
-			if err != nil {
-				return "", "", errors.Wrapf(err, "failed to read reserved size of pool %s", pool.Name())
-			}
-
-			// Make sure adding this filesystem would not bring us over the disk limit
-			if reserved+size > usage.Size {
-				slog.Info().Msgf("Disk does not have enough space left to hold filesystem")
-				continue
-			}
-
-			// existing volume with enough grow its limit
+			// existing volume with enough storage, grow its limit
 			if err := volume.Limit(usage.Size + size); err != nil {
 				return "", "", errors.Wrapf(err, "failed to grow limit of volume %s", volume.Name())
 			}
+
 			slog.Info().
 				Str("volume", volume.Name()).
 				Str("path", volume.Path()).
@@ -126,7 +131,13 @@ func (s *storageModule) Claim(name string, size uint64) error {
 
 	usage, err := v.Usage()
 	if err != nil {
-		return errors.Wrapf(err, "failed to read usage of volume %s", v.Name())
+		return err
+	}
+
+	// limit cannot be 0 cause 0 means not limited
+	limit := usage.Size - size
+	if limit <= 0 {
+		limit = 1
 	}
 
 	// shrink the limit
diff --git a/pkg/upgrade/boot.go b/pkg/upgrade/boot.go
index 8d9e541c..f44cbf01 100644
--- a/pkg/upgrade/boot.go
+++ b/pkg/upgrade/boot.go
@@ -19,6 +19,7 @@ const (
 
 	nameFile = "/tmp/flist.name"
 	infoFile = "/tmp/flist.info"
+	binsFile = "/tmp/bins.info"
 )
 
 // BootMethod defines the node boot method
@@ -60,6 +61,29 @@ func (b *Boot) Name() string {
 	return strings.TrimSpace(string(data))
 }
 
+//CurrentBins returns a list of current binaries installed
+func (b *Boot) CurrentBins() (map[string]RepoFList, error) {
+	f, err := os.Open(binsFile)
+	if err != nil {
+		return nil, err
+	}
+	dec := json.NewDecoder(f)
+
+	var result map[string]RepoFList
+	err = dec.Decode(&result)
+	return result, err
+}
+
+//SetBins sets the current list of binaries in boot files
+func (b *Boot) SetBins(current map[string]RepoFList) error {
+	f, err := os.Create(binsFile)
+	if err != nil {
+		return err
+	}
+	enc := json.NewEncoder(f)
+	return enc.Encode(current)
+}
+
 // Current returns current flist information
 func (b *Boot) Current() (FListEvent, error) {
 	name := b.Name()
diff --git a/pkg/upgrade/hub.go b/pkg/upgrade/hub.go
index 7d08901d..1e795395 100644
--- a/pkg/upgrade/hub.go
+++ b/pkg/upgrade/hub.go
@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
@@ -67,17 +68,52 @@ func (h *hubClient) Info(flist string) (info flistInfo, err error) {
 	return info, err
 }
 
-// flistInfo reflects node boot information (flist + version)
-type flistInfo struct {
+func (h *hubClient) List(repo string) ([]listFListInfo, error) {
+	u, err := url.Parse(hubBaseURL)
+	if err != nil {
+		panic("invalid base url")
+	}
+
+	u.Path = filepath.Join("api", "flist", repo)
+	response, err := http.Get(u.String())
+	if err != nil {
+		return nil, err
+	}
+
+	defer response.Body.Close()
+	defer ioutil.ReadAll(response.Body)
+
+	if response.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to get repository listing: %s", response.Status)
+	}
+
+	dec := json.NewDecoder(response.Body)
+
+	var result []listFListInfo
+	err = dec.Decode(&result)
+
+	for i := range result {
+		result[i].Repository = repo
+	}
+
+	return result, err
+}
+
+type listFListInfo struct {
 	Name       string `json:"name"`
 	Target     string `json:"target"`
 	Type       string `json:"type"`
 	Updated    uint64 `json:"updated"`
-	Hash       string `json:"hash"`
-	Size       uint64 `json:"size"`
 	Repository string `json:"-"`
 }
 
+// flistInfo reflects node boot information (flist + version)
+type flistInfo struct {
+	listFListInfo
+	Hash string `json:"hash"`
+	Size uint64 `json:"size"`
+}
+
 // fileInfo is the file of an flist
 type fileInfo struct {
 	Path string `json:"path"`
@@ -96,7 +132,11 @@ func (b *flistInfo) Commit(path string) error {
 	return enc.Encode(b)
 }
 
-func (b *flistInfo) extractVersion(name string) (ver semver.Version, err error) {
+func (b *listFListInfo) Fqdn() string {
+	return path.Join(b.Repository, b.Name)
+}
+
+func (b *listFListInfo) extractVersion(name string) (ver semver.Version, err error) {
 	// name is suppose to be as follows
 	// <name>:<version>.flist
 	parts := strings.Split(name, ":")
@@ -107,7 +147,7 @@ func (b *flistInfo) extractVersion(name string) (ver semver.Version, err error)
 }
 
 // Version returns the version of the flist
-func (b *flistInfo) Version() (semver.Version, error) {
+func (b *listFListInfo) Version() (semver.Version, error) {
 	// computing the version is tricky because it's part of the name
 	// of the flist (or the target) depends on the type of the flist
 
@@ -119,7 +159,7 @@ func (b *flistInfo) Version() (semver.Version, error) {
 }
 
 // Absolute returns the actual flist name
-func (b *flistInfo) Absolute() string {
+func (b *listFListInfo) Absolute() string {
 	name := b.Name
 	if b.Type == "symlink" {
 		name = b.Target
@@ -129,7 +169,7 @@ func (b *flistInfo) Absolute() string {
 }
 
 // Files gets the list of the files of an flist
-func (b *flistInfo) Files() ([]fileInfo, error) {
+func (b *listFListInfo) Files() ([]fileInfo, error) {
 	flist := b.Absolute()
 	if len(flist) == 0 {
 		return nil, fmt.Errorf("invalid flist info")
diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go
index ae3c66de..45c4e04c 100644
--- a/pkg/upgrade/upgrade.go
+++ b/pkg/upgrade/upgrade.go
@@ -49,7 +49,77 @@ type Upgrader struct {
 // instead the upgraded daemon will be completely stopped
 func (u *Upgrader) Upgrade(from, to FListEvent) error {
 	return u.applyUpgrade(from, to)
+}
+
+// InstallBinary from a single flist.
+func (u *Upgrader) InstallBinary(flist RepoFList) error {
+	log.Info().Str("flist", flist.Fqdn()).Msg("start applying upgrade")
+
+	flistRoot, err := u.FLister.Mount(u.hub.MountURL(flist.Fqdn()), u.hub.StorageURL(), pkg.ReadOnlyMountOptions)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err := u.FLister.Umount(flistRoot); err != nil {
+			log.Error().Err(err).Msgf("fail to umount flist at %s: %v", flistRoot, err)
+		}
+	}()
 
+	if err := copyRecursive(flistRoot, "/"); err != nil {
+		return errors.Wrapf(err, "failed to install flist: %s", flist.Fqdn())
+	}
+
+	p := filepath.Join(flistRoot, "etc", "zinit")
+	log.Debug().Str("path", p).Msg("checking for zinit unit files")
+	files, err := ioutil.ReadDir(p)
+	if os.IsNotExist(err) {
+		log.Debug().Err(err).Msg("/etc/zinit not found on flist")
+		return nil
+	} else if err != nil {
+		return errors.Wrap(err, "failed to list package services")
+	}
+
+	var services []string
+	for _, file := range files {
+		if !strings.HasSuffix(file.Name(), ".yaml") {
+			continue
+		}
+
+		service := strings.TrimSuffix(file.Name(), ".yaml")
+		services = append(services, service)
+	}
+
+	return u.ensureRestarted(services...)
+}
+
+func (u *Upgrader) ensureRestarted(service ...string) error {
+	log.Debug().Strs("services", service).Msg("ensure services")
+	if len(service) == 0 {
+		return nil
+	}
+
+	log.Debug().Strs("services", service).Msg("restarting services")
+	if err := u.stopMultiple(20*time.Second, service...); err != nil {
+		return err
+	}
+
+	for _, name := range service {
+		if err := u.Zinit.Forget(name); err != nil {
+			log.Warn().Err(err).Str("service", name).Msg("could not forget service")
+		}
+
+		if err := u.Zinit.Monitor(name); err != nil {
+			log.Error().Err(err).Str("service", name).Msg("could not monitor service")
+		}
+	}
+
+	return nil
+}
+
+// UninstallBinary  from a single flist.
+func (u *Upgrader) UninstallBinary(flist RepoFList) error {
+	return u.uninstall(flist.listFListInfo)
 }
 
 func (u Upgrader) stopMultiple(timeout time.Duration, service ...string) error {
@@ -157,12 +227,10 @@ func (u *Upgrader) upgradeSelf(root string) error {
 	return ErrRestartNeeded
 }
 
-func (u *Upgrader) uninstall(from FListEvent) error {
-	current := from
-
-	files, err := current.Files()
+func (u *Upgrader) uninstall(flist listFListInfo) error {
+	files, err := flist.Files()
 	if err != nil {
-		return errors.Wrapf(err, "failed to get list of current installed files for '%s'", current.Absolute())
+		return errors.Wrapf(err, "failed to get list of current installed files for '%s'", flist.Absolute())
 	}
 
 	//stop all services names
@@ -209,6 +277,7 @@ func (u *Upgrader) uninstall(from FListEvent) error {
 			log.Debug().Err(err).Str("file", file.Path).Msg("failed to check file")
 			continue
 		}
+
 		if stat.IsDir() {
 			continue
 		}
@@ -239,7 +308,7 @@ func (u *Upgrader) applyUpgrade(from, to FListEvent) error {
 		return err
 	}
 
-	if err := u.uninstall(from); err != nil {
+	if err := u.uninstall(from.listFListInfo); err != nil {
 		log.Error().Err(err).Msg("failed to unistall current flist. Upgraded anyway")
 	}
 
diff --git a/pkg/upgrade/watcher.go b/pkg/upgrade/watcher.go
index c02706fd..2383a9fd 100644
--- a/pkg/upgrade/watcher.go
+++ b/pkg/upgrade/watcher.go
@@ -1,14 +1,13 @@
 package upgrade
 
-import "time"
+import (
+	"context"
+	"time"
 
-import "github.com/blang/semver"
-
-import "context"
-
-import "github.com/rs/zerolog/log"
-
-import "path"
+	"github.com/blang/semver"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+)
 
 /**
 Watcher unifies the upgrade pipeline by making sure we can watch
@@ -48,11 +47,6 @@ func (f *FListEvent) EventType() EventType {
 	return FList
 }
 
-//Fqdn return full name of the flist (with repo)
-func (f *FListEvent) Fqdn() string {
-	return path.Join(f.Repository, f.Name)
-}
-
 //TryVersion will try to parse the version from flist
 //other wise return empty version 0.0.0
 func (f *FListEvent) TryVersion() semver.Version {
@@ -71,9 +65,11 @@ type FListSemverWatcher struct {
 	client hubClient
 }
 
+var _ Watcher = &FListSemverWatcher{}
+
 // Watch an flist change in version
+// The Event returned by the channel is of concrete type FListEvent
 func (w *FListSemverWatcher) Watch(ctx context.Context) (<-chan Event, error) {
-
 	if w.Duration == time.Duration(0) {
 		//default delay of 5min
 		w.Duration = 600 * time.Second
@@ -126,9 +122,14 @@ func (w *FListSemverWatcher) Watch(ctx context.Context) (<-chan Event, error) {
 			}
 
 			if version.GT(w.Current) {
-				ch <- &FListEvent{
+				select {
+				case ch <- &FListEvent{
 					flistInfo: info,
+				}:
+				case <-ctx.Done():
+					return
 				}
+
 				w.Current = version
 			}
 		}
@@ -136,3 +137,129 @@ func (w *FListSemverWatcher) Watch(ctx context.Context) (<-chan Event, error) {
 
 	return ch, nil
 }
+
+//RepoFList holds information of flist from a repo list operation
+type RepoFList struct {
+	listFListInfo
+}
+
+//RepoEvent is returned by the repo watcher
+type RepoEvent struct {
+	Repo  string
+	ToAdd []RepoFList
+	ToDel []RepoFList
+}
+
+//EventType returns event type
+func (e *RepoEvent) EventType() EventType {
+	return Repo
+}
+
+//FListRepoWatcher type
+type FListRepoWatcher struct {
+	Repo     string
+	Current  map[string]RepoFList
+	Duration time.Duration
+
+	client hubClient
+}
+
+func (w *FListRepoWatcher) list() (map[string]RepoFList, error) {
+	packages, err := w.client.List(w.Repo)
+	if err != nil {
+		return nil, err
+	}
+
+	result := make(map[string]RepoFList)
+	for _, pkg := range packages {
+		flist := RepoFList{pkg}
+		result[flist.Fqdn()] = flist
+	}
+
+	return result, nil
+}
+
+func (w *FListRepoWatcher) diff(packages map[string]RepoFList) (toAdd, toDel []RepoFList) {
+	for name, pkg := range packages {
+		current, ok := w.Current[name]
+		if !ok || pkg.Updated != current.Updated {
+			toAdd = append(toAdd, pkg)
+		}
+	}
+
+	for name := range w.Current {
+		current, ok := packages[name]
+		if !ok {
+			toDel = append(toDel, current)
+		}
+	}
+
+	return
+}
+
+// Watch watches a full repo for changes. Event is always of concrete type RepoEvent
+func (w *FListRepoWatcher) Watch(ctx context.Context) (<-chan Event, error) {
+	if w.Duration == time.Duration(0) {
+		//default delay of 5min
+		w.Duration = 600 * time.Second
+	}
+
+	packages, err := w.list()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get available packages")
+	}
+
+	toAdd, toDel := w.diff(packages)
+
+	ch := make(chan Event, 1)
+
+	if len(toAdd) > 0 || len(toDel) > 0 {
+		ch <- &RepoEvent{
+			Repo:  w.Repo,
+			ToAdd: toAdd,
+			ToDel: toDel,
+		}
+
+		w.Current = packages
+	}
+
+	ticker := time.NewTicker(w.Duration)
+
+	go func() {
+		defer close(ch)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-ticker.C:
+			case <-ctx.Done():
+				return
+			}
+
+			packages, err := w.list()
+			if err != nil {
+				log.Error().Err(err).Str("repo", w.Repo).Msg("failed to list repo flists")
+				continue
+			}
+
+			toAdd, toDel := w.diff(packages)
+
+			if len(toAdd) > 0 || len(toDel) > 0 {
+				select {
+				case ch <- &RepoEvent{
+					Repo:  w.Repo,
+					ToAdd: toAdd,
+					ToDel: toDel,
+				}:
+				case <-ctx.Done():
+					return
+				}
+
+				w.Current = packages
+			}
+		}
+
+	}()
+
+	return ch, nil
+}
diff --git a/qemu/overlay/etc/cni/net.d/10-bridge.conf b/qemu/overlay/etc/cni/net.d/10-bridge.conf
deleted file mode 100644
index 01d659e3..00000000
--- a/qemu/overlay/etc/cni/net.d/10-bridge.conf
+++ /dev/null
@@ -1,14 +0,0 @@
-{
-    "cniVersion": "0.2.0",
-	"name": "container_bridge",
-	"type": "bridge",
-	"bridge": "cont0",
-	"isDefaultGateway": true,
-	"forceAddress": false,
-	"ipMasq": true,
-	"hairpinMode": true,
-	"ipam": {
-		"type": "host-local",
-		"subnet": "10.10.0.0/16"
-	}
-}
\ No newline at end of file
diff --git a/qemu/overlay/etc/cni/net.d/10-mynet.conf b/qemu/overlay/etc/cni/net.d/10-mynet.conf
deleted file mode 100644
index 2cc0b474..00000000
--- a/qemu/overlay/etc/cni/net.d/10-mynet.conf
+++ /dev/null
@@ -1,15 +0,0 @@
-{
-"cniVersion": "0.2.0",
-"name": "mynet",
-"type": "bridge",
-"bridge": "cni0",
-"isGateway": true,
-"ipMasq": true,
-"ipam": {
-"type": "host-local",
-"subnet": "10.22.0.0/16",
-"routes": [
-{ "dst": "0.0.0.0/0" }
-]
-}
-}
diff --git a/qemu/overlay/etc/cni/net.d/10-myptp.conf b/qemu/overlay/etc/cni/net.d/10-myptp.conf
deleted file mode 100644
index baa5213b..00000000
--- a/qemu/overlay/etc/cni/net.d/10-myptp.conf
+++ /dev/null
@@ -1,1 +0,0 @@
-{"cniVersion":"0.4.0","name":"myptp","type":"ptp","ipMasq":true,"ipam":{"type":"host-local","subnet":"172.16.29.0/24","routes":[{"dst":"0.0.0.0/0"}]}}
diff --git a/qemu/overlay/etc/cni/net.d/99-loopback.conf b/qemu/overlay/etc/cni/net.d/99-loopback.conf
deleted file mode 100644
index 72024575..00000000
--- a/qemu/overlay/etc/cni/net.d/99-loopback.conf
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-"cniVersion": "0.2.0",
-"name": "lo",
-"type": "loopback"
-}
diff --git a/qemu/overlay/etc/containerd/config.toml b/qemu/overlay/etc/containerd/config.toml
deleted file mode 100644
index d081e5cc..00000000
--- a/qemu/overlay/etc/containerd/config.toml
+++ /dev/null
@@ -1,44 +0,0 @@
-root = "/var/lib/containerd"
-state = "/run/containerd"
-oom_score = 0
-disabled_plugins = ["cri"]
-
-[grpc]
-  address = "/run/containerd/containerd.sock"
-  uid = 0
-  gid = 0
-  max_recv_message_size = 16777216
-  max_send_message_size = 16777216
-
-[debug]
-  address = ""
-  uid = 0
-  gid = 0
-  level = "debug"
-
-[metrics]
-  address = ""
-  grpc_histogram = false
-
-[cgroup]
-  path = ""
-
-[plugins]
-  [plugins.diff-service]
-    default = ["walking"]
-  [plugins.linux]
-    shim = "containerd-shim"
-    runtime = "runc"
-    runtime_root = ""
-    no_shim = false
-    shim_debug = false
-  [plugins.opt]
-    path = "/opt/containerd"
-  [plugins.restart]
-    interval = "10s"
-  [plugins.scheduler]
-    pause_threshold = 0.02
-    deletion_threshold = 0
-    mutation_threshold = 100
-    schedule_delay = "0s"
-    startup_delay = "100ms"
diff --git a/qemu/overlay/etc/zinit/containerd.yaml b/qemu/overlay/etc/zinit/containerd.yaml
deleted file mode 120000
index 9f52e359..00000000
--- a/qemu/overlay/etc/zinit/containerd.yaml
+++ /dev/null
@@ -1,1 +0,0 @@
-../../../../etc/zinit/containerd.yaml
\ No newline at end of file
diff --git a/qemu/overlay/etc/zinit/internet.yaml b/qemu/overlay/etc/zinit/internet.yaml
index e865fcfe..d485ff54 120000
--- a/qemu/overlay/etc/zinit/internet.yaml
+++ b/qemu/overlay/etc/zinit/internet.yaml
@@ -1,1 +1,1 @@
-../../../../etc/zinit/internet.yaml
\ No newline at end of file
+../../../../bootstrap/etc/zinit/internet.yaml
\ No newline at end of file
diff --git a/qemu/overlay/etc/zinit/promtail-config.yaml b/qemu/overlay/etc/zinit/promtail-config.yaml
deleted file mode 120000
index c85fb185..00000000
--- a/qemu/overlay/etc/zinit/promtail-config.yaml
+++ /dev/null
@@ -1,1 +0,0 @@
-../../../../etc/zinit/promtail-config.yaml
\ No newline at end of file
diff --git a/qemu/overlay/etc/zinit/promtail.yaml b/qemu/overlay/etc/zinit/promtail.yaml
deleted file mode 120000
index c22f0940..00000000
--- a/qemu/overlay/etc/zinit/promtail.yaml
+++ /dev/null
@@ -1,1 +0,0 @@
-../../../../etc/zinit/promtail.yaml
\ No newline at end of file
diff --git a/qemu/vm.sh b/qemu/vm.sh
index 18a21b25..758aedae 100755
--- a/qemu/vm.sh
+++ b/qemu/vm.sh
@@ -5,6 +5,8 @@ debug=0
 reset=0
 image=zos.efi
 kernelargs=""
+bridge=zos
+graphics="-nographic -nodefaults"
 
 usage() {
    cat <<EOF
@@ -14,6 +16,8 @@ Usage: vm -n $name [ -r ] [ -d ]
    -r: reset vm (recreate disks)
    -c: Kernel command
    -i: kernel image to boot
+   -b: bridge for network (default zos)
+   -g: open GUI
    -h: help
 
 EOF
@@ -21,13 +25,15 @@ EOF
 }
 
 
-while getopts "c:n:i:rd" opt; do
+while getopts "c:n:i:rdb:g" opt; do
    case $opt in
    i )  image=$OPTARG ;;
    r )  reset=1 ;;
    d )  debug=1 ;;
    n )  name=$OPTARG ;;
    c )  kernelargs=$OPTARG ;;
+   b )  bridge=$OPTARG ;;
+   g )  graphics="" ;;
    h )  usage ; exit 0 ;;
    \?)  usage ; exit 1 ;;
    esac
@@ -62,15 +68,16 @@ if ps -eaf | grep -v grep | grep "$uuid" > /dev/null; then
 fi
 
 echo "boot $image"
+
 qemu-system-x86_64 -kernel $image \
     -m 3072 -enable-kvm -cpu host \
     -uuid $uuid \
-    -netdev bridge,id=zos0,br=zos0 -device virtio-net-pci,netdev=zos0,mac="${basemac}1" \
+    -netdev bridge,id=zos0,br=${bridge} -device virtio-net-pci,netdev=zos0,mac="${basemac}1" \
     -drive file=fat:rw:$basepath/overlay,format=raw \
     -append "${cmdline} console=ttyS1,115200" \
     -drive file=$vmdir/vda.qcow2,if=virtio -drive file=$vmdir/vdb.qcow2,if=virtio \
     -drive file=$vmdir/vdc.qcow2,if=virtio -drive file=$vmdir/vdd.qcow2,if=virtio \
     -drive file=$vmdir/vde.qcow2,if=virtio \
     -serial null -serial mon:stdio \
-    -nographic -nodefaults
+    ${graphics}
 


Test patch:
diff --git a/pkg/flist/flist_test.go b/pkg/flist/flist_test.go
index e253a29e..9465601f 100644
--- a/pkg/flist/flist_test.go
+++ b/pkg/flist/flist_test.go
@@ -2,6 +2,7 @@ package flist
 
 import (
 	"bytes"
+	"context"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -9,6 +10,7 @@ import (
 	"strings"
 	"testing"
 	"text/template"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
@@ -207,3 +209,47 @@ func TestDownloadFlist(t *testing.T) {
 	require.NoError(err)
 	assert.Equal(info1.ModTime(), info2.ModTime())
 }
+
+func TestWaitPIDFileExists(t *testing.T) {
+	require := require.New(t)
+	const testFile = "/tmp/wait.exists.test"
+	os.Remove(testFile)
+
+	out := make(chan error)
+	go func(out chan<- error) {
+		out <- waitPidFile(2*time.Second, testFile, true)
+	}(out)
+
+	os.Create(testFile)
+	err := <-out
+	require.NoError(err)
+}
+
+func TestWaitPIDFileDeleted(t *testing.T) {
+	require := require.New(t)
+	const testFile = "/tmp/wait.deleted.test"
+	os.Create(testFile)
+
+	out := make(chan error)
+	go func(out chan<- error) {
+		out <- waitPidFile(2*time.Second, testFile, false)
+	}(out)
+
+	os.Remove(testFile)
+	err := <-out
+	require.NoError(err)
+}
+
+func TestWaitPIDFileTimeout(t *testing.T) {
+	require := require.New(t)
+	const testFile = "/tmp/wait.deleted.test"
+	os.Create(testFile)
+
+	out := make(chan error)
+	go func(out chan<- error) {
+		out <- waitPidFile(1*time.Second, testFile, false)
+	}(out)
+
+	err := <-out
+	require.Equal(context.DeadlineExceeded, err)
+}
diff --git a/pkg/provision/container_test.go b/pkg/provision/container_test.go
index 5c864da4..aa6afe27 100644
--- a/pkg/provision/container_test.go
+++ b/pkg/provision/container_test.go
@@ -233,3 +233,118 @@ func TestContainerProvisionWithMounts(t *testing.T) {
 	require.Equal("reservation-id", result.ID)
 	require.Equal("192.168.1.1", result.IPv4)
 }
+
+func TestContainerDecomissionExists(t *testing.T) {
+	require := require.New(t)
+
+	var client TestClient
+	var cache TestOwnerCache
+
+	ctx := context.Background()
+	ctx = WithZBus(ctx, &client)
+	ctx = WithOwnerCache(ctx, &cache)
+
+	// const module = "network"
+	// version := zbus.ObjectID{Name: "network", Version: "0.0.1"}
+
+	container := Container{
+		FList: "https://hub.grid.tf/thabet/redis.flist",
+		Network: Network{
+			NetworkID: pkg.NetID("net1"),
+			IPs: []net.IP{
+				net.ParseIP("192.168.1.1"),
+			},
+		},
+	}
+
+	reservation := Reservation{
+		ID:   "reservation-id",
+		User: "user",
+		Type: ContainerReservation,
+		Data: MustMarshal(t, container),
+	}
+	rootFS := "/mnt/flist_root"
+
+	// first, the decomission will inspect the container
+	client.On("Request",
+		"container", zbus.ObjectID{Name: "container", Version: "0.0.1"},
+		"Inspect",
+		fmt.Sprintf("ns%s", reservation.User),
+		pkg.ContainerID(reservation.ID)).
+		Return(pkg.Container{Name: reservation.ID, RootFS: rootFS}, nil)
+
+	// second, the decomission will delete the container
+	client.On("Request",
+		"container", zbus.ObjectID{Name: "container", Version: "0.0.1"},
+		"Delete",
+		fmt.Sprintf("ns%s", reservation.User),
+		pkg.ContainerID(reservation.ID)).
+		Return(nil)
+
+	// third, unmount the flist of the container
+	client.On("Request",
+		"flist", zbus.ObjectID{Name: "flist", Version: "0.0.1"},
+		"Umount",
+		rootFS).
+		Return(nil)
+
+	netID := networkID(reservation.User, string(container.Network.NetworkID))
+	// fourth, leave the container network namespace
+	client.On("Request",
+		"network", zbus.ObjectID{Name: "network", Version: "0.0.1"},
+		"Leave",
+		netID,
+		reservation.ID).
+		Return(nil)
+
+	err := containerDecommission(ctx, &reservation)
+	require.NoError(err)
+}
+
+func TestContainerDecomissionContainerGone(t *testing.T) {
+	require := require.New(t)
+
+	var client TestClient
+	var cache TestOwnerCache
+
+	ctx := context.Background()
+	ctx = WithZBus(ctx, &client)
+	ctx = WithOwnerCache(ctx, &cache)
+
+	container := Container{
+		FList: "https://hub.grid.tf/thabet/redis.flist",
+		Network: Network{
+			NetworkID: pkg.NetID("net1"),
+			IPs: []net.IP{
+				net.ParseIP("192.168.1.1"),
+			},
+		},
+	}
+
+	reservation := Reservation{
+		ID:   "reservation-id",
+		User: "user",
+		Type: ContainerReservation,
+		Data: MustMarshal(t, container),
+	}
+
+	// first, the decomission will inspect the container
+	client.On("Request",
+		"container", zbus.ObjectID{Name: "container", Version: "0.0.1"},
+		"Inspect",
+		fmt.Sprintf("ns%s", reservation.User),
+		pkg.ContainerID(reservation.ID)).
+		Return(nil, fmt.Errorf("container not found"))
+
+	netID := networkID(reservation.User, string(container.Network.NetworkID))
+	// fourth, leave the container network namespace
+	client.On("Request",
+		"network", zbus.ObjectID{Name: "network", Version: "0.0.1"},
+		"Leave",
+		netID,
+		reservation.ID).
+		Return(nil)
+
+	err := containerDecommission(ctx, &reservation)
+	require.NoError(err)
+}
diff --git a/pkg/upgrade/watcher_test.go b/pkg/upgrade/watcher_test.go
index 9668bd88..425cefc3 100644
--- a/pkg/upgrade/watcher_test.go
+++ b/pkg/upgrade/watcher_test.go
@@ -1,10 +1,11 @@
 package upgrade
 
-import "testing"
+import (
+	"context"
+	"testing"
 
-import "context"
-
-import "github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/require"
+)
 
 func TestFListWatcher(t *testing.T) {
 	require := require.New(t)
@@ -27,3 +28,26 @@ func TestFListWatcher(t *testing.T) {
 	require.Equal("zos:development:latest.flist", info.Name)
 	require.Equal("symlink", info.Type)
 }
+
+func TestRepoWatcher(t *testing.T) {
+	require := require.New(t)
+
+	watcher := FListRepoWatcher{
+		Repo: "tf-zos-bins",
+	}
+
+	ctx := context.Background()
+
+	ch, err := watcher.Watch(ctx)
+	require.NoError(err)
+
+	event := <-ch
+	require.Equal(Repo, event.EventType())
+
+	info, ok := event.(*RepoEvent)
+	require.True(ok)
+
+	require.Equal("tf-zos-bins", info.Repo)
+	require.Len(info.ToDel, 0) // we starting from empty Current
+	require.True(len(info.ToAdd) > 0)
+}