Repository: threefoldtech/zos
Pull request #425 (2019-11-29 11:02:27)

Install/update binaries from tf-zos-bins
This PR in draft for the following reasons
- bootstrap need to also make sure binaries are installed from the tf-zos-bins repository 
- we need to agree on a script hook mechanism to make sure actions are taken when new binaries are installed.

Suggestions:
- An action files in the flist (yaml may be) with a limit set of operations that need to be taken during upgrade (like restart service for example)

Concerns:
i think we will (at some point) will need to have a better tracking of installed packages (files) in a package manager similar fashion. The current code operation might cause some package deletions to remove files that belong to other packages. This is why package managers actually track each package files, to make sure no files are deleted by accident from different packages (for example if both packages install the same file)


Patch:
diff --git a/bootstrap/Makefile b/bootstrap/Makefile
index 3840443e..aa44d11c 100644
--- a/bootstrap/Makefile
+++ b/bootstrap/Makefile
@@ -6,12 +6,10 @@ install: build
 	mkdir -p ${ROOT}/etc/zinit/
 	mkdir -p ${ROOT}/bin
 	mkdir -p ${ROOT}/sbin
-	
-	# install interent 
-	cp ../etc/zinit/internet.yaml ${ROOT}/etc/zinit/
+
+	# install interent
 	cp ../bin/internet ${ROOT}/bin
-	
+
 	# install bootstrap
 	cp -a etc ${ROOT}
 	cp bootstrap/target/x86_64-unknown-linux-musl/release/bootstrap ${ROOT}/sbin/
-	
diff --git a/bootstrap/bootstrap/src/bootstrap.rs b/bootstrap/bootstrap/src/bootstrap.rs
index 51cb8959..3ea1f9f3 100644
--- a/bootstrap/bootstrap/src/bootstrap.rs
+++ b/bootstrap/bootstrap/src/bootstrap.rs
@@ -1,6 +1,5 @@
 use super::config;
 use super::hub;
-use super::kparams;
 use super::workdir::WorkDir;
 use super::zfs::Zfs;
 use super::zinit;
@@ -9,51 +8,18 @@ use failure::Error;
 use retry;
 
 const FLIST_REPO: &str = "tf-zos";
+const BIN_REPO: &str = "tf-zos-bins";
 const FLIST_INFO_FILE: &str = "/tmp/flist.info";
 const FLIST_NAME_FILE: &str = "/tmp/flist.name";
 const WORKDIR: &str = "/tmp/bootstrap";
 
 type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug)]
-enum RunMode {
-    Prod,
-    Test,
-    Dev,
-}
-
-fn runmode() -> Result<RunMode> {
-    let params = kparams::params()?;
-    let mode = match params.get("runmode") {
-        Some(mode) => match mode {
-            Some(mode) => match mode.as_ref() {
-                "prod" => RunMode::Prod,
-                "dev" => RunMode::Dev,
-                "test" => RunMode::Test,
-                m => {
-                    bail!("unknown runmode: {}", m);
-                }
-            },
-            None => {
-                //that's an error because runmode was
-                //provided as a kernel argumet but with no
-                //value
-                bail!("missing runmode value");
-            }
-        },
-        // runmode was not provided in cmdline
-        // so default is prod
-        None => RunMode::Prod,
-    };
-
-    Ok(mode)
-}
-
-fn boostrap_zos(mode: RunMode) -> Result<()> {
-    let flist = match mode {
-        RunMode::Prod => "zos:production:latest.flist",
-        RunMode::Dev => "zos:development:latest.flist",
-        RunMode::Test => "zos:testing:latest.flist",
+fn boostrap_zos(cfg: &config::Config) -> Result<()> {
+    let flist = match cfg.runmode {
+        config::RunMode::Prod => "zos:production:latest.flist",
+        config::RunMode::Dev => "zos:development:latest.flist",
+        config::RunMode::Test => "zos:testing:latest.flist",
     };
 
     debug!("using flist: {}/{}", FLIST_REPO, flist);
@@ -80,85 +46,14 @@ fn boostrap_zos(mode: RunMode) -> Result<()> {
     flist.write(FLIST_INFO_FILE)?;
     std::fs::write(FLIST_NAME_FILE, format!("{}/{}", FLIST_REPO, flist.name))?;
 
-    let result = retry::retry(retry::delay::Exponential::from_millis(200), || {
-        info!("download flist: {}", flist.name);
-
-        // the entire point of this match is the
-        // logging of the error.
-        match flist.download("machine.flist") {
-            Ok(ok) => Ok(ok),
-            Err(err) => {
-                error!("failed to download flist: {}", err);
-                bail!("failed to download flist: {}", err);
-            }
-        }
-    });
-
-    // I can't use the ? because error from retry
-    // is not compatible with failure::Error for
-    // some reason.
-    match result {
-        Err(err) => bail!("{:?}", err),
-        _ => (),
-    };
-
-    let fs = Zfs::mount("backend", "machine.flist", "root")?;
-    debug!("zfs started, now copying all files");
-    //copy everything to root
-    match fs.copy("/") {
-        Ok(_) => {}
-        Err(err) => bail!("failed to copy files to rootfs: {}", err),
-    };
-
-    // we need to find all yaml files under /etc/zinit to start monitoring them
-    let mut cfg = std::path::PathBuf::new();
-    cfg.push(&fs);
-    cfg.push("etc");
-    cfg.push("zinit");
-    let services = match std::fs::read_dir(&cfg) {
-        Ok(services) => services,
-        Err(err) => bail!("failed to read directory '{:?}': {}", cfg, err),
-    };
-    for service in services {
-        let service = service?;
-        let path = service.path();
-
-        if !path.is_file() {
-            continue;
-        }
-        let name = match path.file_name() {
-            Some(name) => match name.to_str() {
-                Some(name) => name,
-                None => {
-                    warn!("failed to process name: {:?}", path);
-                    continue;
-                }
-            },
-            None => continue,
-        };
-
-        match name.rfind(".yaml") {
-            None => continue,
-            Some(idx) => {
-                let service = &name[0..idx];
-                match zinit::monitor(service) {
-                    Ok(_) => {}
-                    Err(err) => {
-                        warn!("failed to monitor service '{}': {}", service, err);
-                    }
-                }
-            }
-        }
-    }
-    Ok(())
+    install_package(&flist)
 }
 
 /// bootstrap stage install and starts all zos daemons
-pub fn bootstrap(_: &config::Config) -> Result<()> {
-    let mode = runmode()?;
-    debug!("runmode: {:?}", mode);
+pub fn bootstrap(cfg: &config::Config) -> Result<()> {
+    debug!("runmode: {:?}", cfg.runmode);
     let result = WorkDir::run(WORKDIR, || -> Result<()> {
-        boostrap_zos(mode)?;
+        boostrap_zos(cfg)?;
         Ok(())
     })?;
 
@@ -207,16 +102,81 @@ fn update_bootstrap(debug: bool) -> Result<()> {
         Err(e) => bail!("failed to download flist: {:?}", e),
     };
 
+    // this trick here to allow overriding
+    // the current running bootstrap binary
+    let bin: Vec<String> = std::env::args().take(1).collect();
+    std::fs::rename(&bin[0], format!("{}.bak", &bin[0]))?;
+
+    install_package(&flist)
+}
+
+///install installs all binaries from the tf-zos-bins repo
+pub fn install(cfg: &config::Config) -> Result<()> {
+    let result = WorkDir::run(WORKDIR, || -> Result<()> {
+        install_packages(cfg)?;
+        Ok(())
+    })?;
+
+    result
+}
+
+fn install_packages(cfg: &config::Config) -> Result<()> {
+    let repo = match cfg.runmode {
+        config::RunMode::Prod => BIN_REPO.to_owned(),
+        config::RunMode::Dev => format!("{}.dev", BIN_REPO),
+        config::RunMode::Test => format!("{}.test", BIN_REPO),
+    };
+
+    let client = hub::Repo::new(&repo);
+    let packages = retry::retry(retry::delay::Exponential::from_millis(200), || {
+        info!("list packages in: {}", BIN_REPO);
+        //the full point of this match is the logging.
+        let packages = match client.list() {
+            Ok(info) => info,
+            Err(err) => {
+                error!("failed to list repo '{}': {}", BIN_REPO, err);
+                bail!("failed to list repo '{}': {}", BIN_REPO, err);
+            }
+        };
+
+        Ok(packages)
+    });
+
+    let packages = match packages {
+        Ok(packages) => packages,
+        Err(err) => bail!("failed to list '{}': {:?}", BIN_REPO, err),
+    };
+
+    let mut map = std::collections::HashMap::new();
+    for package in packages.iter() {
+        match install_package(package) {
+            Ok(_) => {}
+            Err(err) => warn!("failed to install package '{}': {}", package.url, err),
+        };
+
+        map.insert(format!("{}/{}", repo, package.name), package.clone());
+    }
+
+    let output = std::fs::OpenOptions::new()
+        .create(true)
+        .write(true)
+        .open("/tmp/bins.info")?;
+    serde_json::to_writer(&output, &map)?;
+
+    Ok(())
+}
+
+fn install_package(flist: &hub::Flist) -> Result<()> {
     let result = retry::retry(retry::delay::Exponential::from_millis(200), || {
         info!("download flist: {}", flist.name);
 
         // the entire point of this match is the
         // logging of the error.
-        match flist.download("bootstrap.flist") {
+        match flist.download(&flist.name) {
             Ok(ok) => Ok(ok),
             Err(err) => {
-                error!("failed to download flist: {}", err);
-                bail!("failed to download flist: {}", err);
+                error!("failed to download flist '{}': {}", flist.url, err);
+                bail!("failed to download flist '{}': {}", flist.url, err);
             }
         }
     });
@@ -229,13 +189,60 @@ fn update_bootstrap(debug: bool) -> Result<()> {
         _ => (),
     };
 
-    let fs = Zfs::mount("backend", "bootstrap.flist", "root")?;
+    let fs = Zfs::mount("backend", &flist.name, "root")?;
     debug!("zfs started, now copying all files");
 
-    // this trick here to allow overriding
-    // the current running bootstrap binary
-    let bin: Vec<String> = std::env::args().take(1).collect();
-    std::fs::rename(&bin[0], format!("{}.bak", &bin[0]))?;
+    fs.copy("/")?;
+
+    run_all(&fs)
+}
 
-    fs.copy("/")
+// run_all tries to run all services from an flist.
+// it will still try to run all other services defined
+// in the list of one or more failed. Returns error only
+// if failed to read the zinit directory
+fn run_all(fs: &Zfs) -> Result<()> {
+    let mut cfg = std::path::PathBuf::new();
+    cfg.push(fs);
+    cfg.push("etc");
+    cfg.push("zinit");
+    let services = match std::fs::read_dir(&cfg) {
+        Ok(services) => services,
+        Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(()),
+        Err(err) => bail!("failed to read directory '{:?}': {}", cfg, err),
+    };
+
+    for service in services {
+        let service = service?;
+        let path = service.path();
+
+        if !path.is_file() {
+            continue;
+        }
+        let name = match path.file_name() {
+            Some(name) => match name.to_str() {
+                Some(name) => name,
+                None => {
+                    warn!("failed to process name: {:?}", path);
+                    continue;
+                }
+            },
+            None => continue,
+        };
+
+        match name.rfind(".yaml") {
+            None => continue,
+            Some(idx) => {
+                let service = &name[0..idx];
+                match zinit::monitor(service) {
+                    Ok(_) => {}
+                    Err(err) => {
+                        warn!("failed to monitor service '{}': {}", service, err);
+                    }
+                }
+            }
+        }
+    }
+
+    Ok(())
 }
diff --git a/bootstrap/bootstrap/src/config.rs b/bootstrap/bootstrap/src/config.rs
index a6a5db63..63d04e87 100644
--- a/bootstrap/bootstrap/src/config.rs
+++ b/bootstrap/bootstrap/src/config.rs
@@ -1,11 +1,47 @@
+use super::kparams;
 use clap::{App, Arg};
 use failure::Error;
 
 type Result<T> = std::result::Result<T, Error>;
 
+#[derive(Debug)]
+pub enum RunMode {
+    Prod,
+    Test,
+    Dev,
+}
+
+fn runmode() -> Result<RunMode> {
+    let params = kparams::params()?;
+    let mode = match params.get("runmode") {
+        Some(mode) => match mode {
+            Some(mode) => match mode.as_ref() {
+                "prod" => RunMode::Prod,
+                "dev" => RunMode::Dev,
+                "test" => RunMode::Test,
+                m => {
+                    bail!("unknown runmode: {}", m);
+                }
+            },
+            None => {
+                //that's an error because runmode was
+                //provided as a kernel argumet but with no
+                //value
+                bail!("missing runmode value");
+            }
+        },
+        // runmode was not provided in cmdline
+        // so default is prod
+        None => RunMode::Prod,
+    };
+
+    Ok(mode)
+}
+
 pub struct Config {
     pub stage: u32,
     pub debug: bool,
+    pub runmode: RunMode,
 }
 
 impl Config {
@@ -44,6 +80,7 @@ impl Config {
         Ok(Config {
             stage: stage,
             debug: matches.occurrences_of("debug") > 0,
+            runmode: runmode()?,
         })
     }
 }
diff --git a/bootstrap/bootstrap/src/hub.rs b/bootstrap/bootstrap/src/hub.rs
index a5f38144..7849ddf8 100644
--- a/bootstrap/bootstrap/src/hub.rs
+++ b/bootstrap/bootstrap/src/hub.rs
@@ -15,19 +15,19 @@ pub struct Repo {
     name: String,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct Flist {
     #[serde(rename = "type")]
     pub kind: String,
     pub updated: u64,
-    pub size: u64,
+    #[serde(default)]
     pub md5: String,
     pub name: String,
     #[serde(default)]
     pub target: String,
 
     #[serde(skip)]
-    url: String,
+    pub url: String,
 }
 
 impl Repo {
@@ -41,6 +41,21 @@ impl Repo {
         }
     }
 
+    pub fn list(&self) -> Result<Vec<Flist>> {
+        let url = format!("{}/api/flist/{}", self.base, self.name,);
+
+        let mut response = get(&url)?;
+        let mut info: Vec<Flist> = match response.status() {
+            StatusCode::OK => response.json()?,
+            s => bail!("failed to get flist info: {}", s),
+        };
+        for flist in info.iter_mut() {
+            flist.url = format!("{}/{}/{}", self.base, self.name, flist.name);
+        }
+
+        Ok(info)
+    }
+
     pub fn get<T>(&self, flist: T) -> Result<Flist>
     where
         T: AsRef<str>,
@@ -121,4 +136,24 @@ mod tests {
         let _ = std::fs::remove_file(temp);
         Ok(())
     }
+
+    #[test]
+    fn test_list_repo() -> Result<()> {
+        let repo = Repo::new(String::from("azmy"));
+        let lists = repo.list()?;
+
+        let mut found: Option<&Flist> = None;
+        for flist in lists.iter() {
+            if flist.name == "test.flist" {
+                found = Some(flist);
+                break;
+            }
+        }
+
+        let found = found.unwrap();
+        assert_eq!(found.name, "test.flist");
+        assert_eq!(found.url, "https://hub.grid.tf/azmy/test.flist");
+
+        Ok(())
+    }
 }
diff --git a/bootstrap/bootstrap/src/main.rs b/bootstrap/bootstrap/src/main.rs
index 734ad35f..b053941d 100644
--- a/bootstrap/bootstrap/src/main.rs
+++ b/bootstrap/bootstrap/src/main.rs
@@ -29,7 +29,17 @@ fn app() -> Result<()> {
 
     // configure available stage
     let stages: Vec<fn(cfg: &Config) -> Result<()>> = vec![
-        |cfg| -> Result<()> { bootstrap::update(cfg) },
+        // self update
+        |cfg| -> Result<()> {
+            if cfg.debug {
+                // if debug is set, do not upgrade self.
+                return Ok(());
+            }
+            bootstrap::update(cfg)
+        },
+        // install all system binaries
+        |cfg| -> Result<()> { bootstrap::install(cfg) },
+        // install and start all services
         |cfg| -> Result<()> { bootstrap::bootstrap(cfg) },
     ];
 
diff --git a/etc/zinit/internet.yaml b/bootstrap/etc/zinit/internet.yaml
similarity index 100%
rename from etc/zinit/internet.yaml
rename to bootstrap/etc/zinit/internet.yaml
diff --git a/cmds/identityd/main.go b/cmds/identityd/main.go
index 37406f23..3f6f0d47 100644
--- a/cmds/identityd/main.go
+++ b/cmds/identityd/main.go
@@ -6,18 +6,20 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
+	"syscall"
 	"time"
 
 	"github.com/pkg/errors"
 	"github.com/threefoldtech/zos/pkg/app"
+	"github.com/threefoldtech/zos/pkg/flist"
 	"github.com/threefoldtech/zos/pkg/gedis"
 	"github.com/threefoldtech/zos/pkg/geoip"
 	"github.com/threefoldtech/zos/pkg/stubs"
+	"github.com/threefoldtech/zos/pkg/upgrade"
 
 	"github.com/cenkalti/backoff/v3"
 	"github.com/threefoldtech/zos/pkg"
 	"github.com/threefoldtech/zos/pkg/environment"
-	"github.com/threefoldtech/zos/pkg/flist"
 	"github.com/threefoldtech/zos/pkg/identity"
 
 	"github.com/threefoldtech/zos/pkg/zinit"
@@ -26,7 +28,6 @@ import (
 
 	"github.com/rs/zerolog/log"
 	"github.com/threefoldtech/zbus"
-	"github.com/threefoldtech/zos/pkg/upgrade"
 	"github.com/threefoldtech/zos/pkg/utils"
 	"github.com/threefoldtech/zos/pkg/version"
 )
@@ -115,15 +116,6 @@ func main() {
 		log.Fatal().Err(err).Str("root", root).Msg("failed to create root directory")
 	}
 
-	zinit, err := zinit.New(zinitSocket)
-	if err != nil {
-		log.Fatal().Err(err).Msg("failed to connect to zinit")
-	}
-
-	// with volume allocation is set to nil this flister can be
-	// only used for RO mounts. Otherwise it will panic
-	flister := flist.New(root, nil)
-
 	boot := upgrade.Boot{}
 
 	bootMethod := boot.DetectBootMethod()
@@ -169,9 +161,9 @@ func main() {
 		return registerNode(nodeID, farmID, v, idStore, loc)
 	}
 
-	if err := backoff.Retry(func() error {
+	if err := backoff.RetryNotify(func() error {
 		return register(current)
-	}, backoff.NewExponentialBackOff()); err != nil {
+	}, backoff.NewExponentialBackOff(), retryNotify); err != nil {
 		log.Error().Err(err).Msg("failed to register node")
 	} else {
 		log.Info().Str("version", current).Msg("node registered successfully")
@@ -211,26 +203,101 @@ func main() {
 	// 4. Start watcher for new version
 	log.Info().Msg("start upgrade daemon")
 
+	upgradeLoop(ctx, &boot, root, debug, register)
+}
+
+func getBinsRepo() string {
+	env, _ := environment.Get()
+
+	switch env.RunningMode {
+	case environment.RunningDev:
+		return "tf-zos-bins.dev"
+	case environment.RunningTest:
+		return "tf-zos-bins.test"
+	default:
+		return "tf-zos-bins"
+	}
+}
+
+// allow reinstall if receive signal USR1
+// only allowed in debug mode
+func debugReinstall(boot *upgrade.Boot, up *upgrade.Upgrader) {
+	c := make(chan os.Signal)
+	signal.Notify(c, syscall.SIGUSR1)
+
+	go func() {
+		for range c {
+			current, err := boot.Current()
+			if err != nil {
+				log.Error().Err(err).Msg("couldn't get current flist info")
+				continue
+			}
+
+			up.Upgrade(current, current)
+		}
+	}()
+}
+func upgradeLoop(
+	ctx context.Context,
+	boot *upgrade.Boot,
+	root string,
+	debug bool,
+	register func(string) error) {
+
+	zinit, err := zinit.New(zinitSocket)
+	if err != nil {
+		log.Fatal().Err(err).Msg("failed to connect to zinit")
+	}
+
+	// with volume allocation is set to nil this flister can be
+	// only used for RO mounts. Otherwise it will panic
+	flister := flist.New(root, nil)
+
 	upgrader := upgrade.Upgrader{
 		FLister:      flister,
 		Zinit:        zinit,
 		NoSelfUpdate: debug,
 	}
 
-	watcher := upgrade.FListSemverWatcher{
+	if debug {
+		debugReinstall(boot, &upgrader)
+	}
+
+	flistWatcher := upgrade.FListSemverWatcher{
 		FList:    boot.Name(),
 		Current:  boot.MustVersion(), //if we are here version must be valid
 		Duration: 600 * time.Second,
 	}
 
-	events, err := watcher.Watch(ctx)
+	flistEvents, err := flistWatcher.Watch(ctx)
+	if err != nil {
+		log.Fatal().Err(err).Str("flist", flistWatcher.FList).Msg("failed to watch flist")
+	}
+
+	bins, err := boot.CurrentBins()
+	if err != nil {
+		log.Warn().Err(err).Msg("could not load current binaries list")
+	}
+
+	repoWatcher := upgrade.FListRepoWatcher{
+		Repo:    getBinsRepo(),
+		Current: bins,
+	}
+
+	repoEvents, err := repoWatcher.Watch(ctx)
 	if err != nil {
-		log.Fatal().Err(err).Str("flist", boot.Name()).Msg("failed to watch flist")
+		log.Fatal().Err(err).Str("repo", repoWatcher.Repo).Msg("failed to watch repo")
 	}
 
-	for event := range events {
-		switch event := event.(type) {
-		case *upgrade.FListEvent:
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case e := <-flistEvents:
+			if e == nil {
+				continue
+			}
+			event := e.(*upgrade.FListEvent)
 			// new flist available
 			version, err := event.Version()
 			if err != nil {
@@ -261,17 +328,43 @@ func main() {
 				log.Error().Err(err).Msg("failed to update boot information")
 			}
 
-			if err := backoff.Retry(func() error {
+			if err := backoff.RetryNotify(func() error {
 				return register(version.String())
-			}, backoff.NewExponentialBackOff()); err != nil {
+			}, backoff.NewExponentialBackOff(), retryNotify); err != nil {
 				log.Error().Err(err).Msg("failed to register node")
 			} else {
 				log.Info().Str("version", version.String()).Msg("node registered successfully")
 			}
+		case e := <-repoEvents:
+			if e == nil {
+				continue
+			}
+			event := e.(*upgrade.RepoEvent)
+			for _, bin := range event.ToDel {
+				if err := upgrader.UninstallBinary(bin); err != nil {
+					log.Error().Err(err).Str("flist", bin.Fqdn()).Msg("failed to uninstall flist")
+				}
+			}
+
+			for _, bin := range event.ToAdd {
+				if err := upgrader.InstallBinary(bin); err != nil {
+					log.Error().Err(err).Str("flist", bin.Fqdn()).Msg("failed to install flist")
+				}
+			}
+
+			if err := boot.SetBins(repoWatcher.Current); err != nil {
+				log.Error().Err(err).Msg("failed to update local db of installed binaries")
+			}
+
+			log.Debug().Msg("finish processing binary updates")
 		}
 	}
 }
 
+func retryNotify(err error, d time.Duration) {
+	log.Warn().Err(err).Str("sleep", d.String()).Msg("registration failed")
+}
+
 func identityMgr(root string) (pkg.IdentityManager, error) {
 	seedPath := filepath.Join(root, seedName)
 
diff --git a/etc/promtail/promtail.yaml b/etc/promtail/promtail.yaml
deleted file mode 100644
index ae933074..00000000
--- a/etc/promtail/promtail.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
-server:
-  http_listen_port: 80
-  grpc_listen_port: 9095
-
-positions:
-  filename: /tmp/positions.yaml
-
-scrape_configs:
-- job_name: zos
-  static_configs:
-    - targets:
-        - localhost
-      labels:
-        job: zinit
-        node: "NODE-ID"
-        __path__: /var/cache/log/system.log
-  pipeline_stages:
-    - regex:
-        expression: '^\[.\]\s(?P<module>[^:]+):\s*(?P<body>.*)'
-    - regex:
-        # this might match other parts of the message.
-        expression: '(?P<level>info|error|debug|warn|fatal|panic)'
-        source: body
-    - regex:
-        expression: '(?P<time>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z(?:\d{2}\:\d{2})?)'
-        source: body
-    - timestamp:
-        source: time
-        format: RFC3339
-    - labels:
-        level:
-        module:
diff --git a/etc/zinit/promtail-config.yaml b/etc/zinit/promtail-config.yaml
deleted file mode 100644
index 81ab9366..00000000
--- a/etc/zinit/promtail-config.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-exec: |
-  sh -c 
-    'sed "s/NODE-ID/$(identityd -id)/g" -i /etc/promtail/promtail.yaml'
-oneshot: true
-after:
-  - boot
\ No newline at end of file
diff --git a/etc/zinit/promtail.yaml b/etc/zinit/promtail.yaml
deleted file mode 100644
index e964c52a..00000000
--- a/etc/zinit/promtail.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
-exec: |
-  promtail 
-  -config.file /etc/promtail/promtail.yaml
-  -client.url http://explorer.devnet.grid.tf:3100/loki/api/v1/push
-log: stdout
-after:
-  - promtail-config
-  - logger
diff --git a/pkg/flist/flist.go b/pkg/flist/flist.go
index 3b5fc5a4..2c2939cf 100644
--- a/pkg/flist/flist.go
+++ b/pkg/flist/flist.go
@@ -235,6 +235,7 @@ func (f *flistModule) Umount(path string) error {
 	if err := syscall.Unmount(path, syscall.MNT_DETACH); err != nil {
 		log.Error().Err(err).Str("path", path).Msg("fail to umount flist")
 	}
+
 	if err := waitPidFile(time.Second*2, pidPath, false); err != nil {
 		log.Error().Err(err).Str("path", path).Msg("0-fs daemon did not stop properly")
 		return err
@@ -349,36 +350,31 @@ func random() (string, error) {
 // else it waits for the file to be deleted
 func waitPidFile(timeout time.Duration, path string, exists bool) error {
 	const delay = time.Millisecond * 100
-	cErr := make(chan error)
 
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 
-	go func() {
-		for {
-			select {
-			case <-ctx.Done():
-				cErr <- ctx.Err()
-			default:
-				_, err := os.Stat(path)
-				if exists {
-					if err != nil {
-						time.Sleep(delay)
-						continue
-					}
-					cErr <- nil
-				} else {
-					if err == nil {
-						time.Sleep(delay)
-						continue
-					}
-					cErr <- nil
-				}
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+			_, err := os.Stat(path)
+			// check exist but the file is not there yet,
+			// or check NOT exist but it is still there
+			// we try again
+			if (exists && os.IsNotExist(err)) || (!exists && err == nil) {
+				break //the select statement
+			} else if err != nil && !os.IsNotExist(err) {
+				//another error that is NOT IsNotExist.
+				return err
+			} else {
+				return nil
 			}
 		}
-	}()
 
-	return <-cErr
+		<-time.After(delay)
+	}
 }
 
 func waitMountedLog(timeout time.Duration, logfile string) error {
diff --git a/pkg/network/bootstrap.go b/pkg/network/bootstrap.go
index 73f30851..9b4d9af8 100644
--- a/pkg/network/bootstrap.go
+++ b/pkg/network/bootstrap.go
@@ -77,7 +77,7 @@ func Bootstrap() error {
 		}
 
 		log.Info().Str("interface", device.Name).Msg("attach interface to default bridge")
-		if err := bridge.AttachNicWithMac(device, br); err != nil {
+		if err := bridge.AttachNic(device, br); err != nil {
 			log.Warn().Err(err).
 				Str("device", device.Name).
 				Str("bridge", br.Name).
diff --git a/pkg/network/bridge/bridge.go b/pkg/network/bridge/bridge.go
index 40cf6128..36e9d4dd 100644
--- a/pkg/network/bridge/bridge.go
+++ b/pkg/network/bridge/bridge.go
@@ -87,18 +87,6 @@ func AttachNic(link netlink.Link, bridge *netlink.Bridge) error {
 	return netlink.LinkSetMaster(link, bridge)
 }
 
-// AttachNicWithMac attaches an interface to a bridge and sets
-// the MAC of the bridge to the same of the NIC
-func AttachNicWithMac(link netlink.Link, bridge *netlink.Bridge) error {
-	hwaddr := link.Attrs().HardwareAddr
-
-	err := netlink.LinkSetHardwareAddr(bridge, hwaddr)
-	if err != nil {
-		return err
-	}
-	return netlink.LinkSetMaster(link, bridge)
-}
-
 // DetachNic detaches an interface from a bridge
 func DetachNic(bridge *netlink.Bridge) error {
 	return netlink.LinkSetNoMaster(bridge)
diff --git a/pkg/network/nr/net_resource.go b/pkg/network/nr/net_resource.go
index ce52642a..5adc0c14 100644
--- a/pkg/network/nr/net_resource.go
+++ b/pkg/network/nr/net_resource.go
@@ -318,19 +318,26 @@ func (nr *NetResource) createNetNS() error {
 func (nr *NetResource) createVethPair() error {
 
 	nsName, err := nr.Namespace()
-	nrVethName := fmt.Sprintf("nr-%s", nr.id[:12])
-
-	netNS, err := namespace.GetByName(nsName)
+	vethName, err := ifaceutil.RandomName("nr-")
 	if err != nil {
-		return fmt.Errorf("network namespace %s does not exits", nsName)
+		return err
 	}
-	defer netNS.Close()
-
 	bridgeName, err := nr.BridgeName()
 	if err != nil {
 		return err
 	}
 
+	// check if the veth already exists
+	if _, err := netlink.LinkByName(vethName); err == nil {
+		return nil
+	}
+
+	netNS, err := namespace.GetByName(nsName)
+	if err != nil {
+		return fmt.Errorf("network namespace %s does not exits", nsName)
+	}
+	defer netNS.Close()
+
 	brLink, err := bridge.Get(bridgeName)
 	if err != nil {
 		return fmt.Errorf("bridge %s does not exits", bridgeName)
@@ -343,7 +350,7 @@ func (nr *NetResource) createVethPair() error {
 			return err
 		}
 
-		if _, err := netlink.LinkByName(nrVethName); err == nil {
+		if _, err := netlink.LinkByName(vethName); err == nil {
 			exists = true
 			return nil
 		}
@@ -354,16 +361,16 @@ func (nr *NetResource) createVethPair() error {
 
 		log.Info().
 			Str("namespace", nsName).
-			Str("veth", nrVethName).
+			Str("veth", vethName).
 			Msg("Create veth pair in net namespace")
 
-		hostVeth, nrVeth, err := ip.SetupVeth(nrVethName, 1500, hostNS)
+		hostVeth, containerVeth, err := ip.SetupVeth(vethName, 1500, hostNS)
 		if err != nil {
 			return err
 		}
 		hostIface = hostVeth.Name
 
-		link, err := netlink.LinkByName(nrVeth.Name)
+		link, err := netlink.LinkByName(containerVeth.Name)
 		if err != nil {
 			return err
 		}
@@ -397,13 +404,12 @@ func (nr *NetResource) createVethPair() error {
 	}
 
 	log.Info().
-		Str("net resource veth", nrVethName).
-		Str("host veth", hostIface).
+		Str("veth", vethName).
 		Str("bridge", bridgeName).
 		Msg("attach veth to bridge")
 
 	if err := bridge.AttachNic(hostVeth, brLink); err != nil {
-		return errors.Wrapf(err, "failed to attach veth %s to bridge %s", nrVethName, bridgeName)
+		return errors.Wrapf(err, "failed to attach veth %s to bridge %s", vethName, bridgeName)
 	}
 	return nil
 }
diff --git a/pkg/storage/filesystem/btrfs.go b/pkg/storage/filesystem/btrfs.go
index 3c674e2e..ee2030fe 100644
--- a/pkg/storage/filesystem/btrfs.go
+++ b/pkg/storage/filesystem/btrfs.go
@@ -8,7 +8,6 @@ import (
 	"strings"
 	"syscall"
 
-	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 
 	"github.com/threefoldtech/zos/pkg"
@@ -175,10 +174,6 @@ func (p *btrfsPool) mounted(fs *Btrfs) (string, bool) {
 	return "", false
 }
 
-func (p *btrfsPool) ID() int {
-	return 0
-}
-
 func (p *btrfsPool) Name() string {
 	return p.name
 }
@@ -187,16 +182,6 @@ func (p *btrfsPool) Path() string {
 	return filepath.Join("/mnt", p.name)
 }
 
-// Limit on a pool is not supported yet
-func (p *btrfsPool) Limit(size uint64) error {
-	return fmt.Errorf("not implemented")
-}
-
-// FsType of the filesystem of this volume
-func (p *btrfsPool) FsType() string {
-	return "btrfs"
-}
-
 // Mount mounts the pool in it's default mount location under /mnt/name
 func (p *btrfsPool) Mount() (string, error) {
 	ctx := context.Background()
@@ -302,7 +287,6 @@ func (p *btrfsPool) Volumes() ([]Volume, error) {
 
 	for _, sub := range subs {
 		volumes = append(volumes, newBtrfsVolume(
-			sub.ID,
 			filepath.Join(mnt, sub.Path),
 			p.utils,
 		))
@@ -312,17 +296,11 @@ func (p *btrfsPool) Volumes() ([]Volume, error) {
 }
 
 func (p *btrfsPool) addVolume(root string) (*btrfsVolume, error) {
-	ctx := context.Background()
-	if err := p.utils.SubvolumeAdd(ctx, root); err != nil {
-		return nil, err
-	}
-
-	volume, err := p.utils.SubvolumeInfo(ctx, root)
-	if err != nil {
+	if err := p.utils.SubvolumeAdd(context.Background(), root); err != nil {
 		return nil, err
 	}
 
-	return newBtrfsVolume(volume.ID, root, p.utils), nil
+	return newBtrfsVolume(root, p.utils), nil
 }
 
 func (p *btrfsPool) AddVolume(name string) (Volume, error) {
@@ -336,23 +314,7 @@ func (p *btrfsPool) AddVolume(name string) (Volume, error) {
 }
 
 func (p *btrfsPool) removeVolume(root string) error {
-	ctx := context.Background()
-
-	info, err := p.utils.SubvolumeInfo(ctx, root)
-	if err != nil {
-		return err
-	}
-
-	if err := p.utils.SubvolumeRemove(ctx, root); err != nil {
-		return err
-	}
-
-	qgroupID := fmt.Sprintf("0/%d", info.ID)
-	if err := p.utils.QGroupDestroy(ctx, qgroupID, p.Path()); err != nil {
-		return errors.Wrapf(err, "failed to delete qgroup %s", qgroupID)
-	}
-
-	return nil
+	return p.utils.SubvolumeRemove(context.Background(), root)
 }
 
 func (p *btrfsPool) RemoveVolume(name string) error {
@@ -395,6 +357,16 @@ func (p *btrfsPool) Usage() (usage Usage, err error) {
 	return Usage{Size: totalSize / raidSizeDivisor[du.Data.Profile], Used: uint64(fsi[0].Used)}, nil
 }
 
+// Limit on a pool is not supported yet
+func (p *btrfsPool) Limit(size uint64) error {
+	return fmt.Errorf("not implemented")
+}
+
+// FsType of the filesystem of this volume
+func (p *btrfsPool) FsType() string {
+	return "btrfs"
+}
+
 // Type of the physical storage used for this pool
 func (p *btrfsPool) Type() pkg.DeviceType {
 	// We only create heterogenous pools for now
@@ -421,71 +393,49 @@ func (p *btrfsPool) Reserved() (uint64, error) {
 	return total, nil
 }
 
-func (p *btrfsPool) Maintenance() error {
-	// this method cleans up all the unused
-	// qgroups that could exists on a filesystem
-
-	volumes, err := p.Volumes()
-	if err != nil {
-		return err
-	}
-	subVolsIDs := map[string]struct{}{}
-	for _, volume := range volumes {
-		// use the 0/X notation to match the qgroup IDs format
-		subVolsIDs[fmt.Sprintf("0/%d", volume.ID())] = struct{}{}
-	}
-
-	ctx := context.Background()
-	qgroups, err := p.utils.QGroupList(ctx, p.Path())
-	if err != nil {
-		return err
-	}
-
-	for qgroupID := range qgroups {
-		// for all qgroup that doesn't have an linked
-		// volume, delete the qgroup
-		_, ok := subVolsIDs[qgroupID]
-		if !ok {
-			log.Debug().Str("id", qgroupID).Msg("destroy qgroup")
-			if err := p.utils.QGroupDestroy(ctx, qgroupID, p.Path()); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
 type btrfsVolume struct {
-	id    int
 	path  string
 	utils *BtrfsUtil
 }
 
-func newBtrfsVolume(ID int, path string, utils *BtrfsUtil) *btrfsVolume {
+func newBtrfsVolume(path string, utils *BtrfsUtil) *btrfsVolume {
 	return &btrfsVolume{
-		id:    ID,
 		path:  path,
 		utils: utils,
 	}
 }
 
-func (v *btrfsVolume) ID() int {
-	return v.id
-}
-
 func (v *btrfsVolume) Path() string {
 	return v.path
 }
 
-// Name of the filesystem
-func (v *btrfsVolume) Name() string {
-	return filepath.Base(v.Path())
+func (v *btrfsVolume) Volumes() ([]Volume, error) {
+	var volumes []Volume
+
+	subs, err := v.utils.SubvolumeList(context.Background(), v.Path())
+	if err != nil {
+		return nil, err
+	}
+
+	for _, sub := range subs {
+		volumes = append(volumes, newBtrfsVolume(filepath.Join(v.Path(), sub.Path), v.utils))
+	}
+
+	return volumes, nil
 }
 
-// FsType of the filesystem
-func (v *btrfsVolume) FsType() string {
-	return "btrfs"
+func (v *btrfsVolume) AddVolume(name string) (Volume, error) {
+	mnt := filepath.Join(v.Path(), name)
+	if err := v.utils.SubvolumeAdd(context.Background(), mnt); err != nil {
+		return nil, err
+	}
+
+	return newBtrfsVolume(mnt, v.utils), nil
+}
+
+func (v *btrfsVolume) RemoveVolume(name string) error {
+	mnt := filepath.Join(v.Path(), name)
+	return v.utils.SubvolumeRemove(context.Background(), mnt)
 }
 
 // Usage return the volume usage
@@ -520,3 +470,13 @@ func (v *btrfsVolume) Limit(size uint64) error {
 
 	return v.utils.QGroupLimit(ctx, size, v.Path())
 }
+
+// Name of the filesystem
+func (v *btrfsVolume) Name() string {
+	return filepath.Base(v.Path())
+}
+
+// FsType of the filesystem
+func (v *btrfsVolume) FsType() string {
+	return "btrfs"
+}
diff --git a/pkg/storage/filesystem/btrfs_utils.go b/pkg/storage/filesystem/btrfs_utils.go
index 3be7742f..52187db4 100644
--- a/pkg/storage/filesystem/btrfs_utils.go
+++ b/pkg/storage/filesystem/btrfs_utils.go
@@ -174,13 +174,6 @@ func (u *BtrfsUtil) QGroupLimit(ctx context.Context, size uint64, path string) e
 	return err
 }
 
-// QGroupDestroy deletes a qgroup on a subvol
-func (u *BtrfsUtil) QGroupDestroy(ctx context.Context, id, path string) error {
-	_, err := u.run(ctx, "btrfs", "qgroup", "destroy", id, path)
-
-	return err
-}
-
 // GetDiskUsage get btrfs usage
 func (u *BtrfsUtil) GetDiskUsage(ctx context.Context, path string) (usage BtrfsDiskUsage, err error) {
 	output, err := u.run(ctx, "btrfs", "filesystem", "df", "--raw", path)
diff --git a/pkg/storage/filesystem/filesystem.go b/pkg/storage/filesystem/filesystem.go
index f047319c..bd2ae42e 100644
--- a/pkg/storage/filesystem/filesystem.go
+++ b/pkg/storage/filesystem/filesystem.go
@@ -14,10 +14,14 @@ type Usage struct {
 
 // Volume represents a logical volume in the pool. Volumes can be nested
 type Volume interface {
-	// Volume ID
-	ID() int
 	// Path of the volume
 	Path() string
+	// Volumes are all subvolumes of this volume
+	Volumes() ([]Volume, error)
+	// AddVolume adds a new subvolume with the given name
+	AddVolume(name string) (Volume, error)
+	// RemoveVolume removes a subvolume with the given name
+	RemoveVolume(name string) error
 	// Usage reports the current usage of the volume
 	Usage() (Usage, error)
 	// Limit the maximum size of the volume
@@ -46,19 +50,8 @@ type Pool interface {
 	Type() pkg.DeviceType
 	// Reserved is reserved size of the devices in bytes
 	Reserved() (uint64, error)
-	// Maintenance is a routine that is called at boot
-	// and that all implementer can use to do some clean up and
-	// other maintenance on the pool
-	Maintenance() error
 
 	// Health() ?
-
-	// Volumes are all subvolumes of this volume
-	Volumes() ([]Volume, error)
-	// AddVolume adds a new subvolume with the given name
-	AddVolume(name string) (Volume, error)
-	// RemoveVolume removes a subvolume with the given name
-	RemoveVolume(name string) error
 }
 
 // Filter closure for Filesystem list
diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go
index d0ac2925..342b93f1 100644
--- a/pkg/storage/storage.go
+++ b/pkg/storage/storage.go
@@ -60,10 +60,6 @@ func New() (pkg.StorageModule, error) {
 		log.Info().Msgf("Finished initializing storage module")
 	}
 
-	if err := s.Maintenance(); err != nil {
-		log.Error().Err(err).Msg("storage devices maintenance failed")
-	}
-
 	return s, err
 }
 
@@ -215,26 +211,6 @@ func (s *storageModule) initialize(policy pkg.StoragePolicy) error {
 	return s.ensureCache()
 }
 
-func (s *storageModule) Maintenance() error {
-
-	for _, pool := range s.volumes {
-		log.Info().
-			Str("pool", pool.Name()).
-			Msg("start storage pool maintained")
-		if err := pool.Maintenance(); err != nil {
-			log.Error().
-				Err(err).
-				Str("pool", pool.Name()).
-				Msg("error during maintainace")
-			return err
-		}
-		log.Info().
-			Str("pool", pool.Name()).
-			Msg("finished storage pool maintained")
-	}
-	return nil
-}
-
 // CreateFilesystem with the given size in a storage pool.
 func (s *storageModule) CreateFilesystem(name string, size uint64, poolType pkg.DeviceType) (string, error) {
 	log.Info().Msgf("Creating new volume with size %d", size)
diff --git a/pkg/upgrade/boot.go b/pkg/upgrade/boot.go
index 8d9e541c..f44cbf01 100644
--- a/pkg/upgrade/boot.go
+++ b/pkg/upgrade/boot.go
@@ -19,6 +19,7 @@ const (
 
 	nameFile = "/tmp/flist.name"
 	infoFile = "/tmp/flist.info"
+	binsFile = "/tmp/bins.info"
 )
 
 // BootMethod defines the node boot method
@@ -60,6 +61,29 @@ func (b *Boot) Name() string {
 	return strings.TrimSpace(string(data))
 }
 
+//CurrentBins returns a list of current binaries installed
+func (b *Boot) CurrentBins() (map[string]RepoFList, error) {
+	f, err := os.Open(binsFile)
+	if err != nil {
+		return nil, err
+	}
+	dec := json.NewDecoder(f)
+
+	var result map[string]RepoFList
+	err = dec.Decode(&result)
+	return result, err
+}
+
+//SetBins sets the current list of binaries in boot files
+func (b *Boot) SetBins(current map[string]RepoFList) error {
+	f, err := os.Create(binsFile)
+	if err != nil {
+		return err
+	}
+	enc := json.NewEncoder(f)
+	return enc.Encode(current)
+}
+
 // Current returns current flist information
 func (b *Boot) Current() (FListEvent, error) {
 	name := b.Name()
diff --git a/pkg/upgrade/hub.go b/pkg/upgrade/hub.go
index 7d08901d..1e795395 100644
--- a/pkg/upgrade/hub.go
+++ b/pkg/upgrade/hub.go
@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
@@ -67,17 +68,52 @@ func (h *hubClient) Info(flist string) (info flistInfo, err error) {
 	return info, err
 }
 
-// flistInfo reflects node boot information (flist + version)
-type flistInfo struct {
+func (h *hubClient) List(repo string) ([]listFListInfo, error) {
+	u, err := url.Parse(hubBaseURL)
+	if err != nil {
+		panic("invalid base url")
+	}
+
+	u.Path = filepath.Join("api", "flist", repo)
+	response, err := http.Get(u.String())
+	if err != nil {
+		return nil, err
+	}
+
+	defer response.Body.Close()
+	defer ioutil.ReadAll(response.Body)
+
+	if response.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to get repository listing: %s", response.Status)
+	}
+
+	dec := json.NewDecoder(response.Body)
+
+	var result []listFListInfo
+	err = dec.Decode(&result)
+
+	for i := range result {
+		result[i].Repository = repo
+	}
+
+	return result, err
+}
+
+type listFListInfo struct {
 	Name       string `json:"name"`
 	Target     string `json:"target"`
 	Type       string `json:"type"`
 	Updated    uint64 `json:"updated"`
-	Hash       string `json:"hash"`
-	Size       uint64 `json:"size"`
 	Repository string `json:"-"`
 }
 
+// flistInfo reflects node boot information (flist + version)
+type flistInfo struct {
+	listFListInfo
+	Hash string `json:"hash"`
+	Size uint64 `json:"size"`
+}
+
 // fileInfo is the file of an flist
 type fileInfo struct {
 	Path string `json:"path"`
@@ -96,7 +132,11 @@ func (b *flistInfo) Commit(path string) error {
 	return enc.Encode(b)
 }
 
-func (b *flistInfo) extractVersion(name string) (ver semver.Version, err error) {
+func (b *listFListInfo) Fqdn() string {
+	return path.Join(b.Repository, b.Name)
+}
+
+func (b *listFListInfo) extractVersion(name string) (ver semver.Version, err error) {
 	// name is suppose to be as follows
 	// <name>:<version>.flist
 	parts := strings.Split(name, ":")
@@ -107,7 +147,7 @@ func (b *flistInfo) extractVersion(name string) (ver semver.Version, err error)
 }
 
 // Version returns the version of the flist
-func (b *flistInfo) Version() (semver.Version, error) {
+func (b *listFListInfo) Version() (semver.Version, error) {
 	// computing the version is tricky because it's part of the name
 	// of the flist (or the target) depends on the type of the flist
 
@@ -119,7 +159,7 @@ func (b *flistInfo) Version() (semver.Version, error) {
 }
 
 // Absolute returns the actual flist name
-func (b *flistInfo) Absolute() string {
+func (b *listFListInfo) Absolute() string {
 	name := b.Name
 	if b.Type == "symlink" {
 		name = b.Target
@@ -129,7 +169,7 @@ func (b *flistInfo) Absolute() string {
 }
 
 // Files gets the list of the files of an flist
-func (b *flistInfo) Files() ([]fileInfo, error) {
+func (b *listFListInfo) Files() ([]fileInfo, error) {
 	flist := b.Absolute()
 	if len(flist) == 0 {
 		return nil, fmt.Errorf("invalid flist info")
diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go
index ae3c66de..45c4e04c 100644
--- a/pkg/upgrade/upgrade.go
+++ b/pkg/upgrade/upgrade.go
@@ -49,7 +49,77 @@ type Upgrader struct {
 // instead the upgraded daemon will be completely stopped
 func (u *Upgrader) Upgrade(from, to FListEvent) error {
 	return u.applyUpgrade(from, to)
+}
+
+// InstallBinary from a single flist.
+func (u *Upgrader) InstallBinary(flist RepoFList) error {
+	log.Info().Str("flist", flist.Fqdn()).Msg("start applying upgrade")
+
+	flistRoot, err := u.FLister.Mount(u.hub.MountURL(flist.Fqdn()), u.hub.StorageURL(), pkg.ReadOnlyMountOptions)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err := u.FLister.Umount(flistRoot); err != nil {
+			log.Error().Err(err).Msgf("fail to umount flist at %s: %v", flistRoot, err)
+		}
+	}()
 
+	if err := copyRecursive(flistRoot, "/"); err != nil {
+		return errors.Wrapf(err, "failed to install flist: %s", flist.Fqdn())
+	}
+
+	p := filepath.Join(flistRoot, "etc", "zinit")
+	log.Debug().Str("path", p).Msg("checking for zinit unit files")
+	files, err := ioutil.ReadDir(p)
+	if os.IsNotExist(err) {
+		log.Debug().Err(err).Msg("/etc/zinit not found on flist")
+		return nil
+	} else if err != nil {
+		return errors.Wrap(err, "failed to list package services")
+	}
+
+	var services []string
+	for _, file := range files {
+		if !strings.HasSuffix(file.Name(), ".yaml") {
+			continue
+		}
+
+		service := strings.TrimSuffix(file.Name(), ".yaml")
+		services = append(services, service)
+	}
+
+	return u.ensureRestarted(services...)
+}
+
+func (u *Upgrader) ensureRestarted(service ...string) error {
+	log.Debug().Strs("services", service).Msg("ensure services")
+	if len(service) == 0 {
+		return nil
+	}
+
+	log.Debug().Strs("services", service).Msg("restarting services")
+	if err := u.stopMultiple(20*time.Second, service...); err != nil {
+		return err
+	}
+
+	for _, name := range service {
+		if err := u.Zinit.Forget(name); err != nil {
+			log.Warn().Err(err).Str("service", name).Msg("could not forget service")
+		}
+
+		if err := u.Zinit.Monitor(name); err != nil {
+			log.Error().Err(err).Str("service", name).Msg("could not monitor service")
+		}
+	}
+
+	return nil
+}
+
+// UninstallBinary  from a single flist.
+func (u *Upgrader) UninstallBinary(flist RepoFList) error {
+	return u.uninstall(flist.listFListInfo)
 }
 
 func (u Upgrader) stopMultiple(timeout time.Duration, service ...string) error {
@@ -157,12 +227,10 @@ func (u *Upgrader) upgradeSelf(root string) error {
 	return ErrRestartNeeded
 }
 
-func (u *Upgrader) uninstall(from FListEvent) error {
-	current := from
-
-	files, err := current.Files()
+func (u *Upgrader) uninstall(flist listFListInfo) error {
+	files, err := flist.Files()
 	if err != nil {
-		return errors.Wrapf(err, "failed to get list of current installed files for '%s'", current.Absolute())
+		return errors.Wrapf(err, "failed to get list of current installed files for '%s'", flist.Absolute())
 	}
 
 	//stop all services names
@@ -209,6 +277,7 @@ func (u *Upgrader) uninstall(from FListEvent) error {
 			log.Debug().Err(err).Str("file", file.Path).Msg("failed to check file")
 			continue
 		}
+
 		if stat.IsDir() {
 			continue
 		}
@@ -239,7 +308,7 @@ func (u *Upgrader) applyUpgrade(from, to FListEvent) error {
 		return err
 	}
 
-	if err := u.uninstall(from); err != nil {
+	if err := u.uninstall(from.listFListInfo); err != nil {
 		log.Error().Err(err).Msg("failed to unistall current flist. Upgraded anyway")
 	}
 
diff --git a/pkg/upgrade/watcher.go b/pkg/upgrade/watcher.go
index c02706fd..2383a9fd 100644
--- a/pkg/upgrade/watcher.go
+++ b/pkg/upgrade/watcher.go
@@ -1,14 +1,13 @@
 package upgrade
 
-import "time"
+import (
+	"context"
+	"time"
 
-import "github.com/blang/semver"
-
-import "context"
-
-import "github.com/rs/zerolog/log"
-
-import "path"
+	"github.com/blang/semver"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+)
 
 /**
 Watcher unifies the upgrade pipeline by making sure we can watch
@@ -48,11 +47,6 @@ func (f *FListEvent) EventType() EventType {
 	return FList
 }
 
-//Fqdn return full name of the flist (with repo)
-func (f *FListEvent) Fqdn() string {
-	return path.Join(f.Repository, f.Name)
-}
-
 //TryVersion will try to parse the version from flist
 //other wise return empty version 0.0.0
 func (f *FListEvent) TryVersion() semver.Version {
@@ -71,9 +65,11 @@ type FListSemverWatcher struct {
 	client hubClient
 }
 
+var _ Watcher = &FListSemverWatcher{}
+
 // Watch an flist change in version
+// The Event returned by the channel is of concrete type FListEvent
 func (w *FListSemverWatcher) Watch(ctx context.Context) (<-chan Event, error) {
-
 	if w.Duration == time.Duration(0) {
 		//default delay of 5min
 		w.Duration = 600 * time.Second
@@ -126,9 +122,14 @@ func (w *FListSemverWatcher) Watch(ctx context.Context) (<-chan Event, error) {
 			}
 
 			if version.GT(w.Current) {
-				ch <- &FListEvent{
+				select {
+				case ch <- &FListEvent{
 					flistInfo: info,
+				}:
+				case <-ctx.Done():
+					return
 				}
+
 				w.Current = version
 			}
 		}
@@ -136,3 +137,129 @@ func (w *FListSemverWatcher) Watch(ctx context.Context) (<-chan Event, error) {
 
 	return ch, nil
 }
+
+//RepoFList holds information of flist from a repo list operation
+type RepoFList struct {
+	listFListInfo
+}
+
+//RepoEvent is returned by the repo watcher
+type RepoEvent struct {
+	Repo  string
+	ToAdd []RepoFList
+	ToDel []RepoFList
+}
+
+//EventType returns event type
+func (e *RepoEvent) EventType() EventType {
+	return Repo
+}
+
+//FListRepoWatcher type
+type FListRepoWatcher struct {
+	Repo     string
+	Current  map[string]RepoFList
+	Duration time.Duration
+
+	client hubClient
+}
+
+func (w *FListRepoWatcher) list() (map[string]RepoFList, error) {
+	packages, err := w.client.List(w.Repo)
+	if err != nil {
+		return nil, err
+	}
+
+	result := make(map[string]RepoFList)
+	for _, pkg := range packages {
+		flist := RepoFList{pkg}
+		result[flist.Fqdn()] = flist
+	}
+
+	return result, nil
+}
+
+func (w *FListRepoWatcher) diff(packages map[string]RepoFList) (toAdd, toDel []RepoFList) {
+	for name, pkg := range packages {
+		current, ok := w.Current[name]
+		if !ok || pkg.Updated != current.Updated {
+			toAdd = append(toAdd, pkg)
+		}
+	}
+
+	for name := range w.Current {
+		current, ok := packages[name]
+		if !ok {
+			toDel = append(toDel, current)
+		}
+	}
+
+	return
+}
+
+// Watch watches a full repo for changes. Event is always of concrete type RepoEvent
+func (w *FListRepoWatcher) Watch(ctx context.Context) (<-chan Event, error) {
+	if w.Duration == time.Duration(0) {
+		//default delay of 5min
+		w.Duration = 600 * time.Second
+	}
+
+	packages, err := w.list()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get available packages")
+	}
+
+	toAdd, toDel := w.diff(packages)
+
+	ch := make(chan Event, 1)
+
+	if len(toAdd) > 0 || len(toDel) > 0 {
+		ch <- &RepoEvent{
+			Repo:  w.Repo,
+			ToAdd: toAdd,
+			ToDel: toDel,
+		}
+
+		w.Current = packages
+	}
+
+	ticker := time.NewTicker(w.Duration)
+
+	go func() {
+		defer close(ch)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-ticker.C:
+			case <-ctx.Done():
+				return
+			}
+
+			packages, err := w.list()
+			if err != nil {
+				log.Error().Err(err).Str("repo", w.Repo).Msg("failed to list repo flists")
+				continue
+			}
+
+			toAdd, toDel := w.diff(packages)
+
+			if len(toAdd) > 0 || len(toDel) > 0 {
+				select {
+				case ch <- &RepoEvent{
+					Repo:  w.Repo,
+					ToAdd: toAdd,
+					ToDel: toDel,
+				}:
+				case <-ctx.Done():
+					return
+				}
+
+				w.Current = packages
+			}
+		}
+
+	}()
+
+	return ch, nil
+}
diff --git a/qemu/overlay/etc/zinit/internet.yaml b/qemu/overlay/etc/zinit/internet.yaml
index e865fcfe..d485ff54 120000
--- a/qemu/overlay/etc/zinit/internet.yaml
+++ b/qemu/overlay/etc/zinit/internet.yaml
@@ -1,1 +1,1 @@
-../../../../etc/zinit/internet.yaml
\ No newline at end of file
+../../../../bootstrap/etc/zinit/internet.yaml
\ No newline at end of file
diff --git a/qemu/overlay/etc/zinit/promtail-config.yaml b/qemu/overlay/etc/zinit/promtail-config.yaml
deleted file mode 120000
index c85fb185..00000000
--- a/qemu/overlay/etc/zinit/promtail-config.yaml
+++ /dev/null
@@ -1,1 +0,0 @@
-../../../../etc/zinit/promtail-config.yaml
\ No newline at end of file
diff --git a/qemu/overlay/etc/zinit/promtail.yaml b/qemu/overlay/etc/zinit/promtail.yaml
deleted file mode 120000
index c22f0940..00000000
--- a/qemu/overlay/etc/zinit/promtail.yaml
+++ /dev/null
@@ -1,1 +0,0 @@
-../../../../etc/zinit/promtail.yaml
\ No newline at end of file
diff --git a/qemu/vm.sh b/qemu/vm.sh
index 18a21b25..758aedae 100755
--- a/qemu/vm.sh
+++ b/qemu/vm.sh
@@ -5,6 +5,8 @@ debug=0
 reset=0
 image=zos.efi
 kernelargs=""
+bridge=zos
+graphics="-nographic -nodefaults"
 
 usage() {
    cat <<EOF
@@ -14,6 +16,8 @@ Usage: vm -n $name [ -r ] [ -d ]
    -r: reset vm (recreate disks)
    -c: Kernel command
    -i: kernel image to boot
+   -b: bridge for network (default zos)
+   -g: open GUI
    -h: help
 
 EOF
@@ -21,13 +25,15 @@ EOF
 }
 
 
-while getopts "c:n:i:rd" opt; do
+while getopts "c:n:i:rdb:g" opt; do
    case $opt in
    i )  image=$OPTARG ;;
    r )  reset=1 ;;
    d )  debug=1 ;;
    n )  name=$OPTARG ;;
    c )  kernelargs=$OPTARG ;;
+   b )  bridge=$OPTARG ;;
+   g )  graphics="" ;;
    h )  usage ; exit 0 ;;
    \?)  usage ; exit 1 ;;
    esac
@@ -62,15 +68,16 @@ if ps -eaf | grep -v grep | grep "$uuid" > /dev/null; then
 fi
 
 echo "boot $image"
+
 qemu-system-x86_64 -kernel $image \
     -m 3072 -enable-kvm -cpu host \
     -uuid $uuid \
-    -netdev bridge,id=zos0,br=zos0 -device virtio-net-pci,netdev=zos0,mac="${basemac}1" \
+    -netdev bridge,id=zos0,br=${bridge} -device virtio-net-pci,netdev=zos0,mac="${basemac}1" \
     -drive file=fat:rw:$basepath/overlay,format=raw \
     -append "${cmdline} console=ttyS1,115200" \
     -drive file=$vmdir/vda.qcow2,if=virtio -drive file=$vmdir/vdb.qcow2,if=virtio \
     -drive file=$vmdir/vdc.qcow2,if=virtio -drive file=$vmdir/vdd.qcow2,if=virtio \
     -drive file=$vmdir/vde.qcow2,if=virtio \
     -serial null -serial mon:stdio \
-    -nographic -nodefaults
+    ${graphics}
 


Test patch:
diff --git a/pkg/flist/flist_test.go b/pkg/flist/flist_test.go
index e253a29e..9465601f 100644
--- a/pkg/flist/flist_test.go
+++ b/pkg/flist/flist_test.go
@@ -2,6 +2,7 @@ package flist
 
 import (
 	"bytes"
+	"context"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -9,6 +10,7 @@ import (
 	"strings"
 	"testing"
 	"text/template"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
@@ -207,3 +209,47 @@ func TestDownloadFlist(t *testing.T) {
 	require.NoError(err)
 	assert.Equal(info1.ModTime(), info2.ModTime())
 }
+
+func TestWaitPIDFileExists(t *testing.T) {
+	require := require.New(t)
+	const testFile = "/tmp/wait.exists.test"
+	os.Remove(testFile)
+
+	out := make(chan error)
+	go func(out chan<- error) {
+		out <- waitPidFile(2*time.Second, testFile, true)
+	}(out)
+
+	os.Create(testFile)
+	err := <-out
+	require.NoError(err)
+}
+
+func TestWaitPIDFileDeleted(t *testing.T) {
+	require := require.New(t)
+	const testFile = "/tmp/wait.deleted.test"
+	os.Create(testFile)
+
+	out := make(chan error)
+	go func(out chan<- error) {
+		out <- waitPidFile(2*time.Second, testFile, false)
+	}(out)
+
+	os.Remove(testFile)
+	err := <-out
+	require.NoError(err)
+}
+
+func TestWaitPIDFileTimeout(t *testing.T) {
+	require := require.New(t)
+	const testFile = "/tmp/wait.deleted.test"
+	os.Create(testFile)
+
+	out := make(chan error)
+	go func(out chan<- error) {
+		out <- waitPidFile(1*time.Second, testFile, false)
+	}(out)
+
+	err := <-out
+	require.Equal(context.DeadlineExceeded, err)
+}
diff --git a/pkg/network/bridge/bridge_test.go b/pkg/network/bridge/bridge_test.go
index bd228c87..dc85fea1 100644
--- a/pkg/network/bridge/bridge_test.go
+++ b/pkg/network/bridge/bridge_test.go
@@ -70,32 +70,3 @@ func TestAttachBridge(t *testing.T) {
 	err = AttachNic(dummy, br)
 	assert.NoError(t, err)
 }
-
-func TestAttachBridgeWithMac(t *testing.T) {
-	const ifName = "bro0"
-	br, err := New(ifName)
-	require.NoError(t, err)
-
-	err = netlink.LinkAdd(&netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "dummy0"}})
-	require.NoError(t, err)
-
-	dummy, err := netlink.LinkByName("dummy0")
-	require.NoError(t, err)
-
-	defer func() {
-		_ = netlink.LinkDel(br)
-		_ = netlink.LinkDel(dummy)
-	}()
-
-	addrBefore := br.Attrs().HardwareAddr
-	err = AttachNicWithMac(dummy, br)
-	require.NoError(t, err)
-
-	l, err := netlink.LinkByName("bro0")
-	require.NoError(t, err)
-	br = l.(*netlink.Bridge)
-	addrAfter := br.Attrs().HardwareAddr
-
-	assert.NotEqual(t, addrBefore, addrAfter)
-	assert.Equal(t, dummy.Attrs().HardwareAddr, addrAfter)
-}
diff --git a/pkg/storage/filesystem/btrfs_ci_test.go b/pkg/storage/filesystem/btrfs_ci_test.go
index 5a6272d1..60d95410 100644
--- a/pkg/storage/filesystem/btrfs_ci_test.go
+++ b/pkg/storage/filesystem/btrfs_ci_test.go
@@ -162,6 +162,13 @@ func basePoolTest(t *testing.T, pool Pool) {
 		assert.Equal(t, uint64(1024*1024*1024), usage.Size)
 	})
 
+	t.Run("test subvolume list no subvolumes", func(t *testing.T) {
+		volumes, err := volume.Volumes()
+		require.NoError(t, err)
+
+		assert.Empty(t, volumes)
+	})
+
 	t.Run("test limit subvolume", func(t *testing.T) {
 		usage, err := volume.Usage()
 		require.NoError(t, err)
@@ -294,60 +301,3 @@ func TestBtrfsListCI(t *testing.T) {
 	ok := assert.Len(t, names, 0)
 	assert.True(t, ok, "not all pools were listed")
 }
-
-func TestCLeanUpQgroupsCI(t *testing.T) {
-	if SkipCITests {
-		t.Skip("test requires ability to create loop devices")
-	}
-
-	devices, err := SetupDevices(1)
-	require.NoError(t, err, "failed to initialize devices")
-	defer devices.Destroy()
-
-	loops := devices.Loops()
-	fs := NewBtrfs(&TestDeviceManager{loops})
-
-	names := make(map[string]struct{})
-	for idx := range loops {
-		loop := &loops[idx]
-		name := fmt.Sprintf("test-list-%d", idx)
-		names[name] = struct{}{}
-		_, err := fs.Create(context.Background(), name, pkg.Single, loop)
-		require.NoError(t, err)
-	}
-	pools, err := fs.List(context.Background(), func(p Pool) bool {
-		return strings.HasPrefix(p.Name(), "test-")
-	})
-	pool := pools[0]
-
-	_, err = pool.Mount()
-	require.NoError(t, err)
-	defer pool.UnMount()
-
-	volume, err := pool.AddVolume("vol1")
-	require.NoError(t, err)
-	t.Logf("volume ID: %v\n", volume.ID())
-
-	err = volume.Limit(256 * 1024 * 1024)
-	require.NoError(t, err)
-
-	btrfsVol, ok := volume.(*btrfsVolume)
-	require.True(t, ok, "volume should be a btrfsVolume")
-
-	qgroups, err := btrfsVol.utils.QGroupList(context.TODO(), pool.Path())
-	require.NoError(t, err)
-	assert.Equal(t, 2, len(qgroups))
-	t.Logf("qgroups before delete: %v", qgroups)
-
-	_, ok = qgroups[fmt.Sprintf("0/%d", btrfsVol.id)]
-	assert.True(t, ok, "qgroups should contains a qgroup linked to the subvolume")
-
-	err = pool.RemoveVolume("vol1")
-	require.NoError(t, err)
-
-	qgroups, err = btrfsVol.utils.QGroupList(context.TODO(), pool.Path())
-	require.NoError(t, err)
-
-	t.Logf("remaining qgroups: %+v", qgroups)
-	assert.Equal(t, 1, len(qgroups), "qgroups should have been deleted with the subvolume")
-}
diff --git a/pkg/upgrade/watcher_test.go b/pkg/upgrade/watcher_test.go
index 9668bd88..425cefc3 100644
--- a/pkg/upgrade/watcher_test.go
+++ b/pkg/upgrade/watcher_test.go
@@ -1,10 +1,11 @@
 package upgrade
 
-import "testing"
+import (
+	"context"
+	"testing"
 
-import "context"
-
-import "github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/require"
+)
 
 func TestFListWatcher(t *testing.T) {
 	require := require.New(t)
@@ -27,3 +28,26 @@ func TestFListWatcher(t *testing.T) {
 	require.Equal("zos:development:latest.flist", info.Name)
 	require.Equal("symlink", info.Type)
 }
+
+func TestRepoWatcher(t *testing.T) {
+	require := require.New(t)
+
+	watcher := FListRepoWatcher{
+		Repo: "tf-zos-bins",
+	}
+
+	ctx := context.Background()
+
+	ch, err := watcher.Watch(ctx)
+	require.NoError(err)
+
+	event := <-ch
+	require.Equal(Repo, event.EventType())
+
+	info, ok := event.(*RepoEvent)
+	require.True(ok)
+
+	require.Equal("tf-zos-bins", info.Repo)
+	require.Len(info.ToDel, 0) // we starting from empty Current
+	require.True(len(info.ToAdd) > 0)
+}