Repository: threefoldtech/zos
Pull request #318 (2019-10-09 09:50:27)

contd: read environment variable from .stratup.toml if it exists and ensure /etc/resolv.conf

codecov[bot]: # [Codecov](https://codecov.io/gh/threefoldtech/zos/pull/318?src=pr&el=h1) Report
> Merging [#318](https://codecov.io/gh/threefoldtech/zos/pull/318?src=pr&el=desc) into [master](https://codecov.io/gh/threefoldtech/zos/commit/3c3777ffde399e63b2cd6adea60da9e1d5b58e35?src=pr&el=desc) will **decrease** coverage by `5.34%`.
> The diff coverage is `30.47%`.

[![Impacted file tree graph](https://codecov.io/gh/threefoldtech/zos/pull/318/graphs/tree.svg?width=650&token=f7Vmq1OAEm&height=150&src=pr)](https://codecov.io/gh/threefoldtech/zos/pull/318?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master     #318      +/-   ##
==========================================
- Coverage   48.56%   43.21%   -5.35%     
==========================================
  Files          37       28       -9     
  Lines        2432     1511     -921     
==========================================
- Hits         1181      653     -528     
+ Misses       1114      804     -310     
+ Partials      137       54      -83
```


| [Impacted Files](https://codecov.io/gh/threefoldtech/zos/pull/318?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [pkg/container/spec\_opts.go](https://codecov.io/gh/threefoldtech/zos/pull/318/diff?src=pr&el=tree#diff-cGtnL2NvbnRhaW5lci9zcGVjX29wdHMuZ28=) | `0% <0%> (ø)` | |
| [pkg/container/container.go](https://codecov.io/gh/threefoldtech/zos/pull/318/diff?src=pr&el=tree#diff-cGtnL2NvbnRhaW5lci9jb250YWluZXIuZ28=) | `16.08% <71.11%> (ø)` | |
| [pkg/gedis/commands\_network.go](https://codecov.io/gh/threefoldtech/zos/pull/318/diff?src=pr&el=tree#diff-cGtnL2dlZGlzL2NvbW1hbmRzX25ldHdvcmsuZ28=) | | |
| [pkg/storage/filesystem/utils.go](https://codecov.io/gh/threefoldtech/zos/pull/318/diff?src=pr&el=tree#diff-cGtnL3N0b3JhZ2UvZmlsZXN5c3RlbS91dGlscy5nbw==) | | |
| [pkg/storage/filesystem/btrfs.go](https://codecov.io/gh/threefoldtech/zos/pull/318/diff?src=pr&el=tree#diff-cGtnL3N0b3JhZ2UvZmlsZXN5c3RlbS9idHJmcy5nbw==) | | |
| [pkg/storage/filesystem/device.go](https://codecov.io/gh/threefoldtech/zos/pull/318/diff?src=pr&el=tree#diff-cGtnL3N0b3JhZ2UvZmlsZXN5c3RlbS9kZXZpY2UuZ28=) | | |
| [pkg/gedis/gedis.go](https://codecov.io/gh/threefoldtech/zos/pull/318/diff?src=pr&el=tree#diff-cGtnL2dlZGlzL2dlZGlzLmdv) | | |
| [pkg/storage/filesystem/filesystem.go](https://codecov.io/gh/threefoldtech/zos/pull/318/diff?src=pr&el=tree#diff-cGtnL3N0b3JhZ2UvZmlsZXN5c3RlbS9maWxlc3lzdGVtLmdv) | | |
| [pkg/gedis/errors.go](https://codecov.io/gh/threefoldtech/zos/pull/318/diff?src=pr&el=tree#diff-cGtnL2dlZGlzL2Vycm9ycy5nbw==) | | |
| [pkg/gedis/commands\_provision.go](https://codecov.io/gh/threefoldtech/zos/pull/318/diff?src=pr&el=tree#diff-cGtnL2dlZGlzL2NvbW1hbmRzX3Byb3Zpc2lvbi5nbw==) | | |
| ... and [6 more](https://codecov.io/gh/threefoldtech/zos/pull/318/diff?src=pr&el=tree-more) | |

------

[Continue to review full report at Codecov](https://codecov.io/gh/threefoldtech/zos/pull/318?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/threefoldtech/zos/pull/318?src=pr&el=footer). Last update [3c3777f...1da0675](https://codecov.io/gh/threefoldtech/zos/pull/318?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/cmds/upgraded/main.go b/cmds/identityd/main.go
similarity index 70%
rename from cmds/upgraded/main.go
rename to cmds/identityd/main.go
index a3b404c0..d141bcc3 100644
--- a/cmds/upgraded/main.go
+++ b/cmds/identityd/main.go
@@ -34,9 +34,8 @@ const (
 )
 
 const (
-	module       = "identityd"
-	identityRoot = "/var/cache/modules/identityd"
-	seedName     = "seed.txt"
+	module   = "identityd"
+	seedName = "seed.txt"
 )
 
 // setup is a sanity check function, the whole purpose of this
@@ -77,13 +76,17 @@ func SafeUpgrade(upgrader *upgrade.Upgrader) error {
 func main() {
 	var (
 		broker   string
+		root     string
 		interval int
 		ver      bool
+		debug    bool
 	)
 
+	flag.StringVar(&root, "root", "/var/cache/modules/identityd", "root working directory of the module")
 	flag.StringVar(&broker, "broker", redisSocket, "connection string to broker")
 	flag.IntVar(&interval, "interval", 600, "interval in seconds between update checks, default to 600")
 	flag.BoolVar(&ver, "v", false, "show version and exit")
+	flag.BoolVar(&debug, "d", false, "when set, no self update is done before upgradeing")
 
 	log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
 
@@ -97,43 +100,25 @@ func main() {
 		log.Fatal().Err(err).Msg("failed to connect to zinit")
 	}
 
-	zbusClient, err := zbus.NewRedisClient(broker)
+	cl, err := zbus.NewRedisClient(broker)
 	if err != nil {
 		log.Error().Err(err).Msg("fail to connect to broker")
 		return
 	}
 
-	flister := stubs.NewFlisterStub(zbusClient)
+	flister := stubs.NewFlisterStub(cl)
 
 	upgrader := upgrade.Upgrader{
-		FLister: flister,
-		Zinit:   zinit,
+		FLister:      flister,
+		Zinit:        zinit,
+		NoSelfUpdate: debug,
 	}
 
 	bootMethod := upgrade.DetectBootMethod()
 
-	if bootMethod == upgrade.BootMethodFList {
-
-		// 1. Do upgrade to latest version
-		if err := SafeUpgrade(&upgrader); err == upgrade.ErrRestartNeeded {
-			log.Info().Msg("restarting upgraded")
-			return
-		} else if err != nil {
-			log.Fatal().Err(err).Msg("upgrade failed")
-		}
-
-		// recover procedure to make sure upgrade always has what it needs
-		// to work
-		if err := setup(zinit); err != nil {
-			log.Fatal().Err(err).Msg("upgraded setup failed")
-		}
-	} else {
-		log.Info().Msg("not booted with an flist. life upgrade is not supported")
-	}
-
 	// 2. Register the node to BCDB
 	// at this point we are running latest version
-	idMgr, err := identityMgr(identityRoot)
+	idMgr, err := identityMgr(root)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to create identity manager")
 	}
@@ -143,15 +128,18 @@ func main() {
 		log.Fatal().Err(err).Msg("failed to create identity client")
 	}
 
-	var version string
+	var current string
 	if bootMethod == upgrade.BootMethodFList {
 		v, err := upgrader.Version()
 		if err != nil {
-			log.Fatal().Err(err).Msg("failed to read current version")
+			//NOTE: this is set to error intentionally (not fatal)
+			//this will cause version to be 0.0.0 and will force an
+			//immediate update of the flist to latest
+			log.Error().Err(err).Msg("failed to read current version")
 		}
-		version = v.String()
+		current = v.String()
 	} else {
-		version = "not booted from flist"
+		current = "not booted from flist"
 	}
 
 	nodeID := idMgr.NodeID()
@@ -160,11 +148,16 @@ func main() {
 		log.Fatal().Err(err).Msg("failed to read farm ID")
 	}
 
-	f := func() error {
-		return registerNode(nodeID, farmID, version, idStore)
+	register := func(v string) error {
+		return registerNode(nodeID, farmID, v, idStore)
 	}
-	if err := backoff.Retry(f, backoff.NewExponentialBackOff()); err == nil {
-		log.Info().Msg("node registered successfully")
+
+	if err := backoff.Retry(func() error {
+		return register(current)
+	}, backoff.NewExponentialBackOff()); err != nil {
+		log.Error().Err(err).Msg("failed to register node")
+	} else {
+		log.Info().Str("version", current).Msg("node registered successfully")
 	}
 
 	// 3. start zbus server to serve identity interface
@@ -172,7 +165,8 @@ func main() {
 	if err != nil {
 		log.Fatal().Msgf("fail to connect to message broker server: %v\n", err)
 	}
-	server.Register(zbus.ObjectID{Name: module, Version: "0.0.1"}, &idMgr)
+
+	server.Register(zbus.ObjectID{Name: "manager", Version: "0.0.1"}, idMgr)
 
 	ctx, cancel := utils.WithSignal(context.Background())
 	// register the cancel function with defer if the process stops because of a update
@@ -188,39 +182,51 @@ func main() {
 		log.Info().Msg("shutting down")
 	})
 
-	// 4. Start watcher for new version
 	if bootMethod != upgrade.BootMethodFList {
+		log.Info().Msg("node is not booted from an flist. upgrade is not supported")
 		<-ctx.Done()
-	} else {
-		log.Info().Msg("start upgrade daemon")
-		ticker := time.NewTicker(time.Second * time.Duration(interval))
-
-		for {
-			err := SafeUpgrade(&upgrader)
-			if err == upgrade.ErrRestartNeeded {
-				log.Info().Msg("restarting upgraded")
-				return
-			} else if err != nil {
-				//TODO: crash or continue!
-				log.Error().Err(err).Msg("upgrade failed")
-			}
+		return
+	}
 
-			version, err := upgrader.Version()
-			if err != nil {
-				log.Fatal().Err(err).Msg("failed to read current version")
-			}
+	// 4. Start watcher for new version
+	log.Info().Msg("start upgrade daemon")
+	ticker := time.NewTicker(time.Second * time.Duration(interval))
+	defer ticker.Stop()
 
-			log.Info().Str("version", version.String()).Msg("new version installed")
+	for {
+		err := SafeUpgrade(&upgrader)
+		if err == upgrade.ErrRestartNeeded {
+			log.Info().Msg("restarting upgraded")
+			return
+		} else if err != nil {
+			//TODO: crash or continue!
+			log.Error().Err(err).Msg("upgrade failed")
+		}
 
-			if _, err = idStore.RegisterNode(nodeID, farmID, version.String()); err != nil {
-				log.Error().Err(err).Msg("fail to register node identity")
-			}
+		version, err := upgrader.Version()
+		if err != nil {
+			log.Fatal().Err(err).Msg("failed to read current version")
+		}
+
+		if version.String() != current {
+			log.Info().Str("version", version.String()).Msg("new version")
 
-			select {
-			case <-ticker.C:
-			case <-ctx.Done():
-				break
+			if err := backoff.Retry(func() error {
+				return register(version.String())
+			}, backoff.NewExponentialBackOff()); err != nil {
+				log.Error().Err(err).Msg("failed to register node")
+			} else {
+				log.Info().Str("version", version.String()).Msg("node registered successfully")
 			}
+
+			current = version.String()
+		}
+
+		select {
+		case <-ticker.C:
+			log.Debug().Msg("checking for updates")
+		case <-ctx.Done():
+			return
 		}
 	}
 }
@@ -270,7 +276,6 @@ func registerNode(nodeID, farmID pkg.Identifier, version string, store identity.
 
 	_, err := store.RegisterNode(nodeID, farmID, version)
 	if err != nil {
-		log.Error().Err(err).Msg("fail to register node identity")
 		return err
 	}
 	return nil
diff --git a/go.mod b/go.mod
index 962def7b..fe8b270b 100644
--- a/go.mod
+++ b/go.mod
@@ -3,6 +3,7 @@ module github.com/threefoldtech/zos
 go 1.13
 
 require (
+	github.com/BurntSushi/toml v0.3.1
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412
 	github.com/alexflint/go-filemutex v0.0.0-20171028004239-d358565f3c3f
diff --git a/pkg/container.go b/pkg/container.go
index ee3b9527..a76801cf 100644
--- a/pkg/container.go
+++ b/pkg/container.go
@@ -34,6 +34,8 @@ type Container struct {
 	RootFS string
 	// Env env variables to container in format {'KEY=VALUE', 'KEY2=VALUE2'}
 	Env []string
+	// WorkingDir of the entrypoint command
+	WorkingDir string
 	// Network network info for container
 	Network NetworkInfo
 	// Mounts extra mounts for container
diff --git a/pkg/container/container.go b/pkg/container/container.go
index 9d6f30b1..33b1b21a 100644
--- a/pkg/container/container.go
+++ b/pkg/container/container.go
@@ -3,6 +3,7 @@ package container
 import (
 	"context"
 
+	"github.com/BurntSushi/toml"
 	"github.com/pkg/errors"
 
 	"fmt"
@@ -62,12 +63,7 @@ func New(root string, containerd string) pkg.ContainerModule {
 }
 
 // Run creates and starts a container
-// THIS IS A WIP Create action and it's not fully implemented atm
 func (c *containerModule) Run(ns string, data pkg.Container) (id pkg.ContainerID, err error) {
-	log.Info().
-		Str("namesapce", ns).
-		Str("data", fmt.Sprintf("%+v", data)).
-		Msgf("create new container")
 	// create a new client connected to the default socket path for containerd
 	client, err := containerd.New(c.containerd)
 	if err != nil {
@@ -85,8 +81,8 @@ func (c *containerModule) Run(ns string, data pkg.Container) (id pkg.ContainerID
 		return id, ErrEmptyRootFS
 	}
 
-	if err := setResolvConf(data.RootFS); err != nil {
-		return id, errors.Wrap(err, "failed to set resolv.conf")
+	if err := applyStartup(&data, filepath.Join(data.RootFS, ".startup.toml")); err != nil {
+		errors.Wrap(err, "error updating environment variable from startup file")
 	}
 
 	if data.Interactive {
@@ -113,8 +109,14 @@ func (c *containerModule) Run(ns string, data pkg.Container) (id pkg.ContainerID
 		oci.WithRootFSPath(data.RootFS),
 		oci.WithProcessArgs(args...),
 		oci.WithEnv(data.Env),
+		oci.WithHostResolvconf,
 		removeRunMount(),
 	}
+
+	if data.WorkingDir != "" {
+		opts = append(opts, oci.WithProcessCwd(data.WorkingDir))
+	}
+
 	if data.Interactive {
 		opts = append(
 			opts,
@@ -152,6 +154,11 @@ func (c *containerModule) Run(ns string, data pkg.Container) (id pkg.ContainerID
 		)
 	}
 
+	log.Info().
+		Str("namespace", ns).
+		Str("data", fmt.Sprintf("%+v", data)).
+		Msgf("create new container")
+
 	container, err := client.NewContainer(
 		ctx,
 		data.Name,
@@ -166,6 +173,7 @@ func (c *containerModule) Run(ns string, data pkg.Container) (id pkg.ContainerID
 		return id, err
 	}
 	log.Info().Msgf("args %+v", spec.Process.Args)
+	log.Info().Msgf("env %+v", spec.Process.Env)
 	log.Info().Msgf("root %+v", spec.Root)
 	for _, linxNS := range spec.Linux.Namespaces {
 		log.Info().Msgf("namespace %+v", linxNS.Type)
@@ -298,3 +306,46 @@ func (c *containerModule) Delete(ns string, id pkg.ContainerID) error {
 
 	return container.Delete(ctx)
 }
+
+func (c *containerModule) ensureNamespace(ctx context.Context, client *containerd.Client, namespace string) error {
+	service := client.NamespaceService()
+	namespaces, err := service.List(ctx)
+	if err != nil {
+		return err
+	}
+
+	for _, ns := range namespaces {
+		if ns == namespace {
+			return nil
+		}
+	}
+
+	return service.Create(ctx, namespace, nil)
+}
+
+func applyStartup(data *pkg.Container, path string) error {
+	f, err := os.Open(path)
+	if err == nil {
+		defer f.Close()
+		log.Info().Msg("startup file found")
+
+		startup := startup{}
+		if _, err := toml.DecodeReader(f, &startup); err != nil {
+			return err
+		}
+
+		entry, ok := startup.Entries["entry"]
+		if !ok {
+			return nil
+		}
+
+		data.Env = mergeEnvs(entry.Envs(), data.Env)
+		if data.Entrypoint == "" && entry.Entrypoint() != "" {
+			data.Entrypoint = entry.Entrypoint()
+		}
+		if data.WorkingDir == "" && entry.WorkingDir() != "" {
+			data.WorkingDir = entry.WorkingDir()
+		}
+	}
+	return nil
+}
diff --git a/pkg/container/spec_opts.go b/pkg/container/opts.go
similarity index 62%
rename from pkg/container/spec_opts.go
rename to pkg/container/opts.go
index f1b0e6da..f0ceb3ce 100644
--- a/pkg/container/spec_opts.go
+++ b/pkg/container/opts.go
@@ -2,14 +2,13 @@ package container
 
 import (
 	"context"
-	"os"
+
 	"path"
-	"path/filepath"
 
-	"github.com/containerd/containerd"
+	"github.com/opencontainers/runtime-spec/specs-go"
+
 	"github.com/containerd/containerd/containers"
 	"github.com/containerd/containerd/oci"
-	"github.com/opencontainers/runtime-spec/specs-go"
 )
 
 // withNetworkNamespace set the named network namespace to use for the container
@@ -57,22 +56,6 @@ func withAddedCapabilities(caps []string) oci.SpecOpts {
 	}
 }
 
-func (c *containerModule) ensureNamespace(ctx context.Context, client *containerd.Client, namespace string) error {
-	service := client.NamespaceService()
-	namespaces, err := service.List(ctx)
-	if err != nil {
-		return err
-	}
-
-	for _, ns := range namespaces {
-		if ns == namespace {
-			return nil
-		}
-	}
-
-	return service.Create(ctx, namespace, nil)
-}
-
 func removeRunMount() oci.SpecOpts {
 	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error {
 		for i, mount := range s.Mounts {
@@ -84,29 +67,3 @@ func removeRunMount() oci.SpecOpts {
 		return nil
 	}
 }
-
-func setResolvConf(root string) error {
-	const tmp = "nameserver 1.1.1.1\nnameserver 1.0.0.1\n2606:4700:4700::1111\nnameserver 2606:4700:4700::1001\n"
-
-	path := filepath.Join(root, "etc/resolv.conf")
-	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 644)
-	if err != nil && !os.IsNotExist(err) {
-		return err
-	}
-
-	defer f.Close()
-
-	if os.IsNotExist(err) {
-		_, err = f.WriteString(tmp)
-		return err
-	}
-
-	info, err := f.Stat()
-	if err != nil {
-		return err
-	}
-	if info.Size() == 0 {
-		_, err = f.WriteString(tmp)
-	}
-	return err
-}
diff --git a/pkg/container/startup.go b/pkg/container/startup.go
new file mode 100644
index 00000000..fa85f45b
--- /dev/null
+++ b/pkg/container/startup.go
@@ -0,0 +1,63 @@
+package container
+
+import (
+	"fmt"
+	"strings"
+)
+
+type startup struct {
+	Entries map[string]entry `toml:"startup"`
+}
+
+type entry struct {
+	Name string
+	Args args
+}
+
+type args struct {
+	Name string
+	Dir  string
+	Env  map[string]string
+}
+
+func (e entry) Entrypoint() string {
+	if e.Name == "core.system" ||
+		e.Name == "core.base" && e.Args.Name != "" {
+		return e.Args.Name
+	}
+	return ""
+}
+
+func (e entry) WorkingDir() string {
+	return e.Args.Dir
+}
+
+func (e entry) Envs() []string {
+	envs := make([]string, 0, len(e.Args.Env))
+	for k, v := range e.Args.Env {
+		envs = append(envs, fmt.Sprintf("%s=%s", k, v))
+	}
+	return envs
+}
+
+// mergeEnvs merge a into b
+// all the key from a will endup in b
+// if a key is present in both, key from a are kept
+func mergeEnvs(a, b []string) []string {
+	m := make(map[string]string, len(a)+len(b))
+
+	for _, s := range b {
+		ss := strings.SplitN(s, "=", 2)
+		m[ss[0]] = ss[1]
+	}
+	for _, s := range a {
+		ss := strings.SplitN(s, "=", 2)
+		m[ss[0]] = ss[1]
+	}
+
+	result := make([]string, 0, len(m))
+	for k, v := range m {
+		result = append(result, fmt.Sprintf("%s=%s", k, v))
+	}
+	return result
+}
diff --git a/pkg/upgrade/hub.go b/pkg/upgrade/hub.go
index 698b0420..c2751c84 100644
--- a/pkg/upgrade/hub.go
+++ b/pkg/upgrade/hub.go
@@ -14,19 +14,19 @@ import (
 )
 
 const (
-	// HubBaseURL base hub url
-	HubBaseURL = "https://hub.grid.tf/"
+	// hubBaseURL base hub url
+	hubBaseURL = "https://hub.grid.tf/"
 
-	// HubStorage default hub db
-	HubStorage = "zdb://hub.grid.tf:9900"
+	// hubStorage default hub db
+	hubStorage = "zdb://hub.grid.tf:9900"
 )
 
-// Hub API for f-list
-type Hub struct{}
+// hubClient API for f-list
+type hubClient struct{}
 
 // MountURL returns the full url of given flist.
-func (h *Hub) MountURL(flist string) string {
-	url, err := url.Parse(HubBaseURL)
+func (h *hubClient) MountURL(flist string) string {
+	url, err := url.Parse(hubBaseURL)
 	if err != nil {
 		panic("invalid base url")
 	}
@@ -35,17 +35,19 @@ func (h *Hub) MountURL(flist string) string {
 }
 
 // StorageURL return hub storage url
-func (h *Hub) StorageURL() string {
-	return HubStorage
+func (h *hubClient) StorageURL() string {
+	return hubStorage
 }
 
 // Info gets flist info from hub
-func (h *Hub) Info(flist string) (info FListInfo, err error) {
-	u, err := url.Parse(HubBaseURL)
+func (h *hubClient) Info(flist string) (info flistInfo, err error) {
+	u, err := url.Parse(hubBaseURL)
 	if err != nil {
 		panic("invalid base url")
 	}
 
+	info.Repository = filepath.Dir(flist)
+
 	u.Path = filepath.Join("api", "flist", flist, "light")
 	response, err := http.Get(u.String())
 	if err != nil {
@@ -65,8 +67,9 @@ func (h *Hub) Info(flist string) (info FListInfo, err error) {
 	return info, err
 }
 
-// LoadInfo get boot info set by bootstrap process
-func LoadInfo(path string) (info FListInfo, err error) {
+// loadInfo get boot info set by bootstrap process
+func loadInfo(fqn string, path string) (info flistInfo, err error) {
+	info.Repository = filepath.Dir(fqn)
 	f, err := os.Open(path)
 	if err != nil {
 		return info, err
@@ -79,18 +82,25 @@ func LoadInfo(path string) (info FListInfo, err error) {
 	return info, err
 }
 
-// FListInfo reflects node boot information (flist + version)
-type FListInfo struct {
-	Name    string `json:"name"`
-	Target  string `json:"target"`
-	Type    string `json:"type"`
-	Updated uint64 `json:"updated"`
-	Hash    string `json:"hash"`
-	Size    uint64 `json:"size"`
+// flistInfo reflects node boot information (flist + version)
+type flistInfo struct {
+	Name       string `json:"name"`
+	Target     string `json:"target"`
+	Type       string `json:"type"`
+	Updated    uint64 `json:"updated"`
+	Hash       string `json:"hash"`
+	Size       uint64 `json:"size"`
+	Repository string `json:"-"`
+}
+
+// fileInfo is the file of an flist
+type fileInfo struct {
+	Path string `json:"path"`
+	Size uint64 `json:"size"`
 }
 
 // Commit write version to version file
-func (b *FListInfo) Commit(path string) error {
+func (b *flistInfo) Commit(path string) error {
 	f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0400)
 	if err != nil {
 		return err
@@ -101,7 +111,7 @@ func (b *FListInfo) Commit(path string) error {
 	return enc.Encode(b)
 }
 
-func (b *FListInfo) extractVersion(name string) (ver semver.Version, err error) {
+func (b *flistInfo) extractVersion(name string) (ver semver.Version, err error) {
 	// name is suppose to be as follows
 	// <name>:<version>.flist
 	parts := strings.Split(name, ":")
@@ -110,7 +120,7 @@ func (b *FListInfo) extractVersion(name string) (ver semver.Version, err error)
 }
 
 // Version returns the version of the flist
-func (b *FListInfo) Version() (semver.Version, error) {
+func (b *flistInfo) Version() (semver.Version, error) {
 	// computing the version is tricky because it's part of the name
 	// of the flist (or the target) depends on the type of the flist
 
@@ -120,3 +130,48 @@ func (b *FListInfo) Version() (semver.Version, error) {
 
 	return b.extractVersion(b.Name)
 }
+
+// Absolute returns the actual flist name
+func (b *flistInfo) Absolute() string {
+	name := b.Name
+	if b.Type == "symlink" {
+		name = b.Target
+	}
+
+	return filepath.Join(b.Repository, name)
+}
+
+// Files gets the list of the files of an flist
+func (b *flistInfo) Files() ([]fileInfo, error) {
+	flist := b.Absolute()
+	if len(flist) == 0 {
+		return nil, fmt.Errorf("invalid flist info")
+	}
+
+	var content struct {
+		Content []fileInfo `json:"content"`
+	}
+
+	u, err := url.Parse(hubBaseURL)
+	if err != nil {
+		panic("invalid base url")
+	}
+
+	u.Path = filepath.Join("api", "flist", flist)
+	response, err := http.Get(u.String())
+	if err != nil {
+		return nil, err
+	}
+
+	defer response.Body.Close()
+	defer ioutil.ReadAll(response.Body)
+
+	if response.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to get flist info: %s", response.Status)
+	}
+
+	dec := json.NewDecoder(response.Body)
+
+	err = dec.Decode(&content)
+	return content.Content, err
+}
diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go
index f37edf0a..3aae5d3b 100644
--- a/pkg/upgrade/upgrade.go
+++ b/pkg/upgrade/upgrade.go
@@ -36,8 +36,8 @@ var (
 	// ErrRestartNeeded is returned if upgraded requires a restart
 	ErrRestartNeeded = fmt.Errorf("restart needed")
 
-	// services that can't be updated with normal procedure
-	protected = []string{"upgraded", "redis"}
+	// services that can't be uninstalled with normal procedure
+	protected = []string{"identityd", "redis"}
 )
 
 // BootMethod defines the node boot method
@@ -72,10 +72,10 @@ func DetectBootMethod() BootMethod {
 // Upgrader is the component that is responsible
 // to keep 0-OS up to date
 type Upgrader struct {
-	FLister pkg.Flister
-	Zinit   *zinit.Client
-
-	hub Hub
+	FLister      pkg.Flister
+	Zinit        *zinit.Client
+	NoSelfUpdate bool
+	hub          hubClient
 }
 
 // Name always return name of the boot flist. If name file
@@ -85,9 +85,19 @@ func (u *Upgrader) Name() string {
 	return strings.TrimSpace(string(data))
 }
 
+// currentInfo returns current flist information
+func (u *Upgrader) currentInfo() (flistInfo, error) {
+	name := u.Name()
+	if len(name) == 0 {
+		return flistInfo{}, fmt.Errorf("flist name is not known")
+	}
+
+	return loadInfo(name, infoFile)
+}
+
 // Version always returns curent version of flist
 func (u *Upgrader) Version() (semver.Version, error) {
-	info, err := LoadInfo(infoFile)
+	info, err := u.currentInfo()
 	if err != nil {
 		return semver.Version{}, errors.Wrap(err, "failed to load flist info")
 	}
@@ -123,7 +133,7 @@ func (u *Upgrader) Upgrade() error {
 	return nil
 }
 
-func (u Upgrader) stopMultiple(timeout time.Duration, service ...string) ([]string, error) {
+func (u Upgrader) stopMultiple(timeout time.Duration, service ...string) error {
 	services := make(map[string]struct{})
 	for _, name := range service {
 		log.Info().Str("service", name).Msg("stopping service")
@@ -136,20 +146,20 @@ func (u Upgrader) stopMultiple(timeout time.Duration, service ...string) ([]stri
 	}
 
 	deadline := time.After(timeout)
-	var stopped []string
 
 	for len(services) > 0 {
+		var stopped []string
 		for service := range services {
 			status, err := u.Zinit.Status(service)
 			if err != nil {
-				return stopped, err
+				return err
 			}
 
 			if status.Target != zinit.ServiceTargetDown {
 				// it means some other entity (another client or command line)
 				// has set the service back to up. I think we should immediately return
 				// with an error instead.
-				return stopped, fmt.Errorf("expected service target should be DOWN. found UP")
+				return fmt.Errorf("expected service '%s' target should be DOWN. found UP", service)
 			}
 
 			if status.State.Exited() {
@@ -171,13 +181,14 @@ func (u Upgrader) stopMultiple(timeout time.Duration, service ...string) ([]stri
 		select {
 		case <-deadline:
 			for service := range services {
+				log.Warn().Str("service", service).Msg("service didn't stop in time. use SIGKILL")
 				u.Zinit.Kill(service, syscall.SIGKILL)
 			}
 		case <-time.After(1 * time.Second):
 		}
 	}
 
-	return stopped, nil
+	return nil
 }
 
 // upgradeSelf will try to check if the flist has
@@ -186,6 +197,11 @@ func (u Upgrader) stopMultiple(timeout time.Duration, service ...string) ([]stri
 // next time this method is called, it will match the flist
 // revision, and hence will continue updating all the other daemons
 func (u *Upgrader) upgradeSelf(root string) error {
+	if u.NoSelfUpdate {
+		log.Debug().Msg("skipping self upgrade")
+		return nil
+	}
+
 	current := currentRevision()
 	log.Debug().Str("revision", current).Msg("current revision")
 
@@ -222,7 +238,74 @@ func (u *Upgrader) upgradeSelf(root string) error {
 	return ErrRestartNeeded
 }
 
-func (u *Upgrader) applyUpgrade(version semver.Version, info FListInfo) error {
+func (u *Upgrader) uninstall() error {
+	current, err := u.currentInfo()
+	if err != nil {
+		return errors.Wrap(err, "failed to get current flist info")
+	}
+
+	files, err := current.Files()
+	if err != nil {
+		return errors.Wrapf(err, "failed to get list of current installed files for '%s'", current.Absolute())
+	}
+
+	//stop all services names
+	var names []string
+	for _, file := range files {
+		dir := filepath.Dir(file.Path)
+		if dir != "/etc/zinit" {
+			continue
+		}
+
+		name := filepath.Base(file.Path)
+		if !strings.HasSuffix(name, ".yaml") {
+			continue
+		}
+
+		name = strings.TrimSuffix(name, ".yaml")
+		// skip self and redis
+		if isIn(name, protected) {
+			continue
+		}
+
+		names = append(names, name)
+	}
+
+	log.Debug().Strs("services", names).Msg("stopping services")
+
+	if err = u.stopMultiple(20*time.Second, names...); err != nil {
+		return errors.Wrapf(err, "failed to stop services")
+	}
+
+	// we do a forget so any changes of the zinit config
+	// themselves get reflected once monitored again
+	for _, name := range names {
+		if err := u.Zinit.Forget(name); err != nil {
+			log.Error().Err(err).Str("service", name).Msg("error on zinit forget")
+		}
+	}
+
+	// now delete ALL files, ignore what doesn't delete
+	for _, file := range files {
+		log.Debug().Str("file", file.Path).Msg("deleting file")
+		stat, err := os.Stat(file.Path)
+		if err != nil {
+			log.Debug().Err(err).Str("file", file.Path).Msg("failed to check file")
+			continue
+		}
+		if stat.IsDir() {
+			continue
+		}
+
+		if err := os.Remove(file.Path); err != nil {
+			log.Error().Err(err).Str("file", file.Path).Msg("failed to remove file")
+		}
+	}
+
+	return nil
+}
+
+func (u *Upgrader) applyUpgrade(version semver.Version, info flistInfo) error {
 	log.Info().Str("flist", u.Name()).Str("version", version.String()).Msg("start applying upgrade")
 
 	flistRoot, err := u.FLister.Mount(u.hub.MountURL(u.Name()), u.hub.StorageURL())
@@ -240,6 +323,11 @@ func (u *Upgrader) applyUpgrade(version semver.Version, info FListInfo) error {
 		return err
 	}
 
+	if err := u.uninstall(); err != nil {
+		log.Error().Err(err).Msg("failed to unistall current flist. Upgraded anyway")
+	}
+
+	log.Info().Msg("clean up complete, copying new files")
 	// once the flist is mounted we can inspect
 	// it for all zinit config files.
 	files, err := ioutil.ReadDir(filepath.Join(flistRoot, "etc", "zinit"))
@@ -254,32 +342,16 @@ func (u *Upgrader) applyUpgrade(version semver.Version, info FListInfo) error {
 			continue
 		}
 
-		name = strings.TrimSuffix(name, ".yaml")
-		// skip self and redis
-		if isIn(name, protected) {
-			continue
-		}
-
-		names = append(names, name)
-	}
-
-	stopped, err := u.stopMultiple(10*time.Second, names...)
-	if err != nil {
-		return errors.Wrapf(err, "failed to stop services: %+v", names)
+		names = append(names, strings.TrimSuffix(name, ".yaml"))
 	}
 
-	// we do a forget so any changes of the zinit config
-	// themselves get reflected once monitored again
-	for _, stopped := range stopped {
-		if err := u.Zinit.Forget(stopped); err != nil {
-			log.Error().Err(err).Str("service", stopped).Msg("error on zinit forget")
-		}
-	}
+	log.Debug().Strs("services", names).Msg("new services")
 
-	if err := copyRecursive(flistRoot, "/", currentBinPath()); err != nil {
+	if err := copyRecursive(flistRoot, "/"); err != nil {
 		return err
 	}
 
+	log.Debug().Msg("copying files complete")
 	// start all services in the flist
 	for _, name := range names {
 		if err := u.Zinit.Monitor(name); err != nil {
diff --git a/qemu/overlay/etc/zinit/identityd.yaml b/qemu/overlay/etc/zinit/identityd.yaml
new file mode 100644
index 00000000..cca32db2
--- /dev/null
+++ b/qemu/overlay/etc/zinit/identityd.yaml
@@ -0,0 +1,4 @@
+exec: identityd -d
+after:
+  - boot
+  - flistd
diff --git a/qemu/overlay/etc/zinit/upgraded.yaml b/qemu/overlay/etc/zinit/upgraded.yaml
deleted file mode 120000
index 7fa33745..00000000
--- a/qemu/overlay/etc/zinit/upgraded.yaml
+++ /dev/null
@@ -1,1 +0,0 @@
-../../../../zinit/upgraded.yaml
\ No newline at end of file
diff --git a/zinit/identityd.yaml b/zinit/identityd.yaml
new file mode 100644
index 00000000..91a9d641
--- /dev/null
+++ b/zinit/identityd.yaml
@@ -0,0 +1,4 @@
+exec: identityd
+after:
+  - boot
+  - flistd
\ No newline at end of file
diff --git a/zinit/upgraded.yaml b/zinit/upgraded.yaml
deleted file mode 100644
index 8b453933..00000000
--- a/zinit/upgraded.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-exec: upgraded
-after:
-  - boot
-  - flistd
\ No newline at end of file


Test patch:
diff --git a/pkg/container/startup_test.go b/pkg/container/startup_test.go
new file mode 100644
index 00000000..3313c358
--- /dev/null
+++ b/pkg/container/startup_test.go
@@ -0,0 +1,96 @@
+package container
+
+import (
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/BurntSushi/toml"
+	"github.com/stretchr/testify/require"
+)
+
+var _startup = `[startup]
+
+[startup.entry]
+name = "core.system"
+
+[startup.entry.args]
+name = "/start"
+dir = "/data"
+
+[startup.entry.args.env]
+DIFFICULTY = "easy"
+LEVEL = "world"
+SERVER_PORT = "25565"
+`
+
+func TestParseStartup(t *testing.T) {
+	r := strings.NewReader(_startup)
+	e := startup{}
+	_, err := toml.DecodeReader(r, &e)
+	require.NoError(t, err)
+
+	entry, ok := e.Entries["entry"]
+	require.True(t, ok)
+	assert.Equal(t, "core.system", entry.Name)
+	assert.Equal(t, "/start", entry.Args.Name)
+	assert.Equal(t, "/data", entry.Args.Dir)
+	assert.Equal(t, map[string]string{
+		"DIFFICULTY":  "easy",
+		"LEVEL":       "world",
+		"SERVER_PORT": "25565",
+	}, entry.Args.Env)
+}
+
+func TestStartupEntrypoint(t *testing.T) {
+	r := strings.NewReader(_startup)
+	e := startup{}
+	_, err := toml.DecodeReader(r, &e)
+	require.NoError(t, err)
+
+	entry, ok := e.Entries["entry"]
+	require.True(t, ok)
+	assert.Equal(t, entry.Entrypoint(), "/start")
+}
+
+func TestStartupEnvs(t *testing.T) {
+	r := strings.NewReader(_startup)
+	e := startup{}
+	_, err := toml.DecodeReader(r, &e)
+	require.NoError(t, err)
+
+	entry, ok := e.Entries["entry"]
+	require.True(t, ok)
+	actual := entry.Envs()
+	sort.Strings(actual)
+	expected := []string{
+		"DIFFICULTY=easy",
+		"LEVEL=world",
+		"SERVER_PORT=25565",
+	}
+	assert.Equal(t, expected, actual)
+}
+
+func TestStartupWorkingDir(t *testing.T) {
+	r := strings.NewReader(_startup)
+	e := startup{}
+	_, err := toml.DecodeReader(r, &e)
+	require.NoError(t, err)
+
+	entry, ok := e.Entries["entry"]
+	require.True(t, ok)
+	assert.Equal(t, entry.WorkingDir(), "/data")
+}
+
+func TestMergeEnvs(t *testing.T) {
+	actual := mergeEnvs(
+		[]string{"FOO=BAR", "HELLO=WORLD"},
+		[]string{"HELLO=HELLO"},
+	)
+
+	expected := []string{"FOO=BAR", "HELLO=WORLD"}
+	sort.Strings(actual)
+	assert.Equal(t, expected, actual)
+}
diff --git a/pkg/upgrade/hub_test.go b/pkg/upgrade/hub_test.go
index 178b6233..3dec38c5 100644
--- a/pkg/upgrade/hub_test.go
+++ b/pkg/upgrade/hub_test.go
@@ -11,7 +11,7 @@ import (
 func TestInfo(t *testing.T) {
 	const input = `{"size": 0, "name": "zos:development:latest.flist", "target": "zos:development:0.2.0.flist", "type": "symlink", "updated": 1569924782, "md5": "9798ef9b930b49ab18c45953cf1d2369"}`
 
-	var info FListInfo
+	var info flistInfo
 	err := json.Unmarshal([]byte(input), &info)
 	require.NoError(t, err)
 
@@ -20,3 +20,17 @@ func TestInfo(t *testing.T) {
 
 	require.Equal(t, semver.MustParse("0.2.0"), ver)
 }
+
+func TestInfoGet(t *testing.T) {
+	const flist = "tf-zos/zos:0.1.0-rc1.flist"
+
+	var hub hubClient
+	info, err := hub.Info(flist)
+	require.NoError(t, err)
+	require.Equal(t, flist, info.Absolute())
+
+	files, err := info.Files()
+	require.NoError(t, err)
+
+	require.NotEmpty(t, files)
+}