Repository: threefoldtech/zos
Pull request #1599 (2022-02-22 10:50:35)

Support multiple substrate urls per env
This is in draft state until we have more urls to add


Patch:
diff --git a/cmds/identityd/main.go b/cmds/identityd/main.go
index fbaab49c..7bf74186 100644
--- a/cmds/identityd/main.go
+++ b/cmds/identityd/main.go
@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"math/rand"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -96,7 +97,7 @@ func main() {
 		log.Fatal().Err(err).Str("root", root).Msg("failed to create root directory")
 	}
 
-	boot := upgrade.Boot{}
+	var boot upgrade.Boot
 
 	bootMethod := boot.DetectBootMethod()
 
@@ -185,7 +186,7 @@ func installBinaries(boot *upgrade.Boot, upgrader *upgrade.Upgrader) {
 
 	env, _ := environment.Get()
 
-	repoWatcher := upgrade.FListRepoWatcher{
+	repoWatcher := upgrade.FListRepo{
 		Repo:    env.BinRepo,
 		Current: bins,
 	}
@@ -222,117 +223,72 @@ func upgradeLoop(
 		debugReinstall(boot, upgrader)
 	}
 
-	flistWatcher := upgrade.FListSemverWatcher{
-		FList:    boot.Name(),
-		Current:  boot.MustVersion(), //if we are here version must be valid
-		Duration: 600 * time.Second,
-	}
-
-	// make sure we push the current version to monitor
-	monitor.C <- boot.MustVersion()
-
-	flistEvents, err := flistWatcher.Watch(ctx)
-	if err != nil {
-		log.Fatal().Err(err).Str("flist", flistWatcher.FList).Msg("failed to watch flist")
-	}
-
-	bins, err := boot.CurrentBins()
-	if err != nil {
-		log.Warn().Err(err).Msg("could not load current binaries list")
-	}
-
-	env, _ := environment.Get()
-
-	repoWatcher := upgrade.FListRepoWatcher{
-		Repo:    env.BinRepo,
-		Current: bins,
-	}
-
-	repoEvents, err := repoWatcher.Watch(ctx)
-	if err != nil {
-		log.Fatal().Err(err).Str("repo", repoWatcher.Repo).Msg("failed to watch repo")
-	}
+	var hub upgrade.HubClient
 
+	flist := boot.Name()
+	//current := boot.MustVersion()
 	for {
-		select {
-		case <-ctx.Done():
-			return
-		case e := <-flistEvents:
-			if e == nil {
-				continue
-			}
-			event := e.(*upgrade.FListEvent)
-			// new flist available
-			version, err := event.Version()
-			if err != nil {
-				log.Error().Err(err).Msg("failed to parse new version")
-				continue
-			}
-
-			from, err := boot.Current()
-			if err != nil {
-				log.Fatal().Err(err).Msg("failed to load current boot information")
-				return
-			}
+		// delay in case of error
+		<-time.After(5 * time.Second)
 
-			exp := backoff.NewExponentialBackOff()
-			exp.MaxInterval = 3 * time.Minute
-			exp.MaxElapsedTime = 60 * time.Minute
-			err = backoff.Retry(func() error {
-				log.Debug().Str("version", version.String()).Msg("trying to update")
+		current, err := boot.Current()
+		if err != nil {
+			log.Fatal().Err(err).Msg("cannot get current boot flist information")
+		}
 
-				err := Safe(func() error {
-					return upgrader.Upgrade(from, *event)
-				})
+		latest, err := hub.Info(flist)
+		if err != nil {
+			log.Error().Err(err).Msg("failed to get flist info")
+			continue
+		}
 
-				if err == upgrade.ErrRestartNeeded {
-					return backoff.Permanent(err)
-				}
+		if !latest.TryVersion().GT(current.TryVersion()) {
+			// We wanted to use the node id to actually calculate the delay to wait but it's not
+			// possible to get the numeric node id from the identityd
+			next := time.Duration(60+rand.Intn(60)) * time.Minute
+			log.Info().Dur("wait", next).Msg("checking for update after milliseconds")
+			<-time.After(next)
+			continue
+		}
 
-				if err != nil {
-					log.Error().Err(err).Msg("update failure. retrying")
-				}
-				return err
-			}, exp)
+		installBinaries(boot, upgrader)
 
-			if err == upgrade.ErrRestartNeeded {
-				log.Info().Msg("restarting upgraded")
-				return
-			} else if err != nil {
-				//TODO: crash or continue!
-				log.Error().Err(err).Msg("upgrade failed")
-				continue
-			} else {
-				log.Info().Msg("update completed")
-			}
-			if err := boot.Set(*event); err != nil {
-				log.Error().Err(err).Msg("failed to update boot information")
-			}
+		// next check for update
+		exp := backoff.NewExponentialBackOff()
+		exp.MaxInterval = 3 * time.Minute
+		exp.MaxElapsedTime = 60 * time.Minute
+		err = backoff.Retry(func() error {
+			log.Debug().Str("version", latest.TryVersion().String()).Msg("trying to update")
 
-			monitor.C <- version
-		case e := <-repoEvents:
-			if e == nil {
-				continue
-			}
-			event := e.(*upgrade.RepoEvent)
-			for _, bin := range event.ToDel {
-				if err := upgrader.UninstallBinary(bin); err != nil {
-					log.Error().Err(err).Str("flist", bin.Fqdn()).Msg("failed to uninstall flist")
-				}
-			}
+			err := Safe(func() error {
+				return upgrader.Upgrade(current, latest)
+			})
 
-			for _, bin := range event.ToAdd {
-				if err := upgrader.InstallBinary(bin); err != nil {
-					log.Error().Err(err).Str("flist", bin.Fqdn()).Msg("failed to install flist")
-				}
+			if err == upgrade.ErrRestartNeeded {
+				return backoff.Permanent(err)
 			}
 
-			if err := boot.SetBins(repoWatcher.Current); err != nil {
-				log.Error().Err(err).Msg("failed to update local db of installed binaries")
+			if err != nil {
+				log.Error().Err(err).Msg("update failure. retrying")
 			}
+			return err
+		}, exp)
 
-			log.Debug().Msg("finish processing binary updates")
+		if err == upgrade.ErrRestartNeeded {
+			log.Info().Msg("restarting upgraded")
+			return
+		} else if err != nil {
+			//TODO: crash or continue!
+			log.Error().Err(err).Msg("upgrade failed")
+			continue
+		} else {
+			log.Info().Str("version", latest.TryVersion().String()).Msg("update completed")
 		}
+		if err := boot.Set(latest); err != nil {
+			log.Fatal().Err(err).Msg("failed to update boot information")
+		}
+
+		monitor.C <- latest.TryVersion()
 	}
 }
 
diff --git a/cmds/modules/noded/main.go b/cmds/modules/noded/main.go
index 8513e374..0d7a8f9f 100644
--- a/cmds/modules/noded/main.go
+++ b/cmds/modules/noded/main.go
@@ -231,5 +231,6 @@ func action(cli *cli.Context) error {
 		return err
 	}
 
-	return runMsgBus(ctx, env.SubstrateURL, id)
+	// todo: change rmb to support using multiple urls.
+	return runMsgBus(ctx, env.SubstrateURL[0], id)
 }
diff --git a/go.mod b/go.mod
index 0072530b..611aa5d1 100644
--- a/go.mod
+++ b/go.mod
@@ -62,7 +62,7 @@ require (
 	github.com/stretchr/testify v1.7.0
 	github.com/threefoldtech/0-fs v1.3.1-0.20201203163303-d963de9adea7
 	github.com/threefoldtech/go-rmb v0.1.10-0.20220127085422-3c3667e4fc58
-	github.com/threefoldtech/substrate-client v0.0.0-20220218102559-9334d468cc02
+	github.com/threefoldtech/substrate-client v0.0.0-20220222101439-a349d66a36cf
 	github.com/threefoldtech/zbus v0.1.5
 	github.com/tinylib/msgp v1.1.5 // indirect
 	github.com/tyler-smith/go-bip39 v1.1.0
diff --git a/go.sum b/go.sum
index 86c49203..24ece755 100644
--- a/go.sum
+++ b/go.sum
@@ -1031,6 +1031,8 @@ github.com/threefoldtech/substrate-client v0.0.0-20220217104847-011310dd5d24 h1:
 github.com/threefoldtech/substrate-client v0.0.0-20220217104847-011310dd5d24/go.mod h1:kI84UXFN+t0v7I3Qtj+otQwIaGel+AT86L5B0t/zQpI=
 github.com/threefoldtech/substrate-client v0.0.0-20220218102559-9334d468cc02 h1:6J4lQTmIiB4E+3g2bWR0ougtxKi7TSGR+hcxXgAvN3w=
 github.com/threefoldtech/substrate-client v0.0.0-20220218102559-9334d468cc02/go.mod h1:kI84UXFN+t0v7I3Qtj+otQwIaGel+AT86L5B0t/zQpI=
+github.com/threefoldtech/substrate-client v0.0.0-20220222101439-a349d66a36cf h1:S+s5D5QPU4q3/UtWW7Z2TFF9XqyL2ebFTaFTj/PLLE0=
+github.com/threefoldtech/substrate-client v0.0.0-20220222101439-a349d66a36cf/go.mod h1:kI84UXFN+t0v7I3Qtj+otQwIaGel+AT86L5B0t/zQpI=
 github.com/threefoldtech/zbus v0.1.5 h1:S9kXbjejoRRnJw1yKHEXFGF2vqL+Drae2n4vpj0pGHo=
 github.com/threefoldtech/zbus v0.1.5/go.mod h1:ZtiRpcqzEBJetVQDsEbw0p48h/AF3O1kf0tvd30I0BU=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
diff --git a/pkg/environment/config.go b/pkg/environment/config.go
index 382a6cd4..8f318e87 100644
--- a/pkg/environment/config.go
+++ b/pkg/environment/config.go
@@ -37,7 +37,7 @@ func GetConfig() (base Config, err error) {
 	if err != nil {
 		return
 	}
-	base, err = getConfig(env.RunningMode, BaseExtendedURL)
+	base, err = getConfig(env.RunningMode, baseExtendedURL)
 	if err != nil {
 		return
 	}
diff --git a/pkg/environment/environment.go b/pkg/environment/environment.go
index 54d1153b..822d68af 100644
--- a/pkg/environment/environment.go
+++ b/pkg/environment/environment.go
@@ -12,18 +12,7 @@ import (
 )
 
 const (
-	// SubstrateDevURL default substrate url
-	SubstrateDevURL  = "wss://tfchain.dev.grid.tf/"
-	ActivationDevURL = "https://activation.dev.grid.tf/activation/activate"
-
-	// SubstrateDevURL default substrate url
-	SubstrateTestURL  = "wss://tfchain.test.grid.tf/"
-	ActivationTestURL = "https://activation.test.grid.tf/activation/activate"
-
-	SubstrateMainURL  = "wss://tfchain.grid.tf/"
-	ActivationMainURL = "https://activation.grid.tf/activation/activate"
-
-	BaseExtendedURL = "https://raw.githubusercontent.com/threefoldtech/zos-config/main/"
+	baseExtendedURL = "https://raw.githubusercontent.com/threefoldtech/zos-config/main/"
 )
 
 // Environment holds information about running environment of a node
@@ -38,7 +27,7 @@ type Environment struct {
 	Orphan   bool
 
 	FarmSecret    string
-	SubstrateURL  string
+	SubstrateURL  []string
 	ActivationURL string
 
 	ExtendedConfigURL string
@@ -77,9 +66,11 @@ const (
 
 var (
 	envDev = Environment{
-		RunningMode:   RunningDev,
-		SubstrateURL:  SubstrateDevURL,
-		ActivationURL: ActivationDevURL,
+		RunningMode: RunningDev,
+		SubstrateURL: []string{
+			"wss://tfchain.dev.grid.tf/",
+		},
+		ActivationURL: "https://activation.dev.grid.tf/activation/activate",
 		FlistURL:      "redis://hub.grid.tf:9900",
 		BinRepo:       "tf-zos-v3-bins.dev",
 	}
@@ -87,17 +78,24 @@ var (
 	envTest = Environment{
 		RunningMode: RunningTest,
 		// TODO: this should become a different substrate ?
-		SubstrateURL:  SubstrateTestURL,
-		ActivationURL: ActivationTestURL,
+		SubstrateURL: []string{
+			"wss://tfchain.test.grid.tf/",
+		},
+		ActivationURL: "https://activation.test.grid.tf/activation/activate",
 		FlistURL:      "redis://hub.grid.tf:9900",
 		BinRepo:       "tf-zos-v3-bins.test",
 	}
 
 	// same as testnet for now. will be updated the day of the launch of production network
 	envProd = Environment{
-		RunningMode:   RunningMain,
-		SubstrateURL:  SubstrateMainURL,
-		ActivationURL: ActivationMainURL,
+		RunningMode: RunningMain,
+		SubstrateURL: []string{
+			"wss://tfchain.grid.tf/",
+			"wss://02.tfchain.grid.tf/",
+			"wss://03.tfchain.grid.tf/",
+			"wss://04.tfchain.grid.tf/",
+		},
+		ActivationURL: "https://activation.grid.tf/activation/activate",
 		FlistURL:      "redis://hub.grid.tf:9900",
 		BinRepo:       "tf-zos-v3-bins",
 	}
@@ -122,7 +120,7 @@ func Get() (Environment, error) {
 
 // GetSubstrate gets a client to subsrate blockchain
 func (e *Environment) GetSubstrate() (*substrate.Substrate, error) {
-	return substrate.NewSubstrate(e.SubstrateURL)
+	return substrate.NewSubstrate(e.SubstrateURL...)
 }
 
 func getEnvironmentFromParams(params kernel.Params) (Environment, error) {
@@ -158,7 +156,7 @@ func getEnvironmentFromParams(params kernel.Params) (Environment, error) {
 
 	if substrate, ok := params.Get("substrate"); ok {
 		if len(substrate) > 0 {
-			env.SubstrateURL = substrate[len(substrate)-1]
+			env.SubstrateURL = substrate
 		}
 	}
 
@@ -201,7 +199,7 @@ func getEnvironmentFromParams(params kernel.Params) (Environment, error) {
 	// override default settings
 
 	if e := os.Getenv("ZOS_SUBSTRATE_URL"); e != "" {
-		env.SubstrateURL = e
+		env.SubstrateURL = []string{e}
 	}
 
 	if e := os.Getenv("ZOS_FLIST_URL"); e != "" {
diff --git a/pkg/identity/identityd.go b/pkg/identity/identityd.go
index dbb44d34..b7512577 100644
--- a/pkg/identity/identityd.go
+++ b/pkg/identity/identityd.go
@@ -52,7 +52,7 @@ func NewManager(path string) (pkg.IdentityManager, error) {
 		}
 	}
 
-	sub, err := substrate.NewSubstrate(env.SubstrateURL)
+	sub, err := env.GetSubstrate()
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to connect to substrate")
 	}
diff --git a/pkg/upgrade/boot.go b/pkg/upgrade/boot.go
index 771a6599..ca008858 100644
--- a/pkg/upgrade/boot.go
+++ b/pkg/upgrade/boot.go
@@ -88,22 +88,17 @@ func (b *Boot) SetBins(current map[string]FListInfo) error {
 }
 
 // Current returns current flist information
-func (b *Boot) Current() (FListEvent, error) {
+func (b *Boot) Current() (FullFListInfo, error) {
 	name := b.Name()
 	if len(name) == 0 {
-		return FListEvent{}, fmt.Errorf("flist name is not known")
+		return FullFListInfo{}, fmt.Errorf("flist name is not known")
 	}
 
-	info, err := loadInfo(name, FlistInfoFile)
-	if err != nil {
-		return FListEvent{}, err
-	}
-
-	return FListEvent{info}, nil
+	return loadInfo(name, FlistInfoFile)
 }
 
 // Set updates the stored flist info
-func (b *Boot) Set(c FListEvent) error {
+func (b *Boot) Set(c FullFListInfo) error {
 	return c.Commit(FlistInfoFile)
 }
 
diff --git a/pkg/upgrade/hub.go b/pkg/upgrade/hub.go
index a0ac59ec..d936714c 100644
--- a/pkg/upgrade/hub.go
+++ b/pkg/upgrade/hub.go
@@ -230,6 +230,11 @@ func (b *FListInfo) Version() (semver.Version, error) {
 	return b.extractVersion(b.Name)
 }
 
+func (b *FListInfo) TryVersion() semver.Version {
+	ver, _ := b.Version()
+	return ver
+}
+
 // Absolute returns the actual flist name
 func (b *FListInfo) Absolute() string {
 	name := b.Name
diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go
index 1ab1ef18..17c8a830 100644
--- a/pkg/upgrade/upgrade.go
+++ b/pkg/upgrade/upgrade.go
@@ -112,7 +112,7 @@ func (u *Upgrader) flistCache() string {
 // if yes, applies the upgrade
 // on a successfully update, upgrade WILL NOT RETURN
 // instead the upgraded daemon will be completely stopped
-func (u *Upgrader) Upgrade(from, to FListEvent) error {
+func (u *Upgrader) Upgrade(from, to FullFListInfo) error {
 	return u.applyUpgrade(from, to)
 }
 
@@ -401,7 +401,7 @@ func (u *Upgrader) uninstall(flist FListInfo) error {
 	return nil
 }
 
-func (u *Upgrader) applyUpgrade(from, to FListEvent) error {
+func (u *Upgrader) applyUpgrade(from, to FullFListInfo) error {
 	log.Info().Str("flist", to.Fqdn()).Str("version", to.TryVersion().String()).Msg("start applying upgrade")
 
 	store, err := u.getFlist(to.Fqdn())
diff --git a/pkg/upgrade/watcher.go b/pkg/upgrade/watcher.go
index 0d6fb8b2..bb5e87d0 100644
--- a/pkg/upgrade/watcher.go
+++ b/pkg/upgrade/watcher.go
@@ -1,12 +1,7 @@
 package upgrade
 
 import (
-	"context"
-	"time"
-
-	"github.com/blang/semver"
 	"github.com/pkg/errors"
-	"github.com/rs/zerolog/log"
 )
 
 /**
@@ -17,149 +12,15 @@ one kind of update is applied at a time.
 This to prevent updates to step on each other toes.
 */
 
-//EventType of the watcher
-type EventType string
-
-const (
-	//FList event type
-	FList EventType = "flist"
-	//Repo event type
-	Repo EventType = "repo"
-)
-
-//Event interface
-type Event interface {
-	EventType() EventType
-}
-
-//Watcher interface
-type Watcher interface {
-	Watch(ctx context.Context) (<-chan Event, error)
-}
-
-//FListEvent struct
-type FListEvent struct {
-	FullFListInfo
-}
-
-//EventType of the event
-func (f *FListEvent) EventType() EventType {
-	return FList
-}
-
-//TryVersion will try to parse the version from flist
-//other wise return empty version 0.0.0
-func (f *FListEvent) TryVersion() semver.Version {
-	version, _ := f.Version()
-	return version
-}
-
-//FListSemverWatcher watches a single FList for changes in it's semver
-//the semver to change without the flist name itself changes, means
-//that this flist is mostly a symlink
-type FListSemverWatcher struct {
-	FList    string
-	Duration time.Duration
-	Current  semver.Version
+//FListRepo type
+type FListRepo struct {
+	Repo    string
+	Current map[string]FListInfo
 
 	client HubClient
 }
 
-var _ Watcher = &FListSemverWatcher{}
-
-// Watch an flist change in version
-// The Event returned by the channel is of concrete type FListEvent
-func (w *FListSemverWatcher) Watch(ctx context.Context) (<-chan Event, error) {
-	if w.Duration == time.Duration(0) {
-		//default delay of 5min
-		w.Duration = 600 * time.Second
-	}
-
-	info, err := w.client.Info(w.FList)
-	if err != nil {
-		return nil, err
-	}
-
-	version, err := info.Version()
-	if err != nil {
-		return nil, err
-	}
-
-	ch := make(chan Event, 1)
-
-	if version.GT(w.Current) {
-		ch <- &FListEvent{
-			FullFListInfo: info,
-		}
-		w.Current = version
-	}
-
-	ticker := time.NewTicker(w.Duration)
-
-	go func() {
-		defer ticker.Stop()
-		defer close(ch)
-
-		for {
-			// wait for next tick
-			select {
-			case <-ticker.C:
-			case <-ctx.Done():
-				return
-			}
-
-			log.Debug().Str("flist", w.FList).Msg("check updates")
-
-			info, err := w.client.Info(w.FList)
-			if err != nil {
-				log.Error().Err(err).Str("flist", w.FList).Msg("failed to get flist info")
-				continue
-			}
-			version, err := info.Version()
-			if err != nil {
-				log.Error().Err(err).Str("flist", w.FList).Msg("failed to get flist version")
-				continue
-			}
-
-			if version.GT(w.Current) {
-				select {
-				case ch <- &FListEvent{
-					FullFListInfo: info,
-				}:
-				case <-ctx.Done():
-					return
-				}
-
-				w.Current = version
-			}
-		}
-	}()
-
-	return ch, nil
-}
-
-//RepoEvent is returned by the repo watcher
-type RepoEvent struct {
-	Repo  string
-	ToAdd []FListInfo
-	ToDel []FListInfo
-}
-
-//EventType returns event type
-func (e *RepoEvent) EventType() EventType {
-	return Repo
-}
-
-//FListRepoWatcher type
-type FListRepoWatcher struct {
-	Repo     string
-	Current  map[string]FListInfo
-	Duration time.Duration
-
-	client HubClient
-}
-
-func (w *FListRepoWatcher) list() (map[string]FListInfo, error) {
+func (w *FListRepo) list() (map[string]FListInfo, error) {
 	packages, err := w.client.List(w.Repo)
 	if err != nil {
 		return nil, err
@@ -174,7 +35,7 @@ func (w *FListRepoWatcher) list() (map[string]FListInfo, error) {
 	return result, nil
 }
 
-func (w *FListRepoWatcher) diff(packages map[string]FListInfo) (toAdd, toDel []FListInfo) {
+func (w *FListRepo) diff(packages map[string]FListInfo) (toAdd, toDel []FListInfo) {
 	for name, pkg := range packages {
 		current, ok := w.Current[name]
 		if !ok || pkg.Updated != current.Updated {
@@ -193,7 +54,7 @@ func (w *FListRepoWatcher) diff(packages map[string]FListInfo) (toAdd, toDel []F
 }
 
 // Diff return the remote changes related to current list of packages
-func (w *FListRepoWatcher) Diff() (all map[string]FListInfo, toAdd, toDell []FListInfo, err error) {
+func (w *FListRepo) Diff() (all map[string]FListInfo, toAdd, toDell []FListInfo, err error) {
 	all, err = w.list()
 	if err != nil {
 		return all, nil, nil, errors.Wrap(err, "failed to get available packages")
@@ -202,66 +63,3 @@ func (w *FListRepoWatcher) Diff() (all map[string]FListInfo, toAdd, toDell []FLi
 	toAdd, toDell = w.diff(all)
 	return
 }
-
-// Watch watches a full repo for changes. Event is always of concrete type RepoEvent
-func (w *FListRepoWatcher) Watch(ctx context.Context) (<-chan Event, error) {
-	if w.Duration == time.Duration(0) {
-		//default delay of 5min
-		w.Duration = 600 * time.Second
-	}
-
-	packages, toAdd, toDel, err := w.Diff()
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to get available packages")
-	}
-
-	ch := make(chan Event, 1)
-
-	if len(toAdd) > 0 || len(toDel) > 0 {
-		ch <- &RepoEvent{
-			Repo:  w.Repo,
-			ToAdd: toAdd,
-			ToDel: toDel,
-		}
-
-		w.Current = packages
-	}
-
-	ticker := time.NewTicker(w.Duration)
-
-	go func() {
-		defer close(ch)
-		defer ticker.Stop()
-
-		for {
-			select {
-			case <-ticker.C:
-			case <-ctx.Done():
-				return
-			}
-
-			packages, toAdd, toDel, err := w.Diff()
-			if err != nil {
-				log.Error().Err(err).Str("repo", w.Repo).Msg("failed to list repo flists")
-				continue
-			}
-
-			if len(toAdd) > 0 || len(toDel) > 0 {
-				select {
-				case ch <- &RepoEvent{
-					Repo:  w.Repo,
-					ToAdd: toAdd,
-					ToDel: toDel,
-				}:
-				case <-ctx.Done():
-					return
-				}
-
-				w.Current = packages
-			}
-		}
-
-	}()
-
-	return ch, nil
-}


Test patch:
diff --git a/pkg/environment/environment_test.go b/pkg/environment/environment_test.go
index 359e50d2..a4eb58ad 100644
--- a/pkg/environment/environment_test.go
+++ b/pkg/environment/environment_test.go
@@ -57,5 +57,5 @@ func TestEnvironmentOverride(t *testing.T) {
 	value, err := getEnvironmentFromParams(params)
 	require.NoError(t, err)
 
-	assert.Equal(t, value.SubstrateURL, "localhost:1234")
+	assert.Equal(t, []string{"localhost:1234"}, value.SubstrateURL)
 }
diff --git a/pkg/upgrade/watcher_test.go b/pkg/upgrade/watcher_test.go
deleted file mode 100644
index 425cefc3..00000000
--- a/pkg/upgrade/watcher_test.go
+++ /dev/null
@@ -1,53 +0,0 @@
-package upgrade
-
-import (
-	"context"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestFListWatcher(t *testing.T) {
-	require := require.New(t)
-
-	watcher := FListSemverWatcher{
-		FList: "tf-zos/zos:development:latest.flist",
-	}
-
-	ctx := context.Background()
-
-	ch, err := watcher.Watch(ctx)
-	require.NoError(err)
-
-	event := <-ch
-	require.Equal(FList, event.EventType())
-
-	info, ok := event.(*FListEvent)
-	require.True(ok)
-
-	require.Equal("zos:development:latest.flist", info.Name)
-	require.Equal("symlink", info.Type)
-}
-
-func TestRepoWatcher(t *testing.T) {
-	require := require.New(t)
-
-	watcher := FListRepoWatcher{
-		Repo: "tf-zos-bins",
-	}
-
-	ctx := context.Background()
-
-	ch, err := watcher.Watch(ctx)
-	require.NoError(err)
-
-	event := <-ch
-	require.Equal(Repo, event.EventType())
-
-	info, ok := event.(*RepoEvent)
-	require.True(ok)
-
-	require.Equal("tf-zos-bins", info.Repo)
-	require.Len(info.ToDel, 0) // we starting from empty Current
-	require.True(len(info.ToAdd) > 0)
-}