Repository: hyperledger/firefly
Pull request #884 (2022-07-06 01:13:35)

Stop orchestrator for ff_system when moving to network V2
Part of [FIR-12](https://github.com/hyperledger/firefly-fir/pull/12)
~~In a chain with #883~~
~~In a chain with #890~~

Background: The legacy `ff_system` namespace is created (ie an orchestrator and managers are initialized) whenever the default namespace is a V1 multiparty network. The `ff_system` namespace is initialized with an exact duplicate of the default namespace config (same plugins, same contract addresses, etc).

With this PR, if the default namespace (and therefore `ff_system`) migrates to V2 rules, the `ff_system` orchestrator will be stopped and removed, along with all its children.

This should allow legacy networks to cleanly migrate to V2 rules and retire `ff_system`. Notable caveat: once the default namespace is on V2, no other V1 namespaces will be supported.

codecov-commenter: # [Codecov](https://codecov.io/gh/hyperledger/firefly/pull/884?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperledger) Report
> Merging [#884](https://codecov.io/gh/hyperledger/firefly/pull/884?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperledger) (c2a8ff6) into [main](https://codecov.io/gh/hyperledger/firefly/commit/51d6f6311220a12880ff87f0ad7804a83969a1ba?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperledger) (51d6f63) will **decrease** coverage by `0.00%`.
> The diff coverage is `100.00%`.

> :exclamation: Current head c2a8ff6 differs from pull request most recent head 813b4e6. Consider uploading reports for the commit 813b4e6 to get more accurate results

```diff
@@            Coverage Diff             @@
##             main     #884      +/-   ##
==========================================
- Coverage   99.96%   99.96%   -0.01%     
==========================================
  Files         302      299       -3     
  Lines       19736    19621     -115     
==========================================
- Hits        19730    19615     -115     
  Misses          5        5              
  Partials        1        1              
```


| [Impacted Files](https://codecov.io/gh/hyperledger/firefly/pull/884?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperledger) | Coverage Δ | |
|---|---|---|
| [internal/blockchain/ethereum/ethereum.go](https://codecov.io/gh/hyperledger/firefly/pull/884/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperledger#diff-aW50ZXJuYWwvYmxvY2tjaGFpbi9ldGhlcmV1bS9ldGhlcmV1bS5nbw==) | `100.00% <100.00%> (ø)` | |
| [internal/blockchain/ethereum/eventstream.go](https://codecov.io/gh/hyperledger/firefly/pull/884/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperledger#diff-aW50ZXJuYWwvYmxvY2tjaGFpbi9ldGhlcmV1bS9ldmVudHN0cmVhbS5nbw==) | `100.00% <100.00%> (ø)` | |
| [internal/blockchain/fabric/eventstream.go](https://codecov.io/gh/hyperledger/firefly/pull/884/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperledger#diff-aW50ZXJuYWwvYmxvY2tjaGFpbi9mYWJyaWMvZXZlbnRzdHJlYW0uZ28=) | `100.00% <100.00%> (ø)` | |
| [internal/blockchain/fabric/fabric.go](https://codecov.io/gh/hyperledger/firefly/pull/884/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperledger#diff-aW50ZXJuYWwvYmxvY2tjaGFpbi9mYWJyaWMvZmFicmljLmdv) | `100.00% <100.00%> (ø)` | |
| [internal/multiparty/manager.go](https://codecov.io/gh/hyperledger/firefly/pull/884/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperledger#diff-aW50ZXJuYWwvbXVsdGlwYXJ0eS9tYW5hZ2VyLmdv) | `100.00% <100.00%> (ø)` | |
| [internal/namespace/manager.go](https://codecov.io/gh/hyperledger/firefly/pull/884/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperledger#diff-aW50ZXJuYWwvbmFtZXNwYWNlL21hbmFnZXIuZ28=) | `100.00% <100.00%> (ø)` | |
| [internal/orchestrator/orchestrator.go](https://codecov.io/gh/hyperledger/firefly/pull/884/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperledger#diff-aW50ZXJuYWwvb3JjaGVzdHJhdG9yL29yY2hlc3RyYXRvci5nbw==) | `100.00% <100.00%> (ø)` | |
| [internal/data/blobstore.go](https://codecov.io/gh/hyperledger/firefly/pull/884/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperledger#diff-aW50ZXJuYWwvZGF0YS9ibG9ic3RvcmUuZ28=) | `100.00% <0.00%> (ø)` | |
| [internal/apiserver/routes.go](https://codecov.io/gh/hyperledger/firefly/pull/884/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperledger#diff-aW50ZXJuYWwvYXBpc2VydmVyL3JvdXRlcy5nbw==) | `100.00% <0.00%> (ø)` | |
| [internal/apiserver/server.go](https://codecov.io/gh/hyperledger/firefly/pull/884/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperledger#diff-aW50ZXJuYWwvYXBpc2VydmVyL3NlcnZlci5nbw==) | `100.00% <0.00%> (ø)` | |
| ... and [66 more](https://codecov.io/gh/hyperledger/firefly/pull/884/diff?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperledger) | |

------

[Continue to review full report at Codecov](https://codecov.io/gh/hyperledger/firefly/pull/884?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperledger).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperledger)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/hyperledger/firefly/pull/884?src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperledger). Last update [bcc66bb...813b4e6](https://codecov.io/gh/hyperledger/firefly/pull/884?src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperledger). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hyperledger).

awrichar: I think this should go in after #891 in order to leverage the caching logic there. Maybe after #890 as well. Moving to draft until some of these others clear out.

Patch:
diff --git a/cmd/firefly.go b/cmd/firefly.go
index 4a9d37c88..d7b9bf897 100644
--- a/cmd/firefly.go
+++ b/cmd/firefly.go
@@ -118,25 +118,27 @@ func run() error {
 	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
 
 	for {
-		rootCtx, rootCancelCtx := context.WithCancel(ctx)
 		mgr := getRootManager()
 		as := apiserver.NewAPIServer()
-		go startFirefly(rootCtx, rootCancelCtx, mgr, as, errChan)
+		go startFirefly(ctx, mgr, as, errChan)
 		select {
 		case sig := <-sigs:
 			log.L(ctx).Infof("Shutting down due to %s", sig.String())
 			cancelCtx()
 			mgr.WaitStop()
 			return nil
-		case <-rootCtx.Done():
-			log.L(ctx).Infof("Restarting due to configuration change")
-			mgr.WaitStop()
-			// Re-read the configuration
-			coreconfig.Reset()
-			if err := config.ReadConfig(configSuffix, cfgFile); err != nil {
-				cancelCtx()
-				return err
-			}
+		/*
+			// TODO: seems useful, but there's currently no way to trigger this
+			case <-ctx.Done():
+				log.L(ctx).Infof("Restarting due to configuration change")
+				mgr.WaitStop()
+				// Re-read the configuration
+				coreconfig.Reset()
+				if err := config.ReadConfig(configSuffix, cfgFile); err != nil {
+					cancelCtx()
+					return err
+				}
+		*/
 		case err := <-errChan:
 			cancelCtx()
 			return err
@@ -144,7 +146,7 @@ func run() error {
 	}
 }
 
-func startFirefly(ctx context.Context, cancelCtx context.CancelFunc, mgr namespace.Manager, as apiserver.Server, errChan chan error) {
+func startFirefly(ctx context.Context, mgr namespace.Manager, as apiserver.Server, errChan chan error) {
 	var err error
 	// Start debug listener
 	debugPort := config.GetInt(coreconfig.DebugPort)
@@ -161,7 +163,7 @@ func startFirefly(ctx context.Context, cancelCtx context.CancelFunc, mgr namespa
 		log.L(ctx).Debugf("Debug HTTP endpoint listening on localhost:%d", debugPort)
 	}
 
-	if err = mgr.Init(ctx, cancelCtx); err != nil {
+	if err = mgr.Init(ctx); err != nil {
 		errChan <- err
 		return
 	}
diff --git a/internal/multiparty/manager.go b/internal/multiparty/manager.go
index d0711dcc1..b36a7e4b7 100644
--- a/internal/multiparty/manager.go
+++ b/internal/multiparty/manager.go
@@ -82,6 +82,7 @@ type Contract struct {
 
 type multipartyManager struct {
 	ctx            context.Context
+	stop           func()
 	namespace      core.NamespaceRef
 	database       database.Plugin
 	blockchain     blockchain.Plugin
@@ -97,12 +98,13 @@ type multipartyManager struct {
 	}
 }
 
-func NewMultipartyManager(ctx context.Context, ns core.NamespaceRef, config Config, di database.Plugin, bi blockchain.Plugin, om operations.Manager, mm metrics.Manager, th txcommon.Helper) (Manager, error) {
+func NewMultipartyManager(ctx context.Context, stop func(), ns core.NamespaceRef, config Config, di database.Plugin, bi blockchain.Plugin, om operations.Manager, mm metrics.Manager, th txcommon.Helper) (Manager, error) {
 	if di == nil || bi == nil || mm == nil || om == nil || th == nil {
 		return nil, i18n.NewError(ctx, coremsgs.MsgInitializationNilDepError, "MultipartyManager")
 	}
 	mp := &multipartyManager{
 		ctx:        ctx,
+		stop:       stop,
 		namespace:  ns,
 		config:     config,
 		database:   di,
@@ -138,7 +140,13 @@ func (mm *multipartyManager) ConfigureContract(ctx context.Context, contracts *c
 		return err
 	}
 
-	subID, err := mm.blockchain.AddFireflySubscription(ctx, mm.namespace.RemoteName, location, firstEvent)
+	if mm.namespace.LocalName == core.LegacySystemNamespace && version > 1 {
+		// ff_system namespace should stop its orchestrator for network V2+
+		mm.stop()
+		return nil
+	}
+
+	subID, err := mm.blockchain.AddFireflySubscription(ctx, mm.namespace.LocalName, location, firstEvent)
 	if err == nil {
 		mm.activeContract.location = location
 		mm.activeContract.firstEvent = firstEvent
diff --git a/internal/namespace/manager.go b/internal/namespace/manager.go
index 49c864d3b..cffb45632 100644
--- a/internal/namespace/manager.go
+++ b/internal/namespace/manager.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"sync"
 
 	"github.com/hyperledger/firefly-common/pkg/auth"
 	"github.com/hyperledger/firefly-common/pkg/auth/authfactory"
@@ -69,7 +70,7 @@ var (
 )
 
 type Manager interface {
-	Init(ctx context.Context, cancelCtx context.CancelFunc) error
+	Init(ctx context.Context) error
 	Start() error
 	WaitStop()
 
@@ -91,7 +92,7 @@ type namespace struct {
 
 type namespaceManager struct {
 	ctx         context.Context
-	cancelCtx   context.CancelFunc
+	nsMux       sync.Mutex
 	namespaces  map[string]*namespace
 	pluginNames map[string]bool
 	plugins     struct {
@@ -175,9 +176,8 @@ func NewNamespaceManager(withDefaults bool) Manager {
 	return nm
 }
 
-func (nm *namespaceManager) Init(ctx context.Context, cancelCtx context.CancelFunc) (err error) {
+func (nm *namespaceManager) Init(ctx context.Context) (err error) {
 	nm.ctx = ctx
-	nm.cancelCtx = cancelCtx
 
 	if err = nm.loadPlugins(ctx); err != nil {
 		return err
@@ -221,7 +221,7 @@ func (nm *namespaceManager) initNamespace(name string, ns *namespace) error {
 		names := core.NamespaceRef{LocalName: name, RemoteName: ns.remoteName}
 		or = orchestrator.NewOrchestrator(names, ns.config, ns.plugins, nm.metrics)
 	}
-	if err := or.Init(nm.ctx, nm.cancelCtx); err != nil {
+	if err := or.Init(nm.ctx); err != nil {
 		return err
 	}
 	ns.orchestrator = or
@@ -234,8 +234,13 @@ func (nm *namespaceManager) Start() error {
 		metrics.Registry()
 	}
 	// Orchestrators must be started before plugins so as not to miss events
-	for _, ns := range nm.namespaces {
-		if err := ns.orchestrator.Start(); err != nil {
+	for name, ns := range nm.namespaces {
+		onStop := func() {
+			nm.nsMux.Lock()
+			defer nm.nsMux.Unlock()
+			delete(nm.namespaces, name)
+		}
+		if err := ns.orchestrator.Start(onStop); err != nil {
 			return err
 		}
 	}
@@ -258,7 +263,14 @@ func (nm *namespaceManager) Start() error {
 }
 
 func (nm *namespaceManager) WaitStop() {
-	for _, ns := range nm.namespaces {
+	nm.nsMux.Lock()
+	namespaces := make(map[string]*namespace, len(nm.namespaces))
+	for k, v := range nm.namespaces {
+		namespaces[k] = v
+	}
+	nm.nsMux.Unlock()
+
+	for _, ns := range namespaces {
 		ns.orchestrator.WaitStop()
 	}
 	nm.adminEvents.WaitStop()
@@ -917,6 +929,8 @@ func (nm *namespaceManager) SPIEvents() spievents.Manager {
 }
 
 func (nm *namespaceManager) Orchestrator(ns string) orchestrator.Orchestrator {
+	nm.nsMux.Lock()
+	defer nm.nsMux.Unlock()
 	if namespace, ok := nm.namespaces[ns]; ok {
 		return namespace.orchestrator
 	}
@@ -924,6 +938,8 @@ func (nm *namespaceManager) Orchestrator(ns string) orchestrator.Orchestrator {
 }
 
 func (nm *namespaceManager) GetNamespaces(ctx context.Context) ([]*core.Namespace, error) {
+	nm.nsMux.Lock()
+	defer nm.nsMux.Unlock()
 	results := make([]*core.Namespace, 0, len(nm.namespaces))
 	for name, ns := range nm.namespaces {
 		results = append(results, &core.Namespace{
diff --git a/internal/orchestrator/orchestrator.go b/internal/orchestrator/orchestrator.go
index 99fc0e63b..668928763 100644
--- a/internal/orchestrator/orchestrator.go
+++ b/internal/orchestrator/orchestrator.go
@@ -52,8 +52,8 @@ import (
 
 // Orchestrator is the main interface behind the API, implementing the actions
 type Orchestrator interface {
-	Init(ctx context.Context, cancelCtx context.CancelFunc) error
-	Start() error
+	Init(ctx context.Context) error
+	Start(onStop func()) error
 	WaitStop() // The close itself is performed by canceling the context
 
 	MultiParty() multiparty.Manager             // only for multiparty
@@ -176,6 +176,7 @@ type Config struct {
 type orchestrator struct {
 	ctx            context.Context
 	cancelCtx      context.CancelFunc
+	onStop         func()
 	started        bool
 	namespace      core.NamespaceRef
 	config         Config
@@ -211,13 +212,12 @@ func NewOrchestrator(ns core.NamespaceRef, config Config, plugins Plugins, metri
 	return or
 }
 
-func (or *orchestrator) Init(ctx context.Context, cancelCtx context.CancelFunc) (err error) {
+func (or *orchestrator) Init(ctx context.Context) (err error) {
 	namespaceLog := or.namespace.LocalName
 	if or.namespace.RemoteName != or.namespace.LocalName {
 		namespaceLog += "->" + or.namespace.RemoteName
 	}
-	or.ctx = log.WithLogField(ctx, "ns", namespaceLog)
-	or.cancelCtx = cancelCtx
+	or.ctx, or.cancelCtx = context.WithCancel(log.WithLogField(ctx, "ns", namespaceLog))
 	err = or.initPlugins(or.ctx)
 	if err == nil {
 		err = or.initComponents(or.ctx)
@@ -254,7 +254,8 @@ func (or *orchestrator) tokens() map[string]tokens.Plugin {
 	return result
 }
 
-func (or *orchestrator) Start() (err error) {
+func (or *orchestrator) Start(onStop func()) (err error) {
+	or.onStop = onStop
 	if or.config.Multiparty.Enabled {
 		var ns *core.Namespace
 		ns, err = or.database().GetNamespace(or.ctx, or.namespace.LocalName)
@@ -290,6 +291,13 @@ func (or *orchestrator) Start() (err error) {
 	return err
 }
 
+func (or *orchestrator) stop() {
+	or.cancelCtx()
+	if or.onStop != nil {
+		or.onStop()
+	}
+}
+
 func (or *orchestrator) WaitStop() {
 	if !or.started {
 		return
@@ -413,7 +421,7 @@ func (or *orchestrator) initManagers(ctx context.Context) (err error) {
 
 	if or.config.Multiparty.Enabled {
 		if or.multiparty == nil {
-			or.multiparty, err = multiparty.NewMultipartyManager(or.ctx, or.namespace, or.config.Multiparty, or.database(), or.blockchain(), or.operations, or.metrics, or.txHelper)
+			or.multiparty, err = multiparty.NewMultipartyManager(or.ctx, or.stop, or.namespace, or.config.Multiparty, or.database(), or.blockchain(), or.operations, or.metrics, or.txHelper)
 			if err != nil {
 				return err
 			}
diff --git a/mocks/namespacemocks/manager.go b/mocks/namespacemocks/manager.go
index 5105fcdc2..3dfb07189 100644
--- a/mocks/namespacemocks/manager.go
+++ b/mocks/namespacemocks/manager.go
@@ -80,13 +80,13 @@ func (_m *Manager) GetOperationByNamespacedID(ctx context.Context, nsOpID string
 	return r0, r1
 }
 
-// Init provides a mock function with given fields: ctx, cancelCtx
-func (_m *Manager) Init(ctx context.Context, cancelCtx context.CancelFunc) error {
-	ret := _m.Called(ctx, cancelCtx)
+// Init provides a mock function with given fields: ctx
+func (_m *Manager) Init(ctx context.Context) error {
+	ret := _m.Called(ctx)
 
 	var r0 error
-	if rf, ok := ret.Get(0).(func(context.Context, context.CancelFunc) error); ok {
-		r0 = rf(ctx, cancelCtx)
+	if rf, ok := ret.Get(0).(func(context.Context) error); ok {
+		r0 = rf(ctx)
 	} else {
 		r0 = ret.Error(0)
 	}
diff --git a/mocks/orchestratormocks/orchestrator.go b/mocks/orchestratormocks/orchestrator.go
index 38c5afe8f..cddc90c89 100644
--- a/mocks/orchestratormocks/orchestrator.go
+++ b/mocks/orchestratormocks/orchestrator.go
@@ -1129,13 +1129,13 @@ func (_m *Orchestrator) GetTransactions(ctx context.Context, filter database.And
 	return r0, r1, r2
 }
 
-// Init provides a mock function with given fields: ctx, cancelCtx
-func (_m *Orchestrator) Init(ctx context.Context, cancelCtx context.CancelFunc) error {
-	ret := _m.Called(ctx, cancelCtx)
+// Init provides a mock function with given fields: ctx
+func (_m *Orchestrator) Init(ctx context.Context) error {
+	ret := _m.Called(ctx)
 
 	var r0 error
-	if rf, ok := ret.Get(0).(func(context.Context, context.CancelFunc) error); ok {
-		r0 = rf(ctx, cancelCtx)
+	if rf, ok := ret.Get(0).(func(context.Context) error); ok {
+		r0 = rf(ctx)
 	} else {
 		r0 = ret.Error(0)
 	}
@@ -1230,13 +1230,13 @@ func (_m *Orchestrator) RequestReply(ctx context.Context, msg *core.MessageInOut
 	return r0, r1
 }
 
-// Start provides a mock function with given fields:
-func (_m *Orchestrator) Start() error {
-	ret := _m.Called()
+// Start provides a mock function with given fields: onStop
+func (_m *Orchestrator) Start(onStop func()) error {
+	ret := _m.Called(onStop)
 
 	var r0 error
-	if rf, ok := ret.Get(0).(func() error); ok {
-		r0 = rf()
+	if rf, ok := ret.Get(0).(func(func()) error); ok {
+		r0 = rf(onStop)
 	} else {
 		r0 = ret.Error(0)
 	}


Test patch:
diff --git a/cmd/firefly_test.go b/cmd/firefly_test.go
index 167fa389f..abaae93c7 100644
--- a/cmd/firefly_test.go
+++ b/cmd/firefly_test.go
@@ -91,58 +91,6 @@ func TestExecOkExitSIGINT(t *testing.T) {
 	assert.NoError(t, err)
 }
 
-func TestExecOkRestartThenExit(t *testing.T) {
-	o := &namespacemocks.Manager{}
-	var orContext context.Context
-	initCount := 0
-	init := o.On("Init", mock.Anything, mock.Anything, mock.Anything).Return(nil)
-	init.RunFn = func(a mock.Arguments) {
-		orContext = a[0].(context.Context)
-		cancelOrContext := a[1].(context.CancelFunc)
-		initCount++
-		if initCount == 2 {
-			init.ReturnArguments = mock.Arguments{fmt.Errorf("second run")}
-		}
-		cancelOrContext()
-	}
-	o.On("Start").Return(nil)
-	ws := o.On("WaitStop")
-	ws.RunFn = func(a mock.Arguments) {
-		<-orContext.Done()
-	}
-	_utManager = o
-	defer func() { _utManager = nil }()
-
-	os.Chdir(configDir)
-	err := Execute()
-	assert.EqualError(t, err, "second run")
-}
-
-func TestExecOkRestartConfigProblem(t *testing.T) {
-	o := &namespacemocks.Manager{}
-	tmpDir, err := os.MkdirTemp(os.TempDir(), "ut")
-	assert.NoError(t, err)
-	defer os.RemoveAll(tmpDir)
-	var orContext context.Context
-	init := o.On("Init", mock.Anything, mock.Anything, mock.Anything).Return(nil)
-	init.RunFn = func(a mock.Arguments) {
-		orContext = a[0].(context.Context)
-		cancelOrContext := a[1].(context.CancelFunc)
-		cancelOrContext()
-	}
-	o.On("Start").Return(nil)
-	o.On("WaitStop").Run(func(args mock.Arguments) {
-		<-orContext.Done()
-		os.Chdir(tmpDir) // this will mean we fail to read the config
-	})
-	_utManager = o
-	defer func() { _utManager = nil }()
-
-	os.Chdir(configDir)
-	err = Execute()
-	assert.Regexp(t, "Config File.*Not Found", err)
-}
-
 func TestAPIServerError(t *testing.T) {
 	o := &namespacemocks.Manager{}
 	o.On("Init", mock.Anything, mock.Anything).Return(nil)
@@ -151,7 +99,7 @@ func TestAPIServerError(t *testing.T) {
 	as.On("Serve", mock.Anything, o).Return(fmt.Errorf("pop"))
 
 	errChan := make(chan error)
-	go startFirefly(context.Background(), func() {}, o, as, errChan)
+	go startFirefly(context.Background(), o, as, errChan)
 	err := <-errChan
 	assert.EqualError(t, err, "pop")
 }
diff --git a/internal/multiparty/manager_test.go b/internal/multiparty/manager_test.go
index 8acf1fa6e..47a5456a4 100644
--- a/internal/multiparty/manager_test.go
+++ b/internal/multiparty/manager_test.go
@@ -82,7 +82,7 @@ func TestNewMultipartyManager(t *testing.T) {
 		core.OpTypeBlockchainNetworkAction,
 	}).Return()
 	ns := core.NamespaceRef{LocalName: "ns1", RemoteName: "ns1"}
-	nm, err := NewMultipartyManager(context.Background(), ns, config, mdi, mbi, mom, mmi, mth)
+	nm, err := NewMultipartyManager(context.Background(), func() {}, ns, config, mdi, mbi, mom, mmi, mth)
 	assert.NotNil(t, nm)
 	assert.NoError(t, err)
 	assert.Equal(t, "MultipartyManager", nm.Name())
@@ -91,7 +91,7 @@ func TestNewMultipartyManager(t *testing.T) {
 
 func TestInitFail(t *testing.T) {
 	config := Config{Contracts: []Contract{}}
-	_, err := NewMultipartyManager(context.Background(), core.NamespaceRef{}, config, nil, nil, nil, nil, nil)
+	_, err := NewMultipartyManager(context.Background(), func() {}, core.NamespaceRef{}, config, nil, nil, nil, nil, nil)
 	assert.Regexp(t, "FF10128", err)
 }
 
@@ -121,7 +121,7 @@ func TestConfigureContract(t *testing.T) {
 	assert.NoError(t, err)
 }
 
-func TestConfigureContractOldestBlock(t *testing.T) {
+func TestConfigureContractTerminateSystem(t *testing.T) {
 	contracts := make([]Contract, 1)
 	location := fftypes.JSONAnyPtr(fftypes.JSONObject{
 		"address": "0x123",
@@ -134,38 +134,11 @@ func TestConfigureContractOldestBlock(t *testing.T) {
 	contracts[0] = contract
 	mp := newTestMultipartyManager()
 	defer mp.cleanup(t)
+	stopped := false
+	mp.stop = func() { stopped = true }
 	mp.namespace.LocalName = core.LegacySystemNamespace
 
-	mp.mbi.On("GetNetworkVersion", mock.Anything, mock.Anything).Return(1, nil)
-	mp.mbi.On("AddFireflySubscription", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("test", nil)
-	mp.multipartyManager.config.Contracts = contracts
-
-	cf := &core.FireFlyContracts{
-		Active: core.FireFlyContractInfo{Index: 0},
-	}
-
-	err := mp.ConfigureContract(context.Background(), cf)
-	assert.NoError(t, err)
-
-	assert.Equal(t, "0", mp.activeContract.firstEvent)
-}
-
-func TestConfigureContractNewestBlock(t *testing.T) {
-	contracts := make([]Contract, 1)
-	location := fftypes.JSONAnyPtr(fftypes.JSONObject{
-		"address": "0x123",
-	}.String())
-	contract := Contract{
-		FirstEvent: "latest",
-		Location:   location,
-	}
-
-	contracts[0] = contract
-	mp := newTestMultipartyManager()
-	defer mp.cleanup(t)
-
-	mp.mbi.On("GetNetworkVersion", mock.Anything, mock.Anything).Return(1, nil)
-	mp.mbi.On("AddFireflySubscription", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return("test", nil)
+	mp.mbi.On("GetNetworkVersion", mock.Anything, mock.Anything).Return(2, nil)
 	mp.multipartyManager.config.Contracts = contracts
 
 	cf := &core.FireFlyContracts{
@@ -174,8 +147,7 @@ func TestConfigureContractNewestBlock(t *testing.T) {
 
 	err := mp.ConfigureContract(context.Background(), cf)
 	assert.NoError(t, err)
-
-	assert.Equal(t, "latest", mp.activeContract.firstEvent)
+	assert.True(t, stopped)
 }
 
 func TestResolveContractDeprecatedConfig(t *testing.T) {
diff --git a/internal/namespace/manager_test.go b/internal/namespace/manager_test.go
index c4d757169..2b5aeeefc 100644
--- a/internal/namespace/manager_test.go
+++ b/internal/namespace/manager_test.go
@@ -153,8 +153,7 @@ func TestInit(t *testing.T) {
 	nm.mev.On("Init", mock.Anything, mock.Anything).Return(nil)
 	nm.auth.On("Init", mock.Anything, mock.Anything, mock.Anything).Return(nil)
 
-	ctx, cancelCtx := context.WithCancel(context.Background())
-	err := nm.Init(ctx, cancelCtx)
+	err := nm.Init(context.Background())
 	assert.NoError(t, err)
 
 	assert.Equal(t, mo, nm.Orchestrator("default"))
@@ -171,8 +170,7 @@ func TestInitDatabaseFail(t *testing.T) {
 
 	nm.mdi.On("Init", mock.Anything, mock.Anything).Return(fmt.Errorf("pop"))
 
-	ctx, cancelCtx := context.WithCancel(context.Background())
-	err := nm.Init(ctx, cancelCtx)
+	err := nm.Init(context.Background())
 	assert.EqualError(t, err, "pop")
 }
 
@@ -186,8 +184,7 @@ func TestInitBlockchainFail(t *testing.T) {
 	nm.mdi.On("SetHandler", database.GlobalHandler, mock.Anything).Return()
 	nm.mbi.On("Init", mock.Anything, mock.Anything, nm.mmi).Return(fmt.Errorf("pop"))
 
-	ctx, cancelCtx := context.WithCancel(context.Background())
-	err := nm.Init(ctx, cancelCtx)
+	err := nm.Init(context.Background())
 	assert.EqualError(t, err, "pop")
 }
 
@@ -202,8 +199,7 @@ func TestInitDataExchangeFail(t *testing.T) {
 	nm.mbi.On("Init", mock.Anything, mock.Anything, nm.mmi).Return(nil)
 	nm.mdx.On("Init", mock.Anything, mock.Anything).Return(fmt.Errorf("pop"))
 
-	ctx, cancelCtx := context.WithCancel(context.Background())
-	err := nm.Init(ctx, cancelCtx)
+	err := nm.Init(context.Background())
 	assert.EqualError(t, err, "pop")
 }
 
@@ -219,8 +215,7 @@ func TestInitSharedStorageFail(t *testing.T) {
 	nm.mdx.On("Init", mock.Anything, mock.Anything).Return(nil)
 	nm.mps.On("Init", mock.Anything, mock.Anything).Return(fmt.Errorf("pop"))
 
-	ctx, cancelCtx := context.WithCancel(context.Background())
-	err := nm.Init(ctx, cancelCtx)
+	err := nm.Init(context.Background())
 	assert.EqualError(t, err, "pop")
 }
 
@@ -237,8 +232,7 @@ func TestInitTokensFail(t *testing.T) {
 	nm.mps.On("Init", mock.Anything, mock.Anything).Return(nil)
 	nm.mti.On("Init", mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("pop"))
 
-	ctx, cancelCtx := context.WithCancel(context.Background())
-	err := nm.Init(ctx, cancelCtx)
+	err := nm.Init(context.Background())
 	assert.EqualError(t, err, "pop")
 }
 
@@ -256,8 +250,7 @@ func TestInitEventsFail(t *testing.T) {
 	nm.mti.On("Init", mock.Anything, mock.Anything, mock.Anything).Return(nil)
 	nm.mev.On("Init", mock.Anything, mock.Anything).Return(fmt.Errorf("pop"))
 
-	ctx, cancelCtx := context.WithCancel(context.Background())
-	err := nm.Init(ctx, cancelCtx)
+	err := nm.Init(context.Background())
 	assert.EqualError(t, err, "pop")
 }
 
@@ -276,8 +269,7 @@ func TestInitAuthFail(t *testing.T) {
 	nm.mev.On("Init", mock.Anything, mock.Anything).Return(nil)
 	nm.auth.On("Init", mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("pop"))
 
-	ctx, cancelCtx := context.WithCancel(context.Background())
-	err := nm.Init(ctx, cancelCtx)
+	err := nm.Init(context.Background())
 	assert.EqualError(t, err, "pop")
 }
 
@@ -299,8 +291,7 @@ func TestInitOrchestratorFail(t *testing.T) {
 	nm.mev.On("Init", mock.Anything, mock.Anything).Return(nil)
 	nm.auth.On("Init", mock.Anything, mock.Anything, mock.Anything).Return(nil)
 
-	ctx, cancelCtx := context.WithCancel(context.Background())
-	err := nm.Init(ctx, cancelCtx)
+	err := nm.Init(context.Background())
 	assert.EqualError(t, err, "pop")
 }
 
@@ -327,8 +318,7 @@ func TestInitVersion1(t *testing.T) {
 	nm.mev.On("Init", mock.Anything, mock.Anything).Return(nil)
 	nm.auth.On("Init", mock.Anything, mock.Anything, mock.Anything).Return(nil)
 
-	ctx, cancelCtx := context.WithCancel(context.Background())
-	err := nm.Init(ctx, cancelCtx)
+	err := nm.Init(context.Background())
 	assert.NoError(t, err)
 
 	assert.Equal(t, mo, nm.Orchestrator("default"))
@@ -361,8 +351,7 @@ func TestInitVersion1Fail(t *testing.T) {
 	nm.mev.On("Init", mock.Anything, mock.Anything).Return(nil)
 	nm.auth.On("Init", mock.Anything, mock.Anything, mock.Anything).Return(nil)
 
-	ctx, cancelCtx := context.WithCancel(context.Background())
-	err := nm.Init(ctx, cancelCtx)
+	err := nm.Init(context.Background())
 	assert.EqualError(t, err, "pop")
 
 	mo.AssertExpectations(t)
@@ -419,8 +408,7 @@ func TestDatabasePluginBadName(t *testing.T) {
 	config.Set("plugins.database", []fftypes.JSONObject{{}})
 	databaseConfig.AddKnownKey(coreconfig.PluginConfigName, "wrong////")
 	databaseConfig.AddKnownKey(coreconfig.PluginConfigType, "postgres")
-	ctx, cancelCtx := context.WithCancel(context.Background())
-	err := nm.Init(ctx, cancelCtx)
+	err := nm.Init(context.Background())
 	assert.Error(t, err)
 }
 
@@ -453,8 +441,7 @@ func TestIdentityPluginNoType(t *testing.T) {
 	iifactory.InitConfig(identityConfig)
 	identityConfig.AddKnownKey(coreconfig.PluginConfigName, "flapflip")
 	config.Set("plugins.identity", []fftypes.JSONObject{{}})
-	ctx, cancelCtx := context.WithCancel(context.Background())
-	err := nm.Init(ctx, cancelCtx)
+	err := nm.Init(context.Background())
 	assert.Regexp(t, "FF10386.*type", err)
 }
 
@@ -518,8 +505,7 @@ func TestBlockchainPluginBadType(t *testing.T) {
 	config.Set("plugins.blockchain", []fftypes.JSONObject{{}})
 	blockchainConfig.AddKnownKey(coreconfig.PluginConfigName, "flapflip")
 	blockchainConfig.AddKnownKey(coreconfig.PluginConfigType, "wrong//")
-	ctx, cancelCtx := context.WithCancel(context.Background())
-	err := nm.Init(ctx, cancelCtx)
+	err := nm.Init(context.Background())
 	assert.Error(t, err)
 }
 
@@ -572,8 +558,7 @@ func TestSharedStoragePluginBadType(t *testing.T) {
 	config.Set("plugins.sharedstorage", []fftypes.JSONObject{{}})
 	sharedstorageConfig.AddKnownKey(coreconfig.PluginConfigName, "flapflip")
 	sharedstorageConfig.AddKnownKey(coreconfig.PluginConfigType, "wrong//")
-	ctx, cancelCtx := context.WithCancel(context.Background())
-	err := nm.Init(ctx, cancelCtx)
+	err := nm.Init(context.Background())
 	assert.Error(t, err)
 }
 
@@ -626,8 +611,7 @@ func TestDataExchangePluginBadType(t *testing.T) {
 	config.Set("plugins.dataexchange", []fftypes.JSONObject{{}})
 	dataexchangeConfig.AddKnownKey(coreconfig.PluginConfigName, "flapflip")
 	dataexchangeConfig.AddKnownKey(coreconfig.PluginConfigType, "wrong//")
-	ctx, cancelCtx := context.WithCancel(context.Background())
-	err := nm.Init(ctx, cancelCtx)
+	err := nm.Init(context.Background())
 	assert.Error(t, err)
 }
 
@@ -705,8 +689,7 @@ func TestTokensPluginBadType(t *testing.T) {
 	config.Set("plugins.tokens", []fftypes.JSONObject{{}})
 	tokensConfig.AddKnownKey(coreconfig.PluginConfigName, "erc20_erc721")
 	tokensConfig.AddKnownKey(coreconfig.PluginConfigType, "wrong")
-	ctx, cancelCtx := context.WithCancel(context.Background())
-	err := nm.Init(ctx, cancelCtx)
+	err := nm.Init(context.Background())
 	assert.Error(t, err)
 }
 
@@ -751,8 +734,7 @@ func TestAuthPluginBadType(t *testing.T) {
 	config.Set("plugins.auth", []fftypes.JSONObject{{}})
 	authConfig.AddKnownKey(coreconfig.PluginConfigName, "basicauth")
 	authConfig.AddKnownKey(coreconfig.PluginConfigType, "wrong")
-	ctx, cancelCtx := context.WithCancel(context.Background())
-	err := nm.Init(ctx, cancelCtx)
+	err := nm.Init(context.Background())
 	assert.Error(t, err)
 }
 
@@ -773,8 +755,7 @@ func TestEventsPluginBadType(t *testing.T) {
 	defer nm.cleanup(t)
 	nm.plugins.events = nil
 	config.Set(coreconfig.EventTransportsEnabled, []string{"!unknown!"})
-	ctx, cancelCtx := context.WithCancel(context.Background())
-	err := nm.Init(ctx, cancelCtx)
+	err := nm.Init(context.Background())
 	assert.Error(t, err)
 }
 
@@ -802,8 +783,7 @@ func TestInitBadNamespace(t *testing.T) {
     `))
 	assert.NoError(t, err)
 
-	ctx, cancelCtx := context.WithCancel(context.Background())
-	err = nm.Init(ctx, cancelCtx)
+	err = nm.Init(context.Background())
 	assert.Regexp(t, "FF00140", err)
 }
 
@@ -1232,7 +1212,7 @@ func TestStart(t *testing.T) {
 	nm.plugins.tokens = nil
 	nm.metricsEnabled = true
 
-	mo.On("Start").Return(nil)
+	mo.On("Start", mock.Anything).Return(nil)
 
 	err := nm.Start()
 	assert.NoError(t, err)
@@ -1251,7 +1231,7 @@ func TestStartBlockchainFail(t *testing.T) {
 		},
 	}
 
-	mo.On("Start").Return(nil)
+	mo.On("Start", mock.Anything).Return(nil)
 	nm.mbi.On("Start").Return(fmt.Errorf("pop"))
 
 	err := nm.Start()
@@ -1271,7 +1251,7 @@ func TestStartDataExchangeFail(t *testing.T) {
 	}
 	nm.plugins.blockchain = nil
 
-	mo.On("Start").Return(nil)
+	mo.On("Start", mock.Anything).Return(nil)
 	nm.mdx.On("Start").Return(fmt.Errorf("pop"))
 
 	err := nm.Start()
@@ -1292,7 +1272,7 @@ func TestStartTokensFail(t *testing.T) {
 	nm.plugins.blockchain = nil
 	nm.plugins.dataexchange = nil
 
-	mo.On("Start").Return(nil)
+	mo.On("Start", mock.Anything).Return(nil)
 	nm.mti.On("Start").Return(fmt.Errorf("pop"))
 
 	err := nm.Start()
@@ -1309,7 +1289,7 @@ func TestStartOrchestratorFail(t *testing.T) {
 		"ns": {orchestrator: mo},
 	}
 
-	mo.On("Start").Return(fmt.Errorf("pop"))
+	mo.On("Start", mock.Anything).Return(fmt.Errorf("pop"))
 
 	err := nm.Start()
 	assert.EqualError(t, err, "pop")
@@ -1317,6 +1297,31 @@ func TestStartOrchestratorFail(t *testing.T) {
 	mo.AssertExpectations(t)
 }
 
+func TestStartStop(t *testing.T) {
+	nm := newTestNamespaceManager(true)
+	defer nm.cleanup(t)
+
+	mo := &orchestratormocks.Orchestrator{}
+	nm.namespaces = map[string]*namespace{
+		"ns": {orchestrator: mo},
+	}
+	nm.plugins.blockchain = nil
+	nm.plugins.dataexchange = nil
+	nm.plugins.tokens = nil
+
+	mo.On("Start", mock.Anything).Return(nil)
+
+	err := nm.Start()
+	assert.NoError(t, err)
+	assert.Len(t, nm.namespaces, 1)
+
+	onStop := mo.Calls[0].Arguments[0].(func())
+	onStop()
+	assert.Len(t, nm.namespaces, 0)
+
+	mo.AssertExpectations(t)
+}
+
 func TestWaitStop(t *testing.T) {
 	nm := newTestNamespaceManager(true)
 	defer nm.cleanup(t)
diff --git a/internal/orchestrator/orchestrator_test.go b/internal/orchestrator/orchestrator_test.go
index b2c2218c3..4f4f668a1 100644
--- a/internal/orchestrator/orchestrator_test.go
+++ b/internal/orchestrator/orchestrator_test.go
@@ -203,7 +203,7 @@ func TestInitOK(t *testing.T) {
 	or.mps.On("SetHandler", "ns2", mock.Anything).Return()
 	or.mti.On("SetHandler", "ns", mock.Anything).Return(nil)
 	or.mti.On("SetOperationHandler", "ns", mock.Anything).Return(nil)
-	err := or.Init(or.ctx, or.cancelCtx)
+	err := or.Init(or.ctx)
 	assert.NoError(t, err)
 
 	assert.Equal(t, or.mba, or.BatchManager())
@@ -230,7 +230,7 @@ func TestInitTokenListenerFail(t *testing.T) {
 	or.mdx.On("SetNodes", mock.Anything).Return()
 	or.mps.On("SetHandler", "ns", mock.Anything).Return()
 	or.mti.On("SetHandler", "ns", mock.Anything).Return(fmt.Errorf("pop"))
-	err := or.Init(or.ctx, or.cancelCtx)
+	err := or.Init(or.ctx)
 	assert.EqualError(t, err, "pop")
 }
 
@@ -373,7 +373,7 @@ func TestStartBatchFail(t *testing.T) {
 	or.mdi.On("UpsertNamespace", mock.Anything, mock.Anything, true).Return(nil)
 	or.mmp.On("ConfigureContract", mock.Anything, mock.Anything).Return(nil)
 	or.mba.On("Start").Return(fmt.Errorf("pop"))
-	err := or.Start()
+	err := or.Start(func() {})
 	assert.EqualError(t, err, "pop")
 }
 
@@ -383,7 +383,7 @@ func TestStartBlockchainsConfigureFail(t *testing.T) {
 	defer or.cleanup(t)
 	or.mdi.On("GetNamespace", mock.Anything, "ns").Return(nil, nil)
 	or.mmp.On("ConfigureContract", mock.Anything, &core.FireFlyContracts{}).Return(fmt.Errorf("pop"))
-	err := or.Start()
+	err := or.Start(func() {})
 	assert.EqualError(t, err, "pop")
 }
 
@@ -404,7 +404,7 @@ func TestStartStopOk(t *testing.T) {
 	or.mdm.On("WaitStop").Return(nil)
 	or.msd.On("WaitStop").Return(nil)
 	or.mom.On("WaitStop").Return(nil)
-	err := or.Start()
+	err := or.Start(func() {})
 	assert.NoError(t, err)
 	or.WaitStop()
 	or.WaitStop() // swallows dups
@@ -436,6 +436,14 @@ func TestNetworkActionNonMultiparty(t *testing.T) {
 	assert.Regexp(t, "FF10414", err)
 }
 
+func TestStop(t *testing.T) {
+	or := newTestOrchestrator()
+	called := false
+	or.onStop = func() { called = true }
+	or.stop()
+	assert.True(t, called)
+}
+
 func TestAuthorize(t *testing.T) {
 	or := newTestOrchestrator()
 	auth := &authmocks.Plugin{}