Repository: hyperledger/firefly
Pull request #1231 (2023-03-21 15:35:38)

Allow FireFly core to start, even if an individual Namespace cannot initialize
Currently if you specify an invalid batch pin contract address for a namespace, either at startup, or when configuration dynamically reloads, then the whole FireFly core server fails with this error:

```
2023-03-21T11:14:31.573Z]  INFO Resolving FireFly contract at index 0 ns=default pid=41459
[2023-03-21T11:14:31.573Z] DEBUG ==> POST http://127.0.0.1:5102/ breq=Ul7tPf2a pid=41459 proto=ethereum
[2023-03-21T11:14:31.591Z] ERROR <== POST http://127.0.0.1:5102/ [404] (17.56ms) breq=Ul7tPf2a pid=41459 proto=ethereum
Error: FF10111: Error from ethereum connector: FF00164: No result found
```

This same error would occur if there were a temporary error contacting the blockchain through the blockchain connector.

This PR proposes a tweak to this such that this particular namespace will continue to retry its initialization, even if this one namespace cannot be initialized.

To do this the namespace initialization becomes asynchronous, and I had to make a change to the just-in-time creation of the `ff_system` namespace as a result of that (because we can't do the checking on that until after the V1 namespace has initialized, which now becomes async).

While initializing namespaces cannot be used on the API, and are not returned from `/api/v1/namespaces` by default.

However, there is a query to specify you would like to see them:
```
curl -s 'http://localhost:5000/api/v1/namespaces?includeinitializing' | jq
[
  {
    "name": "default",
    "networkName": "default",
    "description": "Default predefined namespace",
    "created": "2023-03-21T15:49:28.333291Z",
    "initializing": true,
    "initializationError": "FF10111: Error from ethereum connector: FF00164: No result found"
  }
```

> Note this PR does **not** change the behavior of FireFly in the case that a **plugin** fails to initialize.


Patch:
diff --git a/docs/reference/config.md b/docs/reference/config.md
index 95225103e..eb544ae78 100644
--- a/docs/reference/config.md
+++ b/docs/reference/config.md
@@ -745,6 +745,14 @@ nav_order: 2
 |key|The signing key allocated to the root organization within this namespace|`string`|`<nil>`
 |name|A short name for the local root organization within this namespace|`string`|`<nil>`
 
+## namespaces.retry
+
+|Key|Description|Type|Default Value|
+|---|-----------|----|-------------|
+|factor|The retry backoff factor|`float32`|`<nil>`
+|initDelay|The initial retry delay|[`time.Duration`](https://pkg.go.dev/time#Duration)|`<nil>`
+|maxDelay|The maximum retry delay|[`time.Duration`](https://pkg.go.dev/time#Duration)|`<nil>`
+
 ## node
 
 |Key|Description|Type|Default Value|
diff --git a/docs/swagger/swagger.yaml b/docs/swagger/swagger.yaml
index fab7f7313..494da9bee 100644
--- a/docs/swagger/swagger.yaml
+++ b/docs/swagger/swagger.yaml
@@ -10269,6 +10269,14 @@ paths:
       description: Gets a list of namespaces
       operationId: getNamespaces
       parameters:
+      - description: When set, the API will return namespaces even if they are not
+          yet initialized, including in error cases where an initializationError is
+          included
+        in: query
+        name: includeinitializing
+        schema:
+          example: "true"
+          type: string
       - description: Server-side request timeout (milliseconds, or set a custom suffix
           like 10s)
         in: header
@@ -10290,6 +10298,13 @@ paths:
                     description:
                       description: A description of the namespace
                       type: string
+                    initializationError:
+                      description: Set to a non-empty string in the case that the
+                        namespace is currently failing to initialize
+                      type: string
+                    initializing:
+                      description: Set to true if the namespace is still initializing
+                      type: boolean
                     name:
                       description: The local namespace name
                       type: string
diff --git a/internal/apiserver/route_get_namespaces.go b/internal/apiserver/route_get_namespaces.go
index 9b4284f75..298bd27bf 100644
--- a/internal/apiserver/route_get_namespaces.go
+++ b/internal/apiserver/route_get_namespaces.go
@@ -1,4 +1,4 @@
-// Copyright © 2022 Kaleido, Inc.
+// Copyright © 2023 Kaleido, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -18,6 +18,7 @@ package apiserver
 
 import (
 	"net/http"
+	"strings"
 
 	"github.com/hyperledger/firefly-common/pkg/ffapi"
 	"github.com/hyperledger/firefly/internal/coremsgs"
@@ -25,19 +26,21 @@ import (
 )
 
 var getNamespaces = &ffapi.Route{
-	Name:            "getNamespaces",
-	Path:            "namespaces",
-	Method:          http.MethodGet,
-	PathParams:      nil,
-	QueryParams:     nil,
+	Name:       "getNamespaces",
+	Path:       "namespaces",
+	Method:     http.MethodGet,
+	PathParams: nil,
+	QueryParams: []*ffapi.QueryParam{
+		{Name: "includeinitializing", Example: "true", Description: coremsgs.APIParamsNSIncludeInitializing, IsBool: true},
+	},
 	FilterFactory:   nil,
 	Description:     coremsgs.APIEndpointsGetNamespaces,
 	JSONInputValue:  nil,
-	JSONOutputValue: func() interface{} { return []*core.Namespace{} },
+	JSONOutputValue: func() interface{} { return []*core.NamespaceWithInitStatus{} },
 	JSONOutputCodes: []int{http.StatusOK},
 	Extensions: &coreExtensions{
 		CoreJSONHandler: func(r *ffapi.APIRequest, cr *coreRequest) (output interface{}, err error) {
-			return cr.mgr.GetNamespaces(cr.ctx)
+			return cr.mgr.GetNamespaces(cr.ctx, strings.EqualFold(r.QP["includeinitializing"], "true"))
 		},
 	},
 }
diff --git a/internal/apiserver/route_spi_get_namespaces.go b/internal/apiserver/route_spi_get_namespaces.go
index 9eefb7130..c9b1e7486 100644
--- a/internal/apiserver/route_spi_get_namespaces.go
+++ b/internal/apiserver/route_spi_get_namespaces.go
@@ -1,4 +1,4 @@
-// Copyright © 2022 Kaleido, Inc.
+// Copyright © 2023 Kaleido, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -18,6 +18,7 @@ package apiserver
 
 import (
 	"net/http"
+	"strings"
 
 	"github.com/hyperledger/firefly-common/pkg/ffapi"
 	"github.com/hyperledger/firefly/internal/coremsgs"
@@ -25,18 +26,20 @@ import (
 )
 
 var spiGetNamespaces = &ffapi.Route{
-	Name:            "spiGetNamespaces",
-	Path:            "namespaces",
-	Method:          http.MethodGet,
-	QueryParams:     nil,
+	Name:   "spiGetNamespaces",
+	Path:   "namespaces",
+	Method: http.MethodGet,
+	QueryParams: []*ffapi.QueryParam{
+		{Name: "includeinitializing", Example: "true", Description: coremsgs.APIParamsNSIncludeInitializing, IsBool: true},
+	},
 	FilterFactory:   nil,
 	Description:     coremsgs.APIEndpointsAdminGetNamespaces,
 	JSONInputValue:  nil,
-	JSONOutputValue: func() interface{} { return []*core.Namespace{} },
+	JSONOutputValue: func() interface{} { return []*core.NamespaceWithInitStatus{} },
 	JSONOutputCodes: []int{http.StatusOK},
 	Extensions: &coreExtensions{
 		CoreJSONHandler: func(r *ffapi.APIRequest, cr *coreRequest) (output interface{}, err error) {
-			return cr.mgr.GetNamespaces(cr.ctx)
+			return cr.mgr.GetNamespaces(cr.ctx, strings.EqualFold(r.QP["includeinitializing"], "true"))
 		},
 	},
 }
diff --git a/internal/apiserver/server.go b/internal/apiserver/server.go
index 7ede20481..e57d0a5c6 100644
--- a/internal/apiserver/server.go
+++ b/internal/apiserver/server.go
@@ -188,7 +188,7 @@ func (as *apiServer) swaggerGenerator(routes []*ffapi.Route, apiBaseURL string)
 func (as *apiServer) contractSwaggerGenerator(mgr namespace.Manager, apiBaseURL string) func(req *http.Request) (*openapi3.T, error) {
 	return func(req *http.Request) (*openapi3.T, error) {
 		vars := mux.Vars(req)
-		or, err := mgr.Orchestrator(req.Context(), vars["ns"])
+		or, err := mgr.Orchestrator(req.Context(), vars["ns"], false)
 		if err != nil {
 			return nil, err
 		}
@@ -213,11 +213,11 @@ func (as *apiServer) contractSwaggerGenerator(mgr namespace.Manager, apiBaseURL
 func getOrchestrator(ctx context.Context, mgr namespace.Manager, tag string, r *ffapi.APIRequest) (or orchestrator.Orchestrator, err error) {
 	switch tag {
 	case routeTagDefaultNamespace:
-		return mgr.Orchestrator(ctx, config.GetString(coreconfig.NamespacesDefault))
+		return mgr.Orchestrator(ctx, config.GetString(coreconfig.NamespacesDefault), false)
 	case routeTagNonDefaultNamespace:
 		vars := mux.Vars(r.Req)
 		if ns, ok := vars["ns"]; ok {
-			return mgr.Orchestrator(ctx, ns)
+			return mgr.Orchestrator(ctx, ns, false)
 		}
 	case routeTagGlobal:
 		return nil, nil
diff --git a/internal/coreconfig/coreconfig.go b/internal/coreconfig/coreconfig.go
index f2220332b..4f5217652 100644
--- a/internal/coreconfig/coreconfig.go
+++ b/internal/coreconfig/coreconfig.go
@@ -276,6 +276,12 @@ var (
 	NamespacesDefault = ffc("namespaces.default")
 	// NamespacesPredefined is a list of namespaces to ensure exists, without requiring a broadcast from the network
 	NamespacesPredefined = ffc("namespaces.predefined")
+	// NamespacesRetryFactor is the retry backoff factor for starting/restarting individual namespaces
+	NamespacesRetryFactor = ffc("namespaces.retry.factor")
+	// NamespacesRetryInitDelay is the retry initial delay for starting/restarting individual namespaces
+	NamespacesRetryInitDelay = ffc("namespaces.retry.initDelay")
+	// NamespacesRetryMaxDelay is the maximum delay between retry attempts
+	NamespacesRetryMaxDelay = ffc("namespaces.retry.maxDelay")
 	// NodeName is the short name for the node
 	NodeName = ffc("node.name")
 	// NodeDescription is a description for the node
@@ -400,6 +406,9 @@ func setDefaults() {
 	viper.SetDefault(string(MessageWriterBatchTimeout), "10ms")
 	viper.SetDefault(string(MessageWriterCount), 5)
 	viper.SetDefault(string(NamespacesDefault), "default")
+	viper.SetDefault(string(NamespacesRetryFactor), 2.0)
+	viper.SetDefault(string(NamespacesRetryMaxDelay), "1m")
+	viper.SetDefault(string(NamespacesRetryInitDelay), "5s")
 	viper.SetDefault(string(OrchestratorStartupAttempts), 5)
 	viper.SetDefault(string(OpUpdateRetryInitDelay), "250ms")
 	viper.SetDefault(string(OpUpdateRetryMaxDelay), "1m")
diff --git a/internal/coremsgs/en_api_translations.go b/internal/coremsgs/en_api_translations.go
index 0b439e00b..74dfe470a 100644
--- a/internal/coremsgs/en_api_translations.go
+++ b/internal/coremsgs/en_api_translations.go
@@ -36,6 +36,7 @@ var (
 	APIParamsContractInterfaceVersion       = ffm("api.params.contractInterfaceVersion", "The version of the contract interface")
 	APIParamsContractInterfaceID            = ffm("api.params.contractInterfaceID", "The ID of the contract interface")
 	APIParamsContractInterfaceFetchChildren = ffm("api.params.contractInterfaceFetchChildren", "When set, the API will return the full FireFly Interface document including all methods, events, and parameters")
+	APIParamsNSIncludeInitializing          = ffm("api.params.nsIncludeInitializing", "When set, the API will return namespaces even if they are not yet initialized, including in error cases where an initializationError is included")
 	APIParamsBlobID                         = ffm("api.params.blobID", "The blob ID")
 	APIParamsDataID                         = ffm("api.params.dataID", "The data item ID")
 	APIParamsDatatypeName                   = ffm("api.params.datatypeName", "The name of the datatype")
diff --git a/internal/coremsgs/en_error_messages.go b/internal/coremsgs/en_error_messages.go
index ee69081a8..d2c15683e 100644
--- a/internal/coremsgs/en_error_messages.go
+++ b/internal/coremsgs/en_error_messages.go
@@ -259,7 +259,7 @@ var (
 	MsgInvalidBatchPinEvent               = ffe("FF10418", "BatchPin event is not valid - %s (%s): %s")
 	MsgDuplicatePluginBroadcastName       = ffe("FF10419", "Invalid %s plugin broadcast name: %s - broadcast names must be unique", 409)
 	MsgInvalidConnectorName               = ffe("FF10420", "Could not find name %s for %s connector")
-	MsgCannotInitLegacyNS                 = ffe("FF10421", "could not initialize legacy '%s' namespace - found conflicting V1 multi-party config in %s and %s")
+	MsgCannotInitLegacyNS                 = ffe("FF10421", "could not initialize legacy '%s' namespace - found conflicting V1 multi-party config in %s")
 	MsgInvalidGroupMember                 = ffe("FF10422", "invalid group member - node '%s' is not owned by '%s' or any of its ancestors")
 	MsgContractListenerStatusInvalid      = ffe("FF10423", "Failed to validate contract listener status: %v", 400)
 	MsgCacheMissSizeLimitKeyInternal      = ffe("FF10424", "could not initialize cache - size limit config key is not provided")
@@ -279,4 +279,5 @@ var (
 	MsgDeprecatedResetWithAutoReload      = ffe("FF10438", "The deprecated reset API cannot be used when dynamic config reload is enabled", 409)
 	MsgConfigArrayVsRawConfigMismatch     = ffe("FF10439", "Error processing configuration - mismatch between raw and processed array lengths")
 	MsgDefaultChannelNotConfigured        = ffe("FF10440", "No default channel configured for this namespace", 400)
+	MsgNamespaceInitializing              = ffe("FF10441", "Namespace '%s' is initializing", 412)
 )
diff --git a/internal/coremsgs/en_struct_descriptions.go b/internal/coremsgs/en_struct_descriptions.go
index 930025d3a..40412f5fb 100644
--- a/internal/coremsgs/en_struct_descriptions.go
+++ b/internal/coremsgs/en_struct_descriptions.go
@@ -412,6 +412,10 @@ var (
 	MultipartyContractInfo         = ffm("MultipartyContract.info", "Additional info about the current status of the multi-party contract")
 	NetworkActionType              = ffm("NetworkAction.type", "The action to be performed")
 
+	// NamespaceWithInitStatus field descriptions
+	NamespaceWithInitStatusInitializing        = ffm("NamespaceWithInitStatus.initializing", "Set to true if the namespace is still initializing")
+	NamespaceWithInitStatusInitializationError = ffm("NamespaceWithInitStatus.initializationError", "Set to a non-empty string in the case that the namespace is currently failing to initialize")
+
 	// NamespaceStatus field descriptions
 	NodeNamespace       = ffm("NamespaceStatus.namespace", "The namespace that this status applies to")
 	NamespaceStatusNode = ffm("NamespaceStatus.node", "Details of the local node")
diff --git a/internal/namespace/configreload.go b/internal/namespace/configreload.go
index 0460e6e0e..9dbfcd999 100644
--- a/internal/namespace/configreload.go
+++ b/internal/namespace/configreload.go
@@ -106,13 +106,6 @@ func (nm *namespaceManager) configReloaded(ctx context.Context) {
 		return
 	}
 
-	// Only initialize the updated namespaces (which includes all that depend on above plugins)
-	if err = nm.initNamespaces(ctx, updatedNamespaces); err != nil {
-		log.L(ctx).Errorf("Failed to initialize namespaces after config reload: %s", err)
-		nm.cancelCtx() // stop the world
-		return
-	}
-
 	// Now finally we can start all the new things
 	if err = nm.startNamespacesAndPlugins(updatedNamespaces, updatedPlugins); err != nil {
 		log.L(ctx).Errorf("Failed to initialize namespaces after config reload: %s", err)
diff --git a/internal/namespace/manager.go b/internal/namespace/manager.go
index 673864ca9..c1857d207 100644
--- a/internal/namespace/manager.go
+++ b/internal/namespace/manager.go
@@ -28,6 +28,7 @@ import (
 	"github.com/hyperledger/firefly-common/pkg/fftypes"
 	"github.com/hyperledger/firefly-common/pkg/i18n"
 	"github.com/hyperledger/firefly-common/pkg/log"
+	"github.com/hyperledger/firefly-common/pkg/retry"
 	"github.com/hyperledger/firefly/internal/blockchain/bifactory"
 	"github.com/hyperledger/firefly/internal/cache"
 	"github.com/hyperledger/firefly/internal/coreconfig"
@@ -60,10 +61,10 @@ type Manager interface {
 	WaitStop()
 	Reset(ctx context.Context) error
 
-	Orchestrator(ctx context.Context, ns string) (orchestrator.Orchestrator, error)
+	Orchestrator(ctx context.Context, ns string, includeInitializing bool) (orchestrator.Orchestrator, error)
 	MustOrchestrator(ns string) orchestrator.Orchestrator
 	SPIEvents() spievents.Manager
-	GetNamespaces(ctx context.Context) ([]*core.Namespace, error)
+	GetNamespaces(ctx context.Context, includeInitializing bool) ([]*core.NamespaceWithInitStatus, error)
 	GetOperationByNamespacedID(ctx context.Context, nsOpID string) (*core.Operation, error)
 	ResolveOperationByNamespacedID(ctx context.Context, nsOpID string, op *core.OperationUpdateDTO) error
 	Authorize(ctx context.Context, authReq *fftypes.AuthReq) error
@@ -79,6 +80,8 @@ type namespace struct {
 	configHash   *fftypes.Bytes32
 	pluginNames  []string
 	plugins      *orchestrator.Plugins
+	started      bool
+	initError    string
 }
 
 type namespaceManager struct {
@@ -95,6 +98,7 @@ type namespaceManager struct {
 	adminEvents         spievents.Manager
 	tokenBroadcastNames map[string]string
 	watchConfig         func() // indirect from viper.WatchConfig for testing
+	nsStartupRetry      *retry.Retry
 
 	orchestratorFactory  func(ns *core.Namespace, config orchestrator.Config, plugins *orchestrator.Plugins, metrics metrics.Manager, cacheManager cache.Manager) orchestrator.Orchestrator
 	blockchainFactory    func(ctx context.Context, pluginType string) (blockchain.Plugin, error)
@@ -168,6 +172,11 @@ func NewNamespaceManager() Manager {
 		identityFactory:      iifactory.GetPlugin,
 		eventsFactory:        eifactory.GetPlugin,
 		authFactory:          authfactory.GetPlugin,
+		nsStartupRetry: &retry.Retry{
+			InitialDelay: config.GetDuration(coreconfig.NamespacesRetryInitDelay),
+			MaximumDelay: config.GetDuration(coreconfig.NamespacesRetryMaxDelay),
+			Factor:       config.GetFloat64(coreconfig.NamespacesRetryFactor),
+		},
 	}
 	return nm
 }
@@ -201,11 +210,6 @@ func (nm *namespaceManager) initComponents(ctx context.Context) (err error) {
 		return err
 	}
 
-	// Initialize all the namespaces on initial startup
-	if err = nm.initNamespaces(ctx, nm.namespaces); err != nil {
-		return err
-	}
-
 	if err := nm.startConfigListener(); err != nil {
 		return err
 	}
@@ -213,59 +217,55 @@ func (nm *namespaceManager) initComponents(ctx context.Context) (err error) {
 	return nil
 }
 
-func (nm *namespaceManager) initNamespaces(ctx context.Context, newNamespaces map[string]*namespace) error {
+func (nm *namespaceManager) startV1NamespaceIfRequired(nsToCheck *namespace) error {
 	// In network version 1, the blockchain plugin and multiparty contract were global and singular.
 	// Therefore, if any namespace was EVER pointed at a V1 contract, that contract and that namespace's plugins
 	// become the de facto configuration for ff_system as well. There can only be one V1 contract in the history
 	// of a given FireFly node, because it's impossible to re-create ff_system against a different contract
 	// or different set of plugins.
-	var v1Namespace *namespace
 	var v1Contract *core.MultipartyContract
+	if nsToCheck.config.Multiparty.Enabled {
+		v1Contract = nm.findV1Contract(nsToCheck)
+	}
 
-	for _, ns := range newNamespaces {
-		if err := nm.initNamespace(ctx, ns); err != nil {
-			return err
-		}
-		multiparty := ns.config.Multiparty.Enabled
-		version := "n/a"
-		if multiparty {
-			version = fmt.Sprintf("%d", ns.Namespace.Contracts.Active.Info.Version)
-		}
-		log.L(ctx).Infof("Initialized namespace '%s' multiparty=%s version=%s", ns.Name, strconv.FormatBool(multiparty), version)
-		if multiparty {
-			contract := nm.findV1Contract(ns)
-			if contract != nil {
-				if v1Namespace == nil {
-					v1Namespace = ns
-					v1Contract = contract
-				} else if !stringSlicesEqual(v1Namespace.pluginNames, ns.pluginNames) ||
-					v1Contract.Location.String() != contract.Location.String() ||
-					v1Contract.FirstEvent != contract.FirstEvent {
-					return i18n.NewError(ctx, coremsgs.MsgCannotInitLegacyNS, core.LegacySystemNamespace, v1Namespace.Name, ns.Name)
-				}
+	if v1Contract != nil {
+		nm.nsMux.Lock()
+		defer nm.nsMux.Unlock()
+
+		existingFFSystemNamespace := nm.namespaces[core.LegacySystemNamespace]
+		if existingFFSystemNamespace != nil {
+			existingV1Contract := nm.findV1Contract(existingFFSystemNamespace)
+			if !stringSlicesEqual(existingFFSystemNamespace.pluginNames, nsToCheck.pluginNames) ||
+				v1Contract.Location.String() != existingV1Contract.Location.String() ||
+				v1Contract.FirstEvent != existingV1Contract.FirstEvent {
+				return i18n.NewError(nm.ctx, coremsgs.MsgCannotInitLegacyNS, core.LegacySystemNamespace, nsToCheck.Name)
 			}
 		}
-	}
 
-	if v1Namespace != nil {
 		systemNS := &namespace{
-			Namespace:   v1Namespace.Namespace,
-			loadTime:    v1Namespace.loadTime,
-			config:      v1Namespace.config,
-			pluginNames: v1Namespace.pluginNames,
-			plugins:     v1Namespace.plugins,
-			configHash:  v1Namespace.configHash,
+			Namespace:   nsToCheck.Namespace,
+			loadTime:    nsToCheck.loadTime,
+			config:      nsToCheck.config,
+			pluginNames: nsToCheck.pluginNames,
+			plugins:     nsToCheck.plugins,
+			configHash:  nsToCheck.configHash,
 		}
 		systemNS.Name = core.LegacySystemNamespace
 		systemNS.NetworkName = core.LegacySystemNamespace
-		newNamespaces[core.LegacySystemNamespace] = systemNS
-		if err := nm.initNamespace(ctx, systemNS); err != nil {
+
+		// Start the namespace synchronously, while holding the nsMux (but without retry), so that
+		// all namespaces can do the above ^^^ check ok
+		if err := nm.initNamespace(systemNS); err != nil {
 			return err
 		}
-		log.L(ctx).Infof("Initialized namespace '%s' as a copy of '%s'", core.LegacySystemNamespace, v1Namespace.Name)
+
+		// Ok - we've now initialized the system NS
+		nm.namespaces[core.LegacySystemNamespace] = systemNS
+		log.L(nm.ctx).Infof("Initialized namespace '%s' as a copy of '%s'", core.LegacySystemNamespace, nsToCheck.Name)
 	}
 
 	return nil
+
 }
 
 func (nm *namespaceManager) findV1Contract(ns *namespace) *core.MultipartyContract {
@@ -280,10 +280,59 @@ func (nm *namespaceManager) findV1Contract(ns *namespace) *core.MultipartyContra
 	return nil
 }
 
-func (nm *namespaceManager) initNamespace(ctx context.Context, ns *namespace) (err error) {
+// namespaceStarter is a routine that attempts to init+start namespace.
+//
+// If it fails initialization, then it will log a suitable error message and exponential backoff retry.
+//
+// This means that an individual namespace, does prevent the whole server from starting successfully.
+//
+// Note that plugins have a separate lifecycle, independent from namespace orchestrators.
+func (nm *namespaceManager) namespaceStarter(ns *namespace) {
+	_ = nm.nsStartupRetry.Do(nm.ctx, fmt.Sprintf("namespace %s", ns.Name), func(attempt int) (retry bool, err error) {
+		err = nm.initAndStartNamespace(ns)
+		// If we started successfully, then all is good
+		if err == nil {
+			log.L(nm.ctx).Infof("Namespace %s started", ns.Name)
+			nm.nsMux.Lock()
+			ns.started = true
+			ns.initError = ""
+			nm.nsMux.Unlock()
+			return false, nil
+		}
+		// Otherwise the back-off retry should retry indefinitely (until the context is closed, which is
+		// the responsibility of the retry library to check)
+		nm.nsMux.Lock()
+		ns.initError = err.Error()
+		nm.nsMux.Unlock()
+		return true, err
+	})
+}
+
+func (nm *namespaceManager) initAndStartNamespace(ns *namespace) error {
+	if err := nm.initNamespace(ns); err != nil {
+		return err
+	}
+	version := "n/a"
+	multiparty := ns.config.Multiparty.Enabled
+	if multiparty {
+		version = fmt.Sprintf("%d", ns.Namespace.Contracts.Active.Info.Version)
+	}
+	log.L(nm.ctx).Infof("Initialized namespace '%s' multiparty=%s version=%s", ns.Name, strconv.FormatBool(multiparty), version)
+
+	// Check if we need to start up a V1 system namespace as a side effect of having initialized this namespace
+	// Note we do that start synchronous to this namespace starting.
+	if err := nm.startV1NamespaceIfRequired(ns); err != nil {
+		return err
+	}
+	// Start this namespace
+	return ns.orchestrator.Start()
+}
+
+func (nm *namespaceManager) initNamespace(ns *namespace) error {
+	bgCtx := nm.ctx
 
 	database := ns.plugins.Database.Plugin
-	existing, err := database.GetNamespace(ctx, ns.Name)
+	existing, err := database.GetNamespace(bgCtx, ns.Name)
 	switch {
 	case err != nil:
 		return err
@@ -291,7 +340,7 @@ func (nm *namespaceManager) initNamespace(ctx context.Context, ns *namespace) (e
 		ns.Created = existing.Created
 		ns.Contracts = existing.Contracts
 		if ns.NetworkName != existing.NetworkName {
-			log.L(ctx).Warnf("Namespace '%s' - network name unexpectedly changed from '%s' to '%s'", ns.Name, existing.NetworkName, ns.NetworkName)
+			log.L(bgCtx).Warnf("Namespace '%s' - network name unexpectedly changed from '%s' to '%s'", ns.Name, existing.NetworkName, ns.NetworkName)
 		}
 	default:
 		ns.Created = fftypes.Now()
@@ -299,12 +348,12 @@ func (nm *namespaceManager) initNamespace(ctx context.Context, ns *namespace) (e
 			Active: &core.MultipartyContract{},
 		}
 	}
-	if err = database.UpsertNamespace(ctx, &ns.Namespace, true); err != nil {
+	if err = database.UpsertNamespace(bgCtx, &ns.Namespace, true); err != nil {
 		return err
 	}
 
 	ns.orchestrator = nm.orchestratorFactory(&ns.Namespace, ns.config, ns.plugins, nm.metrics, nm.cacheManager)
-	ns.ctx, ns.cancelCtx = context.WithCancel(ctx)
+	ns.ctx, ns.cancelCtx = context.WithCancel(bgCtx)
 	if err := ns.orchestrator.Init(ns.ctx, ns.cancelCtx); err != nil {
 		return err
 	}
@@ -328,9 +377,7 @@ func (nm *namespaceManager) Start() error {
 func (nm *namespaceManager) startNamespacesAndPlugins(namespacesToStart map[string]*namespace, pluginsToStart map[string]*plugin) error {
 	// Orchestrators must be started before plugins so as not to miss events
 	for _, ns := range namespacesToStart {
-		if err := ns.orchestrator.Start(); err != nil {
-			return err
-		}
+		go nm.namespaceStarter(ns)
 	}
 	for _, plugin := range pluginsToStart {
 		switch plugin.category {
@@ -1023,10 +1070,14 @@ func (nm *namespaceManager) SPIEvents() spievents.Manager {
 	return nm.adminEvents
 }
 
-func (nm *namespaceManager) Orchestrator(ctx context.Context, ns string) (orchestrator.Orchestrator, error) {
+func (nm *namespaceManager) Orchestrator(ctx context.Context, ns string, includeInitializing bool) (orchestrator.Orchestrator, error) {
 	nm.nsMux.Lock()
 	defer nm.nsMux.Unlock()
+	// Only return started namespaces from this call
 	if namespace, ok := nm.namespaces[ns]; ok && namespace != nil {
+		if !includeInitializing && !namespace.started {
+			return nil, i18n.NewError(ctx, coremsgs.MsgNamespaceInitializing, ns)
+		}
 		return namespace.orchestrator, nil
 	}
 	return nil, i18n.NewError(ctx, coremsgs.MsgUnknownNamespace, ns)
@@ -1034,19 +1085,25 @@ func (nm *namespaceManager) Orchestrator(ctx context.Context, ns string) (orches
 
 // MustOrchestrator must only be called by code that is absolutely sure the orchestrator exists
 func (nm *namespaceManager) MustOrchestrator(ns string) orchestrator.Orchestrator {
-	or, err := nm.Orchestrator(context.Background(), ns)
+	or, err := nm.Orchestrator(context.Background(), ns, true)
 	if err != nil {
 		panic(err)
 	}
 	return or
 }
 
-func (nm *namespaceManager) GetNamespaces(ctx context.Context) ([]*core.Namespace, error) {
+func (nm *namespaceManager) GetNamespaces(ctx context.Context, includeInitializing bool) ([]*core.NamespaceWithInitStatus, error) {
 	nm.nsMux.Lock()
 	defer nm.nsMux.Unlock()
-	results := make([]*core.Namespace, 0, len(nm.namespaces))
+	results := make([]*core.NamespaceWithInitStatus, 0, len(nm.namespaces))
 	for _, ns := range nm.namespaces {
-		results = append(results, &ns.Namespace)
+		if includeInitializing || ns.started {
+			results = append(results, &core.NamespaceWithInitStatus{
+				Namespace:           &ns.Namespace,
+				Initializing:        !ns.started,
+				InitializationError: ns.initError,
+			})
+		}
 	}
 	return results, nil
 }
@@ -1056,7 +1113,7 @@ func (nm *namespaceManager) GetOperationByNamespacedID(ctx context.Context, nsOp
 	if err != nil {
 		return nil, err
 	}
-	or, err := nm.Orchestrator(ctx, ns)
+	or, err := nm.Orchestrator(ctx, ns, true)
 	if err != nil {
 		return nil, err
 	}
@@ -1068,7 +1125,7 @@ func (nm *namespaceManager) ResolveOperationByNamespacedID(ctx context.Context,
 	if err != nil {
 		return err
 	}
-	or, err := nm.Orchestrator(ctx, ns)
+	or, err := nm.Orchestrator(ctx, ns, true)
 	if err != nil {
 		return err
 	}
@@ -1128,7 +1185,7 @@ func (nm *namespaceManager) getAuthPlugin(ctx context.Context, plugins map[strin
 }
 
 func (nm *namespaceManager) Authorize(ctx context.Context, authReq *fftypes.AuthReq) error {
-	or, err := nm.Orchestrator(ctx, authReq.Namespace)
+	or, err := nm.Orchestrator(ctx, authReq.Namespace, true)
 	if err != nil {
 		return err
 	}
diff --git a/mocks/namespacemocks/manager.go b/mocks/namespacemocks/manager.go
index 09499e265..f3558a4ab 100644
--- a/mocks/namespacemocks/manager.go
+++ b/mocks/namespacemocks/manager.go
@@ -34,25 +34,25 @@ func (_m *Manager) Authorize(ctx context.Context, authReq *fftypes.AuthReq) erro
 	return r0
 }
 
-// GetNamespaces provides a mock function with given fields: ctx
-func (_m *Manager) GetNamespaces(ctx context.Context) ([]*core.Namespace, error) {
-	ret := _m.Called(ctx)
+// GetNamespaces provides a mock function with given fields: ctx, includeInitializing
+func (_m *Manager) GetNamespaces(ctx context.Context, includeInitializing bool) ([]*core.NamespaceWithInitStatus, error) {
+	ret := _m.Called(ctx, includeInitializing)
 
-	var r0 []*core.Namespace
+	var r0 []*core.NamespaceWithInitStatus
 	var r1 error
-	if rf, ok := ret.Get(0).(func(context.Context) ([]*core.Namespace, error)); ok {
-		return rf(ctx)
+	if rf, ok := ret.Get(0).(func(context.Context, bool) ([]*core.NamespaceWithInitStatus, error)); ok {
+		return rf(ctx, includeInitializing)
 	}
-	if rf, ok := ret.Get(0).(func(context.Context) []*core.Namespace); ok {
-		r0 = rf(ctx)
+	if rf, ok := ret.Get(0).(func(context.Context, bool) []*core.NamespaceWithInitStatus); ok {
+		r0 = rf(ctx, includeInitializing)
 	} else {
 		if ret.Get(0) != nil {
-			r0 = ret.Get(0).([]*core.Namespace)
+			r0 = ret.Get(0).([]*core.NamespaceWithInitStatus)
 		}
 	}
 
-	if rf, ok := ret.Get(1).(func(context.Context) error); ok {
-		r1 = rf(ctx)
+	if rf, ok := ret.Get(1).(func(context.Context, bool) error); ok {
+		r1 = rf(ctx, includeInitializing)
 	} else {
 		r1 = ret.Error(1)
 	}
@@ -116,25 +116,25 @@ func (_m *Manager) MustOrchestrator(ns string) orchestrator.Orchestrator {
 	return r0
 }
 
-// Orchestrator provides a mock function with given fields: ctx, ns
-func (_m *Manager) Orchestrator(ctx context.Context, ns string) (orchestrator.Orchestrator, error) {
-	ret := _m.Called(ctx, ns)
+// Orchestrator provides a mock function with given fields: ctx, ns, includeInitializing
+func (_m *Manager) Orchestrator(ctx context.Context, ns string, includeInitializing bool) (orchestrator.Orchestrator, error) {
+	ret := _m.Called(ctx, ns, includeInitializing)
 
 	var r0 orchestrator.Orchestrator
 	var r1 error
-	if rf, ok := ret.Get(0).(func(context.Context, string) (orchestrator.Orchestrator, error)); ok {
-		return rf(ctx, ns)
+	if rf, ok := ret.Get(0).(func(context.Context, string, bool) (orchestrator.Orchestrator, error)); ok {
+		return rf(ctx, ns, includeInitializing)
 	}
-	if rf, ok := ret.Get(0).(func(context.Context, string) orchestrator.Orchestrator); ok {
-		r0 = rf(ctx, ns)
+	if rf, ok := ret.Get(0).(func(context.Context, string, bool) orchestrator.Orchestrator); ok {
+		r0 = rf(ctx, ns, includeInitializing)
 	} else {
 		if ret.Get(0) != nil {
 			r0 = ret.Get(0).(orchestrator.Orchestrator)
 		}
 	}
 
-	if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
-		r1 = rf(ctx, ns)
+	if rf, ok := ret.Get(1).(func(context.Context, string, bool) error); ok {
+		r1 = rf(ctx, ns, includeInitializing)
 	} else {
 		r1 = ret.Error(1)
 	}
diff --git a/pkg/core/namespace.go b/pkg/core/namespace.go
index e307e92b1..b29ef3266 100644
--- a/pkg/core/namespace.go
+++ b/pkg/core/namespace.go
@@ -1,4 +1,4 @@
-// Copyright © 2022 Kaleido, Inc.
+// Copyright © 2023 Kaleido, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -35,6 +35,12 @@ type Namespace struct {
 	Contracts   *MultipartyContracts `ffstruct:"Namespace" json:"-"`
 }
 
+type NamespaceWithInitStatus struct {
+	*Namespace
+	Initializing        bool   `ffstruct:"NamespaceWithInitStatus" json:"initializing,omitempty"`
+	InitializationError string `ffstruct:"NamespaceWithInitStatus" json:"initializationError,omitempty"`
+}
+
 // MultipartyContracts represent the currently active and any terminated FireFly multiparty contract(s)
 type MultipartyContracts struct {
 	Active     *MultipartyContract   `ffstruct:"MultipartyContracts" json:"active"`


Test patch:
diff --git a/internal/apiserver/route_get_namespace_test.go b/internal/apiserver/route_get_namespace_test.go
index 4e08870b1..349337c58 100644
--- a/internal/apiserver/route_get_namespace_test.go
+++ b/internal/apiserver/route_get_namespace_test.go
@@ -49,7 +49,7 @@ func TestGetNamespaceInvalid(t *testing.T) {
 	req.Header.Set("Content-Type", "application/json; charset=utf-8")
 	res := httptest.NewRecorder()
 
-	mgr.On("Orchestrator", mock.Anything, "BAD").Return(nil, fmt.Errorf("pop"))
+	mgr.On("Orchestrator", mock.Anything, "BAD", false).Return(nil, fmt.Errorf("pop"))
 	r.ServeHTTP(res, req)
 
 	assert.Equal(t, 500, res.Result().StatusCode)
diff --git a/internal/apiserver/route_get_namespaces_test.go b/internal/apiserver/route_get_namespaces_test.go
index 8e2bf0be3..1c59563f8 100644
--- a/internal/apiserver/route_get_namespaces_test.go
+++ b/internal/apiserver/route_get_namespaces_test.go
@@ -29,12 +29,12 @@ import (
 func TestGetNamespaces(t *testing.T) {
 	mgr, _, as := newTestServer()
 	r := as.createMuxRouter(context.Background(), mgr)
-	req := httptest.NewRequest("GET", "/api/v1/namespaces", nil)
+	req := httptest.NewRequest("GET", "/api/v1/namespaces?includeinitializing", nil)
 	req.Header.Set("Content-Type", "application/json; charset=utf-8")
 	res := httptest.NewRecorder()
 
-	mgr.On("GetNamespaces", mock.Anything).
-		Return([]*core.Namespace{}, nil)
+	mgr.On("GetNamespaces", mock.Anything, true).
+		Return([]*core.NamespaceWithInitStatus{}, nil)
 	r.ServeHTTP(res, req)
 
 	assert.Equal(t, 200, res.Result().StatusCode)
diff --git a/internal/apiserver/route_spi_get_namespaces_test.go b/internal/apiserver/route_spi_get_namespaces_test.go
index 2ab7c33d7..1562cd5b9 100644
--- a/internal/apiserver/route_spi_get_namespaces_test.go
+++ b/internal/apiserver/route_spi_get_namespaces_test.go
@@ -32,8 +32,8 @@ func TestSPIGetNamespaces(t *testing.T) {
 	req.Header.Set("Content-Type", "application/json; charset=utf-8")
 	res := httptest.NewRecorder()
 
-	mgr.On("GetNamespaces", mock.Anything, mock.Anything).
-		Return([]*core.Namespace{}, nil, nil)
+	mgr.On("GetNamespaces", mock.Anything, false).
+		Return([]*core.NamespaceWithInitStatus{}, nil, nil)
 	r.ServeHTTP(res, req)
 
 	assert.Equal(t, 200, res.Result().StatusCode)
diff --git a/internal/apiserver/server_test.go b/internal/apiserver/server_test.go
index 9df25251a..75191e6f7 100644
--- a/internal/apiserver/server_test.go
+++ b/internal/apiserver/server_test.go
@@ -54,9 +54,9 @@ func newTestServer() (*namespacemocks.Manager, *orchestratormocks.Orchestrator,
 	InitConfig()
 	mgr := &namespacemocks.Manager{}
 	o := &orchestratormocks.Orchestrator{}
-	mgr.On("Orchestrator", mock.Anything, "default").Return(o, nil).Maybe()
-	mgr.On("Orchestrator", mock.Anything, "mynamespace").Return(o, nil).Maybe()
-	mgr.On("Orchestrator", mock.Anything, "ns1").Return(o, nil).Maybe()
+	mgr.On("Orchestrator", mock.Anything, "default", false).Return(o, nil).Maybe()
+	mgr.On("Orchestrator", mock.Anything, "mynamespace", false).Return(o, nil).Maybe()
+	mgr.On("Orchestrator", mock.Anything, "ns1", false).Return(o, nil).Maybe()
 	config.Set(coreconfig.APIMaxFilterLimit, 100)
 	as := &apiServer{
 		apiTimeout:    5 * time.Second,
@@ -370,7 +370,7 @@ func TestContractAPISwaggerJSONBadNamespace(t *testing.T) {
 	s := httptest.NewServer(r)
 	defer s.Close()
 
-	mgr.On("Orchestrator", mock.Anything, "BAD").Return(nil, i18n.NewError(context.Background(), coremsgs.MsgUnknownNamespace))
+	mgr.On("Orchestrator", mock.Anything, "BAD", false).Return(nil, i18n.NewError(context.Background(), coremsgs.MsgUnknownNamespace))
 
 	res, err := http.Get(fmt.Sprintf("http://%s/api/v1/namespaces/BAD/apis/my-api/api/swagger.json", s.Listener.Addr()))
 	assert.NoError(t, err)
@@ -396,7 +396,7 @@ func TestJSONBadNamespace(t *testing.T) {
 	s := httptest.NewServer(r)
 	defer s.Close()
 
-	mgr.On("Orchestrator", mock.Anything, "BAD").Return(nil, i18n.NewError(context.Background(), coremsgs.MsgUnknownNamespace))
+	mgr.On("Orchestrator", mock.Anything, "BAD", false).Return(nil, i18n.NewError(context.Background(), coremsgs.MsgUnknownNamespace))
 
 	var b bytes.Buffer
 	req := httptest.NewRequest("GET", "/api/v1/namespaces/BAD/apis", &b)
@@ -414,7 +414,7 @@ func TestFormDataBadNamespace(t *testing.T) {
 	s := httptest.NewServer(r)
 	defer s.Close()
 
-	mgr.On("Orchestrator", mock.Anything, "BAD").Return(nil, i18n.NewError(context.Background(), coremsgs.MsgUnknownNamespace))
+	mgr.On("Orchestrator", mock.Anything, "BAD", false).Return(nil, i18n.NewError(context.Background(), coremsgs.MsgUnknownNamespace))
 
 	var b bytes.Buffer
 	w := multipart.NewWriter(&b)
diff --git a/internal/namespace/configreload_test.go b/internal/namespace/configreload_test.go
index 1263ef090..f6c30f92b 100644
--- a/internal/namespace/configreload_test.go
+++ b/internal/namespace/configreload_test.go
@@ -891,6 +891,50 @@ plugins:
 
 }
 
+func TestConfigReloadStartPluginsFailOnReload(t *testing.T) {
+	logrus.SetLevel(logrus.TraceLevel)
+
+	nm, nmm, cleanup := newTestNamespaceManager(t, false)
+	defer cleanup()
+
+	// Start with empty config
+	for _, mei := range nmm.mei {
+		mei.On("Init", mock.Anything, mock.Anything).Return(nil).Maybe()
+	}
+
+	ctx, cancelCtx := context.WithCancel(context.Background())
+	defer cancelCtx()
+
+	err := nm.Init(ctx, cancelCtx, make(chan bool), func() error { return nil })
+	assert.NoError(t, err)
+
+	err = nm.Start()
+	assert.NoError(t, err)
+
+	coreconfig.Reset()
+	InitConfig()
+	viper.SetConfigType("yaml")
+	err = viper.ReadConfig(strings.NewReader(`
+plugins:
+  blockchain:
+  - name: "badness"
+    type: "ethereum"
+`))
+	assert.NoError(t, err)
+
+	// Drive the config reload
+	nmm.mbi.On("Init", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
+	nmm.mbi.On("Start", mock.Anything, mock.Anything).Return(fmt.Errorf("pop"))
+	nm.configReloaded(nm.ctx)
+
+	// Should terminate
+	<-nm.ctx.Done()
+	nmm.mae.On("WaitStop").Return(nil).Maybe()
+	nmm.mo.On("WaitStop").Return(nil).Maybe()
+	nm.WaitStop()
+
+}
+
 func TestConfigReloadInitNamespacesFailOnReload(t *testing.T) {
 	logrus.SetLevel(logrus.TraceLevel)
 
@@ -928,12 +972,17 @@ namespaces:
 	assert.NoError(t, err)
 
 	// Drive the config reload
+	testDone := make(chan struct{})
 	nmm.mdi.On("Init", mock.Anything, mock.Anything).Return(nil)
 	nmm.mdi.On("SetHandler", database.GlobalHandler, mock.Anything).Return()
-	nmm.mdi.On("GetNamespace", mock.Anything, "default").Return(nil, fmt.Errorf("pop"))
+	nmm.mdi.On("GetNamespace", mock.Anything, "default").Return(nil, fmt.Errorf("pop")).Run(func(args mock.Arguments) {
+		close(testDone)
+		cancelCtx() // only once
+	})
 	nm.configReloaded(nm.ctx)
 
 	// Should terminate
+	<-testDone
 	<-nm.ctx.Done()
 	nmm.mae.On("WaitStop").Return(nil).Maybe()
 	nmm.mo.On("WaitStop").Return(nil).Maybe()
@@ -983,7 +1032,9 @@ namespaces:
 	nmm.mdi.On("GetNamespace", mock.Anything, "default").Return(nil, nil)
 	nmm.mdi.On("UpsertNamespace", mock.Anything, mock.AnythingOfType("*core.Namespace"), true).Return(nil)
 	nmm.mo.On("Init", mock.Anything, mock.Anything).Return(nil)
-	nmm.mo.On("Start", mock.Anything, mock.Anything).Return(fmt.Errorf("pop"))
+	nmm.mo.On("Start", mock.Anything, mock.Anything).Return(fmt.Errorf("pop")).Run(func(args mock.Arguments) {
+		cancelCtx() // only once
+	})
 	nm.configReloaded(nm.ctx)
 
 	// Should terminate
diff --git a/internal/namespace/manager_test.go b/internal/namespace/manager_test.go
index a293be2dc..18e0cce99 100644
--- a/internal/namespace/manager_test.go
+++ b/internal/namespace/manager_test.go
@@ -28,6 +28,7 @@ import (
 	"github.com/hyperledger/firefly-common/pkg/auth/authfactory"
 	"github.com/hyperledger/firefly-common/pkg/config"
 	"github.com/hyperledger/firefly-common/pkg/fftypes"
+	"github.com/hyperledger/firefly-common/pkg/retry"
 	"github.com/hyperledger/firefly/internal/blockchain/bifactory"
 	"github.com/hyperledger/firefly/internal/cache"
 	"github.com/hyperledger/firefly/internal/coreconfig"
@@ -242,6 +243,7 @@ func newTestNamespaceManager(t *testing.T, initConfig bool) (*namespaceManager,
 		namespaces:          make(map[string]*namespace),
 		plugins:             make(map[string]*plugin),
 		tokenBroadcastNames: make(map[string]string),
+		nsStartupRetry:      &retry.Retry{},
 	}
 	nmm := mockPluginFactories(nm)
 	nmm.nm.watchConfig = func() {
@@ -255,17 +257,6 @@ func newTestNamespaceManager(t *testing.T, initConfig bool) (*namespaceManager,
 		err := viper.ReadConfig(strings.NewReader(testBaseConfig))
 		assert.NoError(t, err)
 
-		nmm.mo.On("Init", mock.Anything, mock.Anything).
-			Run(func(args mock.Arguments) {
-				nm.namespaces["default"].Contracts.Active = &core.MultipartyContract{
-					Info: core.MultipartyContractInfo{
-						Version: 2,
-					},
-				}
-			}).
-			Return(nil).
-			Once()
-
 		nmm.mdi.On("Init", mock.Anything, mock.Anything).Return(nil).Once()
 		nmm.mdi.On("SetHandler", database.GlobalHandler, mock.Anything).Return().Once()
 		nmm.mbi.On("Init", mock.Anything, mock.Anything, mock.Anything, nmm.mmi, mock.Anything).Return(nil).Once()
@@ -276,8 +267,6 @@ func newTestNamespaceManager(t *testing.T, initConfig bool) (*namespaceManager,
 		nmm.mei[0].On("Init", mock.Anything, mock.Anything).Return(nil)
 		nmm.mei[1].On("Init", mock.Anything, mock.Anything).Return(nil)
 		nmm.mei[2].On("Init", mock.Anything, mock.Anything).Return(nil)
-		nmm.mdi.On("GetNamespace", mock.Anything, "default").Return(nil, nil).Once()
-		nmm.mdi.On("UpsertNamespace", mock.Anything, mock.AnythingOfType("*core.Namespace"), true).Return(nil).Once()
 		nmm.mai.On("Init", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
 
 		err = nmm.nm.Init(nmm.nm.ctx, nmm.nm.cancelCtx, nmm.nm.reset, nmm.nm.reloadConfig)
@@ -394,14 +383,15 @@ func TestInitTokensFail(t *testing.T) {
 }
 
 func TestInitEventsFail(t *testing.T) {
-	nm, nmm, cleanup := newTestNamespaceManager(t, false)
+	nm, _, cleanup := newTestNamespaceManager(t, true)
 	defer cleanup()
 
-	for _, mei := range nmm.mei {
-		mei.On("Init", mock.Anything, mock.Anything).Return(fmt.Errorf("pop")).Maybe()
-	}
-
-	err := nm.Init(nm.ctx, nm.cancelCtx, nm.reset, nm.reloadConfig)
+	mei := &eventsmocks.Plugin{}
+	mei.On("Init", mock.Anything, mock.Anything).Return(fmt.Errorf("pop")).Maybe()
+	nm.plugins["websockets"].events = mei
+	err := nm.initPlugins(map[string]*plugin{
+		"websockets": nm.plugins["websockets"],
+	})
 	assert.EqualError(t, err, "pop")
 }
 
@@ -428,6 +418,9 @@ func TestInitOrchestratorFail(t *testing.T) {
 	nmm.mdi.On("UpsertNamespace", mock.Anything, mock.AnythingOfType("*core.Namespace"), true).Return(nil)
 
 	err := nm.initComponents(nm.ctx)
+	assert.NoError(t, err)
+
+	err = nm.initNamespace(nm.namespaces["default"])
 	assert.Regexp(t, "pop", err)
 }
 
@@ -452,6 +445,7 @@ func TestInitVersion1(t *testing.T) {
 	nm, nmm, cleanup := newTestNamespaceManager(t, true)
 	defer cleanup()
 
+	nmm.mo.On("Start", mock.Anything).Return(nil)
 	nmm.mo.On("Init", mock.Anything, mock.Anything).
 		Run(func(args mock.Arguments) {
 			nm.namespaces["default"].config.Multiparty.Enabled = true
@@ -467,7 +461,7 @@ func TestInitVersion1(t *testing.T) {
 	nmm.mdi.On("GetNamespace", mock.Anything, core.LegacySystemNamespace).Return(nil, nil)
 	nmm.mdi.On("UpsertNamespace", mock.Anything, mock.AnythingOfType("*core.Namespace"), true).Return(nil)
 
-	err := nm.initNamespaces(nm.ctx, nm.namespaces)
+	err := nm.initAndStartNamespace(nm.namespaces["default"])
 	assert.NoError(t, err)
 
 	assert.Equal(t, nmm.mo, nm.MustOrchestrator("default"))
@@ -481,6 +475,7 @@ func TestInitFFSystemWithTerminatedV1Contract(t *testing.T) {
 	defer cleanup()
 	nm.metricsEnabled = true
 
+	nmm.mo.On("Start", mock.Anything).Return(nil)
 	nmm.mo.On("Init", mock.Anything, mock.Anything).
 		Run(func(args mock.Arguments) {
 			nm.namespaces["default"].config.Multiparty.Enabled = true
@@ -504,7 +499,7 @@ func TestInitFFSystemWithTerminatedV1Contract(t *testing.T) {
 	nmm.mdi.On("GetNamespace", mock.Anything, core.LegacySystemNamespace).Return(nil, nil)
 	nmm.mdi.On("UpsertNamespace", mock.Anything, mock.AnythingOfType("*core.Namespace"), true).Return(nil)
 
-	err := nm.initNamespaces(nm.ctx, nm.namespaces)
+	err := nm.initAndStartNamespace(nm.namespaces["default"])
 	assert.NoError(t, err)
 
 	assert.Equal(t, nmm.mo, nm.MustOrchestrator("default"))
@@ -517,35 +512,24 @@ func TestLegacyNamespaceConflictingPlugins(t *testing.T) {
 	defer cleanup()
 	nm.metricsEnabled = true
 
-	coreconfig.Reset()
-	viper.SetConfigType("yaml")
-	err := viper.ReadConfig(strings.NewReader(`
-  namespaces:
-    default: default
-    predefined:
-    - name: default
-      plugins: [ethereum, postgres, erc721, ipfs, ffdx]
-      multiparty:
-        enabled: true
-        networkNamespace: default
-    - name: ns2
-      plugins: [ethereum, postgres, erc1155, ipfs, ffdx]
-      multiparty:
-        enabled: true
-  `))
-	assert.NoError(t, err)
-
 	nmm.mo.On("Init", mock.Anything, mock.Anything).
 		Run(func(args mock.Arguments) {
-			nm.namespaces["default"].Contracts = &core.MultipartyContracts{
-				Active: &core.MultipartyContract{
-					Location: fftypes.JSONAnyPtr("{}"),
-					Info: core.MultipartyContractInfo{
-						Version: 1,
+			nm.namespaces["ff_system"] = &namespace{
+				Namespace: core.Namespace{
+					Contracts: &core.MultipartyContracts{
+						Active: &core.MultipartyContract{
+							Location: fftypes.JSONAnyPtr("{}"),
+							Info: core.MultipartyContractInfo{
+								Version: 1,
+							},
+						},
 					},
 				},
 			}
-			nm.namespaces["ns2"].Contracts = &core.MultipartyContracts{
+			for range nm.namespaces["default"].pluginNames {
+				nm.namespaces["ff_system"].pluginNames = append(nm.namespaces["ff_system"].pluginNames, "something_else")
+			}
+			nm.namespaces["default"].Contracts = &core.MultipartyContracts{
 				Active: &core.MultipartyContract{
 					Location: fftypes.JSONAnyPtr("{}"),
 					Info: core.MultipartyContractInfo{
@@ -557,19 +541,10 @@ func TestLegacyNamespaceConflictingPlugins(t *testing.T) {
 		Return(nil)
 
 	nmm.mdi.On("GetNamespace", mock.Anything, "default").Return(nil, nil)
-	nmm.mdi.On("GetNamespace", mock.Anything, "ns2").Return(nil, nil)
 	nmm.mdi.On("UpsertNamespace", mock.Anything, mock.AnythingOfType("*core.Namespace"), true).Return(nil)
 
-	nm.namespaces, err = nm.loadNamespaces(nm.ctx, nm.dumpRootConfig(), nm.plugins)
-	assert.Len(t, nm.namespaces, 2)
-	assert.NoError(t, err)
-
-	err = nm.initNamespaces(nm.ctx, nm.namespaces)
+	err := nm.initAndStartNamespace(nm.namespaces["default"])
 	assert.Regexp(t, "FF10421", err)
-
-	assert.Equal(t, nmm.mo, nm.MustOrchestrator("default"))
-	assert.Panics(t, func() { nm.MustOrchestrator("unknown") })
-
 }
 
 func TestLegacyNamespaceConflictingPluginsTooManyPlugins(t *testing.T) {
@@ -577,34 +552,22 @@ func TestLegacyNamespaceConflictingPluginsTooManyPlugins(t *testing.T) {
 	defer cleanup()
 	nm.metricsEnabled = true
 
-	coreconfig.Reset()
-	viper.SetConfigType("yaml")
-	err := viper.ReadConfig(strings.NewReader(`
-  namespaces:
-    default: default
-    predefined:
-    - name: default
-      plugins: [ethereum, postgres, erc721, ipfs, ffdx]
-      multiparty:
-        enabled: true
-    - name: ns2
-      plugins: [ethereum, postgres, erc1155, erc721, ipfs, ffdx]
-      multiparty:
-        enabled: true
-  `))
-	assert.NoError(t, err)
-
 	nmm.mo.On("Init", mock.Anything, mock.Anything).
 		Run(func(args mock.Arguments) {
-			nm.namespaces["default"].Contracts = &core.MultipartyContracts{
-				Active: &core.MultipartyContract{
-					Location: fftypes.JSONAnyPtr("{}"),
-					Info: core.MultipartyContractInfo{
-						Version: 1,
+			nm.namespaces["ff_system"] = &namespace{
+				pluginNames: []string{},
+				Namespace: core.Namespace{
+					Contracts: &core.MultipartyContracts{
+						Active: &core.MultipartyContract{
+							Location: fftypes.JSONAnyPtr("{}"),
+							Info: core.MultipartyContractInfo{
+								Version: 1,
+							},
+						},
 					},
 				},
 			}
-			nm.namespaces["ns2"].Contracts = &core.MultipartyContracts{
+			nm.namespaces["default"].Contracts = &core.MultipartyContracts{
 				Active: &core.MultipartyContract{
 					Location: fftypes.JSONAnyPtr("{}"),
 					Info: core.MultipartyContractInfo{
@@ -616,14 +579,9 @@ func TestLegacyNamespaceConflictingPluginsTooManyPlugins(t *testing.T) {
 		Return(nil)
 
 	nmm.mdi.On("GetNamespace", mock.Anything, "default").Return(nil, nil)
-	nmm.mdi.On("GetNamespace", mock.Anything, "ns2").Return(nil, nil)
 	nmm.mdi.On("UpsertNamespace", mock.Anything, mock.AnythingOfType("*core.Namespace"), true).Return(nil)
 
-	nm.namespaces, err = nm.loadNamespaces(nm.ctx, nm.dumpRootConfig(), nm.plugins)
-	assert.Len(t, nm.namespaces, 2)
-	assert.NoError(t, err)
-
-	err = nm.initNamespaces(nm.ctx, nm.namespaces)
+	err := nm.initAndStartNamespace(nm.namespaces["default"])
 	assert.Regexp(t, "FF10421", err)
 
 	assert.Equal(t, nmm.mo, nm.MustOrchestrator("default"))
@@ -636,34 +594,23 @@ func TestLegacyNamespaceMatchingPlugins(t *testing.T) {
 	defer cleanup()
 	nm.metricsEnabled = true
 
-	coreconfig.Reset()
-	viper.SetConfigType("yaml")
-	err := viper.ReadConfig(strings.NewReader(`
-  namespaces:
-    default: default
-    predefined:
-    - name: default
-      plugins: [ethereum, postgres, erc721, ipfs, ffdx]
-      multiparty:
-        enabled: true
-    - name: ns2
-      plugins: [ethereum, postgres, erc721, ipfs, ffdx]
-      multiparty:
-        enabled: true
-  `))
-	assert.NoError(t, err)
-
+	nmm.mo.On("Start", mock.Anything).Return(nil)
 	nmm.mo.On("Init", mock.Anything, mock.Anything).
 		Run(func(args mock.Arguments) {
-			nm.namespaces["default"].Contracts = &core.MultipartyContracts{
-				Active: &core.MultipartyContract{
-					Location: fftypes.JSONAnyPtr("{}"),
-					Info: core.MultipartyContractInfo{
-						Version: 1,
+			nm.namespaces["ff_system"] = &namespace{
+				pluginNames: []string{"ethereum", "postgres", "ffdx", "ipfs", "erc721", "erc1155", "basicauth"},
+				Namespace: core.Namespace{
+					Contracts: &core.MultipartyContracts{
+						Active: &core.MultipartyContract{
+							Location: fftypes.JSONAnyPtr("{}"),
+							Info: core.MultipartyContractInfo{
+								Version: 1,
+							},
+						},
 					},
 				},
 			}
-			nm.namespaces["ns2"].Contracts = &core.MultipartyContracts{
+			nm.namespaces["default"].Contracts = &core.MultipartyContracts{
 				Active: &core.MultipartyContract{
 					Location: fftypes.JSONAnyPtr("{}"),
 					Info: core.MultipartyContractInfo{
@@ -675,15 +622,10 @@ func TestLegacyNamespaceMatchingPlugins(t *testing.T) {
 		Return(nil)
 
 	nmm.mdi.On("GetNamespace", mock.Anything, "default").Return(nil, nil)
-	nmm.mdi.On("GetNamespace", mock.Anything, "ns2").Return(nil, nil)
 	nmm.mdi.On("GetNamespace", mock.Anything, core.LegacySystemNamespace).Return(nil, nil)
 	nmm.mdi.On("UpsertNamespace", mock.Anything, mock.AnythingOfType("*core.Namespace"), true).Return(nil)
 
-	nm.namespaces, err = nm.loadNamespaces(nm.ctx, nm.dumpRootConfig(), nm.plugins)
-	assert.Len(t, nm.namespaces, 2)
-	assert.NoError(t, err)
-
-	err = nm.initNamespaces(nm.ctx, nm.namespaces)
+	err := nm.initAndStartNamespace(nm.namespaces["default"])
 	assert.NoError(t, err)
 
 	assert.Equal(t, nmm.mo, nm.MustOrchestrator("default"))
@@ -712,7 +654,7 @@ func TestInitVersion1Fail(t *testing.T) {
 	nmm.mdi.On("GetNamespace", mock.Anything, core.LegacySystemNamespace).Return(nil, nil)
 	nmm.mdi.On("UpsertNamespace", mock.Anything, mock.AnythingOfType("*core.Namespace"), true).Return(nil)
 
-	err := nm.initNamespaces(nm.ctx, nm.namespaces)
+	err := nm.initAndStartNamespace(nm.namespaces["default"])
 	assert.EqualError(t, err, "pop")
 
 }
@@ -723,7 +665,7 @@ func TestInitNamespaceQueryFail(t *testing.T) {
 
 	nmm.mdi.On("GetNamespace", mock.Anything, "default").Return(nil, fmt.Errorf("pop"))
 
-	err := nm.initNamespace(context.Background(), nm.namespaces["default"])
+	err := nm.initAndStartNamespace(nm.namespaces["default"])
 	assert.EqualError(t, err, "pop")
 }
 
@@ -738,7 +680,7 @@ func TestInitNamespaceExistingUpsertFail(t *testing.T) {
 	nmm.mdi.On("GetNamespace", mock.Anything, "default").Return(existing, nil)
 	nmm.mdi.On("UpsertNamespace", mock.Anything, mock.AnythingOfType("*core.Namespace"), true).Return(fmt.Errorf("pop"))
 
-	err := nm.initNamespace(context.Background(), nm.namespaces["default"])
+	err := nm.initNamespace(nm.namespaces["default"])
 	assert.EqualError(t, err, "pop")
 }
 
@@ -1345,6 +1287,29 @@ func TestLoadNamespacesReservedName(t *testing.T) {
 	assert.Regexp(t, "FF10388", err)
 }
 
+func TestLoadNamespacesNetworkName(t *testing.T) {
+	nm, _, cleanup := newTestNamespaceManager(t, true)
+	defer cleanup()
+
+	coreconfig.Reset()
+	viper.SetConfigType("yaml")
+	err := viper.ReadConfig(strings.NewReader(`
+  namespaces:
+    default: ns1
+    predefined:
+    - name: ns1
+      multiparty:
+        enabled: true
+        networknamespace: default
+    `))
+	assert.NoError(t, err)
+
+	newNS, err := nm.loadNamespaces(context.Background(), nm.dumpRootConfig(), nm.plugins)
+	assert.NoError(t, err)
+
+	assert.Equal(t, "default", newNS["ns1"].NetworkName)
+}
+
 func TestLoadNamespacesReservedNetworkName(t *testing.T) {
 	nm, _, cleanup := newTestNamespaceManager(t, true)
 	defer cleanup()
@@ -1786,21 +1751,31 @@ func TestStart(t *testing.T) {
 	nm, nmm, cleanup := newTestNamespaceManager(t, true)
 	defer cleanup()
 
+	nsStarted := make(chan struct{})
+
 	nmm.mbi.On("Start", mock.Anything).Return(nil)
 	nmm.mdx.On("Start", mock.Anything).Return(nil)
 	nmm.mti[0].On("Start", mock.Anything).Return(nil)
 	nmm.mti[1].On("Start", mock.Anything).Return(nil)
-	nmm.mo.On("Start", mock.Anything).Return(nil)
+	nmm.mdi.On("GetNamespace", mock.Anything, "default").Return(nil, nil)
+	nmm.mdi.On("UpsertNamespace", mock.Anything, mock.AnythingOfType("*core.Namespace"), true).Return(nil)
+	nmm.mo.On("Init", mock.Anything, mock.Anything).Return(nil)
+	nmm.mo.On("Start", mock.Anything).Return(nil).Run(func(args mock.Arguments) {
+		nm.cancelCtx()
+		close(nsStarted) // this is async
+	})
 
 	err := nm.Start()
 	assert.NoError(t, err)
+
+	<-nsStarted
 }
 
 func TestStartBlockchainFail(t *testing.T) {
 	nm, nmm, cleanup := newTestNamespaceManager(t, true)
 	defer cleanup()
 
-	nmm.mo.On("Start", mock.Anything).Return(nil)
+	nm.namespaces = nil
 	nmm.mbi.On("Start").Return(fmt.Errorf("pop"))
 
 	err := nm.startNamespacesAndPlugins(nm.namespaces, map[string]*plugin{
@@ -1814,7 +1789,7 @@ func TestStartDataExchangeFail(t *testing.T) {
 	nm, nmm, cleanup := newTestNamespaceManager(t, true)
 	defer cleanup()
 
-	nmm.mo.On("Start", mock.Anything).Return(nil)
+	nm.namespaces = nil
 	nmm.mdx.On("Start").Return(fmt.Errorf("pop"))
 
 	err := nm.startNamespacesAndPlugins(nm.namespaces, map[string]*plugin{
@@ -1828,7 +1803,7 @@ func TestStartTokensFail(t *testing.T) {
 	nm, nmm, cleanup := newTestNamespaceManager(t, true)
 	defer cleanup()
 
-	nmm.mo.On("Start", mock.Anything).Return(nil)
+	nm.namespaces = nil
 	nmm.mti[0].On("Start").Return(fmt.Errorf("pop"))
 
 	err := nm.startNamespacesAndPlugins(nm.namespaces, map[string]*plugin{
@@ -1842,19 +1817,42 @@ func TestStartOrchestratorFail(t *testing.T) {
 	nm, nmm, cleanup := newTestNamespaceManager(t, true)
 	defer cleanup()
 
-	nmm.mo.On("Start", mock.Anything).Return(fmt.Errorf("pop"))
+	nsStarted := make(chan struct{})
+	nmm.mo.On("Start", mock.Anything).Return(fmt.Errorf("pop")).Run(func(args mock.Arguments) {
+		nm.cancelCtx()
+		close(nsStarted)
+	})
 
+	nmm.mdi.On("GetNamespace", mock.Anything, "default").Return(nil, nil)
+	nmm.mdi.On("UpsertNamespace", mock.Anything, mock.AnythingOfType("*core.Namespace"), true).Return(nil)
+	nmm.mo.On("Init", mock.Anything, mock.Anything).Return(nil)
 	err := nm.startNamespacesAndPlugins(nm.namespaces, map[string]*plugin{})
-	assert.EqualError(t, err, "pop")
+	assert.NoError(t, err)
+
+	<-nsStarted
 }
 
 func TestWaitStop(t *testing.T) {
 	nm, nmm, cleanup := newTestNamespaceManager(t, true)
 	defer cleanup()
 
+	nmm.mdi.On("GetNamespace", mock.Anything, "default").Return(nil, nil)
+	nmm.mdi.On("UpsertNamespace", mock.Anything, mock.AnythingOfType("*core.Namespace"), true).Return(nil)
+	nmm.mo.On("Init", mock.Anything, mock.Anything).Return(nil)
+	nsStarted := make(chan struct{})
+	nmm.mo.On("Start", mock.Anything).Return(nil).Run(func(args mock.Arguments) {
+		nm.cancelCtx()
+		close(nsStarted)
+	})
+
 	nmm.mo.On("WaitStop").Return()
 	nmm.mae.On("WaitStop").Return()
 
+	err := nm.startNamespacesAndPlugins(nm.namespaces, map[string]*plugin{})
+	assert.NoError(t, err)
+
+	<-nsStarted
+
 	nm.WaitStop()
 
 	nmm.mo.AssertExpectations(t)
@@ -1909,7 +1907,7 @@ func TestGetNamespaces(t *testing.T) {
 	nm, _, cleanup := newTestNamespaceManager(t, true)
 	defer cleanup()
 
-	results, err := nm.GetNamespaces(context.Background())
+	results, err := nm.GetNamespaces(context.Background(), true)
 	assert.Nil(t, err)
 	assert.Len(t, results, 1)
 }
@@ -2058,3 +2056,11 @@ func TestValidateNonMultipartyConfig(t *testing.T) {
 	_, err = nm.loadNamespaces(context.Background(), nm.dumpRootConfig(), nm.plugins)
 	assert.NoError(t, err)
 }
+
+func TestOrchestratorWhileInitializing(t *testing.T) {
+	nm, _, cleanup := newTestNamespaceManager(t, true)
+	defer cleanup()
+
+	_, err := nm.Orchestrator(nm.ctx, "default", false)
+	assert.Regexp(t, "FF10441", err)
+}