Repository: kumahq/kuma (go)

Issue:
Webhooks / Defaulter for default Mesh
### Summary

When Kuma starts it creates a default Mesh
```
apiVersion: kuma.io/v1alpha1
kind: Mesh
metadata:
  name: default
```

The problem is that on Kubernetes we've got webhooks for defaulter and validator that are based on K8S Service.
If the Control Plane is not ready yet but it created a Mesh, webhook fails. After a few reconciliation processes, everything is fine, but one of our users experienced a problem that Mesh was not created.

A couple of Ideas:
* Postpone Mesh creation after we make sure that service is ready
* Create a default mesh with `kumactl install control-plane` (tricky because we have to be careful to not override Mesh that exist)
* Omit hooks when creating default Mesh from the control-plane

Pull request:
feat(kuma-cp) central place for creating defaults
### Summary

Since on the horizon, I see creating default TrafficPermission / TrafficRoute I created a module for creating default values. For now we create there Mesh and Signing Key for Dataplane Tokens.

Previously we were creating a Mesh using Namespace Reconciller. Now there is a central place for creating defaults for both K8S and Universal.

I added retries for the Mesh #882 so on Kubernetes we won't see initial errors (visible only on Debug level).

### Issues resolved

Fix #882

### Documentation

- No docs, Internal only.


Patch:
diff --git a/app/kuma-cp/cmd/run.go b/app/kuma-cp/cmd/run.go
index 7e889848d..cbb7cf7e1 100644
--- a/app/kuma-cp/cmd/run.go
+++ b/app/kuma-cp/cmd/run.go
@@ -13,6 +13,7 @@ import (
 	config_core "github.com/kumahq/kuma/pkg/config/core"
 	"github.com/kumahq/kuma/pkg/core"
 	"github.com/kumahq/kuma/pkg/core/bootstrap"
+	"github.com/kumahq/kuma/pkg/defaults"
 	dns "github.com/kumahq/kuma/pkg/dns/components"
 	"github.com/kumahq/kuma/pkg/gc"
 	kds_global "github.com/kumahq/kuma/pkg/kds/global"
@@ -144,6 +145,11 @@ func newRunCmdWithOpts(opts runCmdOpts) *cobra.Command {
 				return err
 			}
 
+			if err := defaults.Setup(rt); err != nil {
+				runLog.Error(err, "unable to set up Defaults")
+				return err
+			}
+
 			runLog.Info("starting Control Plane", "version", kuma_version.Build.Version)
 			if err := rt.Start(opts.SetupSignalHandler()); err != nil {
 				runLog.Error(err, "problem running Control Plane")
diff --git a/pkg/core/bootstrap/bootstrap.go b/pkg/core/bootstrap/bootstrap.go
index 95d29db9a..b0ec208b3 100644
--- a/pkg/core/bootstrap/bootstrap.go
+++ b/pkg/core/bootstrap/bootstrap.go
@@ -6,7 +6,6 @@ import (
 
 	"github.com/pkg/errors"
 
-	"github.com/kumahq/kuma/api/mesh/v1alpha1"
 	kuma_cp "github.com/kumahq/kuma/pkg/config/app/kuma-cp"
 	config_core "github.com/kumahq/kuma/pkg/config/core"
 	"github.com/kumahq/kuma/pkg/config/core/resources/store"
@@ -33,7 +32,6 @@ import (
 	"github.com/kumahq/kuma/pkg/dns"
 	"github.com/kumahq/kuma/pkg/metrics"
 	metrics_store "github.com/kumahq/kuma/pkg/metrics/store"
-	builtin_issuer "github.com/kumahq/kuma/pkg/tokens/builtin/issuer"
 )
 
 func buildRuntime(cfg kuma_cp.Config) (core_runtime.Runtime, error) {
@@ -136,49 +134,12 @@ func Bootstrap(cfg kuma_cp.Config) (core_runtime.Runtime, error) {
 }
 
 func onStartup(runtime core_runtime.Runtime) error {
-	if err := createDefaultMesh(runtime); err != nil {
-		return err
-	}
-	if err := createDefaultSigningKey(runtime); err != nil {
-		return err
-	}
 	if err := createClusterID(runtime); err != nil {
 		return err
 	}
 	return startReporter(runtime)
 }
 
-func createDefaultSigningKey(runtime core_runtime.Runtime) error {
-	create := false
-	switch runtime.Config().Mode {
-	case config_core.Standalone:
-		create = runtime.Config().Environment == config_core.UniversalEnvironment // Signing Key should be created only on Universal since it is not used on K8S
-	case config_core.Global:
-		create = true // Signing Key with multi-zone should be created on Global even if the Environment is K8S, because we may connect Universal Remote
-	case config_core.Remote:
-		create = false // Signing Key should be synced from Global
-	}
-	if create {
-		return builtin_issuer.CreateDefaultSigningKey(runtime.ResourceManager())
-	}
-	return nil
-}
-
-func createDefaultMesh(runtime core_runtime.Runtime) error {
-	switch env := runtime.Config().Environment; env {
-	case config_core.KubernetesEnvironment:
-		// default Mesh on Kubernetes is managed by the Namespace Controller
-		return nil
-	case config_core.UniversalEnvironment:
-		if runtime.Config().Defaults.SkipMeshCreation {
-			return nil
-		}
-		return mesh_managers.CreateDefaultMesh(runtime.ResourceManager(), v1alpha1.Mesh{})
-	default:
-		return errors.Errorf("unknown environment type %s", env)
-	}
-}
-
 func startReporter(runtime core_runtime.Runtime) error {
 	return runtime.Add(component.ComponentFunc(func(stop <-chan struct{}) error {
 		runtime_reports.Init(runtime, runtime.Config())
diff --git a/pkg/core/managers/apis/mesh/mesh_helpers.go b/pkg/core/managers/apis/mesh/mesh_helpers.go
index dd1df8fd2..63cc73a51 100644
--- a/pkg/core/managers/apis/mesh/mesh_helpers.go
+++ b/pkg/core/managers/apis/mesh/mesh_helpers.go
@@ -3,54 +3,12 @@ package mesh
 import (
 	"context"
 
-	mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1"
-	"github.com/kumahq/kuma/pkg/core"
+	"github.com/pkg/errors"
+
 	core_ca "github.com/kumahq/kuma/pkg/core/ca"
 	mesh_core "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
-	core_manager "github.com/kumahq/kuma/pkg/core/resources/manager"
-	core_model "github.com/kumahq/kuma/pkg/core/resources/model"
-	core_store "github.com/kumahq/kuma/pkg/core/resources/store"
-
-	"github.com/pkg/errors"
 )
 
-// FetchDefaultMeshIfExists will try to get the default mesh, if present, and return a boolean
-// for whether or not it exists
-func FetchDefaultMeshIfExists(resManager core_manager.ResourceManager, mesh *mesh_core.MeshResource) (bool, error) {
-	key := core_model.ResourceKey{Mesh: core_model.DefaultMesh, Name: core_model.DefaultMesh}
-
-	if err := resManager.Get(context.Background(), mesh, core_store.GetBy(key)); err != nil {
-		if core_store.IsResourceNotFound(err) {
-			return false, nil
-		} else {
-			return false, err
-		}
-	}
-
-	return true, nil
-}
-
-func CreateDefaultMesh(resManager core_manager.ResourceManager, template mesh_proto.Mesh) error {
-	defaultMesh := mesh_core.MeshResource{}
-
-	key := core_model.ResourceKey{Mesh: core_model.DefaultMesh, Name: core_model.DefaultMesh}
-	exists, err := FetchDefaultMeshIfExists(resManager, &defaultMesh)
-	if err != nil {
-		return err
-	}
-
-	if !exists {
-		defaultMesh.Spec = template
-		core.Log.WithName("bootstrap").Info("Creating default mesh from the settings", "mesh", defaultMesh.Spec)
-
-		if err := resManager.Create(context.Background(), &defaultMesh, core_store.CreateBy(key)); err != nil {
-			return errors.Wrapf(err, "Failed to create `default` Mesh resource in a given resource store")
-		}
-	}
-
-	return nil
-}
-
 func EnsureEnabledCA(ctx context.Context, caManagers core_ca.Managers, mesh *mesh_core.MeshResource, meshName string) error {
 	if mesh.GetEnabledCertificateAuthorityBackend() != nil {
 		backend := mesh.GetEnabledCertificateAuthorityBackend()
diff --git a/pkg/defaults/components.go b/pkg/defaults/components.go
new file mode 100644
index 000000000..2ea8e61b9
--- /dev/null
+++ b/pkg/defaults/components.go
@@ -0,0 +1,75 @@
+package defaults
+
+import (
+	"context"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/sethvargo/go-retry"
+
+	kuma_cp "github.com/kumahq/kuma/pkg/config/app/kuma-cp"
+	config_core "github.com/kumahq/kuma/pkg/config/core"
+	"github.com/kumahq/kuma/pkg/core"
+	core_manager "github.com/kumahq/kuma/pkg/core/resources/manager"
+	"github.com/kumahq/kuma/pkg/core/resources/store"
+	"github.com/kumahq/kuma/pkg/core/runtime"
+	"github.com/kumahq/kuma/pkg/core/runtime/component"
+)
+
+var log = core.Log.WithName("defaults")
+
+func Setup(runtime runtime.Runtime) error {
+	defaultsComponent := NewDefaultsComponent(runtime.Config().Defaults, runtime.Config().Mode, runtime.Config().Environment, runtime.ResourceManager(), runtime.ResourceStore())
+	return runtime.Add(defaultsComponent)
+}
+
+func NewDefaultsComponent(config *kuma_cp.Defaults, cpMode config_core.CpMode, environment config_core.EnvironmentType, resManager core_manager.ResourceManager, resStore store.ResourceStore) component.Component {
+	return &defaultsComponent{
+		cpMode:      cpMode,
+		environment: environment,
+		config:      config,
+		resManager:  resManager,
+		resStore:    resStore,
+	}
+}
+
+var _ component.Component = &defaultsComponent{}
+
+type defaultsComponent struct {
+	cpMode      config_core.CpMode
+	environment config_core.EnvironmentType
+	config      *kuma_cp.Defaults
+	resManager  core_manager.ResourceManager
+	resStore    store.ResourceStore
+}
+
+func (d *defaultsComponent) NeedLeaderElection() bool {
+	// If you spin many instances without default resources at once, many of them would create them, therefore only leader should create default resources.
+	return true
+}
+
+func (d *defaultsComponent) Start(_ <-chan struct{}) error {
+	// todo(jakubdyszkiewicz) once this https://github.com/kumahq/kuma/issues/1001 is done. Wait for all the components to be ready.
+	if d.config.SkipMeshCreation {
+		log.V(1).Info("skipping default Mesh creation because KUMA_DEFAULTS_SKIP_MESH_CREATION is set to true")
+	} else if err := doWithRetry(d.createMeshIfNotExist); err != nil {
+		// Retry this operation since on Kubernetes Mesh needs to be validated and set default values.
+		// This code can execute before the control plane is ready therefore hooks can fail.
+		return errors.Wrap(err, "could not create the default Mesh")
+	}
+
+	if !d.shouldCreateSigningKey() {
+		log.V(1).Info("skip creating default Signing Key since CP with this mode is not required to have a key", "env", d.environment, "mode", d.cpMode)
+	} else if err := d.createSigningKeyIfNotExist(); err != nil {
+		return errors.Wrap(err, "could not create the default Signing Key")
+	}
+	return nil
+}
+
+func doWithRetry(fn func() error) error {
+	backoff, _ := retry.NewConstant(1 * time.Second)
+	backoff = retry.WithMaxDuration(1*time.Minute, backoff) // if after this time we cannot create a resource - something is wrong and we should return an error which will restart CP.
+	return retry.Do(context.Background(), backoff, func(ctx context.Context) error {
+		return retry.RetryableError(fn()) // retry all errors
+	})
+}
diff --git a/pkg/defaults/mesh.go b/pkg/defaults/mesh.go
new file mode 100644
index 000000000..cac4a3904
--- /dev/null
+++ b/pkg/defaults/mesh.go
@@ -0,0 +1,33 @@
+package defaults
+
+import (
+	"context"
+
+	mesh_core "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
+	core_model "github.com/kumahq/kuma/pkg/core/resources/model"
+	core_store "github.com/kumahq/kuma/pkg/core/resources/store"
+)
+
+var defaultMeshKey = core_model.ResourceKey{
+	Mesh: core_model.DefaultMesh,
+	Name: core_model.DefaultMesh,
+}
+
+func (d *defaultsComponent) createMeshIfNotExist() error {
+	mesh := &mesh_core.MeshResource{}
+	err := d.resManager.Get(context.Background(), mesh, core_store.GetBy(defaultMeshKey))
+	if err == nil {
+		log.V(1).Info("default Mesh already exists. Skip creating default Mesh.")
+		return nil
+	}
+	if !core_store.IsResourceNotFound(err) {
+		return err
+	}
+	log.Info("trying to create default Mesh")
+	if err := d.resManager.Create(context.Background(), &mesh_core.MeshResource{}, core_store.CreateBy(defaultMeshKey)); err != nil {
+		log.V(1).Info("could not create default mesh", "err", err)
+		return err
+	}
+	log.Info("default Mesh created")
+	return nil
+}
diff --git a/pkg/defaults/signing_key.go b/pkg/defaults/signing_key.go
new file mode 100644
index 000000000..e3ca104f1
--- /dev/null
+++ b/pkg/defaults/signing_key.go
@@ -0,0 +1,57 @@
+package defaults
+
+import (
+	"context"
+
+	config_core "github.com/kumahq/kuma/pkg/config/core"
+	core_store "github.com/kumahq/kuma/pkg/core/resources/store"
+	"github.com/kumahq/kuma/pkg/tokens/builtin/issuer"
+)
+
+func (d *defaultsComponent) signingKeyExists() (bool, error) {
+	_, err := issuer.GetSigningKey(d.resManager)
+	switch err {
+	case issuer.SigningKeyNotFound:
+		return false, nil
+	case nil:
+		return true, nil
+	default:
+		return false, err
+	}
+}
+
+func (d *defaultsComponent) createSigningKeyIfNotExist() error {
+	exists, err := d.signingKeyExists()
+	if err != nil {
+		return err
+	}
+	if exists {
+		log.V(1).Info("default Signing Key already exists. Skip creating default Signing Key.")
+	} else {
+		log.Info("trying to create Signing Key")
+		key, err := issuer.CreateSigningKey()
+		if err != nil {
+			return err
+		}
+		// We are using directly Resource Store instead of Resource Manager to bypass Mesh validation. Even if there is no default Mesh, we still want to create signing key.
+		// Drop this if we will switch to have Secrets not scoped to the Mesh.
+		if err := d.resStore.Create(context.Background(), &key, core_store.CreateBy(issuer.SigningKeyResourceKey)); err != nil {
+			log.V(1).Info("could not create Signing Key", "err", err)
+			return err
+		}
+		log.Info("Signing Key for generating Dataplane Token created")
+	}
+	return nil
+}
+
+func (d *defaultsComponent) shouldCreateSigningKey() bool {
+	switch d.cpMode {
+	case config_core.Standalone:
+		return d.environment == config_core.UniversalEnvironment // Signing Key should be created only on Universal since it is not used on K8S
+	case config_core.Global:
+		return true // Signing Key with multi-zone should be created on Global even if the Environment is K8S, because we may connect Universal Remote
+	case config_core.Remote:
+		return false // Signing Key should be synced from Global
+	}
+	return false
+}
diff --git a/pkg/plugins/runtime/k8s/controllers/namespace_controller.go b/pkg/plugins/runtime/k8s/controllers/namespace_controller.go
index 27a030eff..e29454886 100644
--- a/pkg/plugins/runtime/k8s/controllers/namespace_controller.go
+++ b/pkg/plugins/runtime/k8s/controllers/namespace_controller.go
@@ -17,9 +17,7 @@ import (
 	kube_source "sigs.k8s.io/controller-runtime/pkg/source"
 
 	mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1"
-	mesh_managers "github.com/kumahq/kuma/pkg/core/managers/apis/mesh"
 	core_manager "github.com/kumahq/kuma/pkg/core/resources/manager"
-	core_model "github.com/kumahq/kuma/pkg/core/resources/model"
 	mesh_k8s "github.com/kumahq/kuma/pkg/plugins/resources/k8s/native/api/v1alpha1"
 	k8scnicncfio "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/apis/k8s.cni.cncf.io"
 	network_v1 "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/apis/k8s.cni.cncf.io/v1"
@@ -38,11 +36,9 @@ type NamespaceReconciler struct {
 	DefaultMeshTemplate     mesh_proto.Mesh
 }
 
-// Reconcile is in charge for two things:
-//   - create NetworkAttachmentDefinition if CNI enabled and namespace has label 'kuma.io/sidecar-injection: enabled'
-//   - create 'default' mesh for cluster
+// Reconcile is in charge of creating NetworkAttachmentDefinition if CNI enabled and namespace has label 'kuma.io/sidecar-injection: enabled'
 func (r *NamespaceReconciler) Reconcile(req kube_ctrl.Request) (kube_ctrl.Result, error) {
-	if !r.CNIEnabled && req.Name != r.SystemNamespace {
+	if !r.CNIEnabled {
 		return kube_ctrl.Result{}, nil
 	}
 	log := r.Log.WithValues("namespace", req.Name)
@@ -58,28 +54,9 @@ func (r *NamespaceReconciler) Reconcile(req kube_ctrl.Request) (kube_ctrl.Result
 		return kube_ctrl.Result{}, err
 	}
 
-	if req.Name == r.SystemNamespace && !r.SkipDefaultMeshCreation {
-		// Fetch default Mesh instance
-		mesh := &mesh_k8s.Mesh{}
-		name := kube_types.NamespacedName{Name: core_model.DefaultMesh}
-		if err := r.Get(ctx, name, mesh); err != nil {
-			if kube_apierrs.IsNotFound(err) {
-				err := mesh_managers.CreateDefaultMesh(r.ResourceManager, r.DefaultMeshTemplate)
-				if err != nil {
-					log.Error(err, "unable to create default Mesh")
-				}
-				return kube_ctrl.Result{}, err
-			}
-			log.Error(err, "unable to fetch Mesh", "name", name)
-			return kube_ctrl.Result{}, err
-		}
-	}
-
-	if r.CNIEnabled {
-		if label, ok := ns.Labels["kuma.io/sidecar-injection"]; ok && label == "enabled" {
-			err := r.createOrUpdateNetworkAttachmentDefinition(req.Name)
-			return kube_ctrl.Result{}, err
-		}
+	if label, ok := ns.Labels["kuma.io/sidecar-injection"]; ok && label == "enabled" {
+		err := r.createOrUpdateNetworkAttachmentDefinition(req.Name)
+		return kube_ctrl.Result{}, err
 	}
 	return kube_ctrl.Result{}, nil
 }
@@ -110,7 +87,6 @@ func (r *NamespaceReconciler) SetupWithManager(mgr kube_ctrl.Manager) error {
 	}
 	return kube_ctrl.NewControllerManagedBy(mgr).
 		For(&kube_core.Namespace{}).
-		// on Mesh update reconcile Namespace it belongs to (in case default Mesh gets deleted)
 		Watches(&kube_source.Kind{Type: &mesh_k8s.Mesh{}}, &kube_handler.EnqueueRequestsFromMapFunc{
 			ToRequests: kube_handler.ToRequestsFunc(func(obj kube_handler.MapObject) []kube_reconcile.Request {
 				return []kube_reconcile.Request{{
diff --git a/pkg/tokens/builtin/issuer/signing_key.go b/pkg/tokens/builtin/issuer/signing_key.go
index b4938a704..ecf7fc713 100644
--- a/pkg/tokens/builtin/issuer/signing_key.go
+++ b/pkg/tokens/builtin/issuer/signing_key.go
@@ -24,36 +24,12 @@ const defaultRsaBits = 2048
 
 var SigningKeyNotFound = errors.New("there is no Signing Key in the Control Plane. If you run multi-zone setup, make sure Remote is connected to the Global before generating tokens.")
 
-var signingKeyResourceKey = model.ResourceKey{
+var SigningKeyResourceKey = model.ResourceKey{
 	Mesh: "default",
 	Name: "dataplane-token-signing-key",
 }
 
-func CreateDefaultSigningKey(manager manager.ResourceManager) error {
-	key, err := createSigningKey()
-	if err != nil {
-		return err
-	}
-	return storeKeyIfNotExist(manager, key)
-}
-
-func storeKeyIfNotExist(manager manager.ResourceManager, keyResource system.SecretResource) error {
-	ctx := context.Background()
-	resource := system.SecretResource{}
-	if err := manager.Get(ctx, &resource, store.GetBy(signingKeyResourceKey)); err != nil {
-		if store.IsResourceNotFound(err) {
-			log.Info("generating signing key for generating dataplane tokens")
-			if err := manager.Create(ctx, &keyResource, store.CreateBy(signingKeyResourceKey)); err != nil {
-				return errors.Wrap(err, "could not store a private key")
-			}
-		} else {
-			return errors.Wrap(err, "could not check if private key exists")
-		}
-	}
-	return nil
-}
-
-func createSigningKey() (system.SecretResource, error) {
+func CreateSigningKey() (system.SecretResource, error) {
 	res := system.SecretResource{}
 	key, err := rsa.GenerateKey(rand.Reader, defaultRsaBits)
 	if err != nil {
@@ -68,7 +44,7 @@ func createSigningKey() (system.SecretResource, error) {
 }
 func GetSigningKey(manager manager.ReadOnlyResourceManager) ([]byte, error) {
 	resource := system.SecretResource{}
-	if err := manager.Get(context.Background(), &resource, store.GetBy(signingKeyResourceKey)); err != nil {
+	if err := manager.Get(context.Background(), &resource, store.GetBy(SigningKeyResourceKey)); err != nil {
 		if store.IsResourceNotFound(err) {
 			return nil, SigningKeyNotFound
 		}


Test patch:
diff --git a/pkg/core/bootstrap/bootstrap_test.go b/pkg/core/bootstrap/bootstrap_test.go
deleted file mode 100644
index 86c304288..000000000
--- a/pkg/core/bootstrap/bootstrap_test.go
+++ /dev/null
@@ -1,102 +0,0 @@
-package bootstrap
-
-import (
-	"context"
-
-	"github.com/kumahq/kuma/api/mesh/v1alpha1"
-
-	kuma_cp "github.com/kumahq/kuma/pkg/config/app/kuma-cp"
-	mesh_managers "github.com/kumahq/kuma/pkg/core/managers/apis/mesh"
-	"github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
-	core_model "github.com/kumahq/kuma/pkg/core/resources/model"
-	core_store "github.com/kumahq/kuma/pkg/core/resources/store"
-	builtin_issuer "github.com/kumahq/kuma/pkg/tokens/builtin/issuer"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-)
-
-var _ = Describe("Bootstrap", func() {
-
-	It("should create default mesh", func() {
-		// given
-		cfg := kuma_cp.DefaultConfig()
-
-		// when control plane is started
-		rt, err := Bootstrap(cfg)
-		ch := make(chan struct{})
-		defer func() {
-			close(ch)
-		}()
-		go func() {
-			defer GinkgoRecover()
-			err := rt.Start(ch)
-			Expect(err).ToNot(HaveOccurred())
-		}()
-		Expect(err).ToNot(HaveOccurred())
-
-		// then wait until resource is created
-		resManager := rt.ResourceManager()
-		Eventually(func() error {
-			getOpts := core_store.GetByKey(core_model.DefaultMesh, core_model.DefaultMesh)
-			return resManager.Get(context.Background(), &mesh.MeshResource{}, getOpts)
-		}, "5s").Should(Succeed())
-
-		// when
-		getOpts := core_store.GetByKey(core_model.DefaultMesh, core_model.DefaultMesh)
-		defaultMesh := mesh.MeshResource{}
-		err = resManager.Get(context.Background(), &defaultMesh, getOpts)
-		Expect(err).ToNot(HaveOccurred())
-
-		// then
-		meshMeta := defaultMesh.GetMeta()
-		Expect(meshMeta.GetName()).To(Equal("default"))
-		Expect(meshMeta.GetMesh()).To(Equal("default"))
-	})
-
-	It("should skip creating mesh if one already exist", func() {
-		// given
-		cfg := kuma_cp.DefaultConfig()
-		runtime, err := buildRuntime(cfg)
-		Expect(err).ToNot(HaveOccurred())
-
-		template := v1alpha1.Mesh{}
-
-		// when
-		Expect(mesh_managers.CreateDefaultMesh(runtime.ResourceManager(), template)).To(Succeed())
-
-		// then mesh exists
-		getOpts := core_store.GetByKey(core_model.DefaultMesh, core_model.DefaultMesh)
-		err = runtime.ResourceManager().Get(context.Background(), &mesh.MeshResource{}, getOpts)
-		Expect(err).ToNot(HaveOccurred())
-
-		// when createDefaultMesh is called once mesh already exist
-		err = mesh_managers.CreateDefaultMesh(runtime.ResourceManager(), template)
-
-		// then
-		Expect(err).ToNot(HaveOccurred())
-	})
-
-	It("should create a default signing key", func() {
-		// given
-		cfg := kuma_cp.DefaultConfig()
-		rt, err := Bootstrap(cfg)
-		Expect(err).ToNot(HaveOccurred())
-
-		// when
-		key, err := builtin_issuer.GetSigningKey(rt.ResourceManager())
-
-		// then
-		Expect(err).ToNot(HaveOccurred())
-		Expect(key).ToNot(HaveLen(0))
-
-		// when kuma-cp is run again
-		err = onStartup(rt)
-		Expect(err).ToNot(HaveOccurred())
-		key2, err := builtin_issuer.GetSigningKey(rt.ResourceManager())
-
-		// then it should skip creating a new signing key
-		Expect(err).ToNot(HaveOccurred())
-		Expect(key).To(Equal(key2))
-	})
-})
diff --git a/pkg/defaults/components_test.go b/pkg/defaults/components_test.go
new file mode 100644
index 000000000..46b2024da
--- /dev/null
+++ b/pkg/defaults/components_test.go
@@ -0,0 +1,161 @@
+package defaults_test
+
+import (
+	"context"
+
+	"github.com/kumahq/kuma/api/mesh/v1alpha1"
+	kuma_cp "github.com/kumahq/kuma/pkg/config/app/kuma-cp"
+	"github.com/kumahq/kuma/pkg/config/core"
+	mesh_core "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
+	core_manager "github.com/kumahq/kuma/pkg/core/resources/manager"
+	core_store "github.com/kumahq/kuma/pkg/core/resources/store"
+	core_component "github.com/kumahq/kuma/pkg/core/runtime/component"
+	"github.com/kumahq/kuma/pkg/defaults"
+	resources_memory "github.com/kumahq/kuma/pkg/plugins/resources/memory"
+	"github.com/kumahq/kuma/pkg/tokens/builtin/issuer"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/extensions/table"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Defaults Component", func() {
+
+	Describe("when skip mesh creation is set to false", func() {
+
+		var component core_component.Component
+		var manager core_manager.ResourceManager
+
+		BeforeEach(func() {
+			cfg := &kuma_cp.Defaults{
+				SkipMeshCreation: false,
+			}
+			store := resources_memory.NewStore()
+			manager = core_manager.NewResourceManager(store)
+			component = defaults.NewDefaultsComponent(cfg, core.Standalone, core.UniversalEnvironment, manager, store)
+		})
+
+		It("should create default mesh", func() {
+			// when
+			err := component.Start(nil)
+
+			// then
+			Expect(err).ToNot(HaveOccurred())
+			err = manager.Get(context.Background(), &mesh_core.MeshResource{}, core_store.GetByKey("default", "default"))
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("should not override already created mesh", func() {
+			// given
+			mesh := &mesh_core.MeshResource{
+				Spec: v1alpha1.Mesh{
+					Mtls: &v1alpha1.Mesh_Mtls{
+						EnabledBackend: "builtin",
+						Backends: []*v1alpha1.CertificateAuthorityBackend{
+							{
+								Name: "builtin",
+								Type: "builtin",
+							},
+						},
+					},
+				},
+			}
+			err := manager.Create(context.Background(), mesh, core_store.CreateByKey("default", "default"))
+			Expect(err).ToNot(HaveOccurred())
+
+			// when
+			err = component.Start(nil)
+
+			// then
+			mesh = &mesh_core.MeshResource{}
+			Expect(err).ToNot(HaveOccurred())
+			err = manager.Get(context.Background(), mesh, core_store.GetByKey("default", "default"))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(mesh.Spec.Mtls.EnabledBackend).To(Equal("builtin"))
+		})
+	})
+
+	Describe("when skip mesh creation is set to true", func() {
+
+		var component core_component.Component
+		var manager core_manager.ResourceManager
+
+		BeforeEach(func() {
+			cfg := &kuma_cp.Defaults{
+				SkipMeshCreation: true,
+			}
+			store := resources_memory.NewStore()
+			manager = core_manager.NewResourceManager(store)
+			component = defaults.NewDefaultsComponent(cfg, core.Standalone, core.UniversalEnvironment, manager, store)
+		})
+
+		It("should not create default mesh", func() {
+			// when
+			err := component.Start(nil)
+
+			// then
+			Expect(err).ToNot(HaveOccurred())
+			err = manager.Get(context.Background(), &mesh_core.MeshResource{}, core_store.GetByKey("default", "default"))
+			Expect(core_store.IsResourceNotFound(err)).To(BeTrue())
+		})
+	})
+
+	type testCase struct {
+		cpMode       core.CpMode
+		environment  core.EnvironmentType
+		shouldCreate bool
+	}
+	DescribeTable("create signing key",
+		func(given testCase) {
+			// given
+			store := resources_memory.NewStore()
+			manager := core_manager.NewResourceManager(store)
+			component := defaults.NewDefaultsComponent(&kuma_cp.Defaults{}, given.cpMode, given.environment, manager, store)
+			err := manager.Create(context.Background(), &mesh_core.MeshResource{}, core_store.CreateByKey("default", "default"))
+			Expect(err).ToNot(HaveOccurred())
+
+			// when
+			err = component.Start(nil)
+
+			// then
+			Expect(err).ToNot(HaveOccurred())
+			_, err = issuer.GetSigningKey(manager)
+			if given.shouldCreate {
+				Expect(err).To(BeNil())
+			} else {
+				Expect(err).To(Equal(issuer.SigningKeyNotFound))
+			}
+		},
+		Entry("should succeed when mode is global and env is universal", testCase{
+			cpMode:       core.Global,
+			environment:  core.UniversalEnvironment,
+			shouldCreate: true,
+		}),
+		Entry("should succeed when mode is global and env is kubernetes", testCase{
+			cpMode:       core.Global,
+			environment:  core.KubernetesEnvironment,
+			shouldCreate: true,
+		}),
+		Entry("should succeed when mode is standalone and env is universal", testCase{
+			cpMode:       core.Standalone,
+			environment:  core.UniversalEnvironment,
+			shouldCreate: true,
+		}),
+		Entry("should fail when mode is remote and env is universal", testCase{
+			cpMode:       core.Remote,
+			environment:  core.UniversalEnvironment,
+			shouldCreate: false,
+		}),
+		Entry("should fail when mode is remote and env is kubernetes", testCase{
+			cpMode:       core.Remote,
+			environment:  core.KubernetesEnvironment,
+			shouldCreate: false,
+		}),
+		Entry("should fail when mode is standalone and env is kubernetes", testCase{
+			cpMode:       core.Standalone,
+			environment:  core.KubernetesEnvironment,
+			shouldCreate: false,
+		}),
+	)
+
+})
diff --git a/pkg/defaults/defaults_suite_test.go b/pkg/defaults/defaults_suite_test.go
new file mode 100644
index 000000000..cf5e82a3e
--- /dev/null
+++ b/pkg/defaults/defaults_suite_test.go
@@ -0,0 +1,13 @@
+package defaults_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestDefaults(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Defaults")
+}
diff --git a/pkg/plugins/runtime/k8s/controllers/namespace_controller_test.go b/pkg/plugins/runtime/k8s/controllers/namespace_controller_test.go
index 5fef3050d..39cc157fe 100644
--- a/pkg/plugins/runtime/k8s/controllers/namespace_controller_test.go
+++ b/pkg/plugins/runtime/k8s/controllers/namespace_controller_test.go
@@ -14,10 +14,7 @@ import (
 	kube_reconcile "sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"github.com/kumahq/kuma/pkg/core"
-	core_mesh "github.com/kumahq/kuma/pkg/core/managers/apis/mesh"
-	mesh_core "github.com/kumahq/kuma/pkg/core/resources/apis/mesh"
 	"github.com/kumahq/kuma/pkg/core/resources/manager"
-	core_model "github.com/kumahq/kuma/pkg/core/resources/model"
 	"github.com/kumahq/kuma/pkg/plugins/resources/memory"
 	"github.com/kumahq/kuma/pkg/plugins/runtime/k8s/controllers"
 
@@ -58,160 +55,91 @@ var _ = Describe("NamespaceReconciler", func() {
 				},
 			},
 		)
+
+		reconciler = &controllers.NamespaceReconciler{
+			Client:          kubeClient,
+			SystemNamespace: "kuma-system",
+			CNIEnabled:      true,
+			Log:             core.Log.WithName("test"),
+			ResourceManager: resManager,
+		}
 	})
 
-	When("SkipDefaultMeshCreation is not set", func() {
-		BeforeEach(func() {
-
-			reconciler = &controllers.NamespaceReconciler{
-				Client:          kubeClient,
-				SystemNamespace: "kuma-system",
-				CNIEnabled:      true,
-				Log:             core.Log.WithName("test"),
-				ResourceManager: resManager,
-			}
-		})
-
-		It("should create NetworkAttachmentDefinition", func() {
-			// given
-			req := kube_ctrl.Request{
-				NamespacedName: kube_types.NamespacedName{
-					Namespace: "non-system-ns-with-sidecar-injection",
-					Name:      "non-system-ns-with-sidecar-injection",
-				},
-			}
-
-			// when
-			result, err := reconciler.Reconcile(req)
-			// then
-			Expect(err).ToNot(HaveOccurred())
-			// and
-			Expect(result).To(BeZero())
-
-			// when
-			nads := &v1.NetworkAttachmentDefinitionList{}
-			err = kubeClient.List(context.Background(), nads)
-			// then
-			Expect(err).ToNot(HaveOccurred())
-			// and
-			Expect(nads.Items).To(HaveLen(1))
-			Expect(nads.Items[0].Namespace).To(Equal("non-system-ns-with-sidecar-injection"))
-			Expect(nads.Items[0].Name).To(Equal("kuma-cni"))
-		})
-
-		It("should ignore system namespace", func() {
-			// given
-			req := kube_ctrl.Request{
-				NamespacedName: kube_types.NamespacedName{
-					Namespace: "kuma-system",
-					Name:      "kuma-system",
-				},
-			}
-
-			// when
-			result, err := reconciler.Reconcile(req)
-			// then
-			Expect(err).ToNot(HaveOccurred())
-			// and
-			Expect(result).To(BeZero())
-
-			// when
-			nads := &v1.NetworkAttachmentDefinitionList{}
-			err = kubeClient.List(context.Background(), nads)
-			// then
-			Expect(err).ToNot(HaveOccurred())
-			// and
-			Expect(nads.Items).To(HaveLen(0))
-		})
-
-		It("should create the default mesh for the system namespace", func() {
-			// given
-			req := kube_ctrl.Request{
-				NamespacedName: kube_types.NamespacedName{
-					Namespace: "kuma-system",
-					Name:      "kuma-system",
-				},
-			}
-
-			// when
-			result, err := reconciler.Reconcile(req)
-			// then
-			Expect(err).ToNot(HaveOccurred())
-			// and
-			Expect(result).To(BeZero())
-
-			// when
-			mesh := &mesh_core.MeshResource{}
-			exists, err := core_mesh.FetchDefaultMeshIfExists(resManager, mesh)
-			// then
-			Expect(err).ToNot(HaveOccurred())
-			// and
-			Expect(exists).To(BeTrue())
-			Expect(mesh.GetMeta().GetName()).To(Equal(core_model.DefaultMesh))
-		})
-
-		It("should ignore namespace namespaces without label", func() {
-			// given
-			req := kube_ctrl.Request{
-				NamespacedName: kube_types.NamespacedName{
-					Namespace: "non-system-ns-without-sidecar-injection",
-					Name:      "non-system-ns-without-sidecar-injection",
-				},
-			}
-
-			// when
-			result, err := reconciler.Reconcile(req)
-			// then
-			Expect(err).ToNot(HaveOccurred())
-			// and
-			Expect(result).To(BeZero())
-
-			// when
-			nads := &v1.NetworkAttachmentDefinitionList{}
-			err = kubeClient.List(context.Background(), nads)
-			// then
-			Expect(err).ToNot(HaveOccurred())
-			// and
-			Expect(nads.Items).To(HaveLen(0))
-		})
+	It("should create NetworkAttachmentDefinition", func() {
+		// given
+		req := kube_ctrl.Request{
+			NamespacedName: kube_types.NamespacedName{
+				Namespace: "non-system-ns-with-sidecar-injection",
+				Name:      "non-system-ns-with-sidecar-injection",
+			},
+		}
+
+		// when
+		result, err := reconciler.Reconcile(req)
+		// then
+		Expect(err).ToNot(HaveOccurred())
+		// and
+		Expect(result).To(BeZero())
+
+		// when
+		nads := &v1.NetworkAttachmentDefinitionList{}
+		err = kubeClient.List(context.Background(), nads)
+		// then
+		Expect(err).ToNot(HaveOccurred())
+		// and
+		Expect(nads.Items).To(HaveLen(1))
+		Expect(nads.Items[0].Namespace).To(Equal("non-system-ns-with-sidecar-injection"))
+		Expect(nads.Items[0].Name).To(Equal("kuma-cni"))
 	})
 
-	When("SkipDefaultMeshCreation is set", func() {
-		BeforeEach(func() {
-
-			reconciler = &controllers.NamespaceReconciler{
-				Client:                  kubeClient,
-				SystemNamespace:         "kuma-system",
-				SkipDefaultMeshCreation: true,
-				CNIEnabled:              true,
-				Log:                     core.Log.WithName("test"),
-				ResourceManager:         resManager,
-			}
-		})
-
-		It("should skip creating default mesh", func() {
-			// given
-			req := kube_ctrl.Request{
-				NamespacedName: kube_types.NamespacedName{
-					Namespace: "non-system-ns-with-sidecar-injection",
-					Name:      "non-system-ns-with-sidecar-injection",
-				},
-			}
-
-			// when
-			result, err := reconciler.Reconcile(req)
-			// then
-			Expect(err).ToNot(HaveOccurred())
-			// and
-			Expect(result).To(BeZero())
-
-			// when
-			mesh := &mesh_core.MeshResource{}
-			exists, err := core_mesh.FetchDefaultMeshIfExists(resManager, mesh)
-			// then
-			Expect(err).ToNot(HaveOccurred())
-			// and
-			Expect(exists).To(BeFalse())
-		})
+	It("should ignore system namespace", func() {
+		// given
+		req := kube_ctrl.Request{
+			NamespacedName: kube_types.NamespacedName{
+				Namespace: "kuma-system",
+				Name:      "kuma-system",
+			},
+		}
+
+		// when
+		result, err := reconciler.Reconcile(req)
+		// then
+		Expect(err).ToNot(HaveOccurred())
+		// and
+		Expect(result).To(BeZero())
+
+		// when
+		nads := &v1.NetworkAttachmentDefinitionList{}
+		err = kubeClient.List(context.Background(), nads)
+		// then
+		Expect(err).ToNot(HaveOccurred())
+		// and
+		Expect(nads.Items).To(HaveLen(0))
 	})
+
+	It("should ignore namespace namespaces without label", func() {
+		// given
+		req := kube_ctrl.Request{
+			NamespacedName: kube_types.NamespacedName{
+				Namespace: "non-system-ns-without-sidecar-injection",
+				Name:      "non-system-ns-without-sidecar-injection",
+			},
+		}
+
+		// when
+		result, err := reconciler.Reconcile(req)
+		// then
+		Expect(err).ToNot(HaveOccurred())
+		// and
+		Expect(result).To(BeZero())
+
+		// when
+		nads := &v1.NetworkAttachmentDefinitionList{}
+		err = kubeClient.List(context.Background(), nads)
+		// then
+		Expect(err).ToNot(HaveOccurred())
+		// and
+		Expect(nads.Items).To(HaveLen(0))
+	})
+
 })
diff --git a/pkg/sds/server/server_test.go b/pkg/sds/server/server_test.go
index 2622799c9..be6a60c3b 100644
--- a/pkg/sds/server/server_test.go
+++ b/pkg/sds/server/server_test.go
@@ -112,7 +112,9 @@ var _ = Describe("SDS Server", func() {
 		Expect(err).ToNot(HaveOccurred())
 
 		// setup Auth with Dataplane Token
-		err = tokens_issuer.CreateDefaultSigningKey(runtime.ResourceManager())
+		key, err := tokens_issuer.CreateSigningKey()
+		Expect(err).ToNot(HaveOccurred())
+		err = resManager.Create(context.Background(), &key, core_store.CreateBy(tokens_issuer.SigningKeyResourceKey))
 		Expect(err).ToNot(HaveOccurred())
 
 		// retrieve example DP token