Repository: elastic/cloud-on-k8s

Issue:
Introduce constants for string length
We are using some magic numbers in different places of the code for tokens/keys length:

https://github.com/elastic/cloud-on-k8s/blob/da1fb962faa7b84b58038a716f8e6c8b137f80c8/pkg/controller/common/password.go#L11-L14

https://github.com/elastic/cloud-on-k8s/blob/ba0516c3a007955a21c05b891a9444343cbc9870/pkg/controller/kibana/config/settings.go#L163

https://github.com/elastic/cloud-on-k8s/blob/ba0516c3a007955a21c05b891a9444343cbc9870/pkg/controller/enterprisesearch/config.go#L127-L132

Let's introduce some constants and document these choices.

Discussion:

Solution patch:
diff --git a/NOTICE.txt b/NOTICE.txt
index 604cb2a05..a2abf7434 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -1486,6 +1486,36 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+--------------------------------------------------------------------------------
+Module  : github.com/sethvargo/go-password
+Version : v0.1.3
+Time    : 2019-10-10T18:40:17Z
+Licence : MIT
+
+Contents of probable licence file $GOMODCACHE/github.com/sethvargo/go-password@v0.1.3/LICENSE:
+
+Copyright 2017 Seth Vargo <seth@sethvargo.com>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
 --------------------------------------------------------------------------------
 Module  : github.com/spf13/cobra
 Version : v0.0.5
diff --git a/docs/operating-eck/troubleshooting.asciidoc b/docs/operating-eck/troubleshooting.asciidoc
index 1494ca251..8a38a6c1f 100644
--- a/docs/operating-eck/troubleshooting.asciidoc
+++ b/docs/operating-eck/troubleshooting.asciidoc
@@ -13,7 +13,7 @@ Most common issues can be identified and resolved by following these instruction
 - <<{p}-describe-failing-resources,Describe failing resources>>
 - <<{p}-eck-debug-logs,Enable ECK debug logs>>
 - <<{p}-view-logs>>
-- <<{p}-pause-controllers,Pause ECK controllers>>
+- <<{p}-exclude-resource,Exclude a resource from reconciliation>>
 - <<{p}-get-k8s-events,Get Kubernetes events>>
 - <<{p}-exec-into-containers,Exec into containers>>
 - <<{p}-webhook-troubleshooting,Troubleshoot webhook>>
@@ -170,11 +170,10 @@ Due to link:https://github.com/eBay/Kubernetes/blob/master/docs/devel/api-conven
 you can get errors reporting a conflict while updating a resource. You can ignore them, as the update goes through at the next reconciliation attempt, which will happen almost immediately.
 
 
-[id="{p}-pause-controllers"]
-== Pause ECK controllers
+[id="{p}-exclude-resource"]
+== Exclude resources from reconciliation
 
-When debugging Elasticsearch, you night need to "pause" the operator reconciliations, so that no resource gets modified or created in the meantime.
-To do this, set the annotation `common.k8s.elastic.co/pause` to `true` to any resource controlled by the operator:
+For debugging purposes, you might want to temporarily prevent ECK from modifying Kubernetes resources belonging to a particular Elastic Stack resource. To do this, annotate the Elastic object with `eck.k8s.elastic.co/managed=false`. This annotation can be added to any of the following types of objects:
 
 - Elasticsearch
 - Kibana
@@ -184,14 +183,14 @@ To do this, set the annotation `common.k8s.elastic.co/pause` to `true` to any re
 ----
 metadata:
   annotations:
-    common.k8s.elastic.co/pause: "true"
+    eck.k8s.elastic.co/managed: "false"
 ----
 
 Or in one line:
 
 [source,sh]
 ----
-kubectl annotate elasticsearch quickstart --overwrite common.k8s.elastic.co/pause=true
+kubectl annotate elasticsearch quickstart --overwrite eck.k8s.elastic.co/managed=false
 ----
 
 [id="{p}-get-k8s-events"]
diff --git a/docs/reference/dependencies.asciidoc b/docs/reference/dependencies.asciidoc
index cdb175d97..3af0eb656 100644
--- a/docs/reference/dependencies.asciidoc
+++ b/docs/reference/dependencies.asciidoc
@@ -32,6 +32,7 @@ This page lists the third-party dependencies used to build {n}.
 | link:https://github.com/imdario/mergo[$$github.com/imdario/mergo$$] | v0.3.8 | BSD-3-Clause
 | link:https://github.com/magiconair/properties[$$github.com/magiconair/properties$$] | v1.8.1 | BSD-2-Clause
 | link:https://github.com/pkg/errors[$$github.com/pkg/errors$$] | v0.8.1 | BSD-2-Clause
+| link:https://github.com/sethvargo/go-password[$$github.com/sethvargo/go-password$$] | v0.1.3 | MIT
 | link:https://github.com/spf13/cobra[$$github.com/spf13/cobra$$] | v0.0.5 | Apache-2.0
 | link:https://github.com/spf13/pflag[$$github.com/spf13/pflag$$] | v1.0.5 | BSD-3-Clause
 | link:https://github.com/spf13/viper[$$github.com/spf13/viper$$] | v1.4.0 | MIT
diff --git a/go.mod b/go.mod
index 9a8387cd3..96313b420 100644
--- a/go.mod
+++ b/go.mod
@@ -33,6 +33,7 @@ require (
 	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect
 	github.com/prometheus/common v0.7.0 // indirect
 	github.com/prometheus/procfs v0.0.5 // indirect
+	github.com/sethvargo/go-password v0.1.3
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5
diff --git a/go.sum b/go.sum
index 2bbdab8e7..37af87bfb 100644
--- a/go.sum
+++ b/go.sum
@@ -420,6 +420,8 @@ github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIH
 github.com/santhosh-tekuri/jsonschema v1.2.4 h1:hNhW8e7t+H1vgY+1QeEQpveR6D4+OwKPXCfD2aieJis=
 github.com/santhosh-tekuri/jsonschema v1.2.4/go.mod h1:TEAUOeZSmIxTTuHatJzrvARHiuO9LYd+cIxzgEHCQI4=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/sethvargo/go-password v0.1.3 h1:18KkbGDkw8SuzeohAbWqBLNSfRQblVwEHOLbPa0PvWM=
+github.com/sethvargo/go-password v0.1.3/go.mod h1:2tyaaoHK/AlXwh5WWQDYjqQbHcq4cjPj5qb/ciYvu/Q=
 github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
diff --git a/pkg/controller/apmserver/apmserver_controller.go b/pkg/controller/apmserver/apmserver_controller.go
index aaf6f3a12..46bb48ced 100644
--- a/pkg/controller/apmserver/apmserver_controller.go
+++ b/pkg/controller/apmserver/apmserver_controller.go
@@ -41,7 +41,6 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/rand"
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -180,9 +179,9 @@ func (r *ReconcileApmServer) Reconcile(request reconcile.Request) (reconcile.Res
 		return reconcile.Result{}, tracing.CaptureError(ctx, err)
 	}
 
-	if common.IsPaused(as.ObjectMeta) {
-		log.Info("Object is paused. Skipping reconciliation", "namespace", as.Namespace, "as_name", as.Name)
-		return common.PauseRequeue, nil
+	if common.IsUnmanaged(as.ObjectMeta) {
+		log.Info("Object currently not managed by this controller. Skipping reconciliation", "namespace", as.Namespace, "as_name", as.Name)
+		return reconcile.Result{}, nil
 	}
 
 	if compatible, err := r.isCompatible(ctx, &as); err != nil || !compatible {
@@ -280,11 +279,9 @@ func reconcileApmServerToken(c k8s.Client, as *apmv1.ApmServer) (corev1.Secret,
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: as.Namespace,
 			Name:      apmname.SecretToken(as.Name),
-			Labels:    labels.NewLabels(as.Name),
-		},
-		Data: map[string][]byte{
-			SecretTokenKey: []byte(rand.String(24)),
+			Labels:    common.AddCredentialsLabel(labels.NewLabels(as.Name)),
 		},
+		Data: make(map[string][]byte),
 	}
 	// reuse the secret token if it already exists
 	var existingSecret corev1.Secret
@@ -294,6 +291,8 @@ func reconcileApmServerToken(c k8s.Client, as *apmv1.ApmServer) (corev1.Secret,
 	}
 	if token, exists := existingSecret.Data[SecretTokenKey]; exists {
 		expectedApmServerSecret.Data[SecretTokenKey] = token
+	} else {
+		expectedApmServerSecret.Data[SecretTokenKey] = common.RandomBytes(24)
 	}
 
 	return reconciler.ReconcileSecret(c, expectedApmServerSecret, as)
diff --git a/pkg/controller/apmserverelasticsearchassociation/apmserverelasticsearchassociation_controller.go b/pkg/controller/apmserverelasticsearchassociation/apmserverelasticsearchassociation_controller.go
index d625a2373..531d31ab1 100644
--- a/pkg/controller/apmserverelasticsearchassociation/apmserverelasticsearchassociation_controller.go
+++ b/pkg/controller/apmserverelasticsearchassociation/apmserverelasticsearchassociation_controller.go
@@ -144,9 +144,9 @@ func (r *ReconcileApmServerElasticsearchAssociation) Reconcile(request reconcile
 		return reconcile.Result{}, tracing.CaptureError(ctx, err)
 	}
 
-	if common.IsPaused(apmServer.ObjectMeta) {
-		log.Info("Object is paused. Skipping reconciliation", "namespace", apmServer.Namespace, "as_name", apmServer.Name)
-		return common.PauseRequeue, nil
+	if common.IsUnmanaged(apmServer.ObjectMeta) {
+		log.Info("Object is currently not managed by this controller. Skipping reconciliation", "namespace", apmServer.Namespace, "as_name", apmServer.Name)
+		return reconcile.Result{}, nil
 	}
 
 	// ApmServer is being deleted, short-circuit reconciliation and remove artifacts related to the association.
diff --git a/pkg/controller/common/association/user.go b/pkg/controller/common/association/user.go
index 2e1c029d0..4b1c10cd2 100644
--- a/pkg/controller/common/association/user.go
+++ b/pkg/controller/common/association/user.go
@@ -7,20 +7,20 @@ package association
 import (
 	"context"
 
-	"go.elastic.co/apm"
-	"golang.org/x/crypto/bcrypt"
-	corev1 "k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-
 	commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1"
 	esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/tracing"
+	eslabel "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/label"
 	esuser "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/user"
 	"github.com/elastic/cloud-on-k8s/pkg/utils/k8s"
+	"go.elastic.co/apm"
+	"golang.org/x/crypto/bcrypt"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 )
 
 // elasticsearchUserName identifies the associated user in Elasticsearch namespace.
@@ -83,18 +83,21 @@ func ReconcileEsUser(
 	span, _ := apm.StartSpan(ctx, "reconcile_es_user", tracing.SpanTypeApp)
 	defer span.End()
 
+	// Add the Elasticsearch name, this is only intended to help the user to filter on these resources
+	labels[eslabel.ClusterNameLabelName] = es.Name
+
 	secKey := secretKey(associated, userObjectSuffix)
 	usrKey := UserKey(associated, userObjectSuffix)
 	expectedSecret := corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      secKey.Name,
 			Namespace: secKey.Namespace,
-			Labels:    labels,
+			Labels:    common.AddCredentialsLabel(labels),
 		},
 		Data: map[string][]byte{},
 	}
 
-	password := common.RandomPasswordBytes()
+	var password []byte
 	// reuse the existing password if there's one
 	var existingSecret corev1.Secret
 	err := c.Get(k8s.ExtractNamespacedName(&expectedSecret), &existingSecret)
@@ -103,6 +106,8 @@ func ReconcileEsUser(
 	}
 	if existingPassword, exists := existingSecret.Data[usrKey.Name]; exists {
 		password = existingPassword
+	} else {
+		password = common.FixedLengthRandomPasswordBytes()
 	}
 	expectedSecret.Data[usrKey.Name] = password
 
diff --git a/pkg/controller/common/labels.go b/pkg/controller/common/labels.go
index 85c88c45b..c0b53171d 100644
--- a/pkg/controller/common/labels.go
+++ b/pkg/controller/common/labels.go
@@ -4,11 +4,17 @@
 
 package common
 
-import "strconv"
+import (
+	"strconv"
+
+	"github.com/elastic/cloud-on-k8s/pkg/utils/maps"
+)
 
 const (
 	// TypeLabelName used to represent a resource type in k8s resources
 	TypeLabelName = "common.k8s.elastic.co/type"
+
+	credentialsLabel = "eck.k8s.elastic.co/credentials"
 )
 
 // TrueFalseLabel is a label that has a true/false value.
@@ -30,3 +36,8 @@ func (l TrueFalseLabel) AsMap(value bool) map[string]string {
 		string(l): strconv.FormatBool(value),
 	}
 }
+
+// AddCredentialsLabel adds a label used to describe a resource which contains some credentials, either a clear-text password or a token.
+func AddCredentialsLabel(original map[string]string) map[string]string {
+	return maps.Merge(map[string]string{credentialsLabel: "true"}, original)
+}
diff --git a/pkg/controller/common/password.go b/pkg/controller/common/password.go
deleted file mode 100644
index 98bfadb74..000000000
--- a/pkg/controller/common/password.go
+++ /dev/null
@@ -1,14 +0,0 @@
-// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-// or more contributor license agreements. Licensed under the Elastic License;
-// you may not use this file except in compliance with the Elastic License.
-
-package common
-
-import (
-	"k8s.io/apimachinery/pkg/util/rand"
-)
-
-// RandomPasswordBytes generates a random password
-func RandomPasswordBytes() []byte {
-	return []byte(rand.String(24))
-}
diff --git a/pkg/controller/common/pause.go b/pkg/controller/common/pause.go
deleted file mode 100644
index a19af1e95..000000000
--- a/pkg/controller/common/pause.go
+++ /dev/null
@@ -1,50 +0,0 @@
-// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-// or more contributor license agreements. Licensed under the Elastic License;
-// you may not use this file except in compliance with the Elastic License.
-
-package common
-
-import (
-	"strconv"
-	"time"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"sigs.k8s.io/controller-runtime/pkg/reconcile"
-)
-
-const (
-	// PauseAnnotationName annotation
-	PauseAnnotationName = "common.k8s.elastic.co/pause"
-)
-
-var (
-
-	// PauseRequeue is the default requeue result if controller is paused
-	PauseRequeue = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second}
-)
-
-// IsPaused computes if a given controller is paused.
-func IsPaused(meta metav1.ObjectMeta) bool {
-	return getBoolFromAnnotation(meta.Annotations)
-}
-
-// Extract the desired state from the map that contains annotations.
-func getBoolFromAnnotation(annotations map[string]string) bool {
-	if annotations == nil {
-		return false
-	}
-
-	stateAsString, exists := annotations[PauseAnnotationName]
-
-	if !exists {
-		return false
-	}
-
-	expectedState, err := strconv.ParseBool(stateAsString)
-	if err != nil {
-		log.Error(err, "Cannot parse %s as a bool, defaulting to %s: \"false\"", annotations[PauseAnnotationName], PauseAnnotationName)
-		return false
-	}
-
-	return expectedState
-}
diff --git a/pkg/controller/common/random.go b/pkg/controller/common/random.go
new file mode 100644
index 000000000..1f444ce6a
--- /dev/null
+++ b/pkg/controller/common/random.go
@@ -0,0 +1,25 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package common
+
+import (
+	"github.com/sethvargo/go-password/password"
+)
+
+// FixedLengthRandomPasswordBytes generates a random password
+func FixedLengthRandomPasswordBytes() []byte {
+	return RandomBytes(24)
+}
+
+// RandomBytes generates some random bytes that can be used as a token or as a key
+func RandomBytes(length int) []byte {
+	return []byte(password.MustGenerate(
+		length,
+		10,    // number of digits to include in the result
+		0,     // number of symbols to include in the result
+		false, // noUpper
+		true,  // allowRepeat
+	))
+}
diff --git a/pkg/controller/common/unmanaged.go b/pkg/controller/common/unmanaged.go
new file mode 100644
index 000000000..14b6f189d
--- /dev/null
+++ b/pkg/controller/common/unmanaged.go
@@ -0,0 +1,31 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package common
+
+import (
+	"fmt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	// ManagedAnnotation annotation
+	LegacyPauseAnnoation = "common.k8s.elastic.co/pause"
+	ManagedAnnotation    = "eck.k8s.elastic.co/managed"
+)
+
+// IsUnmanaged checks if a given resource is currently unmanaged.
+func IsUnmanaged(meta metav1.ObjectMeta) bool {
+	managed, exists := meta.Annotations[ManagedAnnotation]
+	if exists && managed == "false" {
+		return true
+	}
+
+	paused, exists := meta.Annotations[LegacyPauseAnnoation]
+	if exists {
+		log.Info(fmt.Sprintf("%s is deprecated, please use %s", LegacyPauseAnnoation, ManagedAnnotation), "namespace", meta.Namespace, "name", meta.Name)
+	}
+	return exists && paused == "true"
+}
diff --git a/pkg/controller/elasticsearch/elasticsearch_controller.go b/pkg/controller/elasticsearch/elasticsearch_controller.go
index a3321aecc..4e0879a4b 100644
--- a/pkg/controller/elasticsearch/elasticsearch_controller.go
+++ b/pkg/controller/elasticsearch/elasticsearch_controller.go
@@ -184,9 +184,9 @@ func (r *ReconcileElasticsearch) Reconcile(request reconcile.Request) (reconcile
 		return reconcile.Result{}, tracing.CaptureError(ctx, err)
 	}
 
-	if common.IsPaused(es.ObjectMeta) {
-		log.Info("Object is paused. Skipping reconciliation", "namespace", es.Namespace, "es_name", es.Name)
-		return common.PauseRequeue, nil
+	if common.IsUnmanaged(es.ObjectMeta) {
+		log.Info("Object is currently not managed by this controller. Skipping reconciliation", "namespace", es.Namespace, "es_name", es.Name)
+		return reconcile.Result{}, nil
 	}
 
 	selector := map[string]string{label.ClusterNameLabelName: es.Name}
diff --git a/pkg/controller/elasticsearch/remotecluster/remoteca/controller.go b/pkg/controller/elasticsearch/remotecluster/remoteca/controller.go
index 75a40494f..81c3ea48e 100644
--- a/pkg/controller/elasticsearch/remotecluster/remoteca/controller.go
+++ b/pkg/controller/elasticsearch/remotecluster/remoteca/controller.go
@@ -85,9 +85,9 @@ func (r *ReconcileRemoteCa) Reconcile(request reconcile.Request) (reconcile.Resu
 		return reconcile.Result{}, err
 	}
 
-	if common.IsPaused(es.ObjectMeta) {
-		log.Info("Object is paused. Skipping reconciliation", "namespace", es.Namespace, "es_name", es.Name)
-		return common.PauseRequeue, nil
+	if common.IsUnmanaged(es.ObjectMeta) {
+		log.Info("Object is currently not managed by this controller. Skipping reconciliation", "namespace", es.Namespace, "es_name", es.Name)
+		return reconcile.Result{}, nil
 	}
 
 	return doReconcile(ctx, r, &es)
diff --git a/pkg/controller/elasticsearch/user/predefined.go b/pkg/controller/elasticsearch/user/predefined.go
index 6f3d4dec6..fce053e20 100644
--- a/pkg/controller/elasticsearch/user/predefined.go
+++ b/pkg/controller/elasticsearch/user/predefined.go
@@ -87,7 +87,7 @@ func reconcilePredefinedUsers(
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: secretNsn.Namespace,
 			Name:      secretNsn.Name,
-			Labels:    label.NewLabels(k8s.ExtractNamespacedName(&es)),
+			Labels:    common.AddCredentialsLabel(label.NewLabels(k8s.ExtractNamespacedName(&es))),
 		},
 		Data: secretData,
 	}
@@ -116,7 +116,7 @@ func reuseOrGeneratePassword(c k8s.Client, users users, secretRef types.Namespac
 		if password, exists := secret.Data[u.Name]; exists {
 			users[i].Password = password
 		} else {
-			users[i].Password = common.RandomPasswordBytes()
+			users[i].Password = common.FixedLengthRandomPasswordBytes()
 		}
 	}
 	return users, nil
diff --git a/pkg/controller/enterprisesearch/config.go b/pkg/controller/enterprisesearch/config.go
index ee63ca91c..b97c1f74b 100644
--- a/pkg/controller/enterprisesearch/config.go
+++ b/pkg/controller/enterprisesearch/config.go
@@ -12,10 +12,10 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/rand"
 
 	commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1"
 	entsv1beta1 "github.com/elastic/cloud-on-k8s/pkg/apis/enterprisesearch/v1beta1"
+	"github.com/elastic/cloud-on-k8s/pkg/controller/common"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/association"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/driver"
@@ -59,7 +59,7 @@ func ReconcileConfig(driver driver.Interface, ents entsv1beta1.EnterpriseSearch)
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: ents.Namespace,
 			Name:      name.Config(ents.Name),
-			Labels:    NewLabels(ents.Name),
+			Labels:    common.AddCredentialsLabel(NewLabels(ents.Name)),
 		},
 		Data: map[string][]byte{
 			ConfigFilename: cfgBytes,
@@ -125,10 +125,10 @@ func getOrCreateReusableSettings(c k8s.Client, ents entsv1beta1.EnterpriseSearch
 		return nil, err
 	}
 	if len(e.SecretSession) == 0 {
-		e.SecretSession = rand.String(32)
+		e.SecretSession = string(common.RandomBytes(32))
 	}
 	if len(e.EncryptionKeys) == 0 {
-		e.EncryptionKeys = []string{rand.String(32)}
+		e.EncryptionKeys = []string{string(common.RandomBytes(32))}
 	}
 	return settings.MustCanonicalConfig(e), nil
 }
diff --git a/pkg/controller/enterprisesearch/enterprisesearch_controller.go b/pkg/controller/enterprisesearch/enterprisesearch_controller.go
index fb8300f1b..fbfaa7929 100644
--- a/pkg/controller/enterprisesearch/enterprisesearch_controller.go
+++ b/pkg/controller/enterprisesearch/enterprisesearch_controller.go
@@ -150,9 +150,9 @@ func (r *ReconcileEnterpriseSearch) Reconcile(request reconcile.Request) (reconc
 		return reconcile.Result{}, tracing.CaptureError(ctx, err)
 	}
 
-	if common.IsPaused(ents.ObjectMeta) {
-		log.Info("Object is paused. Skipping reconciliation", "namespace", ents.Namespace, "ents_name", ents.Name)
-		return common.PauseRequeue, nil
+	if common.IsUnmanaged(ents.ObjectMeta) {
+		log.Info("Object is currently not managed by this controller. Skipping reconciliation", "namespace", ents.Namespace, "ents_name", ents.Name)
+		return reconcile.Result{}, nil
 	}
 
 	if compatible, err := r.isCompatible(ctx, &ents); err != nil || !compatible {
diff --git a/pkg/controller/entsearchassociation/association_controller.go b/pkg/controller/entsearchassociation/association_controller.go
index 45947b1e9..e4d36a39b 100644
--- a/pkg/controller/entsearchassociation/association_controller.go
+++ b/pkg/controller/entsearchassociation/association_controller.go
@@ -146,9 +146,9 @@ func (r *ReconcileEnterpriseSearchElasticsearchAssociation) Reconcile(request re
 		return reconcile.Result{}, tracing.CaptureError(ctx, err)
 	}
 
-	if common.IsPaused(entSearch.ObjectMeta) {
-		log.Info("Object is paused. Skipping reconciliation", "namespace", entSearch.Namespace, "ents_name", entSearch.Name)
-		return common.PauseRequeue, nil
+	if common.IsUnmanaged(entSearch.ObjectMeta) {
+		log.Info("Object is currently not managed by this controller. Skipping reconciliation", "namespace", entSearch.Namespace, "ents_name", entSearch.Name)
+		return reconcile.Result{}, nil
 	}
 
 	// EnterpriseSearch is being deleted, short-circuit reconciliation and remove artifacts related to the association.
diff --git a/pkg/controller/kibana/config/reconciler.go b/pkg/controller/kibana/config/reconciler.go
index 2b8351d4a..7a273ada5 100644
--- a/pkg/controller/kibana/config/reconciler.go
+++ b/pkg/controller/kibana/config/reconciler.go
@@ -9,6 +9,7 @@ import (
 
 	"github.com/elastic/cloud-on-k8s/pkg/about"
 	kbv1 "github.com/elastic/cloud-on-k8s/pkg/apis/kibana/v1"
+	"github.com/elastic/cloud-on-k8s/pkg/controller/common"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/tracing"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/kibana/label"
@@ -42,9 +43,9 @@ func ReconcileConfigSecret(
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: kb.Namespace,
 			Name:      SecretName(kb),
-			Labels: map[string]string{
+			Labels: common.AddCredentialsLabel(map[string]string{
 				label.KibanaNameLabelName: kb.Name,
-			},
+			}),
 		},
 		Data: map[string][]byte{
 			SettingsFilename:  settingsYamlBytes,
diff --git a/pkg/controller/kibana/config/settings.go b/pkg/controller/kibana/config/settings.go
index d22f42eb7..26c04c50c 100644
--- a/pkg/controller/kibana/config/settings.go
+++ b/pkg/controller/kibana/config/settings.go
@@ -8,17 +8,16 @@ import (
 	"context"
 	"path"
 
-	"github.com/elastic/go-ucfg"
 	"github.com/pkg/errors"
 	"go.elastic.co/apm"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/rand"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
 	commonv1 "github.com/elastic/cloud-on-k8s/pkg/apis/common/v1"
 	kbv1 "github.com/elastic/cloud-on-k8s/pkg/apis/kibana/v1"
+	"github.com/elastic/cloud-on-k8s/pkg/controller/common"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/association"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/settings"
@@ -48,12 +47,7 @@ func NewConfigSettings(ctx context.Context, client k8s.Client, kb kbv1.Kibana, v
 	span, _ := apm.StartSpan(ctx, "new_config_settings", tracing.SpanTypeApp)
 	defer span.End()
 
-	currentConfig, err := getExistingConfig(client, kb)
-	if err != nil {
-		return CanonicalConfig{}, err
-	}
-
-	filteredCurrCfg, err := filterExistingConfig(currentConfig)
+	reusableSettings, err := getOrCreateReusableSettings(client, kb)
 	if err != nil {
 		return CanonicalConfig{}, err
 	}
@@ -75,7 +69,7 @@ func NewConfigSettings(ctx context.Context, client k8s.Client, kb kbv1.Kibana, v
 	if !kb.RequiresAssociation() {
 		// merge the configuration with userSettings last so they take precedence
 		if err := cfg.MergeWith(
-			filteredCurrCfg,
+			reusableSettings,
 			versionSpecificCfg,
 			kibanaTLSCfg,
 			userSettings); err != nil {
@@ -91,7 +85,7 @@ func NewConfigSettings(ctx context.Context, client k8s.Client, kb kbv1.Kibana, v
 
 	// merge the configuration with userSettings last so they take precedence
 	err = cfg.MergeWith(
-		filteredCurrCfg,
+		reusableSettings,
 		versionSpecificCfg,
 		kibanaTLSCfg,
 		settings.MustCanonicalConfig(elasticsearchTLSSettings(kb)),
@@ -110,6 +104,11 @@ func NewConfigSettings(ctx context.Context, client k8s.Client, kb kbv1.Kibana, v
 	return CanonicalConfig{cfg}, nil
 }
 
+// reusableSettings captures secrets settings in the Kibana configuration that we want to reuse.
+type reusableSettings struct {
+	EncryptionKey string `config:"xpack.security.encryptionKey"`
+}
+
 // getExistingConfig retrieves the canonical config for a given Kibana, if one exists
 func getExistingConfig(client k8s.Client, kb kbv1.Kibana) (*settings.CanonicalConfig, error) {
 	var secret corev1.Secret
@@ -135,23 +134,24 @@ func getExistingConfig(client k8s.Client, kb kbv1.Kibana) (*settings.CanonicalCo
 	return cfg, nil
 }
 
-// filterExistingConfig filters an existing config for only items we want to preserve between spec changes
+// getOrCreateReusableSettings filters an existing config for only items we want to preserve between spec changes
 // because they cannot be generated deterministically, e.g. encryption keys
-func filterExistingConfig(cfg *settings.CanonicalConfig) (*settings.CanonicalConfig, error) {
-	if cfg == nil {
-		return nil, nil
-	}
-	val, err := (*ucfg.Config)(cfg).String(XpackSecurityEncryptionKey, -1, settings.Options...)
+func getOrCreateReusableSettings(c k8s.Client, kb kbv1.Kibana) (*settings.CanonicalConfig, error) {
+	cfg, err := getExistingConfig(c, kb)
 	if err != nil {
-		log.V(1).Info("Current config does not contain key", "key", XpackSecurityEncryptionKey, "error", err)
-		return nil, nil
+		return nil, err
 	}
-	filteredCfg, err := settings.NewSingleValue(XpackSecurityEncryptionKey, val)
-	if err != nil {
-		log.Error(err, "Error filtering current config")
+
+	var r reusableSettings
+	if cfg == nil {
+		r = reusableSettings{}
+	} else if err := cfg.Unpack(&r); err != nil {
 		return nil, err
 	}
-	return filteredCfg, nil
+	if len(r.EncryptionKey) == 0 {
+		r.EncryptionKey = string(common.RandomBytes(64))
+	}
+	return settings.MustCanonicalConfig(r), nil
 }
 
 func baseSettings(kb *kbv1.Kibana) map[string]interface{} {
@@ -159,8 +159,6 @@ func baseSettings(kb *kbv1.Kibana) map[string]interface{} {
 		ServerName: kb.Name,
 		ServerHost: "0",
 		XpackMonitoringUiContainerElasticsearchEnabled: true,
-		// this will get overriden if one already exists or is specified by the user
-		XpackSecurityEncryptionKey: rand.String(64),
 	}
 
 	if kb.RequiresAssociation() {
diff --git a/pkg/controller/kibana/kibana_controller.go b/pkg/controller/kibana/kibana_controller.go
index 52c292920..42d0636fa 100644
--- a/pkg/controller/kibana/kibana_controller.go
+++ b/pkg/controller/kibana/kibana_controller.go
@@ -137,10 +137,9 @@ func (r *ReconcileKibana) Reconcile(request reconcile.Request) (reconcile.Result
 		return reconcile.Result{}, tracing.CaptureError(ctx, err)
 	}
 
-	// skip reconciliation if paused
-	if common.IsPaused(kb.ObjectMeta) {
-		log.Info("Object is paused. Skipping reconciliation", "namespace", kb.Namespace, "kibana_name", kb.Name)
-		return common.PauseRequeue, nil
+	if common.IsUnmanaged(kb.ObjectMeta) {
+		log.Info("Object is currently not managed by this controller. Skipping reconciliation", "namespace", kb.Namespace, "kibana_name", kb.Name)
+		return reconcile.Result{}, nil
 	}
 
 	// check for compatibility with the operator version
diff --git a/pkg/controller/kibanaassociation/association_controller.go b/pkg/controller/kibanaassociation/association_controller.go
index 8b26f00ca..dcc9589e7 100644
--- a/pkg/controller/kibanaassociation/association_controller.go
+++ b/pkg/controller/kibanaassociation/association_controller.go
@@ -141,9 +141,9 @@ func (r *ReconcileAssociation) Reconcile(request reconcile.Request) (reconcile.R
 		return reconcile.Result{}, tracing.CaptureError(ctx, r.onDelete(kbName))
 	}
 
-	if common.IsPaused(kibana.ObjectMeta) {
-		log.Info("Object is paused. Skipping reconciliation", "namespace", kibana.Namespace, "kibana_name", kibana.Name)
-		return common.PauseRequeue, nil
+	if common.IsUnmanaged(kibana.ObjectMeta) {
+		log.Info("Object is currently not managed by this controller. Skipping reconciliation", "namespace", kibana.Namespace, "kibana_name", kibana.Name)
+		return reconcile.Result{}, nil
 	}
 
 	compatible, err := r.isCompatible(ctx, &kibana)



Test patch:
diff --git a/pkg/controller/common/pause_test.go b/pkg/controller/common/pause_test.go
deleted file mode 100644
index 3ae57c193..000000000
--- a/pkg/controller/common/pause_test.go
+++ /dev/null
@@ -1,73 +0,0 @@
-// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-// or more contributor license agreements. Licensed under the Elastic License;
-// you may not use this file except in compliance with the Elastic License.
-
-package common
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-type testcase struct {
-	name string
-
-	// annotationSequence is list of annotations that are simulated.
-	annotationSequence []map[string]string
-
-	// Expected pause status.
-	expectedState []bool
-}
-
-func TestPauseCondition(t *testing.T) {
-	tests := []testcase{
-		{
-			name: "Simple pause/resume simulation (a.k.a the Happy Path)",
-			annotationSequence: []map[string]string{
-				{PauseAnnotationName: "true"},
-				{PauseAnnotationName: "false"},
-				{PauseAnnotationName: "true"},
-				{PauseAnnotationName: "false"},
-			},
-			expectedState: []bool{
-				true,
-				false,
-				true,
-				false,
-			},
-		},
-		{
-			name: "Can't parse or empty annotation",
-			annotationSequence: []map[string]string{
-				{PauseAnnotationName: ""}, // empty annotation
-				{PauseAnnotationName: "true"},
-				{PauseAnnotationName: "XXXX"}, // unable to parse this one
-				{PauseAnnotationName: "1"},    // 1 == true
-				{PauseAnnotationName: "0"},    // 0 == false
-			},
-			expectedState: []bool{
-				false,
-				true,
-				false,
-				true,
-				false,
-			},
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			for i, expectedState := range test.expectedState {
-				meta := v1.ObjectMeta{
-					Name:        "bar",
-					Namespace:   "foo",
-					Annotations: test.annotationSequence[i],
-				}
-				actualPauseState := IsPaused(meta)
-				assert.Equal(t, expectedState, actualPauseState)
-			}
-		})
-	}
-}
diff --git a/pkg/controller/common/unmanaged_test.go b/pkg/controller/common/unmanaged_test.go
new file mode 100644
index 000000000..6af7255b6
--- /dev/null
+++ b/pkg/controller/common/unmanaged_test.go
@@ -0,0 +1,89 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package common
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type testcase struct {
+	name string
+
+	// annotationSequence is list of annotations that are simulated.
+	annotationSequence []map[string]string
+
+	// Expected (un)managed state.
+	expectedState []bool
+}
+
+func TestUnmanagedCondition(t *testing.T) {
+	var tests = []testcase{
+		{
+			name: "Simple unmanaged/managed simulation (a.k.a the Happy Path)",
+			annotationSequence: []map[string]string{
+				{ManagedAnnotation: "true"},
+				{ManagedAnnotation: "false"},
+				{ManagedAnnotation: "true"},
+				{ManagedAnnotation: "false"},
+			},
+			expectedState: []bool{
+				false,
+				true,
+				false,
+				true,
+			},
+		},
+		{
+			name: "Anything but 'false' means managed",
+			annotationSequence: []map[string]string{
+				{ManagedAnnotation: ""}, // empty annotation
+				{ManagedAnnotation: "false"},
+				{ManagedAnnotation: "XXXX"}, // unable to parse these
+				{ManagedAnnotation: "1"},
+				{ManagedAnnotation: "0"},
+			},
+			expectedState: []bool{
+				false,
+				true,
+				false,
+				false,
+				false,
+			},
+		},
+		{
+			name: "Still support legacy annotation",
+			annotationSequence: []map[string]string{
+				{LegacyPauseAnnoation: "true"}, // still support legacy for backwards compatibility
+				{LegacyPauseAnnoation: "false"},
+				{LegacyPauseAnnoation: "foo"},
+				{LegacyPauseAnnoation: "false", ManagedAnnotation: "false"}, // new one takes precedence
+				{LegacyPauseAnnoation: "true", ManagedAnnotation: "true"},   // but legacy is respected if true
+			},
+			expectedState: []bool{
+				true,
+				false,
+				false,
+				true,
+				true,
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			for i, expectedState := range test.expectedState {
+				meta := v1.ObjectMeta{
+					Name:        "bar",
+					Namespace:   "foo",
+					Annotations: test.annotationSequence[i],
+				}
+				actualPauseState := IsUnmanaged(meta)
+				assert.Equal(t, expectedState, actualPauseState, test.annotationSequence[i])
+			}
+		})
+	}
+}
diff --git a/pkg/controller/kibana/config/settings_test.go b/pkg/controller/kibana/config/settings_test.go
index ba3c7a390..fca12780d 100644
--- a/pkg/controller/kibana/config/settings_test.go
+++ b/pkg/controller/kibana/config/settings_test.go
@@ -52,6 +52,71 @@ elasticsearch:
     verificationMode: certificate
 `)
 
+func Test_reuseOrGenerateSecrets(t *testing.T) {
+	defaultKb := mkKibana()
+	type args struct {
+		c      k8s.Client
+		kibana kbv1.Kibana
+	}
+	tests := []struct {
+		name      string
+		args      args
+		assertion func(*testing.T, *settings.CanonicalConfig, error)
+		wantErr   bool
+	}{
+		{
+			name: "Do not override existing encryption keys",
+			args: args{
+				c: k8s.WrappedFakeClient(
+					&corev1.Secret{
+						ObjectMeta: metav1.ObjectMeta{Namespace: defaultKb.Namespace, Name: SecretName(defaultKb)},
+						Data: map[string][]byte{
+							SettingsFilename: defaultConfig,
+						},
+					},
+				),
+				kibana: defaultKb,
+			},
+			assertion: func(t *testing.T, got *settings.CanonicalConfig, err error) {
+				expectedSettings := settings.MustCanonicalConfig(map[string]interface{}{
+					"xpack.security.encryptionKey": "thisismyencryptionkey",
+				})
+				assert.Equal(t, expectedSettings, got)
+			},
+		},
+		{
+			name: "Create new encryption keys",
+			args: args{
+				c: k8s.WrappedFakeClient(
+					&corev1.Secret{
+						ObjectMeta: metav1.ObjectMeta{Namespace: defaultKb.Namespace, Name: SecretName(defaultKb)},
+						Data: map[string][]byte{
+							SettingsFilename: associationConfig,
+						},
+					},
+				),
+				kibana: defaultKb,
+			},
+			assertion: func(t *testing.T, got *settings.CanonicalConfig, err error) {
+				// Unpack the configuration to check that some default reusable settings have been generated
+				var r reusableSettings
+				assert.NoError(t, got.Unpack(&r))
+				assert.Equal(t, len(r.EncryptionKey), 64) // Kibana encryption key length should be 64
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := getOrCreateReusableSettings(tt.args.c, tt.args.kibana)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("getOrCreateReusableSettings() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			tt.assertion(t, got, err)
+		})
+	}
+}
+
 func TestNewConfigSettings(t *testing.T) {
 	defaultKb := mkKibana()
 	existingSecret := &corev1.Secret{
@@ -408,36 +473,3 @@ func Test_getExistingConfig(t *testing.T) {
 		})
 	}
 }
-
-func Test_filterExistingConfig(t *testing.T) {
-	tests := []struct {
-		name      string
-		cfg       *settings.CanonicalConfig
-		want      *settings.CanonicalConfig
-		expectErr bool
-	}{
-		{
-			name: "happy path",
-			cfg: settings.MustCanonicalConfig(map[string]interface{}{
-				XpackSecurityEncryptionKey: "value",
-				"notakey":                  "notavalue",
-			}),
-			want: settings.MustCanonicalConfig(map[string]interface{}{
-				XpackSecurityEncryptionKey: "value",
-			}),
-			expectErr: false,
-		},
-	}
-
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			actual, err := filterExistingConfig(tc.cfg)
-			if tc.expectErr {
-				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
-			}
-			assert.Equal(t, tc.want, actual)
-		})
-	}
-}
diff --git a/test/e2e/test/elasticsearch/pause.go b/test/e2e/test/elasticsearch/pause.go
deleted file mode 100644
index f22ecea33..000000000
--- a/test/e2e/test/elasticsearch/pause.go
+++ /dev/null
@@ -1,42 +0,0 @@
-// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-// or more contributor license agreements. Licensed under the Elastic License;
-// you may not use this file except in compliance with the Elastic License.
-
-package elasticsearch
-
-import (
-	"fmt"
-	"strconv"
-
-	esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1"
-	"github.com/elastic/cloud-on-k8s/pkg/controller/common"
-	"github.com/elastic/cloud-on-k8s/pkg/utils/k8s"
-	"github.com/elastic/cloud-on-k8s/test/e2e/test"
-)
-
-func togglePauseOn(paused bool, es esv1.Elasticsearch, k *test.K8sClient) test.Step {
-	return test.Step{
-		Name: fmt.Sprintf("Should pause reconciliation %v", paused),
-		Test: test.Eventually(func() error {
-			var curr esv1.Elasticsearch
-			if err := k.Client.Get(k8s.ExtractNamespacedName(&es), &curr); err != nil {
-				return err
-			}
-			as := curr.Annotations
-			if as == nil {
-				as = map[string]string{}
-			}
-			as[common.PauseAnnotationName] = strconv.FormatBool(paused)
-			curr.Annotations = as
-			return k.Client.Update(&curr)
-		}),
-	}
-}
-
-func PauseReconciliation(es esv1.Elasticsearch, k *test.K8sClient) test.Step {
-	return togglePauseOn(true, es, k)
-}
-
-func ResumeReconciliation(es esv1.Elasticsearch, k *test.K8sClient) test.Step {
-	return togglePauseOn(false, es, k)
-}