Repository: m3db/m3db-operator
Pull request #241 (2020-10-05 21:24:52)

Add frozen field to cluster spec that will suspend changes
This commit adds a new field, `Frozen`, to the `ClusterSpec` that, when set, will suspend the operator from making any changes to a cluster. This can be useful when troubleshooting a cluster as it can be used to ensure that the operator doesn't modify a cluster while you're investigating it.


codecov[bot]: # [Codecov](https://codecov.io/gh/m3db/m3db-operator/pull/241?src=pr&el=h1) Report
> Merging [#241](https://codecov.io/gh/m3db/m3db-operator/pull/241?src=pr&el=desc) into [master](https://codecov.io/gh/m3db/m3db-operator/commit/12ebef6d79e4f55472bce5918aee252598fb4f0f?el=desc) will **increase** coverage by `0.03%`.
> The diff coverage is `n/a`.


[![Impacted file tree graph](https://codecov.io/gh/m3db/m3db-operator/pull/241/graphs/tree.svg?width=650&height=150&src=pr&token=jtHDyWhsto)](https://codecov.io/gh/m3db/m3db-operator/pull/241?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master     #241      +/-   ##
==========================================
+ Coverage   76.25%   76.29%   +0.03%     
==========================================
  Files          30       30              
  Lines        2165     2168       +3     
==========================================
+ Hits         1651     1654       +3     
  Misses        380      380              
  Partials      134      134              
```


------

[Continue to review full report at Codecov](https://codecov.io/gh/m3db/m3db-operator/pull/241?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/m3db/m3db-operator/pull/241?src=pr&el=footer). Last update [12ebef6...4bb3c1b](https://codecov.io/gh/m3db/m3db-operator/pull/241?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/docs/api.md b/docs/api.md
index 8cbe22d..7b255f0 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -69,6 +69,7 @@ ClusterSpec defines the desired state for a M3 cluster to be converge to.
 | podMetadata | PodMetadata is for any Metadata that is unique to the pods, and does not belong on any other objects, such as Prometheus scrape tags | [metav1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#objectmeta-v1-meta) | false |
 | parallelPodManagement | ParallelPodManagement sets StatefulSets created by the operator to have Parallel pod management instead of OrderedReady. This is an EXPERIMENTAL flag and subject to deprecation in a future release. This has not been tested in production and users should not depend on it without validating it for their own use case. | bool | true |
 | serviceAccountName | To use a non-default service account, specify the name here otherwise the service account \"default\" will be used. This is useful for advanced use-cases such as pod security policies. The service account must exist. This operator will not create it. | string | false |
+| frozen | Frozen is used to stop the operator from taking any further actions on a cluster. This is useful when troubleshooting as it guarantees the operator won't make any changes to the cluster. | bool | false |
 
 [Back to TOC](#table-of-contents)
 
diff --git a/go.mod b/go.mod
index 9b24fff..ccaa272 100644
--- a/go.mod
+++ b/go.mod
@@ -43,6 +43,7 @@ require (
 	github.com/uber-go/tally v3.3.13+incompatible
 	github.com/ultraware/funlen v0.0.2 // indirect
 	github.com/urfave/cli v1.22.2 // indirect
+	go.uber.org/atomic v1.6.0
 	go.uber.org/zap v1.13.0
 	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f
 	golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375 // indirect
diff --git a/pkg/apis/m3dboperator/v1alpha1/cluster.go b/pkg/apis/m3dboperator/v1alpha1/cluster.go
index d76b3d5..d10d360 100644
--- a/pkg/apis/m3dboperator/v1alpha1/cluster.go
+++ b/pkg/apis/m3dboperator/v1alpha1/cluster.go
@@ -306,6 +306,12 @@ type ClusterSpec struct {
 	// use-cases such as pod security policies. The service account must exist.
 	// This operator will not create it.
 	ServiceAccountName string `json:"serviceAccountName,omitempty"`
+
+	// Frozen is used to stop the operator from taking any further actions on a
+	// cluster. This is useful when troubleshooting as it guarantees the operator
+	// won't make any changes to the cluster.
+	// +optional
+	Frozen bool `json:"frozen,omitempty"`
 }
 
 // ExternalCoordinatorConfig defines parameters for using an external
diff --git a/pkg/apis/m3dboperator/v1alpha1/openapi_generated.go b/pkg/apis/m3dboperator/v1alpha1/openapi_generated.go
index 5b012de..78e2f05 100644
--- a/pkg/apis/m3dboperator/v1alpha1/openapi_generated.go
+++ b/pkg/apis/m3dboperator/v1alpha1/openapi_generated.go
@@ -617,6 +617,13 @@ func schema_pkg_apis_m3dboperator_v1alpha1_ClusterSpec(ref common.ReferenceCallb
 							Format:      "",
 						},
 					},
+					"frozen": {
+						SchemaProps: spec.SchemaProps{
+							Description: "Frozen is used to stop the operator from taking any further actions on a cluster. This is useful when troubleshooting as it guarantees the operator won't make any changes to the cluster.",
+							Type:        []string{"boolean"},
+							Format:      "",
+						},
+					},
 				},
 				Required: []string{"parallelPodManagement"},
 			},
diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
index 3cc177a..383426c 100644
--- a/pkg/controller/controller.go
+++ b/pkg/controller/controller.go
@@ -358,6 +358,11 @@ func (c *M3DBController) handleClusterUpdate(cluster *myspec.M3DBCluster) error
 
 	clusterLogger := c.logger.With(zap.String("cluster", cluster.Name))
 
+	if cluster.Spec.Frozen {
+		clusterLogger.Info("cluster is frozen so no changes will be made")
+		return nil
+	}
+
 	// https://v1-12.docs.kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/
 	//
 	// If deletion timestamp is zero (cluster hasn't been deleted), make sure our


Test patch:
diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go
index dc7adea..0768fc9 100644
--- a/pkg/controller/controller_test.go
+++ b/pkg/controller/controller_test.go
@@ -57,6 +57,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/uber-go/tally"
+	"go.uber.org/atomic"
 	"go.uber.org/zap"
 )
 
@@ -784,3 +785,36 @@ func TestHandleUpdateClusterUpdatesStatefulSets(t *testing.T) {
 		})
 	}
 }
+
+func TestHandleUpdateClusterFrozen(t *testing.T) {
+
+	var (
+		clusterMeta = newMeta("cluster1", map[string]string{
+			"foo":                      "bar",
+			"operator.m3db.io/app":     "m3db",
+			"operator.m3db.io/cluster": "cluster1",
+		}, nil)
+		sets = []*metav1.ObjectMeta{
+			newMeta("cluster1-rep0", nil, nil),
+		}
+	)
+	cluster, deps := setupTestCluster(t, *clusterMeta, sets, 3)
+	defer deps.cleanup()
+	controller := deps.newController(t)
+
+	cluster.Spec.Frozen = true
+
+	count := atomic.NewInt64(0)
+	controller.kubeClient.(*kubefake.Clientset).PrependReactor(
+		"*", "*", func(action ktesting.Action) (bool, runtime.Object, error) {
+			count.Inc()
+			return false, nil, nil
+		})
+
+	for i := 0; i < 20; i++ {
+		err := controller.handleClusterUpdate(cluster)
+		require.NoError(t, err)
+	}
+
+	assert.Equal(t, int64(0), count.Load())
+}