Repository: m3db/m3db-operator
Pull request #294 (2021-06-14 14:51:32)

[api] Allow blocking cluster scale down
Scaling down an M3DB cluster is a pretty heavy operation. All nodes in
the cluster must restream data, and it can put unnecessary pressure on a
cluster if accidentally triggered.

This PR adds a flag that allows users to ensure a cluster will not be
scaled down.

Thanks for contributing to the M3DB Operator! We'd love to accept your contribution, but we require that most issues
outside of trivial changes such as typo corrections have an issue associated with the pull request. So please [open an
issue](https://github.com/m3db/m3db-operator/issues/new) first!


codecov[bot]: # [Codecov](https://codecov.io/gh/m3db/m3db-operator/pull/294?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=m3db) Report
> Merging [#294](https://codecov.io/gh/m3db/m3db-operator/pull/294?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=m3db) (628e3e9) into [master](https://codecov.io/gh/m3db/m3db-operator/commit/b117fc6eb98bf28fa6a498fa8ccc75c5a9abd51c?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=m3db) (b117fc6) will **decrease** coverage by `0.37%`.
> The diff coverage is `50.00%`.


[![Impacted file tree graph](https://codecov.io/gh/m3db/m3db-operator/pull/294/graphs/tree.svg?width=650&height=150&src=pr&token=jtHDyWhsto&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=m3db)](https://codecov.io/gh/m3db/m3db-operator/pull/294?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=m3db)

```diff
@@            Coverage Diff             @@
##           master     #294      +/-   ##
==========================================
- Coverage   76.44%   76.06%   -0.38%     
==========================================
  Files          32       32              
  Lines        2551     2557       +6     
==========================================
- Hits         1950     1945       -5     
- Misses        425      433       +8     
- Partials      176      179       +3     
```


------

[Continue to review full report at Codecov](https://codecov.io/gh/m3db/m3db-operator/pull/294?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=m3db).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=m3db)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/m3db/m3db-operator/pull/294?src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=m3db). Last update [b117fc6...628e3e9](https://codecov.io/gh/m3db/m3db-operator/pull/294?src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=m3db). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=m3db).


Patch:
diff --git a/pkg/apis/m3dboperator/v1alpha1/cluster.go b/pkg/apis/m3dboperator/v1alpha1/cluster.go
index 973143f..725803f 100644
--- a/pkg/apis/m3dboperator/v1alpha1/cluster.go
+++ b/pkg/apis/m3dboperator/v1alpha1/cluster.go
@@ -328,6 +328,11 @@ type ClusterSpec struct {
 	// If not set, the default zone of "embedded" will be used.
 	// +optional
 	Zone string `json:"zone,omitempty"`
+
+	// PreventScaleDown will prevent the operator from removing any nodes from a
+	// cluster if set to true.
+	// +optional
+	PreventScaleDown bool `json:"preventScaleDown,omitempty"`
 }
 
 // ExternalCoordinatorConfig defines parameters for using an external
diff --git a/pkg/apis/m3dboperator/v1alpha1/openapi_generated.go b/pkg/apis/m3dboperator/v1alpha1/openapi_generated.go
index 6e3c9cb..f12f4f6 100644
--- a/pkg/apis/m3dboperator/v1alpha1/openapi_generated.go
+++ b/pkg/apis/m3dboperator/v1alpha1/openapi_generated.go
@@ -664,6 +664,13 @@ func schema_pkg_apis_m3dboperator_v1alpha1_ClusterSpec(ref common.ReferenceCallb
 							Format:      "",
 						},
 					},
+					"preventScaleDown": {
+						SchemaProps: spec.SchemaProps{
+							Description: "PreventScaleDown will prevent the operator from removing any nodes from a cluster if set to true.",
+							Type:        []string{"boolean"},
+							Format:      "",
+						},
+					},
 				},
 				Required: []string{"parallelPodManagement"},
 			},
diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
index 4440826..4741f1a 100644
--- a/pkg/controller/controller.go
+++ b/pkg/controller/controller.go
@@ -734,6 +734,10 @@ func (c *M3DBController) handleClusterUpdate(cluster *myspec.M3DBCluster) error
 		if current < desired {
 			newCount = current + 1
 		} else {
+			if cluster.Spec.PreventScaleDown {
+				return pkgerrors.Errorf("cannot shrink statefulset %s/%s, preventScaleDown is true",
+					set.Namespace, set.Name)
+			}
 			newCount = current - 1
 		}
 		setLogger.Info("resizing set, desired != current", zap.Int32("newSize", newCount))
diff --git a/pkg/controller/update_cluster.go b/pkg/controller/update_cluster.go
index 3f69edf..814068a 100644
--- a/pkg/controller/update_cluster.go
+++ b/pkg/controller/update_cluster.go
@@ -515,6 +515,11 @@ func (c *M3DBController) expandPlacementForSet(cluster *myspec.M3DBCluster, set
 // removes the last pod in the StatefulSet from the active placement, enabling
 // the StatefulSet size to be decreased once the remove completes.
 func (c *M3DBController) shrinkPlacementForSet(cluster *myspec.M3DBCluster, set *appsv1.StatefulSet, pl placement.Placement) error {
+	if cluster.Spec.PreventScaleDown {
+		return pkgerrors.Errorf("cannot remove nodes from %s/%s, preventScaleDown is true",
+			cluster.Namespace, cluster.Name)
+	}
+
 	selector := klabels.SelectorFromSet(set.Labels)
 	pods, err := c.podLister.Pods(cluster.Namespace).List(selector)
 	if err != nil {


Test patch:
diff --git a/pkg/controller/update_cluster_test.go b/pkg/controller/update_cluster_test.go
index f36951d..1b18531 100644
--- a/pkg/controller/update_cluster_test.go
+++ b/pkg/controller/update_cluster_test.go
@@ -695,6 +695,27 @@ func TestShrinkPlacementForSet(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestShrinkPlacementForSet_PreventScaleDown(t *testing.T) {
+	cluster := getFixture("cluster-3-zones.yaml", t)
+	cluster.Spec.PreventScaleDown = true
+
+	set, err := m3db.GenerateStatefulSet(cluster, "us-fake1-a", 3)
+	require.NoError(t, err)
+
+	pods := podsForClusterSet(cluster, set, 3)
+	deps := newTestDeps(t, &testOpts{
+		kubeObjects: objectsFromPods(pods...),
+	})
+	controller := deps.newController(t)
+	defer deps.cleanup()
+
+	identifyPods(deps.idProvider, pods, nil)
+	pl := placementFromPods(t, cluster, pods, deps.idProvider)
+
+	err = controller.shrinkPlacementForSet(cluster, set, pl)
+	assert.EqualError(t, err, "cannot remove nodes from fake/cluster-zones, preventScaleDown is true")
+}
+
 func podWithName(name string) *corev1.Pod {
 	return &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{