Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1288 (2022-07-14 00:36:50)

Adding support for buggify in plugin
# Description

As part of this PR I added support for buggification in plugin. This addresses #1202 

## Type of change

*Please select one of the options below.*

- New feature (non-breaking change which adds functionality)


## Discussion

*Are there any design details that you would like to discuss further?*
No

## Testing

*Please describe the tests that you ran to verify your changes. Unit tests?
Manual testing?*
Added test cases and perform testing on local cluster. 

*Do we need to perform additional testing once this is merged, or perform in a larger testing environment?*
No

## Documentation

*Did you update relevant documentation within this repository?*
Added documentation for usage in plugin. 

*If this change is adding new functionality, do we need to describe it in our user manual?*

*If this change is adding or removing subreconcilers, have we updated the core technical design doc to reflect that?*

*If this change is adding new safety checks or new potential failure modes, have we documented and how to debug potential issues?*

## Follow-up

*Are there any follow-up issues that we should pursue in the future?*
No

*Does this introduce new defaults that we should re-evaluate in the future?*


foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: 5d3fb1ad6957d85d8d04efaea4034656a5842b99
* Duration 1:26:49
* Result: :x: **FAILED**
* Error: `Error while executing command: make -C tests -kj prOperator. Reason: exit status 2`
* [Build Logs](https://d1e0l78xbkh3xa.cloudfront.net/build-logs/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/497da3f5-efff-4a5f-ad1e-3a99f7bfae45.gz) (available for 30 days)

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: 3829310cb0ad272459cd4a0d9b58ed42d1378bca
* Duration 0:00:15
* Result: :x: **FAILED**
* Error: `Unexpected EC2 error: InvalidGroup.NotFound`
* [Build Logs](https://d1e0l78xbkh3xa.cloudfront.net) (available for 30 days)

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: 41a5e52a8f1bf9a591cc280291f082d6ac3a36f9
* Duration 0:58:03
* Result: :x: **FAILED**
* Error: `Error while executing command: make -C tests -kj prOperator. Reason: exit status 2`
* [Build Logs](https://d1e0l78xbkh3xa.cloudfront.net/build-logs/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/f01991b1-af29-421a-bc7d-a95940bc9281.gz) (available for 30 days)

foundationdb-ci: 
### Result of fdb-kubernetes-operator-pr on Linux CentOS 7

* Commit ID: 0a41b0031cdaaf48ebd2778b44b8eef50d6b3e2e
* Duration 1:01:22
* Result: :x: **FAILED**
* Error: `Error while executing command: make -C tests -kj prOperator. Reason: exit status 2`
* [Build Logs](https://d1e0l78xbkh3xa.cloudfront.net/build-logs/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/822e8846-ddd3-4e5b-b10a-2b4b9395b957.gz) (available for 30 days)


Patch:
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index 7ba92c3a..1052c49d 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -1989,6 +1989,76 @@ func (cluster *FoundationDBCluster) AddProcessGroupsToRemovalList(processGroupID
 	}
 }
 
+// AddProcessGroupsToNoScheduleList adds the provided process group IDs to the no-schedule list.
+// If a process group ID is already present on that list it won't be added a second time.
+func (cluster *FoundationDBCluster) AddProcessGroupsToNoScheduleList(processGroupIDs []string) {
+	noScheduleProcesses := map[string]None{}
+
+	for _, id := range cluster.Spec.Buggify.NoSchedule {
+		noScheduleProcesses[id] = None{}
+	}
+
+	for _, processGroupID := range processGroupIDs {
+		if _, ok := noScheduleProcesses[processGroupID]; ok {
+			continue
+		}
+
+		cluster.Spec.Buggify.NoSchedule = append(cluster.Spec.Buggify.NoSchedule, processGroupID)
+	}
+}
+
+// RemoveProcessGroupsFromNoScheduleList removes the provided process group IDs from the no-schedule list.
+func (cluster *FoundationDBCluster) RemoveProcessGroupsFromNoScheduleList(processGroupIDs []string) {
+	processGroupIDsToRemove := make(map[string]None)
+	for _, processGroupID := range processGroupIDs {
+		processGroupIDsToRemove[processGroupID] = None{}
+	}
+
+	idx := 0
+	for _, processGroupID := range cluster.Spec.Buggify.NoSchedule {
+		if _, ok := processGroupIDsToRemove[processGroupID]; ok {
+			continue
+		}
+		cluster.Spec.Buggify.NoSchedule[idx] = processGroupID
+		idx++
+	}
+
+	cluster.Spec.Buggify.NoSchedule = cluster.Spec.Buggify.NoSchedule[:idx]
+}
+
+// AddProcessGroupsToCrashLoopList adds the provided process group IDs to the crash-loop list.
+// If a process group ID is already present on that list or all the processes are set into crash-loop
+// it won't be added a second time.
+func (cluster *FoundationDBCluster) AddProcessGroupsToCrashLoopList(processGroupIDs []string) {
+	crashLoop, _ := cluster.GetCrashLoopProcessGroups()
+
+	for _, processGroupID := range processGroupIDs {
+		if _, ok := crashLoop[processGroupID]; ok {
+			continue
+		}
+
+		cluster.Spec.Buggify.CrashLoop = append(cluster.Spec.Buggify.CrashLoop, processGroupID)
+	}
+}
+
+// RemoveProcessGroupsFromCrashLoopList removes the provided process group IDs from the crash-loop list.
+func (cluster *FoundationDBCluster) RemoveProcessGroupsFromCrashLoopList(processGroupIDs []string) {
+	processGroupIDsToRemove := make(map[string]None)
+	for _, processGroupID := range processGroupIDs {
+		processGroupIDsToRemove[processGroupID] = None{}
+	}
+
+	idx := 0
+	for _, processGroupID := range cluster.Spec.Buggify.CrashLoop {
+		if _, ok := processGroupIDsToRemove[processGroupID]; ok {
+			continue
+		}
+		cluster.Spec.Buggify.CrashLoop[idx] = processGroupID
+		idx++
+	}
+	cluster.Spec.Buggify.CrashLoop = cluster.Spec.Buggify.CrashLoop[:idx]
+}
+
 // AddProcessGroupsToRemovalWithoutExclusionList adds the provided process group IDs to the remove without exclusion list.
 // If a process group ID is already present on that list it won't be added a second time.
 func (cluster *FoundationDBCluster) AddProcessGroupsToRemovalWithoutExclusionList(processGroupIDs []string) {
@@ -2024,14 +2094,15 @@ func (cluster *FoundationDBCluster) GetRunningVersion() string {
 // if all process group IDs in a cluster should be crash looping.
 func (cluster *FoundationDBCluster) GetCrashLoopProcessGroups() (map[string]None, bool) {
 	crashLoopPods := make(map[string]None, len(cluster.Spec.Buggify.CrashLoop))
+	crashLoopAll := false
 	for _, processGroupID := range cluster.Spec.Buggify.CrashLoop {
 		if processGroupID == "*" {
-			return nil, true
+			crashLoopAll = true
 		}
 		crashLoopPods[processGroupID] = None{}
 	}
 
-	return crashLoopPods, false
+	return crashLoopPods, crashLoopAll
 }
 
 // Validate checks if all settings in the cluster are valid, if not and error will be returned. If multiple issues are
diff --git a/charts/fdb-operator/Chart.yaml b/charts/fdb-operator/Chart.yaml
index 15073fe0..ebcdccf9 100644
--- a/charts/fdb-operator/Chart.yaml
+++ b/charts/fdb-operator/Chart.yaml
@@ -21,7 +21,7 @@ version: 0.2.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: v1.7.0
+appVersion: v1.8.0
 
 maintainers:
   - name: "foundationdb-ci"
diff --git a/charts/fdb-operator/values.yaml b/charts/fdb-operator/values.yaml
index b9d9dac4..4bf3380e 100644
--- a/charts/fdb-operator/values.yaml
+++ b/charts/fdb-operator/values.yaml
@@ -1,7 +1,7 @@
 ---
 image:
   repository: foundationdb/fdb-kubernetes-operator
-  tag: v1.7.0
+  tag: v1.8.0
   pullPolicy: IfNotPresent
 
 initContainers:
@@ -18,7 +18,7 @@ initContainers:
   7.1:
     image:
       repository: foundationdb/foundationdb-kubernetes-sidecar
-      tag: 7.1.5-1
+      tag: 7.1.15-1
       pullPolicy: IfNotPresent
 
 globalMode:
diff --git a/config/deployment/manager.yaml b/config/deployment/manager.yaml
index df32381e..4c108457 100644
--- a/config/deployment/manager.yaml
+++ b/config/deployment/manager.yaml
@@ -64,7 +64,7 @@ spec:
             - name: fdb-binaries
               mountPath: /var/output-files
         - name: foundationdb-kubernetes-init-7-1
-          image: foundationdb/foundationdb-kubernetes-sidecar:7.1.11-1
+          image: foundationdb/foundationdb-kubernetes-sidecar:7.1.15-1
           args:
             - "--copy-library"
             - "7.1"
@@ -75,7 +75,7 @@ spec:
             - "--copy-binary"
             - "fdbrestore"
             - "--output-dir"
-            - "/var/output-files/7.1.11"
+            - "/var/output-files/7.1.15"
             - "--init-mode"
           volumeMounts:
             - name: fdb-binaries
@@ -83,7 +83,7 @@ spec:
       containers:
       - command:
         - /manager
-        image: foundationdb/fdb-kubernetes-operator:v1.7.0
+        image: foundationdb/fdb-kubernetes-operator:v1.8.0
         name: manager
         env:
           - name: WATCH_NAMESPACE
diff --git a/config/samples/deployment.yaml b/config/samples/deployment.yaml
index 05523bd6..9d52dc69 100644
--- a/config/samples/deployment.yaml
+++ b/config/samples/deployment.yaml
@@ -150,7 +150,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: foundationdb/fdb-kubernetes-operator:v1.7.0
+        image: foundationdb/fdb-kubernetes-operator:v1.8.0
         name: manager
         ports:
         - containerPort: 8080
@@ -218,9 +218,9 @@ spec:
         - --copy-binary
         - fdbrestore
         - --output-dir
-        - /var/output-files/7.1.11
+        - /var/output-files/7.1.15
         - --init-mode
-        image: foundationdb/foundationdb-kubernetes-sidecar:7.1.11-1
+        image: foundationdb/foundationdb-kubernetes-sidecar:7.1.15-1
         name: foundationdb-kubernetes-init-7-1
         volumeMounts:
         - mountPath: /var/output-files
diff --git a/docs/changelog/v1.8.0.md b/docs/changelog/v1.8.0.md
new file mode 100644
index 00000000..6f363963
--- /dev/null
+++ b/docs/changelog/v1.8.0.md
@@ -0,0 +1,15 @@
+# v1.8.0
+
+## Changes
+
+### Operator
+
+- Fix bug where config update is triggered to often [#1280](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1280)
+- Use DB key for connection string [#1262](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1262)
+- Retry flaky test [#1267](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1267)
+
+### Tooling
+
+- Update customization.md [#1290](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1290)
+- Fix setup of deployment [#1282](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1282)
+- Fix how goreleaser fetches the current tag [#1272](https://github.com/FoundationDB/fdb-kubernetes-operator/pull/1272/)
diff --git a/docs/manual/customization.md b/docs/manual/customization.md
index bfc49baa..d9a540be 100644
--- a/docs/manual/customization.md
+++ b/docs/manual/customization.md
@@ -19,7 +19,7 @@ spec:
     storageServersPerPod: 2
 ```
 
-A change to the `storageServersPerPod` will replace all of the storage pods. For more information about this feature read the [multiple storage servers per pod](/docs/design/multiple_storage_per_disk.md) design doc.
+A change to the `storageServersPerPod` will replace all of the storage pods. For more information about this feature read the [multiple storage servers per pod](/docs/design/implemented/multiple_storage_per_disk.md) design doc.
 
 ## Customizing the Volumes
 
diff --git a/kubectl-fdb/cmd/analyze.go b/kubectl-fdb/cmd/analyze.go
index 3848ec19..ca719a34 100644
--- a/kubectl-fdb/cmd/analyze.go
+++ b/kubectl-fdb/cmd/analyze.go
@@ -143,11 +143,11 @@ kubectl fdb -n test-namespace analyze sample-cluster-1
 kubectl fdb analyze --auto-fix sample-cluster-1
 
 # Analyze the cluster "sample-cluster-1" in the current namespace and ignore the IncorrectCommandLine and IncorrectPodSpec condition
-kubectl fdb analyze cluster --ignore-condition=IncorrectCommandLine --ignore-condition=IncorrectPodSpec sample-cluster-1
+kubectl fdb analyze --ignore-condition=IncorrectCommandLine --ignore-condition=IncorrectPodSpec sample-cluster-1
 
 # Per default the plugin will print out how many process groups are marked for removal instead of printing out each process group.
 # This can be disabled by using the ignore-removals flag to print out the details about process groups that are marked for removal.
-kubectl fdb analyze cluster --ignore-removals=false sample-cluster-1
+kubectl fdb analyze --ignore-removals=false sample-cluster-1
 `,
 	}
 	cmd.SetOut(o.Out)
diff --git a/kubectl-fdb/cmd/buggify.go b/kubectl-fdb/cmd/buggify.go
new file mode 100644
index 00000000..ccebc4f9
--- /dev/null
+++ b/kubectl-fdb/cmd/buggify.go
@@ -0,0 +1,80 @@
+/*
+ * buggify.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+
+	"github.com/spf13/cobra"
+)
+
+func newBuggifyCmd(streams genericclioptions.IOStreams) *cobra.Command {
+	o := newFDBOptions(streams)
+
+	cmd := &cobra.Command{
+		Use:   "buggify",
+		Short: "Subcommand to add process groups to buggify list for a given cluster",
+		Long: "Subcommand to add process groups to buggify list for a given cluster. " +
+			"Supported options: crash-loop, no-schedule, empty-monitor-conf.",
+		RunE: func(c *cobra.Command, args []string) error {
+			return c.Help()
+		},
+		Example: `
+kubectl fdb -n <namespace> buggify <option> -c <cluster> pod-1 pod-2
+
+# Add process groups into crash loop state for a cluster in the current namespace
+kubectl fdb buggify crash-loop -c cluster pod-1 pod-2
+
+# Remove process groups from crash loop state from a cluster in the current namespace
+kubectl fdb buggify crash-loop --clear -c cluster pod-1 pod-2
+
+# Clean crash loop list of a cluster in the current namespace
+kubectl fdb buggify crash-loop --clean -c cluster
+
+# Add process groups into no-schedule state for a cluster in the current namespace
+kubectl fdb buggify no-schedule -c cluster pod-1 pod-2
+
+# Remove process groups from no-schedule state from a cluster in the current namespace
+kubectl fdb buggify no-schedule --clear -c cluster pod-1 pod-2
+
+# Clean no-schedule list of a cluster in the current namespace
+kubectl fdb buggify no-schedule  --clean -c cluster
+
+# Setting empty-monitor-conf to true
+kubectl fdb buggify empty-monitor-conf -c cluster
+
+# Setting empty-monitor-conf to false
+kubectl fdb buggify empty-monitor-conf --unset -c cluster
+`,
+	}
+	cmd.SetOut(o.Out)
+	cmd.SetErr(o.ErrOut)
+	cmd.SetIn(o.In)
+
+	cmd.AddCommand(
+		newBuggifyCrashLoop(streams),
+		newBuggifyNoSchedule(streams),
+		newBuggifyEmptyMonitorConf(streams),
+	)
+	o.configFlags.AddFlags(cmd.Flags())
+
+	return cmd
+}
diff --git a/kubectl-fdb/cmd/buggify_crash_loop.go b/kubectl-fdb/cmd/buggify_crash_loop.go
new file mode 100644
index 00000000..3d7a4008
--- /dev/null
+++ b/kubectl-fdb/cmd/buggify_crash_loop.go
@@ -0,0 +1,150 @@
+/*
+ * buggify_crash_loop.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	ctx "context"
+	"fmt"
+	"log"
+
+	"github.com/spf13/cobra"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func newBuggifyCrashLoop(streams genericclioptions.IOStreams) *cobra.Command {
+	o := newFDBOptions(streams)
+
+	cmd := &cobra.Command{
+		Use:   "crash-loop",
+		Short: "Updates the crash-loop list of the given cluster",
+		Long:  "Updates the crash-loop list of the given cluster",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			wait, err := cmd.Root().Flags().GetBool("wait")
+			if err != nil {
+				return err
+			}
+			clear, err := cmd.Flags().GetBool("clear")
+			if err != nil {
+				return err
+			}
+			clean, err := cmd.Flags().GetBool("clean")
+			if err != nil {
+				return err
+			}
+			cluster, err := cmd.Flags().GetString("fdb-cluster")
+			if err != nil {
+				return err
+			}
+
+			kubeClient, err := getKubeClient(o)
+			if err != nil {
+				return err
+			}
+
+			namespace, err := getNamespace(*o.configFlags.Namespace)
+			if err != nil {
+				return err
+			}
+
+			processGroups, err := getProcessGroupIDsFromPod(kubeClient, cluster, args, namespace)
+			if err != nil {
+				return err
+			}
+
+			return updateCrashLoopList(kubeClient, cluster, processGroups, namespace, wait, clear, clean)
+		},
+		Example: `
+# Add process groups into crash loop state for a cluster in the current namespace
+kubectl fdb buggify crash-loop -c cluster pod-1 pod-2
+
+# Remove process groups from crash loop state from a cluster in the current namespace
+kubectl fdb buggify crash-loop --clear -c cluster pod-1 pod-2
+
+# Clean crash loop list of a cluster in the current namespace
+kubectl fdb buggify crash-loop --clean -c cluster
+
+# Add process groups into crash loop state for a cluster in the namespace default
+kubectl fdb -n default buggify crash-loop -c cluster pod-1 pod-2
+`,
+	}
+	cmd.Flags().StringP("fdb-cluster", "c", "", "updates the crash-loop list in the provided cluster.")
+	cmd.Flags().Bool("clear", false, "removes the process groups from the crash-loop list.")
+	cmd.Flags().Bool("clean", false, "removes all process groups from the crash-loop list.")
+	err := cmd.MarkFlagRequired("fdb-cluster")
+	if err != nil {
+		log.Fatal(err)
+	}
+	cmd.SetOut(o.Out)
+	cmd.SetErr(o.ErrOut)
+	cmd.SetIn(o.In)
+
+	o.configFlags.AddFlags(cmd.Flags())
+
+	return cmd
+}
+
+// updateCrashLoopList updates the crash-loop list of the cluster
+func updateCrashLoopList(kubeClient client.Client, clusterName string, processGroups []string, namespace string, wait bool, clear bool, clean bool) error {
+	cluster, err := loadCluster(kubeClient, namespace, clusterName)
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			return fmt.Errorf("could not get cluster: %s/%s", namespace, clusterName)
+		}
+		return err
+	}
+
+	patch := client.MergeFrom(cluster.DeepCopy())
+	if clean {
+		if wait {
+			if !confirmAction(fmt.Sprintf("Clearing crash-loop list from cluster %s/%s", namespace, clusterName)) {
+				return fmt.Errorf("user aborted the removal")
+			}
+		}
+		cluster.Spec.Buggify.CrashLoop = nil
+		return kubeClient.Patch(ctx.TODO(), cluster, patch)
+	}
+
+	if len(processGroups) == 0 {
+		return fmt.Errorf("please provide atleast one pod")
+	}
+
+	if wait {
+		if clear {
+			if !confirmAction(fmt.Sprintf("Removing %v to crash-loop from cluster %s/%s", processGroups, namespace, clusterName)) {
+				return fmt.Errorf("user aborted the removal")
+			}
+		} else {
+			if !confirmAction(fmt.Sprintf("Adding %v to crash-loop from cluster %s/%s", processGroups, namespace, clusterName)) {
+				return fmt.Errorf("user aborted the removal")
+			}
+		}
+	}
+
+	if clear {
+		cluster.RemoveProcessGroupsFromCrashLoopList(processGroups)
+	} else {
+		cluster.AddProcessGroupsToCrashLoopList(processGroups)
+	}
+
+	return kubeClient.Patch(ctx.TODO(), cluster, patch)
+}
diff --git a/kubectl-fdb/cmd/buggify_empty_monitor_conf.go b/kubectl-fdb/cmd/buggify_empty_monitor_conf.go
new file mode 100644
index 00000000..9ae6baf8
--- /dev/null
+++ b/kubectl-fdb/cmd/buggify_empty_monitor_conf.go
@@ -0,0 +1,115 @@
+/*
+ * buggify_empty_monitor_conf.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	ctx "context"
+	"fmt"
+	"log"
+
+	"github.com/spf13/cobra"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func newBuggifyEmptyMonitorConf(streams genericclioptions.IOStreams) *cobra.Command {
+	o := newFDBOptions(streams)
+
+	cmd := &cobra.Command{
+		Use:   "empty-monitor-conf",
+		Short: "Instructs the operator to update all of the fdbmonitor.conf files to have zero fdbserver processes configured",
+		Long:  "Instructs the operator to update all of the fdbmonitor.conf files to have zero fdbserver processes configured",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			wait, err := cmd.Root().Flags().GetBool("wait")
+			if err != nil {
+				return err
+			}
+			set, err := cmd.Flags().GetBool("set-value")
+			if err != nil {
+				return err
+			}
+			cluster, err := cmd.Flags().GetString("fdb-cluster")
+			if err != nil {
+				return err
+			}
+
+			kubeClient, err := getKubeClient(o)
+			if err != nil {
+				return err
+			}
+
+			namespace, err := getNamespace(*o.configFlags.Namespace)
+			if err != nil {
+				return err
+			}
+
+			return updateMonitorConf(kubeClient, cluster, namespace, wait, set)
+		},
+		Example: `
+# Setting empty-monitor-conf to true
+kubectl fdb buggify empty-monitor-conf -c cluster
+
+# Setting empty-monitor-conf to false
+kubectl fdb buggify empty-monitor-conf --set-value=false -c cluster
+`,
+	}
+
+	cmd.Flags().StringP("fdb-cluster", "c", "", "updates the empty-monitor-conf for the cluster.")
+	cmd.Flags().Bool("set-value", true, "set the empty-monitor-conf to true.")
+	err := cmd.MarkFlagRequired("fdb-cluster")
+	if err != nil {
+		log.Fatal(err)
+	}
+	cmd.SetOut(o.Out)
+	cmd.SetErr(o.ErrOut)
+	cmd.SetIn(o.In)
+
+	o.configFlags.AddFlags(cmd.Flags())
+
+	return cmd
+}
+
+// updateMonitorConf updates the monitor conf of the cluster
+func updateMonitorConf(kubeClient client.Client, clusterName string, namespace string, wait bool, set bool) error {
+	cluster, err := loadCluster(kubeClient, namespace, clusterName)
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			return fmt.Errorf("could not get cluster: %s/%s", namespace, clusterName)
+		}
+		return err
+	}
+
+	if cluster.Spec.Buggify.EmptyMonitorConf == set {
+		return fmt.Errorf("empty-monitor-conf is already set to %v", set)
+	}
+
+	patch := client.MergeFrom(cluster.DeepCopy())
+
+	if wait {
+		if !confirmAction(fmt.Sprintf("Setting empty-monitor-conf to %v for cluster %s/%s", set, namespace, clusterName)) {
+			return fmt.Errorf("user aborted the removal")
+		}
+	}
+
+	cluster.Spec.Buggify.EmptyMonitorConf = set
+	return kubeClient.Patch(ctx.TODO(), cluster, patch)
+}
diff --git a/kubectl-fdb/cmd/buggify_no_schedule.go b/kubectl-fdb/cmd/buggify_no_schedule.go
new file mode 100644
index 00000000..4d47af35
--- /dev/null
+++ b/kubectl-fdb/cmd/buggify_no_schedule.go
@@ -0,0 +1,151 @@
+/*
+ * buggify_no_schedule.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	ctx "context"
+	"fmt"
+	"log"
+
+	"github.com/spf13/cobra"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func newBuggifyNoSchedule(streams genericclioptions.IOStreams) *cobra.Command {
+	o := newFDBOptions(streams)
+
+	cmd := &cobra.Command{
+		Use:   "no-schedule",
+		Short: "Updates the no-schedule list of the given cluster",
+		Long:  "Updates the no-schedule list of the given cluster\"",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			wait, err := cmd.Root().Flags().GetBool("wait")
+			if err != nil {
+				return err
+			}
+			clear, err := cmd.Flags().GetBool("clear")
+			if err != nil {
+				return err
+			}
+			clean, err := cmd.Flags().GetBool("clean")
+			if err != nil {
+				return err
+			}
+			cluster, err := cmd.Flags().GetString("fdb-cluster")
+			if err != nil {
+				return err
+			}
+
+			kubeClient, err := getKubeClient(o)
+			if err != nil {
+				return err
+			}
+
+			namespace, err := getNamespace(*o.configFlags.Namespace)
+			if err != nil {
+				return err
+			}
+
+			processGroups, err := getProcessGroupIDsFromPod(kubeClient, cluster, args, namespace)
+			if err != nil {
+				return err
+			}
+
+			return updateNoScheduleList(kubeClient, cluster, processGroups, namespace, wait, clear, clean)
+		},
+		Example: `
+# Add process groups into no-schedule state for a cluster in the current namespace
+kubectl fdb buggify no-schedule -c cluster pod-1 pod-2
+
+# Remove process groups from no-schedule state from a cluster in the current namespace
+kubectl fdb buggify no-schedule --clear -c cluster pod-1 pod-2
+
+# Clean no-schedule list of a cluster in the current namespace
+kubectl fdb buggify no-schedule  --clean -c cluster
+
+# Add process groups into no-schedule state for a cluster in the namespace default
+kubectl fdb -n default buggify no-schedule  -c cluster pod-1 pod-2
+`,
+	}
+
+	cmd.Flags().StringP("fdb-cluster", "c", "", "updates the no-schedule list in the provided cluster.")
+	cmd.Flags().Bool("clear", false, "removes the process groups from the no-schedule list.")
+	cmd.Flags().Bool("clean", false, "removes all process groups from the no-schedule list.")
+	err := cmd.MarkFlagRequired("fdb-cluster")
+	if err != nil {
+		log.Fatal(err)
+	}
+	cmd.SetOut(o.Out)
+	cmd.SetErr(o.ErrOut)
+	cmd.SetIn(o.In)
+
+	o.configFlags.AddFlags(cmd.Flags())
+
+	return cmd
+}
+
+// updateNoScheduleList updates the removal list of the cluster
+func updateNoScheduleList(kubeClient client.Client, clusterName string, processGroups []string, namespace string, wait bool, clear bool, clean bool) error {
+	cluster, err := loadCluster(kubeClient, namespace, clusterName)
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			return fmt.Errorf("could not get cluster: %s/%s", namespace, clusterName)
+		}
+		return err
+	}
+
+	patch := client.MergeFrom(cluster.DeepCopy())
+	if clean {
+		if wait {
+			if !confirmAction(fmt.Sprintf("Clearing no-schedule list from cluster %s/%s", namespace, clusterName)) {
+				return fmt.Errorf("user aborted the removal")
+			}
+		}
+		cluster.Spec.Buggify.NoSchedule = nil
+		return kubeClient.Patch(ctx.TODO(), cluster, patch)
+	}
+
+	if len(processGroups) == 0 {
+		return fmt.Errorf("please provide atleast one pod")
+	}
+
+	if wait {
+		if clear {
+			if !confirmAction(fmt.Sprintf("Removing %v from no-schedule from cluster %s/%s", processGroups, namespace, clusterName)) {
+				return fmt.Errorf("user aborted the removal")
+			}
+		} else {
+			if !confirmAction(fmt.Sprintf("Adding %v from no-schedule from cluster %s/%s", processGroups, namespace, clusterName)) {
+				return fmt.Errorf("user aborted the removal")
+			}
+		}
+	}
+
+	if clear {
+		cluster.RemoveProcessGroupsFromNoScheduleList(processGroups)
+	} else {
+		cluster.AddProcessGroupsToNoScheduleList(processGroups)
+	}
+
+	return kubeClient.Patch(ctx.TODO(), cluster, patch)
+}
diff --git a/kubectl-fdb/cmd/k8s_client.go b/kubectl-fdb/cmd/k8s_client.go
index f9771a80..71714a49 100644
--- a/kubectl-fdb/cmd/k8s_client.go
+++ b/kubectl-fdb/cmd/k8s_client.go
@@ -198,3 +198,31 @@ func getAllPodsFromClusterWithCondition(kubeClient client.Client, clusterName st
 
 	return processes, nil
 }
+
+func getProcessGroupIDsFromPod(kubeClient client.Client, clusterName string, podNames []string, namespace string) ([]string, error) {
+	processGroups := make([]string, 0, len(podNames))
+	// Build a map to filter faster
+	podNameMap := map[string]struct{}{}
+	for _, name := range podNames {
+		podNameMap[name] = struct{}{}
+	}
+
+	cluster, err := loadCluster(kubeClient, namespace, clusterName)
+	if err != nil {
+		return processGroups, err
+	}
+	pods, err := getPodsForCluster(kubeClient, cluster, namespace)
+	if err != nil {
+		return processGroups, err
+	}
+
+	for _, pod := range pods.Items {
+		if _, ok := podNameMap[pod.Name]; !ok {
+			continue
+		}
+
+		processGroups = append(processGroups, pod.Labels[cluster.GetProcessGroupIDLabel()])
+	}
+
+	return processGroups, nil
+}
diff --git a/kubectl-fdb/cmd/remove_process_group.go b/kubectl-fdb/cmd/remove_process_group.go
index 1511799a..797f7209 100644
--- a/kubectl-fdb/cmd/remove_process_group.go
+++ b/kubectl-fdb/cmd/remove_process_group.go
@@ -121,34 +121,6 @@ kubectl fdb -n default remove process-group -c cluster --remove-all-failed
 	return cmd
 }
 
-func getProcessGroupIDsFromPod(kubeClient client.Client, clusterName string, podNames []string, namespace string) ([]string, error) {
-	processGroups := make([]string, 0, len(podNames))
-	// Build a map to filter faster
-	podNameMap := map[string]struct{}{}
-	for _, name := range podNames {
-		podNameMap[name] = struct{}{}
-	}
-
-	cluster, err := loadCluster(kubeClient, namespace, clusterName)
-	if err != nil {
-		return processGroups, err
-	}
-	pods, err := getPodsForCluster(kubeClient, cluster, namespace)
-	if err != nil {
-		return processGroups, err
-	}
-
-	for _, pod := range pods.Items {
-		if _, ok := podNameMap[pod.Name]; !ok {
-			continue
-		}
-
-		processGroups = append(processGroups, pod.Labels[cluster.GetProcessGroupIDLabel()])
-	}
-
-	return processGroups, nil
-}
-
 // replaceProcessGroups adds process groups to the removal list of the cluster
 func replaceProcessGroups(kubeClient client.Client, clusterName string, processGroups []string, namespace string, withExclusion bool, withShrink bool, wait bool, removeAllFailed bool) error {
 	cluster, err := loadCluster(kubeClient, namespace, clusterName)
diff --git a/kubectl-fdb/cmd/root.go b/kubectl-fdb/cmd/root.go
index fb044801..adbd9543 100644
--- a/kubectl-fdb/cmd/root.go
+++ b/kubectl-fdb/cmd/root.go
@@ -25,6 +25,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+
 	"strings"
 
 	"github.com/spf13/viper"
@@ -80,6 +81,7 @@ func NewRootCmd(streams genericclioptions.IOStreams) *cobra.Command {
 		newDeprecationCmd(streams),
 		newFixCoordinatorIPsCmd(streams),
 		newGetCmd(streams),
+		newBuggifyCmd(streams),
 	)
 
 	return cmd


Test patch:
diff --git a/api/v1beta2/foundationdbcluster_types_test.go b/api/v1beta2/foundationdbcluster_types_test.go
index 92d299ef..c1fdde37 100644
--- a/api/v1beta2/foundationdbcluster_types_test.go
+++ b/api/v1beta2/foundationdbcluster_types_test.go
@@ -4628,4 +4628,291 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 			),
 		)
 	})
+
+	When("adding processes to the no-schedule list", func() {
+		var cluster *FoundationDBCluster
+
+		BeforeEach(func() {
+			cluster = &FoundationDBCluster{}
+		})
+
+		When("the no-schedule list is empty", func() {
+			type testCase struct {
+				Instances                     []string
+				ExpectedInstancesInNoSchedule []string
+			}
+
+			DescribeTable("should add all targeted processes to the no-schedule list",
+				func(tc testCase) {
+					cluster.AddProcessGroupsToNoScheduleList(tc.Instances)
+					Expect(cluster.Spec.Buggify.NoSchedule).To(ContainElements(tc.ExpectedInstancesInNoSchedule))
+					Expect(len(cluster.Spec.Buggify.NoSchedule)).To(Equal(len(tc.ExpectedInstancesInNoSchedule)))
+				},
+				Entry("Adding single instance",
+					testCase{
+						Instances:                     []string{"instance-1"},
+						ExpectedInstancesInNoSchedule: []string{"instance-1"},
+					}),
+				Entry("Adding multiple instances",
+					testCase{
+						Instances:                     []string{"instance-1", "instance-2"},
+						ExpectedInstancesInNoSchedule: []string{"instance-1", "instance-2"},
+					}),
+			)
+		})
+
+		When("the no-schedule list is not empty", func() {
+			BeforeEach(func() {
+				cluster.Spec.Buggify.NoSchedule = []string{"instance-1"}
+			})
+
+			type testCase struct {
+				Instances                     []string
+				ExpectedInstancesInNoSchedule []string
+			}
+
+			DescribeTable("should add all targeted processes to no-schedule list",
+				func(tc testCase) {
+					cluster.AddProcessGroupsToNoScheduleList(tc.Instances)
+					Expect(cluster.Spec.Buggify.NoSchedule).To(ContainElements(tc.ExpectedInstancesInNoSchedule))
+					Expect(len(cluster.Spec.Buggify.NoSchedule)).To(Equal(len(tc.ExpectedInstancesInNoSchedule)))
+				},
+				Entry("Adding single instance",
+					testCase{
+						Instances:                     []string{"instance-2"},
+						ExpectedInstancesInNoSchedule: []string{"instance-1", "instance-2"},
+					}),
+				Entry("Adding multiple instances",
+					testCase{
+						Instances:                     []string{"instance-2", "instance-3"},
+						ExpectedInstancesInNoSchedule: []string{"instance-1", "instance-2", "instance-3"},
+					}),
+			)
+		})
+
+	})
+
+	When("removing processes from the no-schedule list", func() {
+		var cluster *FoundationDBCluster
+
+		BeforeEach(func() {
+			cluster = &FoundationDBCluster{
+				Spec: FoundationDBClusterSpec{
+					Buggify: BuggifyConfig{
+						NoSchedule: []string{"instance-1", "instance-2", "instance-3"},
+					},
+				},
+			}
+		})
+
+		type testCase struct {
+			Instances                         []string
+			ExpectedInstancesInNoScheduleList []string
+		}
+
+		DescribeTable("should remove all targeted processes from the no-schedule list",
+			func(tc testCase) {
+				cluster.RemoveProcessGroupsFromNoScheduleList(tc.Instances)
+				Expect(cluster.Spec.Buggify.NoSchedule).To(ContainElements(tc.ExpectedInstancesInNoScheduleList))
+				Expect(len(cluster.Spec.Buggify.NoSchedule)).To(Equal(len(tc.ExpectedInstancesInNoScheduleList)))
+			},
+			Entry("Removing single instance",
+				testCase{
+					Instances:                         []string{"instance-1"},
+					ExpectedInstancesInNoScheduleList: []string{"instance-2", "instance-3"},
+				}),
+			Entry("Removing multiple instances",
+				testCase{
+					Instances:                         []string{"instance-2", "instance-3"},
+					ExpectedInstancesInNoScheduleList: []string{"instance-1"},
+				}),
+		)
+
+	})
+
+	When("adding processes to the crash-loop list", func() {
+		var cluster *FoundationDBCluster
+
+		BeforeEach(func() {
+			cluster = &FoundationDBCluster{}
+		})
+
+		When("the crash-loop list is empty", func() {
+			type testCase struct {
+				Instances                    []string
+				ExpectedInstancesInCrashLoop []string
+			}
+
+			DescribeTable("should add all targeted processes to the crash-loop list",
+				func(tc testCase) {
+					cluster.AddProcessGroupsToCrashLoopList(tc.Instances)
+					Expect(cluster.Spec.Buggify.CrashLoop).To(ContainElements(tc.ExpectedInstancesInCrashLoop))
+					Expect(len(cluster.Spec.Buggify.CrashLoop)).To(Equal(len(tc.ExpectedInstancesInCrashLoop)))
+				},
+				Entry("Adding single instance",
+					testCase{
+						Instances:                    []string{"instance-1"},
+						ExpectedInstancesInCrashLoop: []string{"instance-1"},
+					}),
+				Entry("Adding multiple instances",
+					testCase{
+						Instances:                    []string{"instance-1", "instance-2"},
+						ExpectedInstancesInCrashLoop: []string{"instance-1", "instance-2"},
+					}),
+				Entry("Adding all instances",
+					testCase{
+						Instances:                    []string{"*"},
+						ExpectedInstancesInCrashLoop: []string{"*"},
+					}),
+			)
+		})
+
+		When("the crash-loop list is not empty", func() {
+			BeforeEach(func() {
+				cluster.Spec.Buggify.CrashLoop = []string{"instance-1"}
+			})
+
+			type testCase struct {
+				Instances                    []string
+				ExpectedInstancesInCrashLoop []string
+			}
+
+			DescribeTable("should add all targeted processes to crash-loop list",
+				func(tc testCase) {
+					cluster.AddProcessGroupsToCrashLoopList(tc.Instances)
+					Expect(cluster.Spec.Buggify.CrashLoop).To(ContainElements(tc.ExpectedInstancesInCrashLoop))
+					Expect(len(cluster.Spec.Buggify.CrashLoop)).To(Equal(len(tc.ExpectedInstancesInCrashLoop)))
+				},
+				Entry("Adding single instance",
+					testCase{
+						Instances:                    []string{"instance-2"},
+						ExpectedInstancesInCrashLoop: []string{"instance-1", "instance-2"},
+					}),
+				Entry("Adding multiple instances",
+					testCase{
+						Instances:                    []string{"instance-2", "instance-3"},
+						ExpectedInstancesInCrashLoop: []string{"instance-1", "instance-2", "instance-3"},
+					}),
+				Entry("Adding all instances",
+					testCase{
+						Instances:                    []string{"*"},
+						ExpectedInstancesInCrashLoop: []string{"instance-1", "*"},
+					}),
+			)
+		})
+
+		When("the crash-loop list contains *", func() {
+			BeforeEach(func() {
+				cluster.Spec.Buggify.CrashLoop = []string{"*"}
+			})
+
+			type testCase struct {
+				Instances                    []string
+				ExpectedInstancesInCrashLoop []string
+			}
+
+			DescribeTable("should add all targeted processes to crash-loop list",
+				func(tc testCase) {
+					cluster.AddProcessGroupsToCrashLoopList(tc.Instances)
+					Expect(cluster.Spec.Buggify.CrashLoop).To(ContainElements(tc.ExpectedInstancesInCrashLoop))
+					Expect(len(cluster.Spec.Buggify.CrashLoop)).To(Equal(len(tc.ExpectedInstancesInCrashLoop)))
+				},
+				Entry("Adding single instance",
+					testCase{
+						Instances:                    []string{"instance-1"},
+						ExpectedInstancesInCrashLoop: []string{"instance-1", "*"},
+					}),
+				Entry("Adding multiple instances",
+					testCase{
+						Instances:                    []string{"instance-2", "instance-3"},
+						ExpectedInstancesInCrashLoop: []string{"*", "instance-2", "instance-3"},
+					}),
+				Entry("Adding all instances",
+					testCase{
+						Instances:                    []string{"*"},
+						ExpectedInstancesInCrashLoop: []string{"*"},
+					}),
+			)
+		})
+
+	})
+
+	When("removing processes from the crash-loop list", func() {
+		var cluster *FoundationDBCluster
+
+		When("the crash-loop list does not contain *", func() {
+			BeforeEach(func() {
+				cluster = &FoundationDBCluster{
+					Spec: FoundationDBClusterSpec{
+						Buggify: BuggifyConfig{
+							CrashLoop: []string{"instance-1", "instance-2", "instance-3"},
+						},
+					},
+				}
+			})
+
+			type testCase struct {
+				Instances                    []string
+				ExpectedInstancesInCrashLoop []string
+			}
+
+			DescribeTable("should remove all targeted processes from the crash-loop list",
+				func(tc testCase) {
+					cluster.RemoveProcessGroupsFromCrashLoopList(tc.Instances)
+					Expect(cluster.Spec.Buggify.CrashLoop).To(ContainElements(tc.ExpectedInstancesInCrashLoop))
+					Expect(len(cluster.Spec.Buggify.CrashLoop)).To(Equal(len(tc.ExpectedInstancesInCrashLoop)))
+				},
+				Entry("Removing single instance",
+					testCase{
+						Instances:                    []string{"instance-1"},
+						ExpectedInstancesInCrashLoop: []string{"instance-2", "instance-3"},
+					}),
+				Entry("Removing multiple instances",
+					testCase{
+						Instances:                    []string{"instance-2", "instance-3"},
+						ExpectedInstancesInCrashLoop: []string{"instance-1"},
+					}),
+			)
+		})
+
+		When("the crash-loop list contain *", func() {
+			BeforeEach(func() {
+				cluster = &FoundationDBCluster{
+					Spec: FoundationDBClusterSpec{
+						Buggify: BuggifyConfig{
+							CrashLoop: []string{"*", "instance-1", "instance-2", "instance-3"},
+						},
+					},
+				}
+			})
+
+			type testCase struct {
+				Instances                    []string
+				ExpectedInstancesInCrashLoop []string
+			}
+
+			DescribeTable("should remove all targeted processes from the crash-loop list",
+				func(tc testCase) {
+					cluster.RemoveProcessGroupsFromCrashLoopList(tc.Instances)
+					Expect(cluster.Spec.Buggify.CrashLoop).To(ContainElements(tc.ExpectedInstancesInCrashLoop))
+					Expect(len(cluster.Spec.Buggify.CrashLoop)).To(Equal(len(tc.ExpectedInstancesInCrashLoop)))
+				},
+				Entry("Removing single instance",
+					testCase{
+						Instances:                    []string{"instance-1"},
+						ExpectedInstancesInCrashLoop: []string{"*", "instance-2", "instance-3"},
+					}),
+				Entry("Removing multiple instances",
+					testCase{
+						Instances:                    []string{"instance-2", "instance-3"},
+						ExpectedInstancesInCrashLoop: []string{"*", "instance-1"},
+					}),
+				Entry("Removing *",
+					testCase{
+						Instances:                    []string{"*"},
+						ExpectedInstancesInCrashLoop: []string{"instance-1", "instance-2", "instance-3"},
+					}),
+			)
+		})
+	})
 })
diff --git a/kubectl-fdb/cmd/buggify_crash_loop_test.go b/kubectl-fdb/cmd/buggify_crash_loop_test.go
new file mode 100644
index 00000000..b8809f86
--- /dev/null
+++ b/kubectl-fdb/cmd/buggify_crash_loop_test.go
@@ -0,0 +1,214 @@
+/*
+ * buggify_crash_loop_test.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	ctx "context"
+
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("[plugin] buggify crash-loop instances command", func() {
+	clusterName := "test"
+	namespace := "test"
+
+	var cluster *fdbv1beta2.FoundationDBCluster
+
+	BeforeEach(func() {
+		cluster = &fdbv1beta2.FoundationDBCluster{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      clusterName,
+				Namespace: namespace,
+			},
+			Spec: fdbv1beta2.FoundationDBClusterSpec{
+				ProcessCounts: fdbv1beta2.ProcessCounts{
+					Storage: 1,
+				},
+			},
+		}
+	})
+
+	When("running buggify crash-loop instances command", func() {
+		When("adding instances to crash-loop list from a cluster", func() {
+
+			type testCase struct {
+				Instances                    []string
+				ExpectedInstancesInCrashLoop []string
+			}
+
+			DescribeTable("should add all targeted processes to crash-loop list",
+				func(tc testCase) {
+					scheme := runtime.NewScheme()
+					_ = clientgoscheme.AddToScheme(scheme)
+					_ = fdbv1beta2.AddToScheme(scheme)
+					kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
+
+					err := updateCrashLoopList(kubeClient, clusterName, tc.Instances, namespace, false, false, false)
+					Expect(err).NotTo(HaveOccurred())
+
+					var resCluster fdbv1beta2.FoundationDBCluster
+					err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+						Namespace: namespace,
+						Name:      clusterName,
+					}, &resCluster)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(tc.ExpectedInstancesInCrashLoop).To(ContainElements(resCluster.Spec.Buggify.CrashLoop))
+					Expect(len(tc.ExpectedInstancesInCrashLoop)).To(BeNumerically("==", len(resCluster.Spec.Buggify.CrashLoop)))
+				},
+				Entry("Adding single instance.",
+					testCase{
+						Instances:                    []string{"instance-1"},
+						ExpectedInstancesInCrashLoop: []string{"instance-1"},
+					}),
+				Entry("Adding multiple instances.",
+					testCase{
+						Instances:                    []string{"instance-1", "instance-2"},
+						ExpectedInstancesInCrashLoop: []string{"instance-1", "instance-2"},
+					}),
+			)
+
+			When("a process group was already in crash-loop", func() {
+				var kubeClient client.Client
+
+				BeforeEach(func() {
+					cluster.Spec.Buggify.CrashLoop = []string{"instance-1"}
+					scheme := runtime.NewScheme()
+					_ = clientgoscheme.AddToScheme(scheme)
+					_ = fdbv1beta2.AddToScheme(scheme)
+					kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
+				})
+
+				type testCase struct {
+					Instances                    []string
+					ExpectedInstancesInCrashLoop []string
+				}
+
+				DescribeTable("should add all targeted processes to crash-loop list",
+					func(tc testCase) {
+						err := updateCrashLoopList(kubeClient, clusterName, tc.Instances, namespace, false, false, false)
+						Expect(err).NotTo(HaveOccurred())
+
+						var resCluster fdbv1beta2.FoundationDBCluster
+						err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+							Namespace: namespace,
+							Name:      clusterName,
+						}, &resCluster)
+						Expect(err).NotTo(HaveOccurred())
+						Expect(tc.ExpectedInstancesInCrashLoop).To(ContainElements(resCluster.Spec.Buggify.CrashLoop))
+						Expect(len(tc.ExpectedInstancesInCrashLoop)).To(BeNumerically("==", len(resCluster.Spec.Buggify.CrashLoop)))
+					},
+					Entry("Adding the same instance.",
+						testCase{
+							Instances:                    []string{"instance-1"},
+							ExpectedInstancesInCrashLoop: []string{"instance-1"},
+						}),
+					Entry("Adding different instance.",
+						testCase{
+							Instances:                    []string{"instance-2"},
+							ExpectedInstancesInCrashLoop: []string{"instance-1", "instance-2"},
+						}),
+					Entry("Adding multiple instances.",
+						testCase{
+							Instances:                    []string{"instance-2", "instance-3"},
+							ExpectedInstancesInCrashLoop: []string{"instance-1", "instance-2", "instance-3"},
+						}),
+				)
+			})
+		})
+
+		When("removing instances from crash-loop list from a cluster", func() {
+			var kubeClient client.Client
+
+			BeforeEach(func() {
+				cluster.Spec.Buggify.CrashLoop = []string{"instance-1", "instance-2", "instance-3"}
+				scheme := runtime.NewScheme()
+				_ = clientgoscheme.AddToScheme(scheme)
+				_ = fdbv1beta2.AddToScheme(scheme)
+				kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
+			})
+
+			type testCase struct {
+				Instances                    []string
+				ExpectedInstancesInCrashLoop []string
+			}
+
+			DescribeTable("should remove all targeted processes from the crash-loop list",
+				func(tc testCase) {
+					err := updateCrashLoopList(kubeClient, clusterName, tc.Instances, namespace, false, true, false)
+					Expect(err).NotTo(HaveOccurred())
+
+					var resCluster fdbv1beta2.FoundationDBCluster
+					err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+						Namespace: namespace,
+						Name:      clusterName,
+					}, &resCluster)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(tc.ExpectedInstancesInCrashLoop).To(ContainElements(resCluster.Spec.Buggify.CrashLoop))
+					Expect(len(tc.ExpectedInstancesInCrashLoop)).To(BeNumerically("==", len(resCluster.Spec.Buggify.CrashLoop)))
+				},
+				Entry("Removing single instance.",
+					testCase{
+						Instances:                    []string{"instance-1"},
+						ExpectedInstancesInCrashLoop: []string{"instance-2", "instance-3"},
+					}),
+				Entry("Removing multiple instances.",
+					testCase{
+						Instances:                    []string{"instance-2", "instance-3"},
+						ExpectedInstancesInCrashLoop: []string{"instance-1"},
+					}),
+			)
+
+		})
+
+		When("clearing crash-loop list", func() {
+			var kubeClient client.Client
+
+			BeforeEach(func() {
+				cluster.Spec.Buggify.CrashLoop = []string{"instance-1", "instance-2", "instance-3"}
+				scheme := runtime.NewScheme()
+				_ = clientgoscheme.AddToScheme(scheme)
+				_ = fdbv1beta2.AddToScheme(scheme)
+				kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
+			})
+
+			It("should clear the crash-loop list", func() {
+				err := updateCrashLoopList(kubeClient, clusterName, []string{}, namespace, false, false, true)
+				Expect(err).NotTo(HaveOccurred())
+
+				var resCluster fdbv1beta2.FoundationDBCluster
+				err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+					Namespace: namespace,
+					Name:      clusterName,
+				}, &resCluster)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(resCluster.Spec.Buggify.CrashLoop)).To(Equal(0))
+			})
+		})
+	})
+})
diff --git a/kubectl-fdb/cmd/buggify_empty-monitor_conf_test.go b/kubectl-fdb/cmd/buggify_empty-monitor_conf_test.go
new file mode 100644
index 00000000..703a0d05
--- /dev/null
+++ b/kubectl-fdb/cmd/buggify_empty-monitor_conf_test.go
@@ -0,0 +1,106 @@
+/*
+ * buggify_empty_monitor_conf_test.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	ctx "context"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var _ = Describe("[plugin] buggify empty-monitor-conf instances command", func() {
+	clusterName := "test"
+	namespace := "test"
+
+	var cluster *fdbv1beta2.FoundationDBCluster
+
+	BeforeEach(func() {
+		cluster = &fdbv1beta2.FoundationDBCluster{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      clusterName,
+				Namespace: namespace,
+			},
+			Spec: fdbv1beta2.FoundationDBClusterSpec{
+				ProcessCounts: fdbv1beta2.ProcessCounts{
+					Storage: 1,
+				},
+			},
+		}
+	})
+
+	When("running buggify empty-monitor-conf instances command", func() {
+		var kubeClient client.Client
+
+		When("should update empty-monitor-conf", func() {
+			BeforeEach(func() {
+				cluster.Spec.Buggify.EmptyMonitorConf = false
+				scheme := runtime.NewScheme()
+				_ = clientgoscheme.AddToScheme(scheme)
+				_ = fdbv1beta2.AddToScheme(scheme)
+				kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
+			})
+
+			It("setting monitor config to true", func() {
+				err := updateMonitorConf(kubeClient, clusterName, namespace, false, true)
+				Expect(err).NotTo(HaveOccurred())
+
+				var resCluster fdbv1beta2.FoundationDBCluster
+				err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+					Namespace: namespace,
+					Name:      clusterName,
+				}, &resCluster)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(resCluster.Spec.Buggify.EmptyMonitorConf).To(BeTrue())
+			})
+		})
+
+		When("should update empty-monitor-conf", func() {
+			BeforeEach(func() {
+				cluster.Spec.Buggify.EmptyMonitorConf = true
+				scheme := runtime.NewScheme()
+				_ = clientgoscheme.AddToScheme(scheme)
+				_ = fdbv1beta2.AddToScheme(scheme)
+				kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
+			})
+
+			It("setting monitor config to false", func() {
+				err := updateMonitorConf(kubeClient, clusterName, namespace, false, false)
+				Expect(err).NotTo(HaveOccurred())
+
+				var resCluster fdbv1beta2.FoundationDBCluster
+				err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+					Namespace: namespace,
+					Name:      clusterName,
+				}, &resCluster)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(resCluster.Spec.Buggify.EmptyMonitorConf).To(BeFalse())
+			})
+		})
+	})
+})
diff --git a/kubectl-fdb/cmd/buggify_no_schedule_test.go b/kubectl-fdb/cmd/buggify_no_schedule_test.go
new file mode 100644
index 00000000..448612f5
--- /dev/null
+++ b/kubectl-fdb/cmd/buggify_no_schedule_test.go
@@ -0,0 +1,213 @@
+/*
+ * buggify_no_schedule_test.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	ctx "context"
+
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("[plugin] buggify no-schedule instances command", func() {
+	clusterName := "test"
+	namespace := "test"
+
+	var cluster *fdbv1beta2.FoundationDBCluster
+
+	BeforeEach(func() {
+		cluster = &fdbv1beta2.FoundationDBCluster{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      clusterName,
+				Namespace: namespace,
+			},
+			Spec: fdbv1beta2.FoundationDBClusterSpec{
+				ProcessCounts: fdbv1beta2.ProcessCounts{
+					Storage: 1,
+				},
+			},
+		}
+	})
+
+	When("running buggify no-schedule instances command", func() {
+		When("adding instances to no-schedule list from a cluster", func() {
+			type testCase struct {
+				Instances                     []string
+				ExpectedInstancesInNoSchedule []string
+			}
+
+			DescribeTable("should add all targeted processes to no-schedule list",
+				func(tc testCase) {
+					scheme := runtime.NewScheme()
+					_ = clientgoscheme.AddToScheme(scheme)
+					_ = fdbv1beta2.AddToScheme(scheme)
+					kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
+
+					err := updateNoScheduleList(kubeClient, clusterName, tc.Instances, namespace, false, false, false)
+					Expect(err).NotTo(HaveOccurred())
+
+					var resCluster fdbv1beta2.FoundationDBCluster
+					err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+						Namespace: namespace,
+						Name:      clusterName,
+					}, &resCluster)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(tc.ExpectedInstancesInNoSchedule).To(ContainElements(resCluster.Spec.Buggify.NoSchedule))
+					Expect(len(tc.ExpectedInstancesInNoSchedule)).To(BeNumerically("==", len(resCluster.Spec.Buggify.NoSchedule)))
+				},
+				Entry("Adding single instance.",
+					testCase{
+						Instances:                     []string{"instance-1"},
+						ExpectedInstancesInNoSchedule: []string{"instance-1"},
+					}),
+				Entry("Adding multiple instances.",
+					testCase{
+						Instances:                     []string{"instance-1", "instance-2"},
+						ExpectedInstancesInNoSchedule: []string{"instance-1", "instance-2"},
+					}),
+			)
+
+			When("a process group is already in no-schedule", func() {
+				var kubeClient client.Client
+
+				BeforeEach(func() {
+					cluster.Spec.Buggify.NoSchedule = []string{"instance-1"}
+					scheme := runtime.NewScheme()
+					_ = clientgoscheme.AddToScheme(scheme)
+					_ = fdbv1beta2.AddToScheme(scheme)
+					kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
+				})
+
+				type testCase struct {
+					Instances                     []string
+					ExpectedInstancesInNoSchedule []string
+				}
+
+				DescribeTable("should add all targeted processes to no-schedule list",
+					func(tc testCase) {
+						err := updateNoScheduleList(kubeClient, clusterName, tc.Instances, namespace, false, false, false)
+						Expect(err).NotTo(HaveOccurred())
+
+						var resCluster fdbv1beta2.FoundationDBCluster
+						err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+							Namespace: namespace,
+							Name:      clusterName,
+						}, &resCluster)
+						Expect(err).NotTo(HaveOccurred())
+						Expect(tc.ExpectedInstancesInNoSchedule).To(ContainElements(resCluster.Spec.Buggify.NoSchedule))
+						Expect(len(tc.ExpectedInstancesInNoSchedule)).To(BeNumerically("==", len(resCluster.Spec.Buggify.NoSchedule)))
+					},
+					Entry("Adding the same instance.",
+						testCase{
+							Instances:                     []string{"instance-1"},
+							ExpectedInstancesInNoSchedule: []string{"instance-1"},
+						}),
+					Entry("Adding different instance.",
+						testCase{
+							Instances:                     []string{"instance-2"},
+							ExpectedInstancesInNoSchedule: []string{"instance-1", "instance-2"},
+						}),
+					Entry("Adding multiple instances.",
+						testCase{
+							Instances:                     []string{"instance-2", "instance-3"},
+							ExpectedInstancesInNoSchedule: []string{"instance-1", "instance-2", "instance-3"},
+						}),
+				)
+			})
+		})
+
+		When("removing process group from no-schedule list from a cluster", func() {
+			var kubeClient client.Client
+
+			BeforeEach(func() {
+				cluster.Spec.Buggify.NoSchedule = []string{"instance-1", "instance-2", "instance-3"}
+				scheme := runtime.NewScheme()
+				_ = clientgoscheme.AddToScheme(scheme)
+				_ = fdbv1beta2.AddToScheme(scheme)
+				kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
+			})
+
+			type testCase struct {
+				Instances                     []string
+				ExpectedInstancesInNoSchedule []string
+			}
+
+			DescribeTable("should remove all targeted processes from the no-schedule list",
+				func(tc testCase) {
+					err := updateNoScheduleList(kubeClient, clusterName, tc.Instances, namespace, false, true, false)
+					Expect(err).NotTo(HaveOccurred())
+
+					var resCluster fdbv1beta2.FoundationDBCluster
+					err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+						Namespace: namespace,
+						Name:      clusterName,
+					}, &resCluster)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(tc.ExpectedInstancesInNoSchedule).To(Equal(resCluster.Spec.Buggify.NoSchedule))
+					Expect(len(tc.ExpectedInstancesInNoSchedule)).To(BeNumerically("==", len(resCluster.Spec.Buggify.NoSchedule)))
+				},
+				Entry("Removing single instance.",
+					testCase{
+						Instances:                     []string{"instance-1"},
+						ExpectedInstancesInNoSchedule: []string{"instance-2", "instance-3"},
+					}),
+				Entry("Removing multiple instances.",
+					testCase{
+						Instances:                     []string{"instance-2", "instance-3"},
+						ExpectedInstancesInNoSchedule: []string{"instance-1"},
+					}),
+			)
+
+		})
+
+		When("clearing no-schedule list", func() {
+			var kubeClient client.Client
+
+			BeforeEach(func() {
+				cluster.Spec.Buggify.NoSchedule = []string{"instance-1", "instance-2", "instance-3"}
+				scheme := runtime.NewScheme()
+				_ = clientgoscheme.AddToScheme(scheme)
+				_ = fdbv1beta2.AddToScheme(scheme)
+				kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster).Build()
+			})
+
+			It("should clear the no-schedule list", func() {
+				err := updateNoScheduleList(kubeClient, clusterName, nil, namespace, false, false, true)
+				Expect(err).NotTo(HaveOccurred())
+
+				var resCluster fdbv1beta2.FoundationDBCluster
+				err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+					Namespace: namespace,
+					Name:      clusterName,
+				}, &resCluster)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(len(resCluster.Spec.Buggify.NoSchedule)).To(Equal(0))
+			})
+		})
+	})
+})