Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1125 (2022-03-19 11:16:09)

Run specs in parallel
# Description

Run Ginkgo specs in parallel (not the individual tests):

```bash
time go test -count=1 ./...
?       github.com/FoundationDB/fdb-kubernetes-operator [no test files]
ok      github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta1     0.351s
ok      github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2     0.532s
?       github.com/FoundationDB/fdb-kubernetes-operator/cmd/po-docgen   [no test files]
ok      github.com/FoundationDB/fdb-kubernetes-operator/controllers     90.306s
ok      github.com/FoundationDB/fdb-kubernetes-operator/fdbclient       1.325s
ok      github.com/FoundationDB/fdb-kubernetes-operator/internal        0.964s
ok      github.com/FoundationDB/fdb-kubernetes-operator/internal/removals       0.634s
ok      github.com/FoundationDB/fdb-kubernetes-operator/internal/replacements   1.584s
?       github.com/FoundationDB/fdb-kubernetes-operator/kubectl-fdb     [no test files]
ok      github.com/FoundationDB/fdb-kubernetes-operator/kubectl-fdb/cmd 0.605s
ok      github.com/FoundationDB/fdb-kubernetes-operator/mock-kubernetes-client/client   1.214s
?       github.com/FoundationDB/fdb-kubernetes-operator/pkg/fdbadminclient      [no test files]
?       github.com/FoundationDB/fdb-kubernetes-operator/pkg/podclient   [no test files]
ok      github.com/FoundationDB/fdb-kubernetes-operator/pkg/podmanager  0.907s
?       github.com/FoundationDB/fdb-kubernetes-operator/setup   [no test files]
gotest -count=1 ./...  119.18s user 6.71s system 134% cpu 1:33.82 total
```

```bash
test -count=1 ./... --ginkgo.parallel.total 8
?       github.com/FoundationDB/fdb-kubernetes-operator [no test files]
ok      github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta1     0.609s
ok      github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2     0.355s
?       github.com/FoundationDB/fdb-kubernetes-operator/cmd/po-docgen   [no test files]
ok      github.com/FoundationDB/fdb-kubernetes-operator/controllers     5.447s
ok      github.com/FoundationDB/fdb-kubernetes-operator/fdbclient       1.035s
ok      github.com/FoundationDB/fdb-kubernetes-operator/internal        0.623s
ok      github.com/FoundationDB/fdb-kubernetes-operator/internal/removals       0.902s
ok      github.com/FoundationDB/fdb-kubernetes-operator/internal/replacements   0.785s
?       github.com/FoundationDB/fdb-kubernetes-operator/kubectl-fdb     [no test files]
ok      github.com/FoundationDB/fdb-kubernetes-operator/kubectl-fdb/cmd 0.434s
ok      github.com/FoundationDB/fdb-kubernetes-operator/mock-kubernetes-client/client   0.677s
?       github.com/FoundationDB/fdb-kubernetes-operator/pkg/fdbadminclient      [no test files]
?       github.com/FoundationDB/fdb-kubernetes-operator/pkg/podclient   [no test files]
ok      github.com/FoundationDB/fdb-kubernetes-operator/pkg/podmanager  0.900s
?       github.com/FoundationDB/fdb-kubernetes-operator/setup   [no test files]
go test -count=1 ./... --ginkgo.parallel.total 8  22.74s user 4.98s system 323% cpu 8.558 total
```

On my local machine I got a nice improvement from ~90s runtime to ~8s.

## Type of change

*Please select one of the options below.*

- New feature (non-breaking change which adds functionality)

# Discussion

-

# Testing

Locally.

# Documentation

Here is some additional background for ginkgo parallel: https://onsi.github.io/ginkgo/#spec-parallelization

# Follow-up

-


johscheuer: The test execution in our pipeline takes now ~`2m 37s` instead of `24m 56s`.
foundationdb-ci: 
### AWS CodeBuild CI Report for Linux CentOS 7

* CodeBuild project: fdb-kubernetes-operator-pr
* Commit ID: 50e8b1dc36475aed7ffdbcbee3ad83034a060c42
* Result: FAILED
* Error: `Error while executing command: make -C tests -kj prOperator. Reason: exit status 2`
* [Build Logs](https://d1e0l78xbkh3xa.cloudfront.net/build-logs/fdb-kubernetes-operator/fdb-kubernetes-operator-pr/f9a298d4-fe9a-4fea-8061-2688077e59bf.gz) (available for 30 days)


Patch:
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index f8eaee86..f806d26a 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -12,18 +12,18 @@
 - Documentation
 - Other
 
-# Discussion
+## Discussion
 
 *Are there any design details that you would like to discuss further?*
 
-# Testing
+## Testing
 
 *Please describe the tests that you ran to verify your changes. Unit tests?
 Manual testing?*
 
 *Do we need to perform additional testing once this is merged, or perform in a larger testing environment?*
 
-# Documentation
+## Documentation
 
 *Did you update relevant documentation within this repository?*
 
@@ -33,7 +33,7 @@ Manual testing?*
 
 *If this change is adding new safety checks or new potential failure modes, have we documented and how to debug potential issues?*
 
-# Follow-up
+## Follow-up
 
 *Are there any follow-up issues that we should pursue in the future?*
 
diff --git a/Makefile b/Makefile
index 46b16960..a5ef2b1b 100644
--- a/Makefile
+++ b/Makefile
@@ -31,19 +31,18 @@ CONTROLLER_GEN_PKG?=sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.1
 CONTROLLER_GEN=$(GOBIN)/controller-gen
 KUSTOMIZE_PKG?=sigs.k8s.io/kustomize/kustomize/v3@v3.9.4
 KUSTOMIZE=$(GOBIN)/kustomize
-YQ_PKG?=github.com/mikefarah/yq/v4@v4.13.5
-YQ=$(GOBIN)/yq
 GOLANGCI_LINT_PKG=github.com/golangci/golangci-lint/cmd/golangci-lint@v1.42.1
 GOLANGCI_LINT=$(GOBIN)/golangci-lint
 BUILD_DEPS?=
 BUILDER?="docker"
+GINKGO_PARALLEL=8
 
 define godep
 BUILD_DEPS+=$(1)
 $($2):
 	(export TMP_DIR=$$$$(mktemp -d) ;\
 	cd $$$$TMP_DIR ;\
-	go get $$($2_PKG) ;\
+	go install $$($2_PKG) ;\
 	cd - ;\
 	rm -rf $$$$TMP_DIR )
 $(1): $$($2)
@@ -52,7 +51,6 @@ endef
 $(eval $(call godep,controller-gen,CONTROLLER_GEN))
 $(eval $(call godep,golangci-lint,GOLANGCI_LINT))
 $(eval $(call godep,kustomize,KUSTOMIZE))
-$(eval $(call godep,yq,YQ))
 
 GO_SRC=$(shell find . -name "*.go" -not -name "zz_generated.*.go")
 GENERATED_GO=api/v1beta2/zz_generated.deepcopy.go
@@ -82,17 +80,14 @@ clean:
 clean-deps:
 	@rm $(CONTROLLER_GEN) $(KUSTOMIZE) $(KUBEBUILDER) $(GOLANGCI_LINT)
 
-# Run tests
-test:
-ifneq "$(SKIP_TEST)" "1"
-	go test ${go_test_flags} ./... -coverprofile cover.out
-endif
-
 test_if_changed: cover.out
 
+# Run tests
 cover.out: ${GO_ALL} ${MANIFESTS}
+
+test:
 ifneq "$(SKIP_TEST)" "1"
-	go test ${go_test_flags} ./... -coverprofile cover.out -tags test
+	go test ${go_test_flags} ./... -coverprofile cover.out --ginkgo.parallel.total $(GINKGO_PARALLEL)
 endif
 
 # Build manager binary
diff --git a/README.md b/README.md
index 2dd5360f..8a261031 100644
--- a/README.md
+++ b/README.md
@@ -4,8 +4,7 @@
 ![GitHub](https://img.shields.io/github/license/FoundationDB/fdb-kubernetes-operator)
 [![CI for main branch](https://github.com/FoundationDB/fdb-kubernetes-operator/actions/workflows/pull_request.yml/badge.svg)](https://github.com/FoundationDB/fdb-kubernetes-operator/actions/workflows/pull_request.yml)
 
-This project provides an operator for managing FoundationDB
-clusters on Kubernetes.
+This project provides an operator for managing FoundationDB clusters on Kubernetes.
 
 ## Running the Operator
 
@@ -26,10 +25,10 @@ kubectl apply -f https://raw.githubusercontent.com/foundationdb/fdb-kubernetes-o
 ```
 
 You can see logs from the operator by running
-`kubectl logs -f -l app=fdb-kubernetes-operator-controller-manager --container=manager`. To determine whether the reconciliation has completed, you can run `kubectl get foundationdbcluster my-cluster`. This will show the latest generation of the
+`kubectl logs -f -l app=fdb-kubernetes-operator-controller-manager --container=manager`. To determine whether the reconciliation has completed, you can run `kubectl get foundationdbcluster test-cluster`. This will show the latest generation of the
 spec and the last reconciled generation of the spec. Once reconciliation has completed, these values will be the same.
 
-Once the reconciliation is complete, you can run `kubectl exec -it my-cluster-log-1 -- fdbcli` to open up a CLI on your cluster.
+Once the reconciliation is complete, you can run `kubectl exec -it test-cluster-log-1 -- fdbcli` to open up a CLI on your cluster.
 
 You can also browse the [sample directory](config/samples) for more examples of different resource configurations.
 
@@ -37,36 +36,31 @@ For more information about using the operator, including detailed discussion of
 
 For more information on version compatibility, see our [compatibility guide](docs/compatibility.md).
 
-For more information on the fields you can define on the cluster resource, see
-the [API documentation](docs/cluster_spec.md).
+For more information on the fields you can define on the cluster resource, see the [API documentation](docs/cluster_spec.md).
 
 ## Local Development
 
 ### Environment Set-up
 
-1. Install GO on your machine, see the [Getting Started](https://golang.org/doc/install) guide for more information.
-2. Install KubeBuilder and its dependencies on your machine, see [The KubeBuilder Book](https://book.kubebuilder.io/quick-start.html) for more information (currently version `2.3.2` is used).
-3. Set your $GOPATH, e.x. `/Users/me/Code/go`.
-4. Install [kustomize](https://github.com/kubernetes-sigs/kustomize).
-5. Install the [foundationDB client package](https://www.foundationdb.org/download).
+1. Install Go on your machine, see the [Getting Started](https://golang.org/doc/install) guide for more information.
+1. Install the required dependencies with `make deps`.
+1. Install the [foundationDB client package](https://www.foundationdb.org/download).
 
 ### Running Locally
 
 To get this controller running in a local Kubernetes cluster:
 
-1. Change your current directory to $GOPATH/src/github.com using the
-   command `cd $GOPATH/src/github.com` and run `mkdir foundationdb`
-   to create the directory `foundationdb`.
-2. CD into the newly created directory and clone this github repo inside
-   `$GOPATH/src/github.com/foundationdb`.
-3. Run `config/test-certs/generate_secrets.bash` to set up a secret with
+1. The assumption is that you have local Kubernetes cluster running.
+   Depending on what solution you use some of the following steps might differ.
+1. Clone this repository onto your local machine.
+1. Run `config/test-certs/generate_secrets.bash` to set up a secret with
    self-signed test certs.
-4. Run `make rebuild-operator` to install the operator. By default, the
-   docker image is built for the platform where this command is executed.
-   To override the platform, for example, to build amd64 image on Apple M1,
-   you can set the BUILD_PLATFORM env variable
+1. Run `make rebuild-operator` to install the operator. By default, the
+   container image is built for the platform where this command is executed.
+   To override the platform, for example, to build an amd64 image on Apple M1,
+   you can set the `BUILD_PLATFORM` env variable
    `BUILD_PLATFORM="linux/amd64" make rebuild-operator`.
-5. Run `kubectl apply -k ./config/tests/base`
+1. Run `kubectl apply -k ./config/tests/base`
    to create a new FoundationDB cluster with the operator.
 
 ### Running locally with nerdctl
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index be91a526..840d3223 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -1994,3 +1994,39 @@ func (cluster *FoundationDBCluster) GetUseUnifiedImage() bool {
 func (cluster *FoundationDBCluster) GetIgnoreTerminatingPodsSeconds() int {
 	return pointer.IntDeref(cluster.Spec.AutomationOptions.IgnoreTerminatingPodsSeconds, int((10 * time.Minute).Seconds()))
 }
+
+// AddProcessGroupsToRemovalList adds the provided process group IDs to the remove list.
+// If a process group ID is already present on that list it won't be added a second time.
+func (cluster *FoundationDBCluster) AddProcessGroupsToRemovalList(processGroupIDs []string) {
+	removals := map[string]None{}
+
+	for _, id := range cluster.Spec.ProcessGroupsToRemove {
+		removals[id] = None{}
+	}
+
+	for _, processGroupID := range processGroupIDs {
+		if _, ok := removals[processGroupID]; ok {
+			continue
+		}
+
+		cluster.Spec.ProcessGroupsToRemove = append(cluster.Spec.ProcessGroupsToRemove, processGroupID)
+	}
+}
+
+// AddProcessGroupsToRemovalWithoutExclusionList adds the provided process group IDs to the remove without exclusion list.
+// If a process group ID is already present on that list it won't be added a second time.
+func (cluster *FoundationDBCluster) AddProcessGroupsToRemovalWithoutExclusionList(processGroupIDs []string) {
+	removals := map[string]None{}
+
+	for _, id := range cluster.Spec.ProcessGroupsToRemoveWithoutExclusion {
+		removals[id] = None{}
+	}
+
+	for _, processGroupID := range processGroupIDs {
+		if _, ok := removals[processGroupID]; ok {
+			continue
+		}
+
+		cluster.Spec.ProcessGroupsToRemoveWithoutExclusion = append(cluster.Spec.ProcessGroupsToRemoveWithoutExclusion, processGroupID)
+	}
+}
diff --git a/go.mod b/go.mod
index 571a7d2c..5bd9573a 100644
--- a/go.mod
+++ b/go.mod
@@ -29,12 +29,19 @@ require (
 
 require (
 	cloud.google.com/go v0.54.0 // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest v0.11.12 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.5 // indirect
+	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
+	github.com/Azure/go-autorest/logger v0.2.0 // indirect
+	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/evanphx/json-patch v4.11.0+incompatible // indirect
+	github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-errors/errors v1.0.1 // indirect
 	github.com/go-logr/zapr v0.4.0 // indirect
@@ -85,6 +92,7 @@ require (
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.19.0 // indirect
+	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 // indirect
 	golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 // indirect
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
 	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
diff --git a/go.sum b/go.sum
index 947874b1..e929ae51 100644
--- a/go.sum
+++ b/go.sum
@@ -24,15 +24,22 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
+github.com/Azure/go-autorest/autorest v0.11.12 h1:gI8ytXbxMfI+IVbI9mP2JGCTXIuhHLgRlvQ9X4PsnHE=
 github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
 github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=
+github.com/Azure/go-autorest/autorest/adal v0.9.5 h1:Y3bBUV4rTuxenJJs41HU3qmqsb+auo+a3Lz+PlJPpL0=
 github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
+github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
 github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
 github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk=
 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE=
 github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
+github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -120,6 +127,7 @@ github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
+github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
@@ -550,6 +558,7 @@ golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
diff --git a/kubectl-fdb/cmd/remove_process_group.go b/kubectl-fdb/cmd/remove_process_group.go
index b35036e0..79cbd747 100644
--- a/kubectl-fdb/cmd/remove_process_group.go
+++ b/kubectl-fdb/cmd/remove_process_group.go
@@ -21,23 +21,17 @@
 package cmd
 
 import (
+	ctx "context"
 	"fmt"
 	"log"
 
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
-
+	"github.com/spf13/cobra"
+	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
-
 	"k8s.io/cli-runtime/pkg/genericclioptions"
-
-	corev1 "k8s.io/api/core/v1"
-
-	"github.com/spf13/cobra"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	ctx "context"
-
-	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 )
 
 func newRemoveProcessGroupCmd(streams genericclioptions.IOStreams) *cobra.Command {
@@ -155,18 +149,6 @@ func getProcessGroupIDsFromPod(kubeClient client.Client, clusterName string, pod
 	return processGroups, nil
 }
 
-func filterAlreadyMarkedProcessGroups(cluster *fdbv1beta2.FoundationDBCluster, processGroupIDs []string) []string {
-	res := make([]string, 0, len(processGroupIDs))
-
-	for _, processGroupID := range processGroupIDs {
-		if !cluster.ProcessGroupIsBeingRemoved(processGroupID) {
-			res = append(res, processGroupID)
-		}
-	}
-
-	return res
-}
-
 // replaceProcessGroups adds process groups to the removal list of the cluster
 func replaceProcessGroups(kubeClient client.Client, clusterName string, processGroups []string, namespace string, withExclusion bool, withShrink bool, force bool, removeAllFailed bool) error {
 	cluster, err := loadCluster(kubeClient, namespace, clusterName)
@@ -202,8 +184,18 @@ func replaceProcessGroups(kubeClient client.Client, clusterName string, processG
 
 	patch := client.MergeFrom(cluster.DeepCopy())
 
+	processGroupSet := map[string]fdbv1beta2.None{}
+	for _, processGroup := range processGroups {
+		processGroupSet[processGroup] = fdbv1beta2.None{}
+	}
+
 	if removeAllFailed {
 		for _, processGroupStatus := range cluster.Status.ProcessGroups {
+			// Those are already included so we can skip the check and don't add duplicates
+			if _, ok := processGroupSet[processGroupStatus.ProcessGroupID]; ok {
+				continue
+			}
+
 			needsReplacement, _ := processGroupStatus.NeedsReplacement(0)
 			if needsReplacement {
 				processGroups = append(processGroups, processGroupStatus.ProcessGroupID)
@@ -222,12 +214,10 @@ func replaceProcessGroups(kubeClient client.Client, clusterName string, processG
 		}
 	}
 
-	processGroups = filterAlreadyMarkedProcessGroups(cluster, processGroups)
-
 	if withExclusion {
-		cluster.Spec.ProcessGroupsToRemove = append(cluster.Spec.ProcessGroupsToRemove, processGroups...)
+		cluster.AddProcessGroupsToRemovalList(processGroups)
 	} else {
-		cluster.Spec.ProcessGroupsToRemoveWithoutExclusion = append(cluster.Spec.ProcessGroupsToRemoveWithoutExclusion, processGroups...)
+		cluster.AddProcessGroupsToRemovalWithoutExclusionList(processGroups)
 	}
 
 	return kubeClient.Patch(ctx.TODO(), cluster, patch)
diff --git a/kubectl-fdb/main.go b/kubectl-fdb/main.go
index ae560929..36174454 100644
--- a/kubectl-fdb/main.go
+++ b/kubectl-fdb/main.go
@@ -28,7 +28,7 @@ import (
 	"github.com/FoundationDB/fdb-kubernetes-operator/kubectl-fdb/cmd"
 	"github.com/spf13/pflag"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
-	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
 )
 
 func main() {


Test patch:
diff --git a/api/v1beta2/foundationdbcluster_types_test.go b/api/v1beta2/foundationdbcluster_types_test.go
index 15873852..c7af7856 100644
--- a/api/v1beta2/foundationdbcluster_types_test.go
+++ b/api/v1beta2/foundationdbcluster_types_test.go
@@ -3691,4 +3691,82 @@ var _ = Describe("[api] FoundationDBCluster", func() {
 			),
 		)
 	})
+
+	When("adding a process to the removal list", func() {
+		var cluster *FoundationDBCluster
+
+		BeforeEach(func() {
+			cluster = &FoundationDBCluster{}
+		})
+
+		When("removing with exclusion", func() {
+			When("a np process group is already included in the list", func() {
+				It("should add the process group to the removal list", func() {
+					removals := []string{"test1"}
+					cluster.AddProcessGroupsToRemovalList(removals)
+					Expect(cluster.Spec.ProcessGroupsToRemove).To(ContainElements(removals))
+					Expect(len(cluster.Spec.ProcessGroupsToRemove)).To(Equal(len(removals)))
+					Expect(len(cluster.Spec.ProcessGroupsToRemoveWithoutExclusion)).To(Equal(0))
+				})
+			})
+
+			When("a process group is already included in the list", func() {
+				BeforeEach(func() {
+					cluster.Spec.ProcessGroupsToRemove = append(cluster.Spec.ProcessGroupsToRemove, "test1")
+				})
+
+				It("should only add the missing process groups", func() {
+					Expect(cluster.Spec.ProcessGroupsToRemove).To(ContainElements("test1"))
+					removals := []string{"test1", "test2"}
+					cluster.AddProcessGroupsToRemovalList(removals)
+					Expect(cluster.Spec.ProcessGroupsToRemove).To(ContainElements(removals))
+					Expect(len(cluster.Spec.ProcessGroupsToRemove)).To(Equal(len(removals)))
+					Expect(len(cluster.Spec.ProcessGroupsToRemoveWithoutExclusion)).To(Equal(0))
+				})
+			})
+		})
+
+		When("removing without exclusion", func() {
+			When("a process group is not already included in the list", func() {
+				It("should add the process group to the removal list", func() {
+					removals := []string{"test1"}
+					cluster.AddProcessGroupsToRemovalWithoutExclusionList(removals)
+					Expect(cluster.Spec.ProcessGroupsToRemoveWithoutExclusion).To(ContainElements(removals))
+					Expect(len(cluster.Spec.ProcessGroupsToRemoveWithoutExclusion)).To(Equal(len(removals)))
+					Expect(len(cluster.Spec.ProcessGroupsToRemove)).To(Equal(0))
+				})
+			})
+
+			When("a process group is already included in the list", func() {
+				BeforeEach(func() {
+					cluster.Spec.ProcessGroupsToRemoveWithoutExclusion = append(cluster.Spec.ProcessGroupsToRemoveWithoutExclusion, "test1")
+					Expect(cluster.Spec.ProcessGroupsToRemoveWithoutExclusion).To(ContainElements("test1"))
+				})
+
+				It("should only add the missing process groups", func() {
+					removals := []string{"test1", "test2"}
+					cluster.AddProcessGroupsToRemovalWithoutExclusionList(removals)
+					Expect(cluster.Spec.ProcessGroupsToRemoveWithoutExclusion).To(ContainElements(removals))
+					Expect(len(cluster.Spec.ProcessGroupsToRemoveWithoutExclusion)).To(Equal(len(removals)))
+					Expect(len(cluster.Spec.ProcessGroupsToRemove)).To(Equal(0))
+				})
+			})
+
+			When("a process group is already included in the with exclusion list", func() {
+				BeforeEach(func() {
+					cluster.Spec.ProcessGroupsToRemove = append(cluster.Spec.ProcessGroupsToRemove, "test1")
+					Expect(cluster.Spec.ProcessGroupsToRemove).To(ContainElements("test1"))
+					Expect(len(cluster.Spec.ProcessGroupsToRemoveWithoutExclusion)).To(Equal(0))
+				})
+
+				It("should only add the missing process groups", func() {
+					removals := []string{"test1", "test2"}
+					cluster.AddProcessGroupsToRemovalWithoutExclusionList(removals)
+					Expect(cluster.Spec.ProcessGroupsToRemoveWithoutExclusion).To(ContainElements(removals))
+					Expect(len(cluster.Spec.ProcessGroupsToRemoveWithoutExclusion)).To(Equal(len(removals)))
+					Expect(len(cluster.Spec.ProcessGroupsToRemove)).To(Equal(1))
+				})
+			})
+		})
+	})
 })
diff --git a/kubectl-fdb/cmd/remove_process_group_test.go b/kubectl-fdb/cmd/remove_process_group_test.go
index c37db095..df46e946 100644
--- a/kubectl-fdb/cmd/remove_process_group_test.go
+++ b/kubectl-fdb/cmd/remove_process_group_test.go
@@ -247,6 +247,57 @@ var _ = Describe("[plugin] remove instances command", func() {
 						RemoveAllFailed: true,
 					}),
 			)
+
+			When("a procress group was already marked for removal", func() {
+				var kubeClient client.Client
+
+				BeforeEach(func() {
+					cluster.Spec.ProcessGroupsToRemove = []string{"instance-1"}
+					scheme := runtime.NewScheme()
+					_ = clientgoscheme.AddToScheme(scheme)
+					_ = fdbv1beta2.AddToScheme(scheme)
+					kubeClient = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(cluster, &podList).Build()
+				})
+
+				When("adding the same process group to the removal list without exclusion", func() {
+					It("should add the process group to the removal without exclusion list", func() {
+						removals := []string{"instance-1"}
+						err := replaceProcessGroups(kubeClient, clusterName, removals, namespace, false, false, true, false)
+						Expect(err).NotTo(HaveOccurred())
+
+						var resCluster fdbv1beta2.FoundationDBCluster
+						err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+							Namespace: namespace,
+							Name:      clusterName,
+						}, &resCluster)
+
+						Expect(err).NotTo(HaveOccurred())
+						Expect(resCluster.Spec.ProcessGroupsToRemove).To(ContainElements(removals))
+						Expect(len(resCluster.Spec.ProcessGroupsToRemove)).To(BeNumerically("==", len(removals)))
+						Expect(resCluster.Spec.ProcessGroupsToRemoveWithoutExclusion).To(ContainElements(removals))
+						Expect(len(resCluster.Spec.ProcessGroupsToRemoveWithoutExclusion)).To(BeNumerically("==", len(removals)))
+					})
+				})
+
+				When("adding the same process group to the removal list", func() {
+					It("should add the process group to the removal without exclusion list", func() {
+						removals := []string{"instance-1"}
+						err := replaceProcessGroups(kubeClient, clusterName, removals, namespace, true, false, true, false)
+						Expect(err).NotTo(HaveOccurred())
+
+						var resCluster fdbv1beta2.FoundationDBCluster
+						err = kubeClient.Get(ctx.Background(), client.ObjectKey{
+							Namespace: namespace,
+							Name:      clusterName,
+						}, &resCluster)
+
+						Expect(err).NotTo(HaveOccurred())
+						Expect(resCluster.Spec.ProcessGroupsToRemove).To(ContainElements(removals))
+						Expect(len(resCluster.Spec.ProcessGroupsToRemove)).To(BeNumerically("==", len(removals)))
+						Expect(len(resCluster.Spec.ProcessGroupsToRemoveWithoutExclusion)).To(BeNumerically("==", 0))
+					})
+				})
+			})
 		})
 	})
 })