Repository: FoundationDB/fdb-kubernetes-operator
Pull request #1366 (2022-09-27 17:14:09)

Initial changes to support e2e framework
# Description

fixes: https://github.com/FoundationDB/fdb-kubernetes-operator/issues/1354

## Type of change

*Please select one of the options below.*

- New feature (non-breaking change which adds functionality)

## Discussion

Those are the initial changes for the e2e framework and I will complete the create cluster tests before merging.

## Testing

They currently are not doing to much:

```bash
go test -v ./e2e/... --tags=e2e_test
=== RUN   TestCreateSingleFDBCluster
=== RUN   TestCreateSingleFDBCluster/create_single_cluster_for_6.2
=== RUN   TestCreateSingleFDBCluster/create_single_cluster_for_6.2/it_should_create_an_FDB_cluster_in_6.2
=== RUN   TestCreateSingleFDBCluster/create_single_cluster_for_6.2/Assessment-2
=== RUN   TestCreateSingleFDBCluster/create_single_cluster_for_6.3
=== RUN   TestCreateSingleFDBCluster/create_single_cluster_for_6.3/it_should_create_an_FDB_cluster_in_6.3
=== RUN   TestCreateSingleFDBCluster/create_single_cluster_for_6.3/Assessment-2
=== RUN   TestCreateSingleFDBCluster/create_single_cluster_for_7.1
=== RUN   TestCreateSingleFDBCluster/create_single_cluster_for_7.1/it_should_create_an_FDB_cluster_in_7.1
=== RUN   TestCreateSingleFDBCluster/create_single_cluster_for_7.1/Assessment-2
--- PASS: TestCreateSingleFDBCluster (0.11s)
    --- PASS: TestCreateSingleFDBCluster/create_single_cluster_for_6.2 (0.10s)
        --- PASS: TestCreateSingleFDBCluster/create_single_cluster_for_6.2/it_should_create_an_FDB_cluster_in_6.2 (0.00s)
        --- PASS: TestCreateSingleFDBCluster/create_single_cluster_for_6.2/Assessment-2 (0.00s)
    --- PASS: TestCreateSingleFDBCluster/create_single_cluster_for_6.3 (0.01s)
        --- PASS: TestCreateSingleFDBCluster/create_single_cluster_for_6.3/it_should_create_an_FDB_cluster_in_6.3 (0.00s)
        --- PASS: TestCreateSingleFDBCluster/create_single_cluster_for_6.3/Assessment-2 (0.00s)
    --- PASS: TestCreateSingleFDBCluster/create_single_cluster_for_7.1 (0.01s)
        --- PASS: TestCreateSingleFDBCluster/create_single_cluster_for_7.1/it_should_create_an_FDB_cluster_in_7.1 (0.00s)
        --- PASS: TestCreateSingleFDBCluster/create_single_cluster_for_7.1/Assessment-2 (0.00s)
PASS
ok      github.com/FoundationDB/fdb-kubernetes-operator/e2e     29.725s
?       github.com/FoundationDB/fdb-kubernetes-operator/e2e/helper      [no test files]
```

## Documentation

Will be added once we have the setup.

## Follow-up

Adding more e2e tests :) 



Patch:
diff --git a/api/v1beta2/foundationdb_database_configuration.go b/api/v1beta2/foundationdb_database_configuration.go
index cc30a78a..84f7d482 100644
--- a/api/v1beta2/foundationdb_database_configuration.go
+++ b/api/v1beta2/foundationdb_database_configuration.go
@@ -911,5 +911,20 @@ func (counts ProcessCounts) Diff(currentCounts ProcessCounts) map[ProcessClass]i
 			diff[label] = desired - current
 		}
 	}
+
 	return diff
 }
+
+// Total gets the total number of processes for the cluster configuration.
+func (counts ProcessCounts) Total() int {
+	var total int64
+	desiredValue := reflect.ValueOf(counts)
+	for _, index := range processClassIndices {
+		desired := desiredValue.Field(index).Int()
+		if desired > 0 {
+			total += desired
+		}
+	}
+
+	return int(total)
+}
diff --git a/api/v1beta2/foundationdb_status.go b/api/v1beta2/foundationdb_status.go
index 519fd53b..413334be 100644
--- a/api/v1beta2/foundationdb_status.go
+++ b/api/v1beta2/foundationdb_status.go
@@ -78,6 +78,9 @@ type FoundationDBStatusClusterInfo struct {
 	// FullReplication indicates whether the database is fully replicated.
 	FullReplication bool `json:"full_replication,omitempty"`
 
+	// MaintenanceZone contains current zone under maintenance, if any.
+	MaintenanceZone string `json:"maintenance_zone,omitempty"`
+
 	// Clients provides information about clients that are connected to the
 	// database.
 	Clients FoundationDBStatusClusterClientInfo `json:"clients,omitempty"`
diff --git a/api/v1beta2/foundationdbcluster_types.go b/api/v1beta2/foundationdbcluster_types.go
index 673de9a0..e2463915 100644
--- a/api/v1beta2/foundationdbcluster_types.go
+++ b/api/v1beta2/foundationdbcluster_types.go
@@ -39,6 +39,8 @@ import (
 // +kubebuilder:printcolumn:name="Reconciled",type="integer",JSONPath=".status.generations.reconciled",description="Last reconciled generation of the spec",priority=0
 // +kubebuilder:printcolumn:name="Available",type="boolean",JSONPath=".status.health.available",description="Database available",priority=0
 // +kubebuilder:printcolumn:name="FullReplication",type="boolean",JSONPath=".status.health.fullReplication",description="Database fully replicated",priority=0
+// +kubebuilder:printcolumn:name="ReconciledProcessGroups",type="integer",JSONPath=".status.reconciledProcessGroups",description="Number of reconciled process groups",priority=1
+// +kubebuilder:printcolumn:name="DesiredProcessGroups",type="integer",JSONPath=".status.desiredProcessGroups",description="Desired number of process groups",priority=1
 // +kubebuilder:printcolumn:name="Version",type="string",JSONPath=".status.runningVersion",description="Running version",priority=0
 // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
 // +kubebuilder:storageversion
@@ -279,6 +281,26 @@ type FoundationDBClusterStatus struct {
 
 	// Locks contains information about the locking system.
 	Locks LockSystemStatus `json:"locks,omitempty"`
+
+	// MaintenenanceModeInfo contains information regarding process groups in maintenance mode
+	MaintenanceModeInfo MaintenanceModeInfo `json:"maintenanceModeInfo,omitempty"`
+
+	// DesiredProcessGroups reflects the number of expected running process groups.
+	DesiredProcessGroups int `json:"desiredProcessGroups,omitempty"`
+
+	// ReconciledProcessGroups reflects the number of process groups that have no condition and are not marked for removal.
+	ReconciledProcessGroups int `json:"reconciledProcessGroups,omitempty"`
+}
+
+// MaintenanceModeInfo contains information regarding the zone and process groups that are put
+// into maintenance mode by the operator
+type MaintenanceModeInfo struct {
+	// StartTimestamp provides the timestamp when this zone is put into maintenance mode
+	StartTimestamp *metav1.Time `json:"startTimestamp,omitempty"`
+	// ZoneID that is placed in maintenance mode
+	ZoneID string `json:"zoneID,omitempty"`
+	// ProcessGroups that are placed in maintenance mode
+	ProcessGroups []string `json:"processGroups,omitempty"`
 }
 
 // LockSystemStatus provides a summary of the status of the locking system.
@@ -874,6 +896,20 @@ type FoundationDBClusterAutomationOptions struct {
 	// UseManagementAPI defines if the operator should make use of the management API instead of
 	// using fdbcli to interact with the FoundationDB cluster.
 	UseManagementAPI *bool `json:"useManagementAPI,omitempty"`
+
+	// MaintenanceModeOptions contains options for maintenance mode related settings.
+	MaintenanceModeOptions MaintenanceModeOptions `json:"maintenanceModeOptions,omitempty"`
+}
+
+// MaintenanceModeOptions controls options for placing zones in maintenance mode.
+type MaintenanceModeOptions struct {
+	// UseMaintenanceModeChecker defines whether the operator is allowed to use maintenance mode before updating pods.
+	// Default is false.
+	UseMaintenanceModeChecker *bool `json:"UseMaintenanceModeChecker,omitempty"`
+
+	// MaintenanceModeTimeSeconds provides the duration for the zone to be in maintenance. It will automatically be switched off after the time elapses.
+	// Default is 600.
+	MaintenanceModeTimeSeconds *int `json:"maintenanceModeTimeSeconds,omitempty"`
 }
 
 // AutomaticReplacementOptions controls options for automatically replacing
@@ -1126,7 +1162,6 @@ func (cluster *FoundationDBCluster) CheckReconciliation(log logr.Logger) (bool,
 	}
 
 	cluster.Status.Generations = ClusterGenerationStatus{Reconciled: cluster.Status.Generations.Reconciled}
-
 	for _, processGroup := range cluster.Status.ProcessGroups {
 		if !processGroup.IsMarkedForRemoval() {
 			continue
@@ -1161,8 +1196,15 @@ func (cluster *FoundationDBCluster) CheckReconciliation(log logr.Logger) (bool,
 		}
 	}
 
+	cluster.Status.DesiredProcessGroups = desiredCounts.Total()
+	cluster.Status.ReconciledProcessGroups = 0
+
 	for _, processGroup := range cluster.Status.ProcessGroups {
-		if len(processGroup.ProcessGroupConditions) > 0 && !processGroup.IsMarkedForRemoval() {
+		if processGroup.IsMarkedForRemoval() {
+			continue
+		}
+
+		if len(processGroup.ProcessGroupConditions) > 0 {
 			conditions := make([]ProcessGroupConditionType, 0, len(processGroup.ProcessGroupConditions))
 			for _, condition := range processGroup.ProcessGroupConditions {
 				conditions = append(conditions, condition.ProcessGroupConditionType)
@@ -1171,7 +1213,14 @@ func (cluster *FoundationDBCluster) CheckReconciliation(log logr.Logger) (bool,
 			logger.Info("Has unhealthy process group", "processGroupID", processGroup.ProcessGroupID, "state", "HasUnhealthyProcess", "conditions", conditions)
 			cluster.Status.Generations.HasUnhealthyProcess = cluster.ObjectMeta.Generation
 			reconciled = false
+			continue
 		}
+
+		cluster.Status.ReconciledProcessGroups++
+	}
+
+	if cluster.Status.DesiredProcessGroups != cluster.Status.ReconciledProcessGroups {
+		logger.Info("Not all process groups are reconciled", "desiredProcessGroups", cluster.Status.DesiredProcessGroups, "reconciledProcessGroups", cluster.Status.ReconciledProcessGroups)
 	}
 
 	if !cluster.Status.Health.Available {
@@ -1875,6 +1924,16 @@ func (cluster *FoundationDBCluster) GetWaitBetweenRemovalsSeconds() int {
 	return duration
 }
 
+// UseMaintenaceMode returns true if UseMaintenanceModeChecker is set.
+func (cluster *FoundationDBCluster) UseMaintenaceMode() bool {
+	return pointer.BoolDeref(cluster.Spec.AutomationOptions.MaintenanceModeOptions.UseMaintenanceModeChecker, false)
+}
+
+// GetMaintenaceModeTimeoutSeconds returns the timeout for maintenance zone after which it will be reset.
+func (cluster *FoundationDBCluster) GetMaintenaceModeTimeoutSeconds() int {
+	return pointer.IntDeref(cluster.Spec.AutomationOptions.MaintenanceModeOptions.MaintenanceModeTimeSeconds, 600)
+}
+
 // PodUpdateStrategy defines how Pod spec changes should be applied.
 type PodUpdateStrategy string
 
diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go
index d73d53c7..98e4a512 100644
--- a/api/v1beta2/zz_generated.deepcopy.go
+++ b/api/v1beta2/zz_generated.deepcopy.go
@@ -510,6 +510,7 @@ func (in *FoundationDBClusterAutomationOptions) DeepCopyInto(out *FoundationDBCl
 		*out = new(bool)
 		**out = **in
 	}
+	in.MaintenanceModeOptions.DeepCopyInto(&out.MaintenanceModeOptions)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationDBClusterAutomationOptions.
@@ -676,6 +677,7 @@ func (in *FoundationDBClusterStatus) DeepCopyInto(out *FoundationDBClusterStatus
 		}
 	}
 	in.Locks.DeepCopyInto(&out.Locks)
+	in.MaintenanceModeInfo.DeepCopyInto(&out.MaintenanceModeInfo)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FoundationDBClusterStatus.
@@ -1306,6 +1308,55 @@ func (in *LockSystemStatus) DeepCopy() *LockSystemStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *MaintenanceModeInfo) DeepCopyInto(out *MaintenanceModeInfo) {
+	*out = *in
+	if in.StartTimestamp != nil {
+		in, out := &in.StartTimestamp, &out.StartTimestamp
+		*out = (*in).DeepCopy()
+	}
+	if in.ProcessGroups != nil {
+		in, out := &in.ProcessGroups, &out.ProcessGroups
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MaintenanceModeInfo.
+func (in *MaintenanceModeInfo) DeepCopy() *MaintenanceModeInfo {
+	if in == nil {
+		return nil
+	}
+	out := new(MaintenanceModeInfo)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *MaintenanceModeOptions) DeepCopyInto(out *MaintenanceModeOptions) {
+	*out = *in
+	if in.UseMaintenanceModeChecker != nil {
+		in, out := &in.UseMaintenanceModeChecker, &out.UseMaintenanceModeChecker
+		*out = new(bool)
+		**out = **in
+	}
+	if in.MaintenanceModeTimeSeconds != nil {
+		in, out := &in.MaintenanceModeTimeSeconds, &out.MaintenanceModeTimeSeconds
+		*out = new(int)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MaintenanceModeOptions.
+func (in *MaintenanceModeOptions) DeepCopy() *MaintenanceModeOptions {
+	if in == nil {
+		return nil
+	}
+	out := new(MaintenanceModeOptions)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *None) DeepCopyInto(out *None) {
 	*out = *in
diff --git a/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml b/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
index 4ad9a3e0..71c9e105 100644
--- a/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
+++ b/config/crd/bases/apps.foundationdb.org_foundationdbclusters.yaml
@@ -9828,6 +9828,16 @@ spec:
       jsonPath: .status.health.fullReplication
       name: FullReplication
       type: boolean
+    - description: Number of reconciled process groups
+      jsonPath: .status.reconciledProcessGroups
+      name: ReconciledProcessGroups
+      priority: 1
+      type: integer
+    - description: Desired number of process groups
+      jsonPath: .status.desiredProcessGroups
+      name: DesiredProcessGroups
+      priority: 1
+      type: integer
     - description: Running version
       jsonPath: .status.runningVersion
       name: Version
@@ -9868,6 +9878,13 @@ spec:
                     type: integer
                   killProcesses:
                     type: boolean
+                  maintenanceModeOptions:
+                    properties:
+                      UseMaintenanceModeChecker:
+                        type: boolean
+                      maintenanceModeTimeSeconds:
+                        type: integer
+                    type: object
                   maxConcurrentReplacements:
                     minimum: 0
                     type: integer
@@ -13671,6 +13688,8 @@ spec:
                   usable_regions:
                     type: integer
                 type: object
+              desiredProcessGroups:
+                type: integer
               generations:
                 properties:
                   hasExtraListeners:
@@ -13746,6 +13765,18 @@ spec:
                       type: string
                     type: array
                 type: object
+              maintenanceModeInfo:
+                properties:
+                  processGroups:
+                    items:
+                      type: string
+                    type: array
+                  startTimestamp:
+                    format: date-time
+                    type: string
+                  zoneID:
+                    type: string
+                type: object
               needsNewCoordinators:
                 type: boolean
               processGroups:
@@ -13779,6 +13810,8 @@ spec:
                       type: string
                   type: object
                 type: array
+              reconciledProcessGroups:
+                type: integer
               requiredAddresses:
                 properties:
                   nonTLS:
diff --git a/controllers/admin_client_mock.go b/controllers/admin_client_mock.go
index 1803c8e9..dc44ad40 100644
--- a/controllers/admin_client_mock.go
+++ b/controllers/admin_client_mock.go
@@ -26,6 +26,7 @@ import (
 	"net"
 	"strings"
 	"sync"
+	"time"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
@@ -54,6 +55,9 @@ type mockAdminClient struct {
 	maxZoneFailuresWithoutLosingData         *int
 	maxZoneFailuresWithoutLosingAvailability *int
 	knobs                                    []string
+	maintenanceZone                          string
+	maintenanceZoneStartTimestamp            time.Time
+	uptimeSecondsForMaintenanceZone          float64
 }
 
 // adminClientCache provides a cache of mock admin clients.
@@ -204,6 +208,15 @@ func (client *mockAdminClient) GetStatus() (*fdbv1beta2.FoundationDBStatus, erro
 				locality["process_id"] = fmt.Sprintf("%s-%d", processGroupID, processIndex)
 			}
 
+			var uptimeSeconds float64 = 60000
+			if client.maintenanceZone == pod.Name || client.maintenanceZone == "simulation" {
+				if client.uptimeSecondsForMaintenanceZone != 0.0 {
+					uptimeSeconds = client.uptimeSecondsForMaintenanceZone
+				} else {
+					uptimeSeconds = time.Since(client.maintenanceZoneStartTimestamp).Seconds()
+				}
+			}
+
 			status.Cluster.Processes[fmt.Sprintf("%s-%d", pod.Name, processIndex)] = fdbv1beta2.FoundationDBStatusProcessInfo{
 				Address:       fullAddress,
 				ProcessClass:  internal.GetProcessClassFromMeta(client.Cluster, pod.ObjectMeta),
@@ -211,7 +224,7 @@ func (client *mockAdminClient) GetStatus() (*fdbv1beta2.FoundationDBStatus, erro
 				Excluded:      excluded,
 				Locality:      locality,
 				Version:       client.Cluster.Status.RunningVersion,
-				UptimeSeconds: 60000,
+				UptimeSeconds: uptimeSeconds,
 				Roles:         fdbRoles,
 			}
 		}
@@ -226,13 +239,22 @@ func (client *mockAdminClient) GetStatus() (*fdbv1beta2.FoundationDBStatus, erro
 				locality[key] = value
 			}
 
+			var uptimeSeconds float64 = 60000
+			if client.maintenanceZone == processGroup.ProcessGroupID || client.maintenanceZone == "simulation" {
+				if client.uptimeSecondsForMaintenanceZone != 0.0 {
+					uptimeSeconds = client.uptimeSecondsForMaintenanceZone
+				} else {
+					uptimeSeconds = time.Since(client.maintenanceZoneStartTimestamp).Seconds()
+				}
+			}
+
 			fullAddress := client.Cluster.GetFullAddress(processGroup.Addresses[0], 1)
 			status.Cluster.Processes[processGroup.ProcessGroupID] = fdbv1beta2.FoundationDBStatusProcessInfo{
 				Address:       fullAddress,
 				ProcessClass:  processGroup.ProcessClass,
 				Locality:      locality,
 				Version:       client.Cluster.Status.RunningVersion,
-				UptimeSeconds: 60000,
+				UptimeSeconds: uptimeSeconds,
 			}
 
 		}
@@ -302,15 +324,18 @@ func (client *mockAdminClient) GetStatus() (*fdbv1beta2.FoundationDBStatus, erro
 			status.Cluster.Layers.Backup.Paused = tagStatus.Paused
 		}
 	}
-
+	faultToleranceSubtractor := 0
+	if client.maintenanceZone != "" {
+		faultToleranceSubtractor = 1
+	}
 	if client.maxZoneFailuresWithoutLosingData == nil {
-		status.Cluster.FaultTolerance.MaxZoneFailuresWithoutLosingData = client.Cluster.DesiredFaultTolerance()
+		status.Cluster.FaultTolerance.MaxZoneFailuresWithoutLosingData = client.Cluster.DesiredFaultTolerance() - faultToleranceSubtractor
 	}
 
 	if client.maxZoneFailuresWithoutLosingAvailability == nil {
-		status.Cluster.FaultTolerance.MaxZoneFailuresWithoutLosingAvailability = client.Cluster.DesiredFaultTolerance()
+		status.Cluster.FaultTolerance.MaxZoneFailuresWithoutLosingAvailability = client.Cluster.DesiredFaultTolerance() - faultToleranceSubtractor
 	}
-
+	status.Cluster.MaintenanceZone = client.maintenanceZone
 	return status, nil
 }
 
@@ -711,3 +736,26 @@ func (client *mockAdminClient) GetCoordinatorSet() (map[string]struct{}, error)
 func (client *mockAdminClient) SetKnobs(knobs []string) {
 	client.knobs = knobs
 }
+
+// GetMaintenanceZone gets current maintenance zone, if any
+func (client *mockAdminClient) GetMaintenanceZone() (string, error) {
+	return client.maintenanceZone, nil
+}
+
+// SetMaintenanceZone places zone into maintenance mode
+func (client *mockAdminClient) SetMaintenanceZone(zone string, _ int) error {
+	client.maintenanceZone = zone
+	client.maintenanceZoneStartTimestamp = time.Now()
+	return nil
+}
+
+// Reset maintenance mode
+func (client *mockAdminClient) ResetMaintenanceMode() error {
+	client.maintenanceZone = ""
+	return nil
+}
+
+// Reset maintenance mode
+func (client *mockAdminClient) MockUptimeSecondsForMaintenanceZone(seconds float64) {
+	client.uptimeSecondsForMaintenanceZone = seconds
+}
diff --git a/controllers/cluster_controller.go b/controllers/cluster_controller.go
index 9bc4e2b0..0e5ff5d9 100644
--- a/controllers/cluster_controller.go
+++ b/controllers/cluster_controller.go
@@ -150,6 +150,7 @@ func (r *FoundationDBClusterReconciler) Reconcile(ctx context.Context, request c
 		excludeProcesses{},
 		changeCoordinators{},
 		bounceProcesses{},
+		maintenanceModeChecker{},
 		updatePods{},
 		removeProcessGroups{},
 		removeServices{},
diff --git a/controllers/maintenance_mode_checker.go b/controllers/maintenance_mode_checker.go
new file mode 100644
index 00000000..b86b6e0a
--- /dev/null
+++ b/controllers/maintenance_mode_checker.go
@@ -0,0 +1,115 @@
+/*
+ * maintenance_mode_checker.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+)
+
+// maintenanceModeChecker provides a reconciliation step for clearing the maintenance mode if all the pods in the current maintenance zone are up.
+type maintenanceModeChecker struct{}
+
+// reconcile runs the reconciler's work.
+func (maintenanceModeChecker) reconcile(ctx context.Context, r *FoundationDBClusterReconciler, cluster *fdbv1beta2.FoundationDBCluster) *requeue {
+	logger := log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "reconciler", "maintenanceModeChecker")
+
+	if !cluster.UseMaintenaceMode() {
+		return nil
+	}
+
+	adminClient, err := r.getDatabaseClientProvider().GetAdminClient(cluster, r.Client)
+	if err != nil {
+		return &requeue{curError: err}
+	}
+	defer adminClient.Close()
+
+	maintenanceZone, err := adminClient.GetMaintenanceZone()
+	if err != nil {
+		return &requeue{curError: err}
+	}
+	// Cluster is not in maintenance mode
+	if maintenanceZone == "" {
+		if cluster.Status.MaintenanceModeInfo.ZoneID != "" {
+			cluster.Status.MaintenanceModeInfo = fdbv1beta2.MaintenanceModeInfo{}
+			err = r.updateOrApply(ctx, cluster)
+			if err != nil {
+				return &requeue{curError: err}
+			}
+		}
+		return nil
+	}
+	// FDB Cluster is in maintenance mode but not due to this operator actions
+	if maintenanceZone != cluster.Status.MaintenanceModeInfo.ZoneID {
+		if cluster.Status.MaintenanceModeInfo.ZoneID != "" {
+			cluster.Status.MaintenanceModeInfo = fdbv1beta2.MaintenanceModeInfo{}
+			err = r.updateOrApply(ctx, cluster)
+			if err != nil {
+				return &requeue{curError: err}
+			}
+		}
+		return nil
+	}
+	logger.Info("Cluster in maintenance mode", "zone", maintenanceZone)
+	// FDB Cluster is in maintenance mode due to this operator actions
+	status, err := adminClient.GetStatus()
+	if err != nil {
+		return &requeue{curError: err}
+	}
+	processGroupsToCheck := make(map[string]struct{})
+	for _, id := range cluster.Status.MaintenanceModeInfo.ProcessGroups {
+		processGroupsToCheck[id] = fdbv1beta2.None{}
+	}
+	for _, process := range status.Cluster.Processes {
+		if _, ok := processGroupsToCheck[process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]]; !ok {
+			continue
+		}
+		// TODO: Also include deletion timestamp to make this logic more robust to account for the corner case of the process crash/restarts.
+		if process.UptimeSeconds < time.Since(cluster.Status.MaintenanceModeInfo.StartTimestamp.Time).Seconds() {
+			delete(processGroupsToCheck, process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey])
+		} else {
+			return &requeue{message: fmt.Sprintf("Waiting for pod %s to be updated", process.Locality[fdbv1beta2.FDBLocalityInstanceIDKey]), delayedRequeue: true}
+		}
+	}
+	// Some of the pods are not yet up
+	if len(processGroupsToCheck) != 0 {
+		return &requeue{message: fmt.Sprintf("Waiting for all proceeses in zone %s to be up", maintenanceZone), delayedRequeue: true}
+	}
+	// All the pods for this zone under maintenance are up
+	hasLock, err := r.takeLock(cluster, "maintenance mode check")
+	if !hasLock {
+		return &requeue{curError: err}
+	}
+	logger.Info("Switching off maintenance mode", "zone", maintenanceZone)
+	err = adminClient.ResetMaintenanceMode()
+	if err != nil {
+		return &requeue{curError: err}
+	}
+	cluster.Status.MaintenanceModeInfo = fdbv1beta2.MaintenanceModeInfo{}
+	err = r.updateOrApply(ctx, cluster)
+	if err != nil {
+		return &requeue{curError: err}
+	}
+	return nil
+}
diff --git a/controllers/update_pods.go b/controllers/update_pods.go
index 2a3c2e0a..749c5ca7 100644
--- a/controllers/update_pods.go
+++ b/controllers/update_pods.go
@@ -31,6 +31,7 @@ import (
 	"github.com/FoundationDB/fdb-kubernetes-operator/pkg/podmanager"
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // updatePods provides a reconciliation step for recreating pods with new pod
@@ -227,7 +228,29 @@ func deletePodsForUpdates(ctx context.Context, r *FoundationDBClusterReconciler,
 		}
 	}
 
-	logger.Info("Deleting pods", "zone", zone, "count", len(deletions), "deletionMode", string(deletionMode))
+	if deletionMode == fdbv1beta2.PodUpdateModeZone && cluster.UseMaintenaceMode() {
+		var processGroups []string
+		for _, pod := range deletions {
+			processGroups = append(processGroups, pod.Labels[cluster.GetProcessGroupIDLabel()])
+		}
+
+		logger.Info("Setting maintenance mode", "zone", zone)
+		cluster.Status.MaintenanceModeInfo = fdbv1beta2.MaintenanceModeInfo{
+			StartTimestamp: &metav1.Time{Time: time.Now()},
+			ZoneID:         zone,
+			ProcessGroups:  processGroups,
+		}
+		err = r.updateOrApply(ctx, cluster)
+		if err != nil {
+			return &requeue{curError: err}
+		}
+		err = adminClient.SetMaintenanceZone(zone, cluster.GetMaintenaceModeTimeoutSeconds())
+		if err != nil {
+			return &requeue{curError: err}
+		}
+	}
+
+	logger.Info("Deleting pods", "zone", zone, "count", len(deletions), "deletionMode", string(cluster.Spec.AutomationOptions.DeletionMode))
 	r.Recorder.Event(cluster, corev1.EventTypeNormal, "UpdatingPods", fmt.Sprintf("Recreating pods in zone %s", zone))
 
 	err = r.PodLifecycleManager.UpdatePods(logr.NewContext(ctx, logger), r, cluster, deletions, false)
diff --git a/controllers/update_status.go b/controllers/update_status.go
index fdb0f9f6..2aa6cb24 100644
--- a/controllers/update_status.go
+++ b/controllers/update_status.go
@@ -48,6 +48,8 @@ func (updateStatus) reconcile(ctx context.Context, r *FoundationDBClusterReconci
 	logger := log.WithValues("namespace", cluster.Namespace, "cluster", cluster.Name, "reconciler", "updateStatus")
 	originalStatus := cluster.Status.DeepCopy()
 	status := fdbv1beta2.FoundationDBClusterStatus{}
+	// Pass through Maintenance Mode Info as the maintenance_mode_checker reconciler takes care of updating it
+	originalStatus.MaintenanceModeInfo.DeepCopyInto(&status.MaintenanceModeInfo)
 	status.Generations.Reconciled = cluster.Status.Generations.Reconciled
 
 	// Initialize with the current desired storage servers per Pod
diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md
index 664782ba..cf7d36c1 100644
--- a/docs/cluster_spec.md
+++ b/docs/cluster_spec.md
@@ -22,6 +22,8 @@ This Document documents the types introduced by the FoundationDB Operator to be
 * [LockDenyListEntry](#lockdenylistentry)
 * [LockOptions](#lockoptions)
 * [LockSystemStatus](#locksystemstatus)
+* [MaintenanceModeInfo](#maintenancemodeinfo)
+* [MaintenanceModeOptions](#maintenancemodeoptions)
 * [ProcessGroupCondition](#processgroupcondition)
 * [ProcessGroupStatus](#processgroupstatus)
 * [ProcessSettings](#processsettings)
@@ -165,6 +167,7 @@ FoundationDBClusterAutomationOptions provides flags for enabling or disabling op
 | waitBetweenRemovalsSeconds | WaitBetweenRemovalsSeconds defines how long to wait between the last removal and the next removal. This is only an upper limit if the process group and the according resources are deleted faster than the provided duration the operator will move on with the next removal. The idea is to prevent a race condition were the operator deletes a resource but the Kubernetes API is slower to trigger the actual deletion, and we are running into a situation where the fault tolerance check still includes the already deleted processes. Defaults to 60. | *int | false |
 | podUpdateStrategy | PodUpdateStrategy defines how Pod spec changes are rolled out either by replacing Pods or by deleting Pods. The default for this is ReplaceTransactionSystem. | [PodUpdateStrategy](#podupdatestrategy) | false |
 | useManagementAPI | UseManagementAPI defines if the operator should make use of the management API instead of using fdbcli to interact with the FoundationDB cluster. | *bool | false |
+| maintenanceModeOptions | MaintenanceModeOptions contains options for maintenance mode related settings. | [MaintenanceModeOptions](#maintenancemodeoptions) | false |
 
 [Back to TOC](#table-of-contents)
 
@@ -254,6 +257,9 @@ FoundationDBClusterStatus defines the observed state of FoundationDBCluster
 | imageTypes | ImageTypes defines the kinds of images that are in use in the cluster. If there is more than one value in the slice the reconcile phase is not finished. | [][ImageType](#imagetype) | false |
 | processGroups | ProcessGroups contain information about a process group. This information is used in multiple places to trigger the according action. | []*[ProcessGroupStatus](#processgroupstatus) | false |
 | locks | Locks contains information about the locking system. | [LockSystemStatus](#locksystemstatus) | false |
+| maintenanceModeInfo | MaintenenanceModeInfo contains information regarding process groups in maintenance mode | [MaintenanceModeInfo](#maintenancemodeinfo) | false |
+| desiredProcessGroups | DesiredProcessGroups reflects the number of expected running process groups. | int | false |
+| reconciledProcessGroups | ReconciledProcessGroups reflects the number of process groups that have no condition and are not marked for removal. | int | false |
 
 [Back to TOC](#table-of-contents)
 
@@ -311,6 +317,29 @@ LockSystemStatus provides a summary of the status of the locking system.
 
 [Back to TOC](#table-of-contents)
 
+## MaintenanceModeInfo
+
+MaintenanceModeInfo contains information regarding the zone and process groups that are put into maintenance mode by the operator
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| startTimestamp | StartTimestamp provides the timestamp when this zone is put into maintenance mode | *metav1.Time | false |
+| zoneID | ZoneID that is placed in maintenance mode | string | false |
+| processGroups | ProcessGroups that are placed in maintenance mode | []string | false |
+
+[Back to TOC](#table-of-contents)
+
+## MaintenanceModeOptions
+
+MaintenanceModeOptions controls options for placing zones in maintenance mode.
+
+| Field | Description | Scheme | Required |
+| ----- | ----------- | ------ | -------- |
+| UseMaintenanceModeChecker | UseMaintenanceModeChecker defines whether the operator is allowed to use maintenance mode before updating pods. Default is false. | *bool | false |
+| maintenanceModeTimeSeconds | MaintenanceModeTimeSeconds provides the duration for the zone to be in maintenance. It will automatically be switched off after the time elapses. Default is 600. | *int | false |
+
+[Back to TOC](#table-of-contents)
+
 ## PodUpdateMode
 
 PodUpdateMode defines the deletion mode for the cluster
diff --git a/fdbclient/admin_client.go b/fdbclient/admin_client.go
index db5f2b9e..b3cfb4d4 100644
--- a/fdbclient/admin_client.go
+++ b/fdbclient/admin_client.go
@@ -143,9 +143,9 @@ func (command cliCommand) getClusterFileFlag() string {
 }
 
 // getTimeout returns the timeout for the command
-func (command cliCommand) getTimeout() int {
+func (command cliCommand) getTimeout() time.Duration {
 	if command.timeout != 0 {
-		return int(command.timeout.Seconds())
+		return command.timeout
 	}
 
 	return DefaultCLITimeout
@@ -206,10 +206,10 @@ func (client *cliAdminClient) runCommand(command cliCommand) (string, error) {
 	}
 
 	if command.hasTimeoutArg() {
-		args = append(args, "--timeout", strconv.Itoa(command.getTimeout()))
+		args = append(args, "--timeout", strconv.Itoa(int(command.getTimeout().Seconds())))
 		hardTimeout += command.getTimeout()
 	}
-	timeoutContext, cancelFunction := context.WithTimeout(context.Background(), time.Second*time.Duration(hardTimeout))
+	timeoutContext, cancelFunction := context.WithTimeout(context.Background(), hardTimeout)
 	defer cancelFunction()
 	execCommand := exec.CommandContext(timeoutContext, binary, args...)
 
@@ -247,25 +247,25 @@ func (client *cliAdminClient) runCommand(command cliCommand) (string, error) {
 
 // runCommandWithBackoff is a wrapper around runCommand which allows retrying commands if they hit a timeout.
 func (client *cliAdminClient) runCommandWithBackoff(command string) (string, error) {
-	maxTimeoutInSeconds := 40
-	currentTimeoutInSeconds := DefaultCLITimeout
+	maxTimeout := 40 * time.Second
+	currentTimeout := DefaultCLITimeout
 
 	var rawResult string
 	var err error
 
 	// This method will be retrying to get the status if a timeout is seen. The timeout will be doubled everytime we try
-	// it with the default timeout of 10 we will try it 3 times with the following timeouts: 10s - 20s - 40s. We have
+	// it with the default timeout of 10s we will try it 3 times with the following timeouts: 10s - 20s - 40s. We have
 	// seen that during upgrades of version incompatible version, when not all coordinators are properly restarted that
 	// the response time will be increased.
-	for currentTimeoutInSeconds <= maxTimeoutInSeconds {
-		rawResult, err = client.runCommand(cliCommand{command: command, timeout: time.Duration(currentTimeoutInSeconds) * time.Second})
+	for currentTimeout <= maxTimeout {
+		rawResult, err = client.runCommand(cliCommand{command: command, timeout: currentTimeout})
 		if err == nil {
 			break
 		}
 
 		if _, ok := err.(fdbv1beta2.TimeoutError); ok {
 			client.log.Info("timeout issue will retry with higher timeout")
-			currentTimeoutInSeconds *= 2
+			currentTimeout *= 2
 			continue
 		}
 
@@ -345,6 +345,35 @@ func (client *cliAdminClient) ConfigureDatabase(configuration fdbv1beta2.Databas
 	return err
 }
 
+// GetMaintenanceZone gets current maintenance zone, if any. Returns empty string if maintenance mode is off
+func (client *cliAdminClient) GetMaintenanceZone() (string, error) {
+	// TODO: Use special keyspace to just read the maintenance zone info instead of reading the whole status
+	status, err := client.GetStatus()
+	if err != nil {
+		return "", err
+	}
+	return status.Cluster.MaintenanceZone, nil
+}
+
+// SetMaintenanceZone places zone into maintenance mode
+func (client *cliAdminClient) SetMaintenanceZone(zone string, timeoutSeconds int) error {
+	_, err := client.runCommand(cliCommand{
+		command: fmt.Sprintf(
+			"maintenance on %s %s",
+			zone,
+			strconv.Itoa(timeoutSeconds)),
+	})
+	return err
+}
+
+// ResetMaintenanceMode switches of maintenance mode
+func (client *cliAdminClient) ResetMaintenanceMode() error {
+	_, err := client.runCommand(cliCommand{
+		command: "maintenance off",
+	})
+	return err
+}
+
 // ExcludeProcesses starts evacuating processes so that they can be removed from the database.
 func (client *cliAdminClient) ExcludeProcesses(addresses []fdbv1beta2.ProcessAddress) error {
 	if len(addresses) == 0 {
diff --git a/fdbclient/common.go b/fdbclient/common.go
index 37193909..6b7b9579 100644
--- a/fdbclient/common.go
+++ b/fdbclient/common.go
@@ -23,6 +23,7 @@ package fdbclient
 import (
 	"fmt"
 	"os"
+	"time"
 
 	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
 	"github.com/FoundationDB/fdb-kubernetes-operator/controllers"
@@ -33,11 +34,11 @@ import (
 )
 
 const (
-	defaultTransactionTimeout int64 = 5000
+	defaultTransactionTimeout = 5 * time.Second
 )
 
 // DefaultCLITimeout is the default timeout for CLI commands.
-var DefaultCLITimeout = 10
+var DefaultCLITimeout = 10 * time.Second
 
 // getFDBDatabase opens an FDB database. The result will be cached for
 // subsequent calls, based on the cluster namespace and name.
@@ -64,7 +65,7 @@ func getFDBDatabase(cluster *fdbv1beta2.FoundationDBCluster) (fdb.Database, erro
 		return fdb.Database{}, err
 	}
 
-	err = database.Options().SetTransactionTimeout(defaultTransactionTimeout)
+	err = database.Options().SetTransactionTimeout(defaultTransactionTimeout.Milliseconds())
 	if err != nil {
 		return fdb.Database{}, err
 	}
@@ -72,7 +73,7 @@ func getFDBDatabase(cluster *fdbv1beta2.FoundationDBCluster) (fdb.Database, erro
 	return database, nil
 }
 
-func getValueFromDBUsingKey(cluster *fdbv1beta2.FoundationDBCluster, log logr.Logger, fdbKey string, extraTimeout int64) ([]byte, error) {
+func getValueFromDBUsingKey(cluster *fdbv1beta2.FoundationDBCluster, log logr.Logger, fdbKey string, timeout time.Duration) ([]byte, error) {
 	log.Info("Fetch values from FDB", "namespace", cluster.Namespace, "cluster", cluster.Name, "key", fdbKey)
 	defer func() {
 		log.Info("Done fetching values from FDB", "namespace", cluster.Namespace, "cluster", cluster.Name, "key", fdbKey)
@@ -87,7 +88,7 @@ func getValueFromDBUsingKey(cluster *fdbv1beta2.FoundationDBCluster, log logr.Lo
 		if err != nil {
 			return nil, err
 		}
-		err = transaction.Options().SetTimeout(int64(DefaultCLITimeout) * extraTimeout)
+		err = transaction.Options().SetTimeout(timeout.Milliseconds())
 		if err != nil {
 			return nil, err
 		}
@@ -121,12 +122,12 @@ func getValueFromDBUsingKey(cluster *fdbv1beta2.FoundationDBCluster, log logr.Lo
 
 // getConnectionStringFromDB gets the database's connection string directly from the system key
 func getConnectionStringFromDB(cluster *fdbv1beta2.FoundationDBCluster, log logr.Logger) ([]byte, error) {
-	return getValueFromDBUsingKey(cluster, log, "\xff/coordinators", 1)
+	return getValueFromDBUsingKey(cluster, log, "\xff/coordinators", DefaultCLITimeout)
 }
 
 // getStatusFromDB gets the database's status directly from the system key
 func getStatusFromDB(cluster *fdbv1beta2.FoundationDBCluster, log logr.Logger) ([]byte, error) {
-	return getValueFromDBUsingKey(cluster, log, "\xff\xff/status/json", 1000)
+	return getValueFromDBUsingKey(cluster, log, "\xff\xff/status/json", DefaultCLITimeout)
 }
 
 type realDatabaseClientProvider struct {
diff --git a/go.mod b/go.mod
index b3e4c18e..318090b8 100644
--- a/go.mod
+++ b/go.mod
@@ -7,7 +7,7 @@ require (
 	github.com/apple/foundationdb/fdbkubernetesmonitor v0.0.0-20220513200452-e6fa4d7422d2
 	github.com/fatih/color v1.10.0
 	github.com/go-logr/logr v1.2.0
-	github.com/google/go-cmp v0.5.5
+	github.com/google/go-cmp v0.5.6
 	github.com/hashicorp/go-retryablehttp v0.6.8
 	github.com/onsi/ginkgo/v2 v2.1.3
 	github.com/onsi/gomega v1.18.1
@@ -24,6 +24,7 @@ require (
 	k8s.io/klog/v2 v2.60.1
 	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9
 	sigs.k8s.io/controller-runtime v0.12.3
+	sigs.k8s.io/e2e-framework v0.0.7
 	sigs.k8s.io/yaml v1.3.0
 )
 
@@ -54,7 +55,7 @@ require (
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/btree v1.0.1 // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect
-	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.1.2 // indirect
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
@@ -87,6 +88,7 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/stretchr/testify v1.7.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/vladimirvivien/gexe v0.1.1 // indirect
 	github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect
 	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
diff --git a/go.sum b/go.sum
index e3dafc4e..a8fb5201 100644
--- a/go.sum
+++ b/go.sum
@@ -260,11 +260,13 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
+github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@@ -540,6 +542,8 @@ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/vladimirvivien/gexe v0.1.1 h1:2A0SBaOSKH+cwLVdt6H+KkHZotZWRNLlWygANGw5DxE=
+github.com/vladimirvivien/gexe v0.1.1/go.mod h1:LHQL00w/7gDUKIak24n801ABp8C+ni6eBht9vGVst8w=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI=
@@ -1078,6 +1082,8 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw=
 sigs.k8s.io/controller-runtime v0.12.3 h1:FCM8xeY/FI8hoAfh/V4XbbYMY20gElh9yh+A98usMio=
 sigs.k8s.io/controller-runtime v0.12.3/go.mod h1:qKsk4WE6zW2Hfj0G4v10EnNB2jMG1C+NTb8h+DwCoU0=
+sigs.k8s.io/e2e-framework v0.0.7 h1:nMv2oSPBLWARse2aBoqX5Wq3ox67w8jrhTGWGpccWDQ=
+sigs.k8s.io/e2e-framework v0.0.7/go.mod h1:hdwYGVQg4bvDAah5eidNf2/qkG35qHjzuyMVr2A3oiY=
 sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y=
 sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY=
 sigs.k8s.io/kustomize/api v0.11.4 h1:/0Mr3kfBBNcNPOW5Qwk/3eb8zkswCwnqQxxKtmrTkRo=
diff --git a/pkg/fdbadminclient/admin_client.go b/pkg/fdbadminclient/admin_client.go
index e7825096..ff0054e7 100644
--- a/pkg/fdbadminclient/admin_client.go
+++ b/pkg/fdbadminclient/admin_client.go
@@ -102,4 +102,13 @@ type AdminClient interface {
 
 	// SetKnobs sets the knobs that should be used for the commandline call.
 	SetKnobs([]string)
+
+	// GetMaintenanceZone gets current maintenance zone, if any
+	GetMaintenanceZone() (string, error)
+
+	// SetMaintenanceZone places zone into maintenance mode
+	SetMaintenanceZone(zone string, timeoutSeconds int) error
+
+	// Reset maintenance mode
+	ResetMaintenanceMode() error
 }
diff --git a/setup/setup.go b/setup/setup.go
index b85d82ff..1125250d 100644
--- a/setup/setup.go
+++ b/setup/setup.go
@@ -83,7 +83,7 @@ func (o *Options) BindFlags(fs *flag.FlagSet) {
 		"Apply defaults from the next major version of the operator. This is only intended for use in development.",
 	)
 	fs.StringVar(&o.LogFile, "log-file", "", "The path to a file to write logs to.")
-	fs.IntVar(&o.CliTimeout, "cli-timeout", 10, "The timeout to use for CLI commands.")
+	fs.IntVar(&o.CliTimeout, "cli-timeout", 10, "The timeout to use for CLI commands in seconds.")
 	fs.IntVar(&o.MaxConcurrentReconciles, "max-concurrent-reconciles", 1, "Defines the maximum number of concurrent reconciles for all controllers.")
 	fs.BoolVar(&o.CleanUpOldLogFile, "cleanup-old-cli-logs", true, "Defines if the operator should delete old fdbcli log files.")
 	fs.DurationVar(&o.LogFileMinAge, "log-file-min-age", 5*time.Minute, "Defines the minimum age of fdbcli log files before removing when \"--cleanup-old-cli-logs\" is set.")
@@ -143,7 +143,7 @@ func StartManager(
 	klog.SetLogger(logger)
 
 	setupLog := logger.WithName("setup")
-	fdbclient.DefaultCLITimeout = operatorOpts.CliTimeout
+	fdbclient.DefaultCLITimeout = time.Duration(operatorOpts.CliTimeout) * time.Second
 
 	options := ctrl.Options{
 		Scheme:             scheme,


Test patch:
diff --git a/api/v1beta2/foundationdb_database_configuration_test.go b/api/v1beta2/foundationdb_database_configuration_test.go
index c404be82..ec63884b 100644
--- a/api/v1beta2/foundationdb_database_configuration_test.go
+++ b/api/v1beta2/foundationdb_database_configuration_test.go
@@ -157,4 +157,37 @@ var _ = Describe("DatabaseConfiguration", func() {
 			})
 		})
 	})
+
+	When("using ProcessCounts", func() {
+		When("calculating the total number of processes", func() {
+			var counts ProcessCounts
+
+			When("no negative process counts are specified", func() {
+				BeforeEach(func() {
+					counts = ProcessCounts{
+						Storage: 5,
+						Log:     5,
+					}
+				})
+
+				It("should print the correct number of total processes", func() {
+					Expect(counts.Total()).To(BeNumerically("==", 10))
+				})
+			})
+
+			When("negative process counts are specified", func() {
+				BeforeEach(func() {
+					counts = ProcessCounts{
+						Storage:   5,
+						Log:       5,
+						Stateless: -1,
+					}
+				})
+
+				It("should print the correct number of total processes", func() {
+					Expect(counts.Total()).To(BeNumerically("==", 10))
+				})
+			})
+		})
+	})
 })
diff --git a/controllers/maintenance_mode_checker_test.go b/controllers/maintenance_mode_checker_test.go
new file mode 100644
index 00000000..baf26b8e
--- /dev/null
+++ b/controllers/maintenance_mode_checker_test.go
@@ -0,0 +1,154 @@
+/*
+ * maintenance_mode_checker_test.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controllers
+
+import (
+	"context"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/pointer"
+
+	"github.com/FoundationDB/fdb-kubernetes-operator/internal"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+)
+
+var _ = Describe("maintenance_mode_checker", func() {
+	var cluster *fdbv1beta2.FoundationDBCluster
+	var err error
+	var requeue *requeue
+	var adminClient *mockAdminClient
+
+	BeforeEach(func() {
+		cluster = internal.CreateDefaultCluster()
+		//Set maintenance mode on
+		cluster.Spec.AutomationOptions.MaintenanceModeOptions.UseMaintenanceModeChecker = pointer.Bool(true)
+		err = k8sClient.Create(context.TODO(), cluster)
+		Expect(err).NotTo(HaveOccurred())
+
+		result, err := reconcileCluster(cluster)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(result.Requeue).To(BeFalse())
+
+		generation, err := reloadCluster(cluster)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(generation).To(Equal(int64(1)))
+
+		adminClient, err = newMockAdminClientUncast(cluster, k8sClient)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	JustBeforeEach(func() {
+		requeue = maintenanceModeChecker{}.reconcile(context.TODO(), clusterReconciler, cluster)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = reloadCluster(cluster)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	Context("maintenance mode is off", func() {
+		When("status maintenance is empty", func() {
+			It("should not be requeued", func() {
+				Expect(requeue).To(BeNil())
+				Expect(cluster.Status.MaintenanceModeInfo).To(Equal(fdbv1beta2.MaintenanceModeInfo{}))
+			})
+		})
+		When("status maintenance is not empty", func() {
+			BeforeEach(func() {
+				cluster.Status.MaintenanceModeInfo = fdbv1beta2.MaintenanceModeInfo{ZoneID: "storage-4"}
+			})
+			It("should not be requeued", func() {
+				Expect(requeue).To(BeNil())
+				Expect(cluster.Status.MaintenanceModeInfo).To(Equal(fdbv1beta2.MaintenanceModeInfo{}))
+			})
+		})
+
+	})
+
+	Context("maintenance mode is on", func() {
+		BeforeEach(func() {
+			Expect(adminClient.SetMaintenanceZone("operator-test-1-storage-1", 0)).NotTo(HaveOccurred())
+		})
+
+		When("status maintenance zone does not match", func() {
+			BeforeEach(func() {
+				cluster.Status.MaintenanceModeInfo = fdbv1beta2.MaintenanceModeInfo{ZoneID: "operator-test-1-storage-4"}
+			})
+			It("should not be requeued", func() {
+				Expect(requeue).To(BeNil())
+				Expect(cluster.Status.MaintenanceModeInfo).To(Equal(fdbv1beta2.MaintenanceModeInfo{}))
+			})
+		})
+
+		When("pod hasn't been updated yet", func() {
+			BeforeEach(func() {
+				cluster.Status.MaintenanceModeInfo = fdbv1beta2.MaintenanceModeInfo{
+					StartTimestamp: &metav1.Time{Time: time.Now()},
+					ZoneID:         "operator-test-1-storage-1",
+					ProcessGroups:  []string{"storage-1"},
+				}
+				adminClient.MockUptimeSecondsForMaintenanceZone(3600)
+			})
+			It("should be requeued", func() {
+				Expect(requeue).ToNot(BeNil())
+				Expect(requeue.message).To(Equal("Waiting for pod storage-1 to be updated"))
+			})
+		})
+
+		When("pod is still down", func() {
+			BeforeEach(func() {
+				cluster.Status.MaintenanceModeInfo = fdbv1beta2.MaintenanceModeInfo{
+					StartTimestamp: &metav1.Time{Time: time.Now()},
+					ZoneID:         "operator-test-1-storage-1",
+					ProcessGroups:  []string{"storage-1"},
+				}
+				adminClient.MockMissingProcessGroup("storage-1", true)
+			})
+			It("should be requeued", func() {
+				Expect(requeue).ToNot(BeNil())
+				Expect(requeue.message).To(Equal("Waiting for all proceeses in zone operator-test-1-storage-1 to be up"))
+			})
+		})
+
+		When("reconciliation should be successful", func() {
+			BeforeEach(func() {
+				cluster.Status.MaintenanceModeInfo = fdbv1beta2.MaintenanceModeInfo{
+					StartTimestamp: &metav1.Time{Time: time.Now().Add(-60 * time.Second)},
+					ZoneID:         "operator-test-1-storage-1",
+					ProcessGroups:  []string{"storage-1"},
+				}
+
+			})
+			It("should not be requeued", func() {
+				Expect(requeue).To(BeNil())
+				Expect(cluster.Status.MaintenanceModeInfo).To(Equal(fdbv1beta2.MaintenanceModeInfo{}))
+			})
+		})
+	})
+
+	AfterEach(func() {
+		clearMockAdminClients()
+		k8sClient.Clear()
+	})
+})
diff --git a/e2e/README.md b/e2e/README.md
new file mode 100644
index 00000000..ecfdf66c
--- /dev/null
+++ b/e2e/README.md
@@ -0,0 +1,21 @@
+# e2e tests for FDB Kubernetes operator
+
+This folder contains test suites to run e2e tests against a [kind](https://kind.sigs.k8s.io) or a real cluster.
+
+## Running e2e tests
+
+The current setup assumes that it can use an already running Kubernetes cluster.
+If you don't have a running test cluster you can create one using [kind](https://kind.sigs.k8s.io)
+**Important** if you use an already existing Kubernetes cluster the e2e tests will apply the latest CRDs from the current branch you are running them and afterwards they will be removed.
+
+
+```bash
+$ kind create cluster
+$ kind get kubeconfig > ~/.kube/e2e_test
+```
+
+Now you can run the e2e tests against the local cluster:
+
+```bash
+go test -v ./e2e/... --tags=e2e_test --kubeconfig=${HOME}/.kube/e2e_test
+```
diff --git a/e2e/create_single_fdb_cluster_test.go b/e2e/create_single_fdb_cluster_test.go
new file mode 100644
index 00000000..c8e55b83
--- /dev/null
+++ b/e2e/create_single_fdb_cluster_test.go
@@ -0,0 +1,41 @@
+//go:build e2e_test
+
+/*
+ * create_single_fdb_cluster_test.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package e2e
+
+import (
+	"testing"
+
+	"github.com/FoundationDB/fdb-kubernetes-operator/e2e/helper"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+)
+
+func TestCreateSingleFDBCluster(t *testing.T) {
+	testVersions := helper.GetTestFDBVersions()
+	createSingleClusterFeatures := make([]features.Feature, 0, len(testVersions))
+
+	for _, version := range testVersions {
+		createSingleClusterFeatures = append(createSingleClusterFeatures, helper.CreateSingleClusterTest(version, t))
+	}
+
+	testenv.Test(t, createSingleClusterFeatures...)
+}
diff --git a/e2e/helper/fdb_cluster_helper.go b/e2e/helper/fdb_cluster_helper.go
new file mode 100644
index 00000000..c5c8409a
--- /dev/null
+++ b/e2e/helper/fdb_cluster_helper.go
@@ -0,0 +1,262 @@
+/*
+ * fdb_cluster_helper.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package helper
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	fdbv1beta2 "github.com/FoundationDB/fdb-kubernetes-operator/api/v1beta2"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"sigs.k8s.io/e2e-framework/klient/k8s"
+	"sigs.k8s.io/e2e-framework/klient/k8s/resources"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+)
+
+type contextKey int
+
+const (
+	keyNamespaceID contextKey = iota
+	keyClusterNameID
+)
+
+// GetTestFDBVersions returns a slice that contains all FDB versions we use to do e2e tests.
+func GetTestFDBVersions() []fdbv1beta2.Version {
+	return []fdbv1beta2.Version{
+		{Major: 6, Minor: 2, Patch: 20},
+		{Major: 6, Minor: 3, Patch: 23},
+		{Major: 7, Minor: 1, Patch: 23},
+	}
+}
+
+// CreateSingleClusterTest returns an e2e test that creates a cluster and assess that the cluster was correctly created.
+func CreateSingleClusterTest(version fdbv1beta2.Version, t *testing.T) features.Feature {
+	return features.
+		New("create single cluster for "+version.Compact()).
+		WithLabel("type", "create-cluster").
+		Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			clusterName := envconf.RandomName("fdb-create", 32)
+			namespace := ctx.Value(keyNamespaceID).(string)
+
+			testCluster := createFoundationDBCluster(clusterName, namespace, version.String())
+
+			err := cfg.Client().Resources(namespace).Create(ctx, testCluster)
+			if err != nil {
+				t.Error(err)
+			}
+
+			return context.WithValue(ctx, keyClusterNameID, clusterName)
+		}).
+		Assess("it should create an FDB cluster in "+version.Compact(), func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			namespace := ctx.Value(keyNamespaceID).(string)
+			clusterName := ctx.Value(keyClusterNameID).(string)
+
+			testCluster := &fdbv1beta2.FoundationDBCluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      clusterName,
+					Namespace: namespace,
+				},
+			}
+
+			err := wait.For(conditions.New(cfg.Client().Resources(namespace)).ResourceMatch(testCluster, func(object k8s.Object) bool {
+				cluster := object.(*fdbv1beta2.FoundationDBCluster)
+				return cluster.Status.Generations.Reconciled >= 1
+			}), wait.WithTimeout(time.Minute*3))
+
+			if err != nil {
+				t.Error(err)
+			}
+
+			return ctx
+		}).
+		Assess("it should create all desired Pods", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			namespace := ctx.Value(keyNamespaceID).(string)
+			clusterName := ctx.Value(keyClusterNameID).(string)
+
+			testCluster := &fdbv1beta2.FoundationDBCluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      clusterName,
+					Namespace: namespace,
+				},
+			}
+
+			err := cfg.Client().Resources(namespace).Get(ctx, testCluster.Name, testCluster.Namespace, testCluster)
+			if err != nil {
+				t.Error(err)
+				t.Fail()
+			}
+
+			podList := &corev1.PodList{}
+
+			err = cfg.Client().Resources(namespace).List(ctx, podList, resources.WithLabelSelector(labels.SelectorFromSet(testCluster.GetMatchLabels()).String()))
+			if err != nil {
+				t.Error(err)
+				t.Fail()
+			}
+
+			processCounts, err := testCluster.GetProcessCountsWithDefaults()
+			if err != nil {
+				t.Error(err)
+			}
+
+			var storageCount, statelessCount, logCount int
+			for _, pod := range podList.Items {
+				pClass, ok := pod.GetLabels()[testCluster.GetProcessClassLabel()]
+				if !ok {
+					continue
+				}
+
+				if pClass == string(fdbv1beta2.ProcessClassStorage) {
+					storageCount++
+					continue
+				} else if pClass == string(fdbv1beta2.ProcessClassLog) {
+					logCount++
+					continue
+				} else if pClass == string(fdbv1beta2.ProcessClassStateless) {
+					statelessCount++
+					continue
+				}
+			}
+
+			if storageCount != processCounts.Storage || statelessCount != processCounts.Stateless || logCount != processCounts.Log {
+				t.Errorf(`Expected to have matching process counts
+Expected Storage: %d, got: %d
+Expected Stateless: %d, got: %d
+Expected Log: %d, got: %d
+`,
+					processCounts.Storage, storageCount,
+					processCounts.Stateless, statelessCount,
+					processCounts.Log, logCount,
+				)
+			}
+
+			return ctx
+		}).
+		Teardown(func(ctx context.Context, t *testing.T, config *envconf.Config) context.Context {
+			namespace := ctx.Value(keyNamespaceID).(string)
+			clusterName := ctx.Value(keyClusterNameID).(string)
+
+			testCluster := &fdbv1beta2.FoundationDBCluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      clusterName,
+					Namespace: namespace,
+				},
+			}
+
+			err := config.Client().Resources(namespace).Delete(ctx, testCluster)
+			if err != nil {
+				t.Error(err)
+			}
+
+			return ctx
+		}).Feature()
+}
+
+func createFoundationDBCluster(clusterName string, namespace string, version string) *fdbv1beta2.FoundationDBCluster {
+	return &fdbv1beta2.FoundationDBCluster{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      clusterName,
+			Namespace: namespace,
+		},
+		Spec: fdbv1beta2.FoundationDBClusterSpec{
+			FaultDomain: fdbv1beta2.FoundationDBClusterFaultDomain{
+				Key: "foundationdb.org/none",
+			},
+			MinimumUptimeSecondsForBounce: 60,
+			Version:                       version,
+			Processes: map[fdbv1beta2.ProcessClass]fdbv1beta2.ProcessSettings{
+				fdbv1beta2.ProcessClassGeneral: {
+					CustomParameters: fdbv1beta2.FoundationDBCustomParameters{
+						"knob_disable_posix_kernel_aio=1",
+					},
+					PodTemplate: &corev1.PodTemplateSpec{
+						Spec: corev1.PodSpec{
+							Containers: []corev1.Container{
+								{
+									Name: "foundationdb",
+									Resources: corev1.ResourceRequirements{
+										Requests: corev1.ResourceList{
+											"foundationdb.org/empty": resource.MustParse("0"),
+										},
+									},
+								},
+								{
+									Name: "foundationdb-kubernetes-sidecar",
+									Resources: corev1.ResourceRequirements{
+										Requests: corev1.ResourceList{
+											"foundationdb.org/empty": resource.MustParse("0"),
+										},
+									},
+								},
+							},
+							InitContainers: []corev1.Container{
+								{
+									Name: "foundationdb-kubernetes-init",
+									Resources: corev1.ResourceRequirements{
+										Requests: corev1.ResourceList{
+											"foundationdb.org/empty": resource.MustParse("0"),
+										},
+									},
+								},
+							},
+						},
+					},
+					VolumeClaimTemplate: &corev1.PersistentVolumeClaim{
+						Spec: corev1.PersistentVolumeClaimSpec{
+							Resources: corev1.ResourceRequirements{
+								Requests: corev1.ResourceList{
+									corev1.ResourceStorage: resource.MustParse("2G"),
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+// RegisterFDBScheme registers the fdbv1beta2 scheme to the rest config
+func RegisterFDBScheme(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
+	r, err := resources.New(cfg.Client().RESTConfig())
+	if err != nil {
+		return ctx, err
+	}
+
+	err = fdbv1beta2.AddToScheme(r.GetScheme())
+
+	return ctx, err
+}
+
+// WaitUntilCRDAvailable waits until the CRD is available
+func WaitUntilCRDAvailable(ctx context.Context, _ *envconf.Config) (context.Context, error) {
+	// TODO (johscheuer): Implement this logic instead of sleeping 1 second.
+	time.Sleep(1 * time.Second)
+
+	return ctx, nil
+}
diff --git a/e2e/helper/operator_helper.go b/e2e/helper/operator_helper.go
new file mode 100644
index 00000000..a91d5380
--- /dev/null
+++ b/e2e/helper/operator_helper.go
@@ -0,0 +1,91 @@
+/*
+ * operator_helper.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package helper
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	appsv1 "k8s.io/api/apps/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/e2e-framework/klient/decoder"
+	"sigs.k8s.io/e2e-framework/klient/k8s"
+	"sigs.k8s.io/e2e-framework/klient/k8s/resources"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+)
+
+// InstallOperator is a helper method to install the FDB operator inside the test namespace.
+func InstallOperator(ctx context.Context, cfg *envconf.Config, _ *testing.T) (context.Context, error) {
+	r, err := resources.New(cfg.Client().RESTConfig())
+	if err != nil {
+		return ctx, err
+	}
+
+	ns := ctx.Value(keyNamespaceID).(string)
+
+	// TODO (johscheuer): Make the image a flag that can be passed
+	err = decoder.DecodeEachFile(ctx, os.DirFS("../config/samples"), "deployment.yaml",
+		decoder.CreateHandler(r),
+		decoder.MutateNamespace(ns),
+		mutateOperatorImage("manager", "foundationdb/fdb-kubernetes-operator:latest"),
+	)
+	if err != nil {
+		return ctx, err
+	}
+
+	deploy := &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "fdb-kubernetes-operator-controller-manager",
+			Namespace: ns,
+		},
+	}
+
+	err = wait.For(conditions.New(cfg.Client().Resources(ns)).ResourceMatch(deploy, func(object k8s.Object) bool {
+		d := object.(*appsv1.Deployment)
+		return d.Status.ReadyReplicas >= 1
+	}), wait.WithTimeout(time.Minute*2))
+
+	return ctx, err
+}
+
+func mutateOperatorImage(name string, image string) decoder.DecodeOption {
+	return decoder.MutateOption(func(obj k8s.Object) error {
+		deploy, ok := obj.(*appsv1.Deployment)
+		if !ok {
+			return nil
+		}
+
+		for idx, container := range deploy.Spec.Template.Spec.Containers {
+			if container.Name != name {
+				continue
+			}
+
+			container.Image = image
+			deploy.Spec.Template.Spec.Containers[idx] = container
+		}
+
+		return nil
+	})
+}
diff --git a/e2e/helper/test_setup_helper.go b/e2e/helper/test_setup_helper.go
new file mode 100644
index 00000000..9c055575
--- /dev/null
+++ b/e2e/helper/test_setup_helper.go
@@ -0,0 +1,39 @@
+package helper
+
+import (
+	"context"
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+)
+
+// CreateNamespace ...
+func CreateNamespace(ctx context.Context, cfg *envconf.Config, t *testing.T, runID string) (context.Context, error) {
+	ns := envconf.RandomName(runID, 10)
+	ctx = context.WithValue(ctx, keyNamespaceID, ns)
+
+	t.Logf("Creating NS %v for test %v", ns, t.Name())
+	nsObj := corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: ns,
+		},
+	}
+
+	return ctx, cfg.Client().Resources().Create(ctx, &nsObj)
+}
+
+// DeleteNamespace ...
+func DeleteNamespace(ctx context.Context, cfg *envconf.Config, t *testing.T) (context.Context, error) {
+	ns := ctx.Value(keyNamespaceID).(string)
+	t.Logf("Deleting NS %v for test %v", ns, t.Name())
+
+	nsObj := corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: ns,
+		},
+	}
+
+	return ctx, cfg.Client().Resources().Delete(ctx, &nsObj)
+}
diff --git a/e2e/main_test.go b/e2e/main_test.go
new file mode 100644
index 00000000..f3502128
--- /dev/null
+++ b/e2e/main_test.go
@@ -0,0 +1,73 @@
+//go:build e2e_test
+
+/*
+ * main_test.go
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/FoundationDB/fdb-kubernetes-operator/e2e/helper"
+
+	// If testing with a cloud vendor managed cluster uncomment one of the below dependencies to properly get authorised.
+	//_ "k8s.io/client-go/plugin/pkg/client/auth/azure" // auth for AKS clusters
+	//_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"   // auth for GKE clusters
+	//_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"  // auth for OIDC
+	"os"
+	"testing"
+
+	"sigs.k8s.io/e2e-framework/pkg/env"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/envfuncs"
+)
+
+var testenv env.Environment
+
+func TestMain(m *testing.M) {
+	testenv = env.New()
+
+	cfg, err := envconf.NewFromFlags()
+	if err != nil {
+		_, _ = fmt.Fprintf(os.Stderr, "envconf failed: %s\n", err)
+		os.Exit(1)
+	}
+
+	testenv = env.NewWithConfig(cfg)
+
+	testenv.Setup(
+		envfuncs.SetupCRDs("../config/crd/bases", "*"),
+		helper.RegisterFDBScheme,
+		helper.WaitUntilCRDAvailable,
+	)
+
+	testenv.BeforeEachTest(func(ctx context.Context, cfg *envconf.Config, t *testing.T) (context.Context, error) {
+		return helper.CreateNamespace(ctx, cfg, t, envconf.RandomName("ns", 4))
+	}, helper.InstallOperator)
+
+	testenv.AfterEachTest(helper.DeleteNamespace)
+
+	testenv.Finish(
+		envfuncs.TeardownCRDs("../config/crd/bases", "*"),
+	)
+
+	os.Exit(testenv.Run(m))
+}