Repository: scylladb/scylla-operator
Pull request #358 (2021-02-02 10:25:43)

Helm example


Patch:
diff --git a/config/operator/operator/operator.yaml b/config/operator/operator/operator.yaml
index c2c7dffb1..c942541b9 100644
--- a/config/operator/operator/operator.yaml
+++ b/config/operator/operator/operator.yaml
@@ -41,9 +41,6 @@ spec:
         imagePullPolicy: IfNotPresent
         name: manager
         resources:
-          limits:
-            cpu: 100m
-            memory: 30Mi
           requests:
             cpu: 100m
             memory: 20Mi
diff --git a/docs/source/helm.md b/docs/source/helm.md
new file mode 100644
index 000000000..08d184601
--- /dev/null
+++ b/docs/source/helm.md
@@ -0,0 +1,336 @@
+# Deploying Scylla stack using Helm Charts
+
+In this example we will install Scylla stack on Kubernetes. This includes the following components:
+- Scylla Operator
+- Scylla Manager
+- Scylla
+
+We will use Minikube K8s cluster, but this could be any K8s cluster supported by the Scylla Operator.
+
+### Prerequisites
+
+- Kubernetes 1.16+
+- Helm 3+
+
+### TL;DR
+
+```
+helm repo add scylla https://scylla-operator-charts.storage.googleapis.com
+helm repo update
+kubectl apply -f examples/common/cert-manager.yaml 
+helm install scylla-operator scylla/scylla-operator --create-namespace --namespace scylla-operator
+helm install scylla-manager scylla/scylla-manager --create-namespace --namespace scylla-manager
+helm install scylla scylla/scylla --create-namespace --namespace scylla
+```
+
+### Deploy Cert Manager
+
+This step is optional if you want to use your own certificate. 
+If you don't have one, make sure to not disable autogeneration using Scylla Operator Helm Chart.
+
+First deploy Cert Manager, you can either follow [upsteam instructions](https://cert-manager.io/docs/installation/kubernetes/) or use following command:
+
+```console
+kubectl apply -f examples/common/cert-manager.yaml
+```
+
+Once it's deployed, wait until all Cert Manager pods will enter into Running state:
+
+```console
+kubectl wait -n cert-manager --for=condition=ready pod -l app=cert-manager --timeout=60s
+```
+
+### Helm Chart repository
+
+To install Scylla Helm Chart repository execute the following commands:
+```
+helm repo add scylla https://scylla-operator-charts.storage.googleapis.com
+helm repo update
+```
+
+Then you can search through repository, it should contain at least three Helm charts: 
+```
+helm search repo scylla
+NAME                   CHART VERSION   APP VERSION     DESCRIPTION                                       
+scylla/scylla          1.0.1           v1.0.1          Scylla is a close-to-the-hardware rewrite of Ca...
+scylla/scylla-manager  1.0.1           v1.0.1          Scylla Manager automates database operations.     
+scylla/scylla-operator 1.0.1           v1.0.1          Scylla Operator is a Kubernetes Operator for ma...
+```
+
+All these charts should be installable without any need of customizing (defaults are provided). 
+Although Helm is used for this particular reason, so lets customize them a bit.
+
+### Scylla Operator Chart
+
+This chart is very simple, most interesting customizable fields are `image`, `resources` and `webhook`. 
+All others can be looked up in Chart source in Scylla Operator repository.
+
+#### image
+
+Image allows to define which Scylla Operator image will be used. By default it downloads the image from main 
+Docker Hub repository, using version defined in Helm Chart. 
+You can also change `pullPolicy` if default one does not
+fullfill your needs. In [Kubernetes documentation](https://kubernetes.io/docs/concepts/containers/images/) you 
+can read more about different pull policies.
+
+Image URL will be composed based on these fields in follwing pattern:
+`repository/scylla-operator:tag`
+```yaml
+image:
+  repository: scylladb
+  pullPolicy: IfNotPresent
+  tag: ""
+```
+
+#### resources
+
+You can customize how much resources will be allocated for Operator pods via `resource` field:
+```yaml
+resources:
+  limits:
+    cpu: 100m
+    memory: 128Mi
+  requests:
+    cpu: 100m
+    memory: 32Mi
+```
+
+To read more about resource specification, follow [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/). 
+
+#### webhook
+
+Webhook field allows to decide whether you want to use autogenerated self-signed certificate using Cert Manager or 
+whether you want to provide your own certificate.
+
+`createSelfSignedCertificate` specifies whether a self-signed certificate should be created using Cert Manager
+`certificateSecretName`: name of a secret containing custom certificate.
+
+```yaml
+webhook:
+  createSelfSignedCertificate: true
+  certificateSecretName: ""
+```
+
+#### Customization
+
+You can customize all these fields and others by providing file containing desired values.
+Content of this file will overwrite default values.
+
+You can find an example in Scylla Operator repository under `examples/helm/values.operator.yaml`
+
+#### Installation
+
+To deploy Scylla Operator using customized values file execute the following:
+```
+helm install scylla-operator scylla/scylla-operator --values examples/helm/values.operator.yaml --create-namespace --namespace scylla-operator
+```
+
+### Scylla Helm Chart
+
+Scylla Chart allows to customize and deploy Scylla cluster.
+By default Scylla Helm charts deploys working Scylla cluster, but of course we can customize it.
+
+#### Customization
+
+Versions of images used in the cluster can be set via `scyllaImage` and `agentImage`
+```yaml
+scyllaImage:
+  repository: scylladb/scylla
+  tag: 4.3.0
+
+agentImage:
+  repository: scylladb/scylla-manager-agent
+  tag: 2.2.1
+```
+
+A minimal Scylla cluster can be expressed as:
+```yaml
+datacenter: us-east-1
+racks:
+- name: us-east-1b
+  members: 2
+  storage:
+    capacity: 5G
+  resources:
+    limits:
+      cpu: 1
+      memory: 1Gi
+    requests:
+      cpu: 1
+      memory: 1Gi
+```
+
+Above cluster will use 4.3.0 Scylla, 2.2.1 Scylla Manager Agent sidecar and will have a single rack having 2 nodes. 
+Each node will have a single CPU and 1 GiB of memory.
+
+For other customizable fields, please refer to [ScyllaCluster CRD definition](scylla_cluster_crd.md).
+CRD Rack Spec and Helm Chart Rack should have the same fields.
+
+#### Installation
+
+To deploy Scylla cluster using customzied values file execute the following command:
+```
+helm install scylla scylla/scylla --values examples/helm/values.cluster.yaml --create-namespace --namespace scylla
+```
+
+Scylla Operator will provision this cluster on your K8s environment.
+
+### Scylla Manager Helm Chart
+
+Scylla Manager Chart allows to customize and deploy Scylla Manager in K8s environment.
+Scylla Manager consist of two applications (Scylla Manager itself and Scylla Manager Controller) and additional Scylla cluster.
+
+To read more about Scylla Manager see [Manager guide](manager.md).
+
+#### Scylla Manager
+
+To set version of used Scylla Manager you can use `image` field:
+```yaml
+image:
+  repository: scylladb
+  pullPolicy: IfNotPresent
+  tag: 2.2.1
+```
+To control how many resources are allocated for Scylla Manager use `resource` field:
+```yaml
+resources:
+  limits:
+    cpu: 500m
+    memory: 500Mi
+  requests:
+    cpu: 500m
+    memory: 500Mi
+```
+
+#### Scylla Manager Controller 
+
+Similarly Scylla Manager Controller image can be customized:
+
+```yaml
+controllerImage:
+  repository: scylladb
+  pullPolicy: IfNotPresent
+  tag: ""
+```
+
+And allocated resources:
+```yaml
+controllerResources:
+  limits:
+    cpu: 100m
+    memory: 30Mi
+  requests:
+    cpu: 100m
+    memory: 20Mi
+```
+
+#### Scylla
+
+To customize internal Scylla instance dedicated to Scylla Manager, see guide above customizing Scylla Helm Chart.
+It's definition should land as a `scylla` field.
+
+#### Customization
+
+All others customizable fields can be looked up in Chart source in Scylla Operator repository.
+
+#### Installation
+
+To deploy Scylla Manager using customized values file execute the following command:
+```
+helm install scylla-manager scylla/scylla-manager --values examples/helm/values.manager.yaml --create-namespace --namespace scylla-manager
+```
+
+## Results
+
+Scylla need some time to bootstrap all nodes, but after some time you should be ready to roll. It was simple isn't it?
+You can validate if everything was set up correctly by looking at the all resources created in used namespaces.
+
+Scylla Operator:
+```shell
+$ kubectl -n scylla-operator get all
+
+NAME                    READY   STATUS    RESTARTS   AGE
+pod/scylla-operator-0   1/1     Running   0          4m48s
+
+NAME                      TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)   AGE
+service/scylla-operator   ClusterIP   10.43.231.78   <none>        443/TCP   4m48s
+
+NAME                               READY   AGE
+statefulset.apps/scylla-operator   1/1     4m48s
+```
+
+Operator is running!
+
+Scylla Manager:
+```shell
+$ kubectl -n scylla-manager get all 
+
+NAME                                           READY   STATUS    RESTARTS   AGE
+pod/scylla-manager-767b875bcf-7cppw            1/1     Running   0          7m
+pod/scylla-manager-0                           1/1     Running   0          7m
+pod/scylla-manager-manager-dc-manager-rack-0   2/2     Running   0          6m59s
+
+NAME                                               TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                                                           AGE
+service/scylla-manager-manager-service             ClusterIP   10.43.211.89    <none>        80/TCP                                                            7m1s
+service/scylla-manager-client                      ClusterIP   None            <none>        9180/TCP,5090/TCP                                                 7m
+service/scylla-manager-manager-dc-manager-rack-0   ClusterIP   10.43.239.136   <none>        7000/TCP,7001/TCP,7199/TCP,10001/TCP,9042/TCP,9142/TCP,9160/TCP   6m59s
+
+NAME                             READY   UP-TO-DATE   AVAILABLE   AGE
+deployment.apps/scylla-manager   1/1     1            1           7m1s
+
+NAME                                        DESIRED   CURRENT   READY   AGE
+replicaset.apps/scylla-manager-767b875bcf   1         1         1       7m
+
+NAME                                                      READY   AGE
+statefulset.apps/scylla-manager                           1/1     7m1s
+statefulset.apps/scylla-manager-manager-dc-manager-rack   1/1     7m
+
+```
+
+Good to go, ready to serve!
+
+Scylla itself:
+```shell
+$ kubectl -n scylla get all        
+
+NAME                                READY   STATUS    RESTARTS   AGE
+pod/scylla-us-east-1-us-east-1b-0   2/2     Running   0          5m58s
+pod/scylla-us-east-1-us-east-1b-1   2/2     Running   0          4m29s
+
+NAME                                    TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)                                                           AGE
+service/scylla-client                   ClusterIP   None           <none>        9180/TCP,5090/TCP                                                 5m59s
+service/scylla-us-east-1-us-east-1b-0   ClusterIP   10.43.149.92   <none>        7000/TCP,7001/TCP,7199/TCP,10001/TCP,9042/TCP,9142/TCP,9160/TCP   5m58s
+service/scylla-us-east-1-us-east-1b-1   ClusterIP   10.43.49.0     <none>        7000/TCP,7001/TCP,7199/TCP,10001/TCP,9042/TCP,9142/TCP,9160/TCP   4m29s
+
+NAME                                           READY   AGE
+statefulset.apps/scylla-us-east-1-us-east-1b   2/2     5m59s
+```
+
+Two running nodes, exactly what we were asking for.
+
+## Monitoring
+
+To spin up a Prometheus monitoring refer to [monitoring guide](monitoring.md).
+
+Helm charts can create ServiceMonitors needed to observe Scylla Managger and Scylla. 
+Both of these Helm Charts allows to specify whether you want to create a ServiceMonitor:
+```yaml
+serviceMonitor:
+  create: false
+```
+
+Change `create` to `true` and update your current deployment using:
+```shell
+helm upgrade --install scylla --namespace scylla scylla/scylla -f examples/helm/values.cluster.yaml
+```
+
+Helm should notice the difference, install the ServiceMonitor, and then Prometheous will be able to scrape metrics.
+
+## Cleanup
+
+To remove these applications you can simply uninstall them using Helm CLI:
+```shell
+helm uninstall scylla -n scylla
+helm uninstall scylla-manager -n scylla-manager
+helm uninstall scylla-operator -n scylla-operator
+```
diff --git a/docs/source/index.rst b/docs/source/index.rst
index fb776da06..cc5e0517e 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -9,6 +9,7 @@ Scylla Operator Documentation
    generic
    eks
    gke
+   helm
    manager
    monitoring
    node_operations
@@ -46,11 +47,9 @@ Currently it supports:
 * :doc:`Deploying Scylla on EKS (experimental) <eks>`
 * :doc:`Deploying Scylla on GKE <gke>`
 * :doc:`Deploying Scylla Manager on a Kubernetes Cluster <manager>`
+* :doc:`Deploying Scylla stack using Helm Charts <helm>`
 * :doc:`Setting up Monitoring using Prometheus and Grafana <monitoring>`
 * :doc:`Node operations <node_operations>`
 * :doc:`Upgrade procedures <upgrade>`
 * :doc:`Scylla Cluster Custom Resource Definition (CRD) <scylla_cluster_crd>`
 * :doc:`Contributing to the Scylla Operator Project <contributing>`
-
-
-
diff --git a/examples/common/operator.yaml b/examples/common/operator.yaml
index 191de0ed9..4dcb733c9 100644
--- a/examples/common/operator.yaml
+++ b/examples/common/operator.yaml
@@ -1940,9 +1940,6 @@ spec:
           initialDelaySeconds: 5
           periodSeconds: 10
         resources:
-          limits:
-            cpu: 100m
-            memory: 30Mi
           requests:
             cpu: 100m
             memory: 20Mi
diff --git a/examples/helm/us-east-1b-values.yaml b/examples/helm/us-east-1b-values.yaml
deleted file mode 100644
index 2c32e5c7a..000000000
--- a/examples/helm/us-east-1b-values.yaml
+++ /dev/null
@@ -1,38 +0,0 @@
-scyllaImage:
-  tag: 4.3.0
-agentImage:
-  tag: 2.2.1
-
-cpuset: true
-hostNetworking: true
-
-sysctls:
-- "fs.aio-max-nr=2097152"
-
-datacenter: us-east1-b
-racks:
-- name: us-east1
-  members: 2
-  storage:
-    capacity: 100G
-  resources:
-    limits:
-      cpu: 4
-      memory: 16G
-    requests:
-      cpu: 4
-      memory: 16G
-  placement:
-    nodeAffinity:
-      requiredDuringSchedulingIgnoredDuringExecution:
-        nodeSelectorTerms:
-          - matchExpressions:
-            - key: failure-domain.beta.kubernetes.io/zone
-              operator: In
-              values:
-              - us-east1-b
-    tolerations:
-      - key: role
-        operator: Equal
-        value: scylla-clusters
-        effect: NoSchedule
diff --git a/examples/helm/values.cluster.yaml b/examples/helm/values.cluster.yaml
new file mode 100644
index 000000000..df06656a7
--- /dev/null
+++ b/examples/helm/values.cluster.yaml
@@ -0,0 +1,21 @@
+# Version information
+scyllaImage:
+  tag: 4.3.0
+agentImage:
+  tag: 2.2.1
+
+# Cluster information
+developerMode: true
+datacenter: us-east-1
+racks:
+- name: us-east-1b
+  members: 2
+  storage:
+    capacity: 5Gi
+  resources:
+    limits:
+      cpu: 1
+      memory: 1Gi
+    requests:
+      cpu: 1
+      memory: 1Gi
diff --git a/examples/helm/values.manager.yaml b/examples/helm/values.manager.yaml
new file mode 100644
index 000000000..cfca5a5de
--- /dev/null
+++ b/examples/helm/values.manager.yaml
@@ -0,0 +1,41 @@
+# Scylla Manager image
+image:
+  tag: 2.2.1
+
+# Resources allocated to Scylla Manager pods
+resources:
+  limits:
+    cpu: 200m
+    memory: 256Mi
+  requests:
+    cpu: 200m
+    memory: 32Mi
+# Resources allocated to Scylla Manager Controller pods
+controllerResources:
+  limits:
+    cpu: 200m
+    memory: 128Mi
+  requests:
+    cpu: 200m
+    memory: 32Mi
+
+# Scylla instance for Manager
+scylla:
+  developerMode: true
+  scyllaImage:
+    tag: 4.3.0
+  agentImage:
+    tag: 2.2.1
+  datacenter: manager-dc
+  racks:
+    - name: manager-rack
+      members: 1
+      storage:
+        capacity: 5Gi
+      resources:
+        limits:
+          cpu: 1
+          memory: 200Mi
+        requests:
+          cpu: 1
+          memory: 200Mi
diff --git a/examples/helm/values.operator.yaml b/examples/helm/values.operator.yaml
new file mode 100644
index 000000000..735ce0d72
--- /dev/null
+++ b/examples/helm/values.operator.yaml
@@ -0,0 +1,15 @@
+# Specify desired Scylla Operator image
+image:
+  tag: nightly
+
+# Change log level
+logLevel: info
+
+# Resources allocated to Scylla Operator pods
+resources:
+  limits:
+    cpu: 200m
+    memory: 128Mi
+  requests:
+    cpu: 200m
+    memory: 32Mi
diff --git a/helm/scylla-manager/templates/deployment.yaml b/helm/scylla-manager/templates/deployment.yaml
index 13f772487..863ee96e3 100644
--- a/helm/scylla-manager/templates/deployment.yaml
+++ b/helm/scylla-manager/templates/deployment.yaml
@@ -41,7 +41,7 @@ spec:
           name: scylla-manager-config-volume
       volumes:
       - configMap:
-          name: scylla-manager-manager-config-2tfggm9842
+          name: scylla-manager-manager-config
         name: scylla-manager-config-volume
       {{- with .Values.nodeSelector }}
       nodeSelector:
diff --git a/helm/scylla-manager/templates/manager_config.yaml b/helm/scylla-manager/templates/manager_config.yaml
index 0bb2a72b0..7d4363f56 100644
--- a/helm/scylla-manager/templates/manager_config.yaml
+++ b/helm/scylla-manager/templates/manager_config.yaml
@@ -8,10 +8,10 @@ data:
       hosts:
         {{- range $rack := .Values.scylla.racks }}
         {{- range $idx, $e := until ($rack.members | int) }}
-        - {{ printf "%s-%s-%s-%s-%d.%s.svc" $.Release.Name (default "scylla" $.Values.scylla.fullnameOverride) $.Values.scylla.datacenter $rack.name $idx $.Release.Namespace }}
+        - {{ printf "%s-%s-%s-%d.%s.svc" $.Release.Name $.Values.scylla.datacenter $rack.name $idx $.Release.Namespace }}
         {{- end }}
         {{- end }}
 kind: ConfigMap
 metadata:
-  name: scylla-manager-manager-config-2tfggm9842
+  name: scylla-manager-manager-config
   namespace: {{ .Release.Namespace }}
diff --git a/helm/scylla-manager/values.yaml b/helm/scylla-manager/values.yaml
index 160faa0bf..a9f747cf2 100644
--- a/helm/scylla-manager/values.yaml
+++ b/helm/scylla-manager/values.yaml
@@ -1,6 +1,3 @@
-# Specifies under which namespace Scylla Manager will be installed
-namespace: scylla-manager-system
-
 # Allows to override Scylla Manager name showing up in recommended k8s labels
 nameOverride: ""
 # Allows to override names used in Scylla Manager k8s objects.
@@ -57,7 +54,6 @@ serviceAccount:
 
 scylla:
   developerMode: true
-  namespace: scylla-manager-system
   scyllaImage:
     tag: 4.3.0
   agentImage:
@@ -71,10 +67,10 @@ scylla:
     resources:
       limits:
         cpu: 1
-        memory: 1Gi
+        memory: 200Mi
       requests:
         cpu: 1
-        memory: 1Gi
+        memory: 200Mi
 
 # Whether to create Prometheus ServiceMonitor
 serviceMonitor:
diff --git a/helm/scylla-operator/values.yaml b/helm/scylla-operator/values.yaml
index e7d78ba4e..94bef9a88 100644
--- a/helm/scylla-operator/values.yaml
+++ b/helm/scylla-operator/values.yaml
@@ -1,6 +1,3 @@
-# Specifies under which namespace Scylla Operator will be installed
-namespace: scylla-operator-system
-
 # Allows to override Scylla Operator name showing up in recommended k8s labels
 nameOverride: ""
 # Allows to override names used in Scylla Operator k8s objects.
diff --git a/helm/scylla/values.yaml b/helm/scylla/values.yaml
index 1d1dbb56c..e61f4d273 100644
--- a/helm/scylla/values.yaml
+++ b/helm/scylla/values.yaml
@@ -1,6 +1,3 @@
-# Specifies under which namespace Scylla will be installed
-namespace: scylla
-
 # Allows to override Scylla name showing up in recommended k8s labels
 nameOverride: ""
 # Allows to override names used in Scylla k8s objects.
diff --git a/pkg/cmd/options/common.go b/pkg/cmd/options/common.go
index cb59824c7..2bbf69576 100644
--- a/pkg/cmd/options/common.go
+++ b/pkg/cmd/options/common.go
@@ -11,19 +11,19 @@ import (
 )
 
 // Singleton
-var commonOpts = &commonOptions{}
+var commonOpts = &CommonOptions{}
 
-type commonOptions struct {
+type CommonOptions struct {
 	Name      string
 	Namespace string
 	LogLevel  string
 }
 
-func GetCommonOptions() *commonOptions {
+func GetCommonOptions() *CommonOptions {
 	return commonOpts
 }
 
-func (o *commonOptions) AddFlags(cmd *cobra.Command) {
+func (o *CommonOptions) AddFlags(cmd *cobra.Command) {
 	cmd.Flags().StringVar(&o.Name, "pod-name", os.Getenv(naming.EnvVarEnvVarPodName), "name of the pod")
 	cmd.Flags().StringVar(&o.Namespace, "pod-namespace", os.Getenv(naming.EnvVarPodNamespace), "namespace of the pod")
 	cmd.Flags().StringVar(&o.LogLevel,
@@ -34,7 +34,7 @@ func (o *commonOptions) AddFlags(cmd *cobra.Command) {
 
 }
 
-func (o *commonOptions) Validate() error {
+func (o *CommonOptions) Validate() error {
 	if o.Name == "" {
 		return errors.New("pod-name not set")
 	}
diff --git a/pkg/cmd/options/manager.go b/pkg/cmd/options/manager.go
index 9c8a7a085..834789368 100644
--- a/pkg/cmd/options/manager.go
+++ b/pkg/cmd/options/manager.go
@@ -6,11 +6,11 @@ import (
 
 // Singleton
 var managerOpts = &managerOptions{
-	commonOptions: GetCommonOptions(),
+	CommonOptions: GetCommonOptions(),
 }
 
 type managerOptions struct {
-	*commonOptions
+	*CommonOptions
 }
 
 func GetManagerOptions() *managerOptions {
@@ -18,7 +18,7 @@ func GetManagerOptions() *managerOptions {
 }
 
 func (o *managerOptions) AddFlags(cmd *cobra.Command) {
-	o.commonOptions.AddFlags(cmd)
+	o.CommonOptions.AddFlags(cmd)
 }
 
 func (o *managerOptions) Validate() error {
diff --git a/pkg/cmd/options/operator.go b/pkg/cmd/options/operator.go
index 22eeccda4..3d2578eca 100644
--- a/pkg/cmd/options/operator.go
+++ b/pkg/cmd/options/operator.go
@@ -6,29 +6,29 @@ import (
 )
 
 // Singleton
-var operatorOpts = &operatorOptions{
-	commonOptions: GetCommonOptions(),
+var operatorOpts = &OperatorOptions{
+	CommonOptions: GetCommonOptions(),
 }
 
-type operatorOptions struct {
-	*commonOptions
+type OperatorOptions struct {
+	*CommonOptions
 	Image                  string
 	EnableAdmissionWebhook bool
 }
 
-func GetOperatorOptions() *operatorOptions {
+func GetOperatorOptions() *OperatorOptions {
 	return operatorOpts
 }
 
-func (o *operatorOptions) AddFlags(cmd *cobra.Command) {
-	o.commonOptions.AddFlags(cmd)
+func (o *OperatorOptions) AddFlags(cmd *cobra.Command) {
+	o.CommonOptions.AddFlags(cmd)
 	cmd.Flags().StringVar(&o.Image, "image", "", "image of the operator used")
 	cmd.Flags().BoolVar(&o.EnableAdmissionWebhook, "enable-admission-webhook", true, "enable the admission webhook")
 }
 
-func (o *operatorOptions) Validate() error {
+func (o *OperatorOptions) Validate() error {
 
-	if o.Image == "" && o.commonOptions.Validate() != nil {
+	if o.Image == "" && o.CommonOptions.Validate() != nil {
 		return errors.New("image not set - you must set either image or namespace and name")
 	}
 	return nil
diff --git a/pkg/cmd/options/sidecar.go b/pkg/cmd/options/sidecar.go
index 196bac9c6..da5560d6c 100644
--- a/pkg/cmd/options/sidecar.go
+++ b/pkg/cmd/options/sidecar.go
@@ -10,11 +10,11 @@ import (
 
 // Singleton
 var sidecarOpts = &sidecarOptions{
-	commonOptions: GetCommonOptions(),
+	CommonOptions: GetCommonOptions(),
 }
 
 type sidecarOptions struct {
-	*commonOptions
+	*CommonOptions
 	CPU string
 }
 
@@ -23,12 +23,12 @@ func GetSidecarOptions() *sidecarOptions {
 }
 
 func (o *sidecarOptions) AddFlags(cmd *cobra.Command) {
-	o.commonOptions.AddFlags(cmd)
+	o.CommonOptions.AddFlags(cmd)
 	cmd.Flags().StringVar(&o.CPU, "cpu", os.Getenv(naming.EnvVarCPU), "number of cpus to use")
 }
 
 func (o *sidecarOptions) Validate() error {
-	if err := o.commonOptions.Validate(); err != nil {
+	if err := o.CommonOptions.Validate(); err != nil {
 		return errors.WithStack(err)
 	}
 	if o.CPU == "" {
diff --git a/pkg/controllers/cluster/cluster_controller.go b/pkg/controllers/cluster/cluster_controller.go
index 061862187..0dd336b22 100644
--- a/pkg/controllers/cluster/cluster_controller.go
+++ b/pkg/controllers/cluster/cluster_controller.go
@@ -73,7 +73,7 @@ func New(ctx context.Context, mgr ctrl.Manager, logger log.Logger) (*ClusterReco
 		return nil, errors.Wrap(err, "get dynamic uncached client")
 	}
 
-	operatorImage, err := getOperatorImage(ctx, kubeClient)
+	operatorImage, err := getOperatorImage(ctx, kubeClient, options.GetOperatorOptions())
 	if err != nil {
 		return nil, errors.Wrap(err, "get operator image")
 	}
@@ -211,8 +211,7 @@ func (r *ClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return nil
 }
 
-func getOperatorImage(ctx context.Context, kubeClient kubernetes.Interface) (string, error) {
-	opts := options.GetOperatorOptions()
+func getOperatorImage(ctx context.Context, kubeClient kubernetes.Interface, opts *options.OperatorOptions) (string, error) {
 	if opts.Image != "" {
 		return opts.Image, nil
 	}


Test patch:
diff --git a/pkg/controllers/cluster/cluster_controller_test.go b/pkg/controllers/cluster/cluster_controller_test.go
index dc1c05887..043e016c2 100644
--- a/pkg/controllers/cluster/cluster_controller_test.go
+++ b/pkg/controllers/cluster/cluster_controller_test.go
@@ -21,9 +21,12 @@ var _ = Describe("Cluster controller", func() {
 			namespace = "ns"
 			name      = "name"
 		)
-		opts := options.GetOperatorOptions()
-		opts.Namespace = namespace
-		opts.Name = name
+		opts := &options.OperatorOptions{
+			CommonOptions: &options.CommonOptions{
+				Name:      name,
+				Namespace: namespace,
+			},
+		}
 
 		pod := &corev1.Pod{
 			ObjectMeta: metav1.ObjectMeta{
@@ -38,7 +41,7 @@ var _ = Describe("Cluster controller", func() {
 		}
 
 		kubeClientFake := kubefake.NewSimpleClientset(pod)
-		image, err := getOperatorImage(context.Background(), kubeClientFake)
+		image, err := getOperatorImage(context.Background(), kubeClientFake, opts)
 		if expectedImage == "" {
 			Expect(err).To(HaveOccurred())
 		} else {
diff --git a/pkg/controllers/cluster/controller_integration_test.go b/pkg/controllers/cluster/controller_integration_test.go
index ad44034e6..b2e3882de 100644
--- a/pkg/controllers/cluster/controller_integration_test.go
+++ b/pkg/controllers/cluster/controller_integration_test.go
@@ -6,6 +6,7 @@ package cluster_test
 import (
 	"context"
 	"strings"
+	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -13,6 +14,7 @@ import (
 	"github.com/scylladb/scylla-operator/pkg/naming"
 	"github.com/scylladb/scylla-operator/pkg/test/integration"
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -61,7 +63,6 @@ var _ = Describe("Cluster controller", func() {
 
 	It("When PVC affinity is bound to lost node, node is replaced", func() {
 		scylla := singleNodeCluster(ns)
-		scylla.Spec.AutomaticOrphanedNodeCleanup = true
 
 		Expect(testEnv.Create(ctx, scylla)).To(Succeed())
 		defer func() {
@@ -89,6 +90,10 @@ var _ = Describe("Cluster controller", func() {
 			Expect(sstStub.CreatePVCs(ctx, scylla, pvOption)).To(Succeed())
 		}
 
+		Expect(testEnv.Refresh(ctx, scylla)).To(Succeed())
+		scylla.Spec.AutomaticOrphanedNodeCleanup = true
+		Expect(testEnv.Update(ctx, scylla)).To(Succeed())
+
 		services, err := rackMemberService(ns.Namespace, rack, scylla)
 		Expect(err).To(BeNil())
 		Expect(services).To(Not(BeEmpty()))
@@ -131,6 +136,7 @@ var _ = Describe("Cluster controller", func() {
 		})
 
 		It("replace non seed node", func() {
+			const shortWait = 2 * time.Second
 			rack := scylla.Spec.Datacenter.Racks[0]
 
 			services, err := nonSeedServices(ns.Namespace, rack, scylla)
@@ -152,19 +158,27 @@ var _ = Describe("Cluster controller", func() {
 				return scylla.Status.Racks[rack.Name].ReplaceAddressFirstBoot, nil
 			}).Should(HaveKeyWithValue(serviceToReplace.Name, replacedServiceIP))
 
+			By("Old PVC should be removed")
+			Eventually(func() []corev1.PersistentVolumeClaim {
+				pvcs := &corev1.PersistentVolumeClaimList{}
+
+				Expect(testEnv.List(ctx, pvcs, &client.ListOptions{
+					LabelSelector: naming.RackSelector(rack, scylla),
+				})).To(Succeed())
+
+				return pvcs.Items
+			}, shortWait).Should(HaveLen(int(rack.Members - 1)))
+
 			By("Old Pod should be removed")
-			Eventually(func() (bool, error) {
-				if err := sstStub.SyncStatus(ctx, scylla); err != nil {
-					return false, err
-				}
+			Eventually(func() []corev1.Pod {
+				pods := &corev1.PodList{}
 
-				if err := testEnv.Refresh(ctx, scylla); err != nil {
-					return false, err
-				}
-				readyMembers := scylla.Status.Racks[rack.Name].ReadyMembers
-				expectedMembers := scylla.Spec.Datacenter.Racks[0].Members
-				return readyMembers == expectedMembers-1, nil
-			}).Should(BeTrue())
+				Expect(testEnv.List(ctx, pods, &client.ListOptions{
+					LabelSelector: naming.RackSelector(rack, scylla),
+				})).To(Succeed())
+
+				return pods.Items
+			}, shortWait).Should(HaveLen(int(rack.Members - 1)))
 
 			By("When new pod is scheduled")
 			Expect(sstStub.CreatePods(ctx, scylla)).To(Succeed())
@@ -181,11 +195,14 @@ var _ = Describe("Cluster controller", func() {
 					Name:      serviceToReplace.Name,
 				}
 				if err := testEnv.Get(ctx, key, service); err != nil {
+					if apierrors.IsNotFound(err) {
+						return nil, nil
+					}
 					return nil, err
 				}
 
 				return service.Labels, nil
-			}).Should(HaveKeyWithValue(naming.ReplaceLabel, replacedServiceIP))
+			}, shortWait).Should(HaveKeyWithValue(naming.ReplaceLabel, replacedServiceIP))
 		})
 
 		It("replace seed node", func() {
diff --git a/pkg/test/integration/statefulset.go b/pkg/test/integration/statefulset.go
index 8eac9d82c..52e440ded 100644
--- a/pkg/test/integration/statefulset.go
+++ b/pkg/test/integration/statefulset.go
@@ -208,6 +208,7 @@ func (s *StatefulSetOperatorStub) CreatePVCs(ctx context.Context, cluster *scyll
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      naming.PVCNameForPod(pod.Name),
 					Namespace: pod.Namespace,
+					Labels:    naming.RackLabels(rack, cluster),
 				},
 				Spec: corev1.PersistentVolumeClaimSpec{
 					VolumeName:  pv.Name,
@@ -218,7 +219,7 @@ func (s *StatefulSetOperatorStub) CreatePVCs(ctx context.Context, cluster *scyll
 					},
 				},
 			}
-
+			s.logger.Info(ctx, "Creating/updating PVC", "pvc", pvc.Name, "pod", pod.Name)
 			if _, err := controllerutil.CreateOrUpdate(ctx, s.env, pvc, func() error { return nil }); err != nil {
 				return err
 			}