Repository: k8ssandra/k8ssandra-operator (go)

Issue:
K8SSAND-1247 ⁃ Current anti affinity rules don't apply with multi DC
The anti affinity rules we currently apply to prevent different Cassandra pods from being collocated on the same workers don't apply when multiple DCs get started, as they will be started in different namespaces:

```
  affinity:
    podAntiAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        - labelSelector:
            matchExpressions:
              - key: cassandra.datastax.com/cluster
                operator: Exists
              - key: cassandra.datastax.com/datacenter
                operator: Exists
              - key: cassandra.datastax.com/rack
                operator: Exists
           topologyKey: kubernetes.io/hostname
```

Adding `namespaces: []` would allow the label selector to apply across all namespaces instead of just the current pod's namespace:

```
  affinity:
    podAntiAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        - labelSelector:
            matchExpressions:
              - key: cassandra.datastax.com/cluster
                operator: Exists
              - key: cassandra.datastax.com/datacenter
                operator: Exists
              - key: cassandra.datastax.com/rack
                operator: Exists
           topologyKey: kubernetes.io/hostname
           namespaces: []
```

┆Issue is synchronized with this [Jira Bug](https://k8ssandra.atlassian.net/browse/K8SSAND-1247) by [Unito](https://www.unito.io)
┆fixVersions: k8ssandra-operator-v1.0.0
┆friendlyId: K8SSAND-1247
┆priority: Medium


Pull request description:
Expose CassandraDatacenter Tolerations property
<!--  Thanks for sending a pull request!  Before submitting:

1. Read our CONTRIBUTING.md guide
2. Rebase your PR if it gets out of sync with main
-->

**What this PR does**:
Exposes CassandraDatacenter Tolerations property

**Which issue(s) this PR fixes**:
Fixes #402

**Checklist**
- [ ] Changes manually tested
- [ ] Automated Tests added/updated
- [ ] Documentation added/updated
- [ ] CHANGELOG.md updated (not required for documentation PRs)
- [ ] CLA Signed:  [DataStax CLA](https://cla.datastax.com/)


Patch:
diff --git a/CHANGELOG/CHANGELOG-1.0.md b/CHANGELOG/CHANGELOG-1.0.md
index 99b11f70..ca8a0381 100644
--- a/CHANGELOG/CHANGELOG-1.0.md
+++ b/CHANGELOG/CHANGELOG-1.0.md
@@ -15,6 +15,7 @@ When cutting a new release, update the `unreleased` heading to the tag being gen
 
 ## unreleased
 
+* [FEATURE] [#402](https://github.com/k8ssandra/k8ssandra-operator/issues/402) Expose CassandraDatacenter Tolerations property
 * [BUGFIX] [#381](https://github.com/k8ssandra/k8ssandra-operator/issues/381) Add additionalSeeds property to support migrations
 * [TESTING] [#332](https://github.com/k8ssandra/k8ssandra-operator/issues/332) Configure mocks to avoid panics
 
diff --git a/apis/config/v1beta1/zz_generated.deepcopy.go b/apis/config/v1beta1/zz_generated.deepcopy.go
index 40a35a45..f0bc5e66 100644
--- a/apis/config/v1beta1/zz_generated.deepcopy.go
+++ b/apis/config/v1beta1/zz_generated.deepcopy.go
@@ -1,3 +1,4 @@
+//go:build !ignore_autogenerated
 // +build !ignore_autogenerated
 
 /*
diff --git a/apis/k8ssandra/v1alpha1/k8ssandracluster_types.go b/apis/k8ssandra/v1alpha1/k8ssandracluster_types.go
index 2d196313..7a7320ec 100644
--- a/apis/k8ssandra/v1alpha1/k8ssandracluster_types.go
+++ b/apis/k8ssandra/v1alpha1/k8ssandracluster_types.go
@@ -259,6 +259,10 @@ type CassandraClusterTemplate struct {
 	// the number of nodes required.
 	SoftPodAntiAffinity *bool `json:"softPodAntiAffinity,omitempty"`
 
+	// Tolerations applied to every Cassandra pod.
+	// +optional
+	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
+
 	// Internode encryption stores which are used by Cassandra and Stargate.
 	// +optional
 	ServerEncryptionStores *encryption.Stores `json:"serverEncryptionStores,omitempty"`
@@ -343,6 +347,10 @@ type CassandraDatacenterTemplate struct {
 	// This should normally be false to ensure cluster resilience but may be set true for test/dev scenarios to minimise
 	// the number of nodes required.
 	SoftPodAntiAffinity *bool `json:"softPodAntiAffinity,omitempty"`
+
+	// Tolerations applied to every Cassandra pod.
+	// +optional
+	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
 }
 
 type EmbeddedObjectMeta struct {
diff --git a/apis/k8ssandra/v1alpha1/zz_generated.deepcopy.go b/apis/k8ssandra/v1alpha1/zz_generated.deepcopy.go
index 5969e2df..21482c2d 100644
--- a/apis/k8ssandra/v1alpha1/zz_generated.deepcopy.go
+++ b/apis/k8ssandra/v1alpha1/zz_generated.deepcopy.go
@@ -1,3 +1,4 @@
+//go:build !ignore_autogenerated
 // +build !ignore_autogenerated
 
 /*
@@ -165,6 +166,13 @@ func (in *CassandraClusterTemplate) DeepCopyInto(out *CassandraClusterTemplate)
 		*out = new(bool)
 		**out = **in
 	}
+	if in.Tolerations != nil {
+		in, out := &in.Tolerations, &out.Tolerations
+		*out = make([]v1.Toleration, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	if in.ServerEncryptionStores != nil {
 		in, out := &in.ServerEncryptionStores, &out.ServerEncryptionStores
 		*out = new(encryption.Stores)
@@ -260,6 +268,13 @@ func (in *CassandraDatacenterTemplate) DeepCopyInto(out *CassandraDatacenterTemp
 		*out = new(bool)
 		**out = **in
 	}
+	if in.Tolerations != nil {
+		in, out := &in.Tolerations, &out.Tolerations
+		*out = make([]v1.Toleration, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CassandraDatacenterTemplate.
diff --git a/apis/medusa/v1alpha1/zz_generated.deepcopy.go b/apis/medusa/v1alpha1/zz_generated.deepcopy.go
index a6581894..25b015dc 100644
--- a/apis/medusa/v1alpha1/zz_generated.deepcopy.go
+++ b/apis/medusa/v1alpha1/zz_generated.deepcopy.go
@@ -1,3 +1,4 @@
+//go:build !ignore_autogenerated
 // +build !ignore_autogenerated
 
 /*
diff --git a/apis/reaper/v1alpha1/zz_generated.deepcopy.go b/apis/reaper/v1alpha1/zz_generated.deepcopy.go
index 5bc7a83f..4b243db5 100644
--- a/apis/reaper/v1alpha1/zz_generated.deepcopy.go
+++ b/apis/reaper/v1alpha1/zz_generated.deepcopy.go
@@ -1,3 +1,4 @@
+//go:build !ignore_autogenerated
 // +build !ignore_autogenerated
 
 /*
diff --git a/apis/replication/v1alpha1/zz_generated.deepcopy.go b/apis/replication/v1alpha1/zz_generated.deepcopy.go
index a397bc19..710fcead 100644
--- a/apis/replication/v1alpha1/zz_generated.deepcopy.go
+++ b/apis/replication/v1alpha1/zz_generated.deepcopy.go
@@ -1,3 +1,4 @@
+//go:build !ignore_autogenerated
 // +build !ignore_autogenerated
 
 /*
diff --git a/apis/stargate/v1alpha1/zz_generated.deepcopy.go b/apis/stargate/v1alpha1/zz_generated.deepcopy.go
index 0b2ceb83..aaf636d2 100644
--- a/apis/stargate/v1alpha1/zz_generated.deepcopy.go
+++ b/apis/stargate/v1alpha1/zz_generated.deepcopy.go
@@ -1,3 +1,4 @@
+//go:build !ignore_autogenerated
 // +build !ignore_autogenerated
 
 /*
diff --git a/apis/telemetry/v1alpha1/zz_generated.deepcopy.go b/apis/telemetry/v1alpha1/zz_generated.deepcopy.go
index 7be31b4a..6eb70e8d 100644
--- a/apis/telemetry/v1alpha1/zz_generated.deepcopy.go
+++ b/apis/telemetry/v1alpha1/zz_generated.deepcopy.go
@@ -1,3 +1,4 @@
+//go:build !ignore_autogenerated
 // +build !ignore_autogenerated
 
 /*
diff --git a/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml b/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml
index 156bb815..f4b8bf37 100644
--- a/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml
+++ b/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml
@@ -5866,6 +5866,49 @@ spec:
                                   type: string
                               type: object
                           type: object
+                        tolerations:
+                          description: Tolerations applied to every Cassandra pod.
+                          items:
+                            description: The pod this Toleration is attached to tolerates
+                              any taint that matches the triple <key,value,effect>
+                              using the matching operator <operator>.
+                            properties:
+                              effect:
+                                description: Effect indicates the taint effect to
+                                  match. Empty means match all taint effects. When
+                                  specified, allowed values are NoSchedule, PreferNoSchedule
+                                  and NoExecute.
+                                type: string
+                              key:
+                                description: Key is the taint key that the toleration
+                                  applies to. Empty means match all taint keys. If
+                                  the key is empty, operator must be Exists; this
+                                  combination means to match all values and all keys.
+                                type: string
+                              operator:
+                                description: Operator represents a key's relationship
+                                  to the value. Valid operators are Exists and Equal.
+                                  Defaults to Equal. Exists is equivalent to wildcard
+                                  for value, so that a pod can tolerate all taints
+                                  of a particular category.
+                                type: string
+                              tolerationSeconds:
+                                description: TolerationSeconds represents the period
+                                  of time the toleration (which must be of effect
+                                  NoExecute, otherwise this field is ignored) tolerates
+                                  the taint. By default, it is not set, which means
+                                  tolerate the taint forever (do not evict). Zero
+                                  and negative values will be treated as 0 (evict
+                                  immediately) by the system.
+                                format: int64
+                                type: integer
+                              value:
+                                description: Value is the taint value the toleration
+                                  matches to. If the operator is Exists, the value
+                                  should be empty, otherwise just a regular string.
+                                type: string
+                            type: object
+                          type: array
                       required:
                       - size
                       type: object
@@ -6412,6 +6455,47 @@ spec:
                           TODO: Add other useful fields. apiVersion, kind, uid?'
                         type: string
                     type: object
+                  tolerations:
+                    description: Tolerations applied to every Cassandra pod.
+                    items:
+                      description: The pod this Toleration is attached to tolerates
+                        any taint that matches the triple <key,value,effect> using
+                        the matching operator <operator>.
+                      properties:
+                        effect:
+                          description: Effect indicates the taint effect to match.
+                            Empty means match all taint effects. When specified, allowed
+                            values are NoSchedule, PreferNoSchedule and NoExecute.
+                          type: string
+                        key:
+                          description: Key is the taint key that the toleration applies
+                            to. Empty means match all taint keys. If the key is empty,
+                            operator must be Exists; this combination means to match
+                            all values and all keys.
+                          type: string
+                        operator:
+                          description: Operator represents a key's relationship to
+                            the value. Valid operators are Exists and Equal. Defaults
+                            to Equal. Exists is equivalent to wildcard for value,
+                            so that a pod can tolerate all taints of a particular
+                            category.
+                          type: string
+                        tolerationSeconds:
+                          description: TolerationSeconds represents the period of
+                            time the toleration (which must be of effect NoExecute,
+                            otherwise this field is ignored) tolerates the taint.
+                            By default, it is not set, which means tolerate the taint
+                            forever (do not evict). Zero and negative values will
+                            be treated as 0 (evict immediately) by the system.
+                          format: int64
+                          type: integer
+                        value:
+                          description: Value is the taint value the toleration matches
+                            to. If the operator is Exists, the value should be empty,
+                            otherwise just a regular string.
+                          type: string
+                      type: object
+                    type: array
                 type: object
               medusa:
                 description: Medusa defines the desired deployment characteristics
diff --git a/pkg/cassandra/datacenter.go b/pkg/cassandra/datacenter.go
index 6e69b80c..37b38bbe 100644
--- a/pkg/cassandra/datacenter.go
+++ b/pkg/cassandra/datacenter.go
@@ -108,6 +108,7 @@ type DatacenterConfig struct {
 	PodTemplateSpec          *corev1.PodTemplateSpec
 	MgmtAPIHeap              *resource.Quantity
 	SoftPodAntiAffinity      *bool
+	Tolerations              []corev1.Toleration
 	ServerEncryptionStores   *encryption.Stores
 	ClientEncryptionStores   *encryption.Stores
 	ClientKeystorePassword   string
@@ -183,6 +184,8 @@ func NewDatacenter(klusterKey types.NamespacedName, template *DatacenterConfig)
 		dc.Spec.AllowMultipleNodesPerWorker = *template.SoftPodAntiAffinity
 	}
 
+	dc.Spec.Tolerations = template.Tolerations
+
 	return dc, nil
 }
 
@@ -330,6 +333,13 @@ func Coalesce(clusterName string, clusterTemplate *api.CassandraClusterTemplate,
 	} else {
 		dcConfig.SoftPodAntiAffinity = dcTemplate.SoftPodAntiAffinity
 	}
+
+	if len(dcTemplate.Tolerations) == 0 {
+		dcConfig.Tolerations = clusterTemplate.Tolerations
+	} else {
+		dcConfig.Tolerations = dcTemplate.Tolerations
+	}
+
 	// Client/Server Encryption stores are only defined at the cluster level
 	dcConfig.ServerEncryptionStores = clusterTemplate.ServerEncryptionStores
 	dcConfig.ClientEncryptionStores = clusterTemplate.ClientEncryptionStores
diff --git a/pkg/encryption/zz_generated.deepcopy.go b/pkg/encryption/zz_generated.deepcopy.go
index cb6a0871..db165015 100644
--- a/pkg/encryption/zz_generated.deepcopy.go
+++ b/pkg/encryption/zz_generated.deepcopy.go
@@ -1,3 +1,4 @@
+//go:build !ignore_autogenerated
 // +build !ignore_autogenerated
 
 /*
diff --git a/pkg/images/zz_generated.deepcopy.go b/pkg/images/zz_generated.deepcopy.go
index 10c3b6c1..ba9c8d29 100644
--- a/pkg/images/zz_generated.deepcopy.go
+++ b/pkg/images/zz_generated.deepcopy.go
@@ -1,3 +1,4 @@
+//go:build !ignore_autogenerated
 // +build !ignore_autogenerated
 
 /*


Test patch:
diff --git a/pkg/cassandra/datacenter_test.go b/pkg/cassandra/datacenter_test.go
index 923d1267..533a45ba 100644
--- a/pkg/cassandra/datacenter_test.go
+++ b/pkg/cassandra/datacenter_test.go
@@ -367,6 +367,22 @@ func TestNewDatacenter_AllowMultipleCassPerNodeSet(t *testing.T) {
 	assert.Equal(t, true, dc.Spec.AllowMultipleNodesPerWorker)
 }
 
+func TestNewDatacenter_Tolerations(t *testing.T) {
+	template := GetDatacenterConfig()
+	template.Tolerations = []corev1.Toleration{{
+		Key:      "key1",
+		Operator: corev1.TolerationOpEqual,
+		Value:    "value1",
+		Effect:   corev1.TaintEffectNoSchedule,
+	}}
+	dc, err := NewDatacenter(
+		types.NamespacedName{Name: "testdc", Namespace: "test-namespace"},
+		&template,
+	)
+	assert.NoError(t, err)
+	assert.Equal(t, template.Tolerations, dc.Spec.Tolerations)
+}
+
 // TestNewDatacenter_Fail_NoStorageConfig tests that NewDatacenter fails when no storage config is provided.
 func TestNewDatacenter_Fail_NoStorageConfig(t *testing.T) {
 	template := GetDatacenterConfig()