Repository: k8ssandra/k8ssandra-operator (go)

Issue:
K8SSAND-1853 ⁃ Support additional jvm options for the various option files
Cassandra uses multiple jvm options files (`jvm.options`, `jvm-server.options`, `jvm8-server.options`, `jvm11-server.options`, `cassandra-env.sh`), and we're currently allowing to pass unstructured options to `cassandra-env.sh` only.

We'd need to have new `[]string` additional jvm options fields for each file in the `CassandraConfig.JvmOptions` struct:

```
config:
  jvmOptions:
      additionalJvmServerOptions:
        - '-XX:+UseConcMarkSweepGC'
        - '-XX:+UseCMSInitiatingOccupancyOnly'
        - '-XX:+CMSParallelInitialMarkEnabled'
      additionalJvm8ServerOptions:
        - '-XX:ThreadPriorityPolicy=42'
      additionalJvm11ServerOptions:
        - '-Djdk.attach.allowAttachSelf=true'
```

would map in the cassdc to:

```
  config:
    jvm-server-options:
      additional-jvm-opts:
        - '-XX:+UseConcMarkSweepGC'
        - '-XX:+UseCMSInitiatingOccupancyOnly'
        - '-XX:+CMSParallelInitialMarkEnabled'
    jvm11-server-options:
      additional-jvm-opts:
        - '-Djdk.attach.allowAttachSelf=true'
    jvm8-server-options:
      additional-jvm-opts:
        - '-XX:ThreadPriorityPolicy=42'
```

Additional options for the `cassandra-env.sh` would still be handled by the  `JvmOptions.AdditionalOptions` field as they currently are.


┆Issue is synchronized with this [Jira Task](https://k8ssandra.atlassian.net/browse/K8SSAND-1853) by [Unito](https://www.unito.io)
┆friendlyId: K8SSAND-1853
┆priority: Medium


Pull request description:
Support additional jvm options for the various options files
<!--  Thanks for sending a pull request!  Before submitting:

1. Read our CONTRIBUTING.md guide
2. Rebase your PR if it gets out of sync with main
-->

**What this PR does**:
Adds the `AdditionalJvmServerOptions`, `AdditionalJvm8ServerOptions` and `AdditionalJvm11ServerOptions` fields to the `JvmOptions` struct of the K8ssandraCluster CRD, which map to the respective cassdc config `additional-jvm-opts` entries, allowing to pass arbitrary jvm options targeting each specific file.

**Which issue(s) this PR fixes**:
Fixes #737

**Checklist**
- [x] Changes manually tested
- [x] Automated Tests added/updated
- [ ] Documentation added/updated
- [x] CHANGELOG.md updated (not required for documentation PRs)
- [x] CLA Signed:  [DataStax CLA](https://cla.datastax.com/)


Patch:
diff --git a/CHANGELOG/CHANGELOG-1.4.md b/CHANGELOG/CHANGELOG-1.4.md
index 2811932a..6c64e67e 100644
--- a/CHANGELOG/CHANGELOG-1.4.md
+++ b/CHANGELOG/CHANGELOG-1.4.md
@@ -15,6 +15,7 @@ When cutting a new release, update the `unreleased` heading to the tag being gen
 
 ## unreleased
 
+* [ENHANCEMENT] [#737](https://github.com/k8ssandra/k8ssandra-operator/issues/737) Support additional jvm options for the various options files 
 * [FEATURE] [#599](https://github.com/k8ssandra/k8ssandra-operator/issues/599) Introduce a secrets provider setting in the CRD
 * [FEATURE] [#728](https://github.com/k8ssandra/k8ssandra-operator/issues/728) Add token generation utility
 * [FEATURE] [#724](https://github.com/k8ssandra/k8ssandra-operator/issues/724) Ability to provide per-node configuration
diff --git a/apis/k8ssandra/v1alpha1/cassandraconfig_types.go b/apis/k8ssandra/v1alpha1/cassandraconfig_types.go
index e284c7b7..a873236b 100644
--- a/apis/k8ssandra/v1alpha1/cassandraconfig_types.go
+++ b/apis/k8ssandra/v1alpha1/cassandraconfig_types.go
@@ -572,9 +572,21 @@ type JvmOptions struct {
 	// +optional
 	PreserveFramePointer *bool `json:"debug_preserve_frame_pointer,omitempty" cass-config:">=4.x,dse@>=6.8.x:jvm-server-options/preserve-frame-pointer"`
 
-	// Additional, arbitrary JVM options (advanced).
+	// Additional, arbitrary JVM options which are written into the cassandra-env.sh file.
 	// +optional
 	AdditionalOptions []string `json:"additionalOptions,omitempty" cass-config:"cassandra-env-sh/additional-jvm-opts"`
+
+	// Jvm11ServerOptions are additional options that will be passed on to the jvm11-server-options file.
+	// +optional
+	AdditionalJvm11ServerOptions []string `json:"additionalJvm11ServerOptions,omitempty" cass-config:"jvm11-server-options/additional-jvm-opts"`
+
+	// Jvm8ServerOptions are additional options that will be passed on to the jvm8-server-options file.
+	// +optional
+	AdditionalJvm8ServerOptions []string `json:"additionalJvm8ServerOptions,omitempty" cass-config:"jvm8-server-options/additional-jvm-opts"`
+
+	// JvmServerOptions are additional options that will be passed on to the jvm-server-options file.
+	// +optional
+	AdditionalJvmServerOptions []string `json:"additionalJvmServerOptions,omitempty" cass-config:"jvm-server-options/additional-jvm-opts"`
 }
 
 type ParameterizedClass struct {
diff --git a/apis/k8ssandra/v1alpha1/zz_generated.deepcopy.go b/apis/k8ssandra/v1alpha1/zz_generated.deepcopy.go
index a221247c..f977023b 100644
--- a/apis/k8ssandra/v1alpha1/zz_generated.deepcopy.go
+++ b/apis/k8ssandra/v1alpha1/zz_generated.deepcopy.go
@@ -732,6 +732,21 @@ func (in *JvmOptions) DeepCopyInto(out *JvmOptions) {
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.AdditionalJvm11ServerOptions != nil {
+		in, out := &in.AdditionalJvm11ServerOptions, &out.AdditionalJvm11ServerOptions
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.AdditionalJvm8ServerOptions != nil {
+		in, out := &in.AdditionalJvm8ServerOptions, &out.AdditionalJvm8ServerOptions
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.AdditionalJvmServerOptions != nil {
+		in, out := &in.AdditionalJvmServerOptions, &out.AdditionalJvmServerOptions
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JvmOptions.
diff --git a/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml b/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml
index 2078697f..aa7dd28e 100644
--- a/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml
+++ b/config/crd/bases/k8ssandra.io_k8ssandraclusters.yaml
@@ -218,8 +218,27 @@ spec:
                         x-kubernetes-preserve-unknown-fields: true
                       jvmOptions:
                         properties:
+                          additionalJvm8ServerOptions:
+                            description: Jvm8ServerOptions are additional options
+                              that will be passed on to the jvm8-server-options file.
+                            items:
+                              type: string
+                            type: array
+                          additionalJvm11ServerOptions:
+                            description: Jvm11ServerOptions are additional options
+                              that will be passed on to the jvm11-server-options file.
+                            items:
+                              type: string
+                            type: array
+                          additionalJvmServerOptions:
+                            description: JvmServerOptions are additional options that
+                              will be passed on to the jvm-server-options file.
+                            items:
+                              type: string
+                            type: array
                           additionalOptions:
-                            description: Additional, arbitrary JVM options (advanced).
+                            description: Additional, arbitrary JVM options which are
+                              written into the cassandra-env.sh file.
                             items:
                               type: string
                             type: array
@@ -2041,8 +2060,30 @@ spec:
                               x-kubernetes-preserve-unknown-fields: true
                             jvmOptions:
                               properties:
+                                additionalJvm8ServerOptions:
+                                  description: Jvm8ServerOptions are additional options
+                                    that will be passed on to the jvm8-server-options
+                                    file.
+                                  items:
+                                    type: string
+                                  type: array
+                                additionalJvm11ServerOptions:
+                                  description: Jvm11ServerOptions are additional options
+                                    that will be passed on to the jvm11-server-options
+                                    file.
+                                  items:
+                                    type: string
+                                  type: array
+                                additionalJvmServerOptions:
+                                  description: JvmServerOptions are additional options
+                                    that will be passed on to the jvm-server-options
+                                    file.
+                                  items:
+                                    type: string
+                                  type: array
                                 additionalOptions:
-                                  description: Additional, arbitrary JVM options (advanced).
+                                  description: Additional, arbitrary JVM options which
+                                    are written into the cassandra-env.sh file.
                                   items:
                                     type: string
                                   type: array


Test patch:
diff --git a/pkg/cassandra/config_test.go b/pkg/cassandra/config_test.go
index 2e1fdb75..ca01aae3 100644
--- a/pkg/cassandra/config_test.go
+++ b/pkg/cassandra/config_test.go
@@ -1,9 +1,10 @@
 package cassandra
 
 import (
+	"testing"
+
 	"github.com/Masterminds/semver/v3"
 	"k8s.io/apimachinery/pkg/api/resource"
-	"testing"
 
 	"k8s.io/utils/pointer"
 
@@ -198,7 +199,8 @@ func TestCreateJsonConfig(t *testing.T) {
 			serverType:    api.ServerDistributionDse,
 			cassandraConfig: api.CassandraConfig{
 				JvmOptions: api.JvmOptions{
-					GarbageCollector: pointer.String("ZGC"),
+					GarbageCollector:             pointer.String("ZGC"),
+					AdditionalJvm11ServerOptions: []string{"-XX:+UseConcMarkSweepGC"},
 				},
 				DseYaml: unstructured.Unstructured{
 					"authentication_options": map[string]interface{}{
@@ -208,13 +210,49 @@ func TestCreateJsonConfig(t *testing.T) {
 			},
 			want: `{
              "jvm11-server-options": {
-               "garbage_collector": "ZGC"
+               "garbage_collector": "ZGC",
+							 "additional-jvm-opts": ["-XX:+UseConcMarkSweepGC"]
              },
              "dse-yaml": {
                "authentication_options": {
 				  "enabled": true
                }
              }
+           }`,
+		},
+		{
+			name:          "[DSE 6.8.25] multiple jvm-option files",
+			serverVersion: semver.MustParse("6.8.25"),
+			serverType:    api.ServerDistributionDse,
+			cassandraConfig: api.CassandraConfig{
+				JvmOptions: api.JvmOptions{
+					GarbageCollector:             pointer.String("ZGC"),
+					AdditionalJvm11ServerOptions: []string{"-XX:+UseConcMarkSweepGC"},
+					AdditionalJvm8ServerOptions:  []string{"-XX:ThreadPriorityPolicy=42"},
+					AdditionalJvmServerOptions:   []string{"-Dio.netty.maxDirectMemory=0"},
+				},
+				DseYaml: unstructured.Unstructured{
+					"authentication_options": map[string]interface{}{
+						"enabled": true,
+					},
+				},
+			},
+			want: `{
+				    "jvm-server-options": {
+							"additional-jvm-opts": ["-Dio.netty.maxDirectMemory=0"]
+						},
+						 "jvm8-server-options": {
+							"additional-jvm-opts": ["-XX:ThreadPriorityPolicy=42"]
+						},
+            "jvm11-server-options": {
+              "garbage_collector": "ZGC",
+							"additional-jvm-opts": ["-XX:+UseConcMarkSweepGC"]
+            },
+            "dse-yaml": {
+              "authentication_options": {
+				        "enabled": true
+               }
+             }
            }`,
 		},
 	}
diff --git a/test/testdata/fixtures/gc/4.0-jdk11-CMS/k8ssandra.yaml b/test/testdata/fixtures/gc/4.0-jdk11-CMS/k8ssandra.yaml
index afbb2a2c..9d901584 100644
--- a/test/testdata/fixtures/gc/4.0-jdk11-CMS/k8ssandra.yaml
+++ b/test/testdata/fixtures/gc/4.0-jdk11-CMS/k8ssandra.yaml
@@ -31,6 +31,7 @@ spec:
               - -XX:+CMSParallelInitialMarkEnabled
               - -XX:+CMSEdenChunksRecordAlways
               - -XX:+CMSClassUnloadingEnabled
+            additionalJvm11ServerOptions:
               # user settings
               - -XX:SurvivorRatio=4
               - -XX:MaxTenuringThreshold=2