Repository: grafana/k6-operator
Pull request #142 (2022-08-10 13:51:35)

feat: add use of tolerations for starter and runner
This PR aims to add support for Tolerations for the starter and the different runner pods.

yorugac: And one more thing: I think tolerations should be added to all pods in the operator, i.e. to the initializer pod as well.
spoukke: Hi @yorugac ! I re-generated the manifests.

From what I see in the codebase, the Initializer is of type `*batchv1.Job` so it will directly inherit the filed Tolerations from `corev1.PodTemplate`.

Also, I was wondering why you did not use the `corev1.PoTemplate` for the Runner et Starter fields instead of recreating your own struct. If you do this, you will directly have all the fields form a Pods, and the operator will automatically follow the updates from the Kubernetes API.
spoukke: Hello @yorugac ! I juste understood what you wnted regarding the initializer. Now, the runner tolerations are mapped to the intializer one.

Let me know if there is anything else

Patch:
diff --git a/.github/ISSUE_TEMPLATE/bug.yaml b/.github/ISSUE_TEMPLATE/bug.yaml
index 7ee742a..b88b47c 100644
--- a/.github/ISSUE_TEMPLATE/bug.yaml
+++ b/.github/ISSUE_TEMPLATE/bug.yaml
@@ -16,7 +16,7 @@ body:
       label: k6-operator version or image
     validations:
       required: true
-  - type: input
+  - type: textarea
     attributes:
       label: K6 YAML
     validations:
diff --git a/README.md b/README.md
index 8bcc408..fff14e2 100644
--- a/README.md
+++ b/README.md
@@ -41,19 +41,59 @@ $ kubectl create configmap my-test --from-file /path/to/my/test.js
 
 #### VolumeClaim
 
-If you have a PVC with the name `stress-test` containing your script and any other supporting file(s), you can pass it to the test like this:
+There is a sample avaiable in `config/samples/k6_v1alpha1_k6_with_volumeClaim.yaml` on how to configure to run a test script with a volumeClaim.
+
+If you have a PVC with the name `stress-test-volumeClaim` containing your script and any other supporting file(s), you can pass it to the test like this:
 
 ```
 spec:
   parallelism: 2
   script:
     volumeClaim:
-      name: "stress-test"
+      name: "stress-test-volumeClaim"
       file: "test.js"
 ```
 
 ***Note:*** the pods will expect to find script files in `/test/` folder. If `volumeClaim` fails, it's the first place to check: the latest initializer pod does not generate any logs and when it can't find the file, it will terminate with error. So missing file may not be that obvious and it makes sense to check it manually. See #143 for potential improvements.
 
+##### Example directory structure while using volumeClaim
+
+```
+├── test
+│   ├── requests
+│   │   ├── stress-test.js
+│   ├── test.js
+```
+In the above example, `test.js` imports a function from `stress-test.js` and they would look like this:
+```js
+// stress-test.js
+import stress-test from "./requests/stress-test.js";
+
+export let options = {
+      vus: 50,
+      duration: '10s'
+    };
+
+export default function () {
+      stress-test();
+}
+```
+```js
+// stress-test.js
+import { sleep, check } from 'k6';
+import http from 'k6/http';
+
+
+export default () => {
+  const res = http.get('https://test-api.k6.io');
+  check(res, {
+    'status is 200': () => res.status === 200,
+  });
+  sleep(1);
+};
+
+```
+
 #### LocalFile
 
 There is a sample avaiable in `config/samples/k6_v1alpha1_k6_with_localfile.yaml` on how to configure to run a test script inside the docker image.
diff --git a/api/v1alpha1/k6_types.go b/api/v1alpha1/k6_types.go
index e53838c..2695c11 100644
--- a/api/v1alpha1/k6_types.go
+++ b/api/v1alpha1/k6_types.go
@@ -31,6 +31,7 @@ type Pod struct {
 	Image                        string                      `json:"image,omitempty"`
 	Metadata                     PodMetadata                 `json:"metadata,omitempty"`
 	NodeSelector                 map[string]string           `json:"nodeselector,omitempty"`
+	Tolerations                  []corev1.Toleration         `json:"tolerations,omitempty"`
 	Resources                    corev1.ResourceRequirements `json:"resources,omitempty"`
 	ServiceAccountName           string                      `json:"serviceAccountName,omitempty"`
 	SecurityContext              corev1.PodSecurityContext   `json:"securityContext,omitempty"`
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 4164701..778b742 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -211,6 +211,13 @@ func (in *Pod) DeepCopyInto(out *Pod) {
 	}
 	in.Resources.DeepCopyInto(&out.Resources)
 	in.SecurityContext.DeepCopyInto(&out.SecurityContext)
+	if in.EnvFrom != nil {
+		in, out := &in.EnvFrom, &out.EnvFrom
+		*out = make([]v1.EnvFromSource, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Pod.
diff --git a/config/crd/bases/k6.io_k6s.yaml b/config/crd/bases/k6.io_k6s.yaml
index 88c6665..c5ddc62 100644
--- a/config/crd/bases/k6.io_k6s.yaml
+++ b/config/crd/bases/k6.io_k6s.yaml
@@ -813,7 +813,6 @@ spec:
                       - name
                       type: object
                     type: array
-                  # ref: https://github.com/kubernetes/kubernetes/blob/7e2c02d4c86d73051b6d4f47298d35a7ffb42b6a/api/openapi-spec/v3/apis__apps__v1_openapi.json#L1681
                   envFrom:
                     items:
                       description: EnvFromSource represents the source of a set of
@@ -1022,6 +1021,46 @@ spec:
                     type: object
                   serviceAccountName:
                     type: string
+                  tolerations:
+                    items:
+                      description: The pod this Toleration is attached to tolerates
+                        any taint that matches the triple <key,value,effect> using
+                        the matching operator <operator>.
+                      properties:
+                        effect:
+                          description: Effect indicates the taint effect to match.
+                            Empty means match all taint effects. When specified, allowed
+                            values are NoSchedule, PreferNoSchedule and NoExecute.
+                          type: string
+                        key:
+                          description: Key is the taint key that the toleration applies
+                            to. Empty means match all taint keys. If the key is empty,
+                            operator must be Exists; this combination means to match
+                            all values and all keys.
+                          type: string
+                        operator:
+                          description: Operator represents a key's relationship to
+                            the value. Valid operators are Exists and Equal. Defaults
+                            to Equal. Exists is equivalent to wildcard for value,
+                            so that a pod can tolerate all taints of a particular
+                            category.
+                          type: string
+                        tolerationSeconds:
+                          description: TolerationSeconds represents the period of
+                            time the toleration (which must be of effect NoExecute,
+                            otherwise this field is ignored) tolerates the taint.
+                            By default, it is not set, which means tolerate the taint
+                            forever (do not evict). Zero and negative values will
+                            be treated as 0 (evict immediately) by the system.
+                          format: int64
+                          type: integer
+                        value:
+                          description: Value is the taint value the toleration matches
+                            to. If the operator is Exists, the value should be empty,
+                            otherwise just a regular string.
+                          type: string
+                      type: object
+                    type: array
                 type: object
               script:
                 description: K6Script describes where the script to execute the tests
@@ -2011,6 +2050,46 @@ spec:
                     type: object
                   serviceAccountName:
                     type: string
+                  tolerations:
+                    items:
+                      description: The pod this Toleration is attached to tolerates
+                        any taint that matches the triple <key,value,effect> using
+                        the matching operator <operator>.
+                      properties:
+                        effect:
+                          description: Effect indicates the taint effect to match.
+                            Empty means match all taint effects. When specified, allowed
+                            values are NoSchedule, PreferNoSchedule and NoExecute.
+                          type: string
+                        key:
+                          description: Key is the taint key that the toleration applies
+                            to. Empty means match all taint keys. If the key is empty,
+                            operator must be Exists; this combination means to match
+                            all values and all keys.
+                          type: string
+                        operator:
+                          description: Operator represents a key's relationship to
+                            the value. Valid operators are Exists and Equal. Defaults
+                            to Equal. Exists is equivalent to wildcard for value,
+                            so that a pod can tolerate all taints of a particular
+                            category.
+                          type: string
+                        tolerationSeconds:
+                          description: TolerationSeconds represents the period of
+                            time the toleration (which must be of effect NoExecute,
+                            otherwise this field is ignored) tolerates the taint.
+                            By default, it is not set, which means tolerate the taint
+                            forever (do not evict). Zero and negative values will
+                            be treated as 0 (evict immediately) by the system.
+                          format: int64
+                          type: integer
+                        value:
+                          description: Value is the taint value the toleration matches
+                            to. If the operator is Exists, the value should be empty,
+                            otherwise just a regular string.
+                          type: string
+                      type: object
+                    type: array
                 type: object
             required:
             - parallelism
diff --git a/config/samples/k6_v1alpha1_k6_with_volumeClaim.yaml b/config/samples/k6_v1alpha1_k6_with_volumeClaim.yaml
new file mode 100644
index 0000000..653a23c
--- /dev/null
+++ b/config/samples/k6_v1alpha1_k6_with_volumeClaim.yaml
@@ -0,0 +1,14 @@
+apiVersion: k6.io/v1alpha1
+kind: K6
+metadata:
+  name: k6-sample
+  namespace: load-test
+spec:
+  parallelism: 4
+  script:
+    volumeClaim:
+      name: stress-test-volumeClaim
+      # test.js should exist inside /test/ folder.
+      # And, All the js files and directories test.js
+      # is importing from should be inside the same directory as well.
+      file: test.js
diff --git a/pkg/resources/jobs/initializer.go b/pkg/resources/jobs/initializer.go
index 8178c15..40bf736 100644
--- a/pkg/resources/jobs/initializer.go
+++ b/pkg/resources/jobs/initializer.go
@@ -82,6 +82,7 @@ func NewInitializerJob(k6 *v1alpha1.K6, argLine string) (*batchv1.Job, error) {
 					ServiceAccountName:           serviceAccountName,
 					Affinity:                     k6.Spec.Runner.Affinity,
 					NodeSelector:                 k6.Spec.Runner.NodeSelector,
+					Tolerations:                  k6.Spec.Runner.Tolerations,
 					RestartPolicy:                corev1.RestartPolicyNever,
 					Containers: []corev1.Container{
 						{
diff --git a/pkg/resources/jobs/runner.go b/pkg/resources/jobs/runner.go
index acf5451..ca4773e 100644
--- a/pkg/resources/jobs/runner.go
+++ b/pkg/resources/jobs/runner.go
@@ -68,6 +68,9 @@ func NewRunnerJob(k6 *v1alpha1.K6, index int, testRunId, token string) (*batchv1
 	// Add an instance tag: in case metrics are stored, they need to be distinguished by instance
 	command = append(command, "--tag", fmt.Sprintf("instance_id=%d", index))
 
+	// Add an job tag: in case metrics are stored, they need to be distinguished by job
+	command = append(command, "--tag", fmt.Sprintf("job_name=%s", name))
+
 	command = script.UpdateCommand(command)
 
 	var (
@@ -144,6 +147,7 @@ func NewRunnerJob(k6 *v1alpha1.K6, index int, testRunId, token string) (*batchv1
 					RestartPolicy:                corev1.RestartPolicyNever,
 					Affinity:                     k6.Spec.Runner.Affinity,
 					NodeSelector:                 k6.Spec.Runner.NodeSelector,
+					Tolerations:                  k6.Spec.Runner.Tolerations,
 					SecurityContext:              &k6.Spec.Runner.SecurityContext,
 					Containers: []corev1.Container{{
 						Image:        image,
diff --git a/pkg/resources/jobs/starter.go b/pkg/resources/jobs/starter.go
index 7f433d7..31a6d5d 100644
--- a/pkg/resources/jobs/starter.go
+++ b/pkg/resources/jobs/starter.go
@@ -61,6 +61,7 @@ func NewStarterJob(k6 *v1alpha1.K6, hostname []string) *batchv1.Job {
 					ServiceAccountName:           serviceAccountName,
 					Affinity:                     k6.Spec.Starter.Affinity,
 					NodeSelector:                 k6.Spec.Starter.NodeSelector,
+					Tolerations:                  k6.Spec.Starter.Tolerations,
 					RestartPolicy:                corev1.RestartPolicyNever,
 					SecurityContext:              &k6.Spec.Starter.SecurityContext,
 					Containers: []corev1.Container{


Test patch:
diff --git a/pkg/resources/jobs/runner_test.go b/pkg/resources/jobs/runner_test.go
index 2ba36de..73032a8 100644
--- a/pkg/resources/jobs/runner_test.go
+++ b/pkg/resources/jobs/runner_test.go
@@ -284,12 +284,13 @@ func TestNewRunnerJob(t *testing.T) {
 					SecurityContext:              &corev1.PodSecurityContext{},
 					Affinity:                     nil,
 					NodeSelector:                 nil,
+					Tolerations:                  nil,
 					ServiceAccountName:           "default",
 					AutomountServiceAccountToken: &automountServiceAccountToken,
 					Containers: []corev1.Container{{
 						Image:        "ghcr.io/grafana/operator:latest-runner",
 						Name:         "k6",
-						Command:      []string{"k6", "run", "--quiet", "/test/test.js", "--address=0.0.0.0:6565", "--paused", "--tag", "instance_id=1"},
+						Command:      []string{"k6", "run", "--quiet", "/test/test.js", "--address=0.0.0.0:6565", "--paused", "--tag", "instance_id=1", "--tag", "job_name=test-1"},
 						Env:          []corev1.EnvVar{},
 						Resources:    corev1.ResourceRequirements{},
 						VolumeMounts: script.VolumeMount(),
@@ -393,13 +394,14 @@ func TestNewRunnerJobNoisy(t *testing.T) {
 					RestartPolicy:                corev1.RestartPolicyNever,
 					Affinity:                     nil,
 					NodeSelector:                 nil,
+					Tolerations:                  nil,
 					ServiceAccountName:           "default",
 					AutomountServiceAccountToken: &automountServiceAccountToken,
 					SecurityContext:              &corev1.PodSecurityContext{},
 					Containers: []corev1.Container{{
 						Image:        "ghcr.io/grafana/operator:latest-runner",
 						Name:         "k6",
-						Command:      []string{"k6", "run", "/test/test.js", "--address=0.0.0.0:6565", "--paused", "--tag", "instance_id=1"},
+						Command:      []string{"k6", "run", "/test/test.js", "--address=0.0.0.0:6565", "--paused", "--tag", "instance_id=1", "--tag", "job_name=test-1"},
 						Env:          []corev1.EnvVar{},
 						Resources:    corev1.ResourceRequirements{},
 						VolumeMounts: script.VolumeMount(),
@@ -486,13 +488,14 @@ func TestNewRunnerJobUnpaused(t *testing.T) {
 					RestartPolicy:                corev1.RestartPolicyNever,
 					Affinity:                     nil,
 					NodeSelector:                 nil,
+					Tolerations:                  nil,
 					ServiceAccountName:           "default",
 					AutomountServiceAccountToken: &automountServiceAccountToken,
 					SecurityContext:              &corev1.PodSecurityContext{},
 					Containers: []corev1.Container{{
 						Image:        "ghcr.io/grafana/operator:latest-runner",
 						Name:         "k6",
-						Command:      []string{"k6", "run", "--quiet", "/test/test.js", "--address=0.0.0.0:6565", "--tag", "instance_id=1"},
+						Command:      []string{"k6", "run", "--quiet", "/test/test.js", "--address=0.0.0.0:6565", "--tag", "instance_id=1", "--tag", "job_name=test-1"},
 						Env:          []corev1.EnvVar{},
 						Resources:    corev1.ResourceRequirements{},
 						VolumeMounts: script.VolumeMount(),
@@ -579,13 +582,14 @@ func TestNewRunnerJobArguments(t *testing.T) {
 					RestartPolicy:                corev1.RestartPolicyNever,
 					Affinity:                     nil,
 					NodeSelector:                 nil,
+					Tolerations:                  nil,
 					ServiceAccountName:           "default",
 					AutomountServiceAccountToken: &automountServiceAccountToken,
 					SecurityContext:              &corev1.PodSecurityContext{},
 					Containers: []corev1.Container{{
 						Image:        "ghcr.io/grafana/operator:latest-runner",
 						Name:         "k6",
-						Command:      []string{"k6", "run", "--quiet", "--cool-thing", "/test/test.js", "--address=0.0.0.0:6565", "--paused", "--tag", "instance_id=1"},
+						Command:      []string{"k6", "run", "--quiet", "--cool-thing", "/test/test.js", "--address=0.0.0.0:6565", "--paused", "--tag", "instance_id=1", "--tag", "job_name=test-1"},
 						Env:          []corev1.EnvVar{},
 						Resources:    corev1.ResourceRequirements{},
 						VolumeMounts: script.VolumeMount(),
@@ -673,13 +677,14 @@ func TestNewRunnerJobServiceAccount(t *testing.T) {
 					RestartPolicy:                corev1.RestartPolicyNever,
 					Affinity:                     nil,
 					NodeSelector:                 nil,
+					Tolerations:                  nil,
 					ServiceAccountName:           "test",
 					AutomountServiceAccountToken: &automountServiceAccountToken,
 					SecurityContext:              &corev1.PodSecurityContext{},
 					Containers: []corev1.Container{{
 						Image:        "ghcr.io/grafana/operator:latest-runner",
 						Name:         "k6",
-						Command:      []string{"k6", "run", "--quiet", "/test/test.js", "--address=0.0.0.0:6565", "--paused", "--tag", "instance_id=1"},
+						Command:      []string{"k6", "run", "--quiet", "/test/test.js", "--address=0.0.0.0:6565", "--paused", "--tag", "instance_id=1", "--tag", "job_name=test-1"},
 						Env:          []corev1.EnvVar{},
 						Resources:    corev1.ResourceRequirements{},
 						VolumeMounts: script.VolumeMount(),
@@ -768,13 +773,14 @@ func TestNewRunnerJobIstio(t *testing.T) {
 					RestartPolicy:                corev1.RestartPolicyNever,
 					Affinity:                     nil,
 					NodeSelector:                 nil,
+					Tolerations:                  nil,
 					ServiceAccountName:           "default",
 					AutomountServiceAccountToken: &automountServiceAccountToken,
 					SecurityContext:              &corev1.PodSecurityContext{},
 					Containers: []corev1.Container{{
 						Image:   "ghcr.io/grafana/operator:latest-runner",
 						Name:    "k6",
-						Command: []string{"scuttle", "k6", "run", "--quiet", "/test/test.js", "--address=0.0.0.0:6565", "--paused", "--tag", "instance_id=1"},
+						Command: []string{"scuttle", "k6", "run", "--quiet", "/test/test.js", "--address=0.0.0.0:6565", "--paused", "--tag", "instance_id=1", "--tag", "job_name=test-1"},
 						Env: []corev1.EnvVar{
 							{
 								Name:  "ENVOY_ADMIN_API",
@@ -875,13 +881,14 @@ func TestNewRunnerJobCloud(t *testing.T) {
 					RestartPolicy:                corev1.RestartPolicyNever,
 					Affinity:                     nil,
 					NodeSelector:                 nil,
+					Tolerations:                  nil,
 					ServiceAccountName:           "default",
 					SecurityContext:              &corev1.PodSecurityContext{},
 					AutomountServiceAccountToken: &automountServiceAccountToken,
 					Containers: []corev1.Container{{
 						Image:   "ghcr.io/grafana/operator:latest-runner",
 						Name:    "k6",
-						Command: []string{"k6", "run", "--quiet", "--out", "cloud", "/test/test.js", "--address=0.0.0.0:6565", "--paused", "--tag", "instance_id=1"},
+						Command: []string{"k6", "run", "--quiet", "--out", "cloud", "/test/test.js", "--address=0.0.0.0:6565", "--paused", "--tag", "instance_id=1", "--tag", "job_name=test-1"},
 						Env: []corev1.EnvVar{
 							{
 								Name:  "K6_CLOUD_PUSH_REF_ID",
@@ -973,13 +980,14 @@ func TestNewRunnerJobLocalFile(t *testing.T) {
 					RestartPolicy:                corev1.RestartPolicyNever,
 					Affinity:                     nil,
 					NodeSelector:                 nil,
+					Tolerations:                  nil,
 					ServiceAccountName:           "default",
 					AutomountServiceAccountToken: &automountServiceAccountToken,
 					SecurityContext:              &corev1.PodSecurityContext{},
 					Containers: []corev1.Container{{
 						Image:        "ghcr.io/grafana/operator:latest-runner",
 						Name:         "k6",
-						Command:      []string{"sh", "-c", "if [ ! -f /test/test.js ]; then echo \"LocalFile not found exiting...\"; exit 1; fi;\nk6 run --quiet /test/test.js --address=0.0.0.0:6565 --paused --tag instance_id=1"},
+						Command:      []string{"sh", "-c", "if [ ! -f /test/test.js ]; then echo \"LocalFile not found exiting...\"; exit 1; fi;\nk6 run --quiet /test/test.js --address=0.0.0.0:6565 --paused --tag instance_id=1 --tag job_name=test-1"},
 						Env:          []corev1.EnvVar{},
 						Resources:    corev1.ResourceRequirements{},
 						VolumeMounts: script.VolumeMount(),
diff --git a/pkg/resources/jobs/starter_test.go b/pkg/resources/jobs/starter_test.go
index a40c66d..873d1ce 100644
--- a/pkg/resources/jobs/starter_test.go
+++ b/pkg/resources/jobs/starter_test.go
@@ -45,6 +45,7 @@ func TestNewStarterJob(t *testing.T) {
 					ServiceAccountName:           "default",
 					Affinity:                     nil,
 					NodeSelector:                 nil,
+					Tolerations:                  nil,
 					RestartPolicy:                corev1.RestartPolicyNever,
 					SecurityContext:              &corev1.PodSecurityContext{},
 					Containers: []corev1.Container{
@@ -123,6 +124,7 @@ func TestNewStarterJobIstio(t *testing.T) {
 					ServiceAccountName:           "default",
 					Affinity:                     nil,
 					NodeSelector:                 nil,
+					Tolerations:                  nil,
 					RestartPolicy:                corev1.RestartPolicyNever,
 					SecurityContext:              &corev1.PodSecurityContext{},
 					Containers: []corev1.Container{