Repository: grafana/k6-operator
Pull request #93 (2022-01-17 16:36:35)

Feat: Execution of test using a LocalFile inside the Docker Image
Hello everyone,

Recently i've had some problems running with a configMap due to the size of my test, the company that i work on doesn't like the idea of using VolumeClaim for applications in our clusters, so i made a simple fix to execute a local file inside the docker image.


Patch:
diff --git a/README.md b/README.md
index 2e78d02..88b6b4d 100644
--- a/README.md
+++ b/README.md
@@ -35,15 +35,23 @@ Two samples are available in `config/samples`, one for a test script and one for
 
 ### Adding test scripts
 
-The operator utilises `ConfigMap`s to serve test scripts to the jobs. To upload your own test script, run the following:
+The operator utilises `ConfigMap`s and `LocalFile` to serve test scripts to the jobs. To upload your own test script, run the following command to configure through `ConfigMap`:
 
+#### ConfigMap
 ```bash
 $ kubectl create configmap my-test --from-file /path/to/my/test.js
 ```
 
-***Note: there is a character limit of 1048576 bytes to a single configmap. If you need to have a larger test file, you'll need to use a volumeClaim instead***
+***Note: there is a character limit of 1048576 bytes to a single configmap. If you need to have a larger test file, you'll need to use a volumeClaim or a LocalFile instead***
+
+#### LocalFile
+
+There is a sample avaiable in `config/samples/k6_v1alpha1_k6_with_localfile.yaml` on how to configure to run a test script inside the docker image.
+
+***Note: if there is any limitation on usage of volumeClaim in your cluster you can use this option, but always prefer the usage of volumeClaim.***
 
 ### Executing tests
+
 Tests are executed by applying the custom resource `K6` to a cluster where the operator is running. The properties
 of a test run are few, but allow you to control some key aspects of a distributed execution.
 
@@ -138,8 +146,6 @@ Defines options for the starter pod. This includes:
 * passing in custom image
 * passing in labels and annotations
 
-
-
 ### Cleaning up between test runs
 After completing a test run, you need to clean up the test jobs created. This is done by running the following command:
 ```bash
diff --git a/api/v1alpha1/k6_types.go b/api/v1alpha1/k6_types.go
index 30f7807..a40fb1b 100644
--- a/api/v1alpha1/k6_types.go
+++ b/api/v1alpha1/k6_types.go
@@ -67,6 +67,7 @@ type K6Spec struct {
 type K6Script struct {
 	VolumeClaim K6VolumeClaim `json:"volumeClaim,omitempty"`
 	ConfigMap   K6Configmap   `json:"configMap,omitempty"`
+	LocalFile   string        `json:"localFile,omitempty"`
 }
 
 // K6VolumeClaim describes the volume claim script location
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
index 2a1bfce..bb28003 100644
--- a/api/v1alpha1/zz_generated.deepcopy.go
+++ b/api/v1alpha1/zz_generated.deepcopy.go
@@ -1,4 +1,3 @@
-//go:build !ignore_autogenerated
 // +build !ignore_autogenerated
 
 /*
diff --git a/config/crd/bases/k6.io_k6s.yaml b/config/crd/bases/k6.io_k6s.yaml
index 7eaa32b..87985ea 100644
--- a/config/crd/bases/k6.io_k6s.yaml
+++ b/config/crd/bases/k6.io_k6s.yaml
@@ -868,6 +868,8 @@ spec:
                     required:
                     - name
                     type: object
+                  localFile:
+                    type: string
                   volumeClaim:
                     description: K6VolumeClaim describes the volume claim script location
                     properties:
diff --git a/config/samples/k6_v1alpha1_k6_with_localfile.yaml b/config/samples/k6_v1alpha1_k6_with_localfile.yaml
new file mode 100644
index 0000000..f3378df
--- /dev/null
+++ b/config/samples/k6_v1alpha1_k6_with_localfile.yaml
@@ -0,0 +1,11 @@
+---
+apiVersion: k6.io/v1alpha1
+kind: K6
+metadata:
+  name: k6-sample
+spec:
+  parallelism: 4
+  script:
+    localFile: /test/test.js
+  runner:
+    image: <custom-image>
diff --git a/controllers/k6_start.go b/controllers/k6_start.go
index 4024cb2..358c7ee 100644
--- a/controllers/k6_start.go
+++ b/controllers/k6_start.go
@@ -3,6 +3,7 @@ package controllers
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"time"
 
 	"github.com/go-logr/logr"
@@ -15,6 +16,17 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+func isServiceReady(log logr.Logger, service *v1.Service) bool {
+	resp, err := http.Get(fmt.Sprintf("http://%v.%v.svc.cluster.local:6565/v1/status", service.ObjectMeta.Name, service.ObjectMeta.Namespace))
+
+	if err != nil {
+		log.Error(err, fmt.Sprintf("failed to get status from %v", service.ObjectMeta.Name))
+		return false
+	}
+
+	return resp.StatusCode < 400
+}
+
 // StartJobs in the Ready phase using a curl container
 func StartJobs(ctx context.Context, log logr.Logger, k6 *v1alpha1.K6, r *K6Reconciler) (ctrl.Result, error) {
 	log.Info("Waiting for pods to get ready")
@@ -58,6 +70,11 @@ func StartJobs(ctx context.Context, log logr.Logger, k6 *v1alpha1.K6, r *K6Recon
 
 		for _, service := range sl.Items {
 			hostnames = append(hostnames, service.ObjectMeta.Name)
+
+			if !isServiceReady(log, &service) {
+				log.Info(fmt.Sprintf("%v service is not ready, aborting", service.ObjectMeta.Name))
+				return false, nil
+			}
 		}
 
 		starter := jobs.NewStarterJob(k6, hostnames)
diff --git a/pkg/resources/jobs/runner.go b/pkg/resources/jobs/runner.go
index 846b2e6..15b816d 100644
--- a/pkg/resources/jobs/runner.go
+++ b/pkg/resources/jobs/runner.go
@@ -61,7 +61,7 @@ func NewRunnerJob(k6 *v1alpha1.K6, index int) (*batchv1.Job, error) {
 
 	command = append(
 		command,
-		fmt.Sprintf("/test/%s", script.File),
+		fmt.Sprintf(script.File),
 		"--address=0.0.0.0:6565")
 
 	paused := true
@@ -73,6 +73,8 @@ func NewRunnerJob(k6 *v1alpha1.K6, index int) (*batchv1.Job, error) {
 		command = append(command, "--paused")
 	}
 
+	command = appendFileCheckerCommand(script, command)
+
 	var zero int64 = 0
 
 	image := "ghcr.io/grafana/operator:latest-runner"
@@ -131,16 +133,13 @@ func NewRunnerJob(k6 *v1alpha1.K6, index int) (*batchv1.Job, error) {
 					Affinity:                     k6.Spec.Runner.Affinity,
 					NodeSelector:                 k6.Spec.Runner.NodeSelector,
 					Containers: []corev1.Container{{
-						Image:     image,
-						Name:      "k6",
-						Command:   command,
-						Env:       env,
-						Resources: k6.Spec.Runner.Resources,
-						VolumeMounts: []corev1.VolumeMount{{
-							Name:      "k6-test-volume",
-							MountPath: "/test",
-						}},
-						Ports: ports,
+						Image:        image,
+						Name:         "k6",
+						Command:      command,
+						Env:          env,
+						Resources:    k6.Spec.Runner.Resources,
+						VolumeMounts: newVolumeMountSpec(script),
+						Ports:        ports,
 					}},
 					TerminationGracePeriodSeconds: &zero,
 					Volumes:                       newVolumeSpec(script),
@@ -220,8 +219,19 @@ func newAntiAffinity() *corev1.Affinity {
 	}
 }
 
+func newVolumeMountSpec(s *Script) []corev1.VolumeMount {
+	if s.Type == "LocalFile" {
+		return []corev1.VolumeMount{}
+	}
+	return []corev1.VolumeMount{{
+		Name:      "k6-test-volume",
+		MountPath: "/test",
+	}}
+}
+
 func newVolumeSpec(s *Script) []corev1.Volume {
-	if s.Type == "VolumeClaim" {
+	switch s.Type {
+	case "VolumeClaim":
 		return []corev1.Volume{{
 			Name: "k6-test-volume",
 			VolumeSource: corev1.VolumeSource{
@@ -230,18 +240,20 @@ func newVolumeSpec(s *Script) []corev1.Volume {
 				},
 			},
 		}}
-	}
-
-	return []corev1.Volume{{
-		Name: "k6-test-volume",
-		VolumeSource: corev1.VolumeSource{
-			ConfigMap: &corev1.ConfigMapVolumeSource{
-				LocalObjectReference: corev1.LocalObjectReference{
-					Name: s.Name,
+	case "ConfigMap":
+		return []corev1.Volume{{
+			Name: "k6-test-volume",
+			VolumeSource: corev1.VolumeSource{
+				ConfigMap: &corev1.ConfigMapVolumeSource{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: s.Name,
+					},
 				},
 			},
-		},
-	}}
+		}}
+	default:
+		return []corev1.Volume{}
+	}
 }
 
 func newScript(spec v1alpha1.K6Spec) (*Script, error) {
@@ -254,6 +266,7 @@ func newScript(spec v1alpha1.K6Spec) (*Script, error) {
 			s.File = spec.Script.VolumeClaim.File
 		}
 
+		s.File = fmt.Sprintf("/test/%s", s.File)
 		s.Type = "VolumeClaim"
 		return s, nil
 	}
@@ -264,10 +277,26 @@ func newScript(spec v1alpha1.K6Spec) (*Script, error) {
 		if spec.Script.ConfigMap.File != "" {
 			s.File = spec.Script.ConfigMap.File
 		}
-
+		s.File = fmt.Sprintf("/test/%s", s.File)
 		s.Type = "ConfigMap"
 		return s, nil
 	}
 
-	return nil, errors.New("ConfigMap or VolumeClaim not provided in script definition")
+	if spec.Script.LocalFile != "" {
+		s.Name = "LocalFile"
+		s.File = spec.Script.LocalFile
+		s.Type = "LocalFile"
+		return s, nil
+	}
+
+	return nil, errors.New("ConfigMap, VolumeClaim or LocalFile not provided in script definition")
+}
+
+func appendFileCheckerCommand(s *Script, cmd []string) []string {
+	if s.Type == "LocalFile" {
+		joincmd := strings.Join(cmd, " ")
+		checkCommand := []string{"sh", "-c", fmt.Sprintf("if [ ! -f %v ]; then echo \"LocalFile not found exiting...\"; exit 1; fi;\n%v", s.File, joincmd)}
+		return checkCommand
+	}
+	return cmd
 }


Test patch:
diff --git a/pkg/resources/jobs/runner_test.go b/pkg/resources/jobs/runner_test.go
index 2312a43..9ab8793 100644
--- a/pkg/resources/jobs/runner_test.go
+++ b/pkg/resources/jobs/runner_test.go
@@ -16,7 +16,7 @@ func TestNewScriptVolumeClaim(t *testing.T) {
 
 	expectedOutcome := &Script{
 		Name: "Test",
-		File: "thing.js",
+		File: "/test/thing.js",
 		Type: "VolumeClaim",
 	}
 
@@ -42,7 +42,7 @@ func TestNewScriptConfigMap(t *testing.T) {
 
 	expectedOutcome := &Script{
 		Name: "Test",
-		File: "thing.js",
+		File: "/test/thing.js",
 		Type: "ConfigMap",
 	}
 
@@ -64,17 +64,41 @@ func TestNewScriptConfigMap(t *testing.T) {
 	}
 }
 
+func TestNewScriptLocalFile(t *testing.T) {
+
+	expectedOutcome := &Script{
+		Name: "LocalFile",
+		File: "/custom/my_test.js",
+		Type: "LocalFile",
+	}
+
+	k6 := v1alpha1.K6Spec{
+		Script: v1alpha1.K6Script{
+			LocalFile: "/custom/my_test.js",
+		},
+	}
+
+	script, err := newScript(k6)
+	if err != nil {
+		t.Errorf("NewScript with LocalFile errored, got: %v, want: %v", err, expectedOutcome)
+	}
+	if !reflect.DeepEqual(script, expectedOutcome) {
+		t.Errorf("NewScript with LocalFile failed to return expected output, got: %v, expected: %v", script, expectedOutcome)
+	}
+}
+
 func TestNewScriptNoScript(t *testing.T) {
 
 	k6 := v1alpha1.K6Spec{}
 
 	script, err := newScript(k6)
 	if err == nil && script != nil {
-		t.Errorf("Expected Error from NewScript, got: %v, want: %v", err, errors.New("configMap or VolumeClaim not provided in script definition"))
+		t.Errorf("Expected Error from NewScript, got: %v, want: %v", err, errors.New("configMap, VolumeClaim or LocalFile not provided in script definition"))
 	}
 }
 
 func TestNewVolumeSpecVolumeClaim(t *testing.T) {
+
 	expectedOutcome := []corev1.Volume{{
 		Name: "k6-test-volume",
 		VolumeSource: corev1.VolumeSource{
@@ -119,16 +143,7 @@ func TestNewVolumeSpecConfigMap(t *testing.T) {
 }
 
 func TestNewVolumeSpecNoType(t *testing.T) {
-	expectedOutcome := []corev1.Volume{{
-		Name: "k6-test-volume",
-		VolumeSource: corev1.VolumeSource{
-			ConfigMap: &corev1.ConfigMapVolumeSource{
-				LocalObjectReference: corev1.LocalObjectReference{
-					Name: "test",
-				},
-			},
-		},
-	}}
+	expectedOutcome := []corev1.Volume{}
 
 	k6 := &Script{
 		Name: "test",
@@ -809,3 +824,94 @@ func TestNewRunnerJobIstio(t *testing.T) {
 		t.Errorf("NewRunnerJob returned unexpected data, diff: %s", diff)
 	}
 }
+
+func TestNewRunnerJobLocalFile(t *testing.T) {
+	script := &Script{
+		Name: "test",
+		File: "/test/test.js",
+		Type: "LocalFile",
+	}
+
+	var zero int64 = 0
+	automountServiceAccountToken := true
+
+	expectedOutcome := &batchv1.Job{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-1",
+			Namespace: "test",
+			Labels: map[string]string{
+				"app":    "k6",
+				"k6_cr":  "test",
+				"label1": "awesome",
+			},
+			Annotations: map[string]string{
+				"awesomeAnnotation": "dope",
+			},
+		},
+		Spec: batchv1.JobSpec{
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"app":    "k6",
+						"k6_cr":  "test",
+						"label1": "awesome",
+					},
+					Annotations: map[string]string{
+						"awesomeAnnotation": "dope",
+					},
+				},
+				Spec: corev1.PodSpec{
+					Hostname:                     "test-1",
+					RestartPolicy:                corev1.RestartPolicyNever,
+					Affinity:                     nil,
+					NodeSelector:                 nil,
+					ServiceAccountName:           "default",
+					AutomountServiceAccountToken: &automountServiceAccountToken,
+					Containers: []corev1.Container{{
+						Image:        "ghcr.io/grafana/operator:latest-runner",
+						Name:         "k6",
+						Command:      []string{"sh", "-c", "if [ ! -f /test/test.js ]; then echo \"LocalFile not found exiting...\"; exit 1; fi;\nk6 run --quiet /test/test.js --address=0.0.0.0:6565 --paused"},
+						Env:          []corev1.EnvVar{},
+						Resources:    corev1.ResourceRequirements{},
+						VolumeMounts: []corev1.VolumeMount{},
+						Ports:        []corev1.ContainerPort{{ContainerPort: 6565}},
+					}},
+					TerminationGracePeriodSeconds: &zero,
+					Volumes:                       newVolumeSpec(script),
+				},
+			},
+		},
+	}
+	k6 := &v1alpha1.K6{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test",
+			Namespace: "test",
+		},
+		Spec: v1alpha1.K6Spec{
+			Scuttle: v1alpha1.K6Scuttle{
+				Enabled: "false",
+			},
+			Script: v1alpha1.K6Script{
+				LocalFile: "/test/test.js",
+			},
+			Runner: v1alpha1.Pod{
+				Metadata: v1alpha1.PodMetadata{
+					Labels: map[string]string{
+						"label1": "awesome",
+					},
+					Annotations: map[string]string{
+						"awesomeAnnotation": "dope",
+					},
+				},
+			},
+		},
+	}
+
+	job, err := NewRunnerJob(k6, 1)
+	if err != nil {
+		t.Errorf("NewRunnerJob errored, got: %v", err)
+	}
+	if diff := deep.Equal(job, expectedOutcome); diff != nil {
+		t.Errorf("NewRunnerJob returned unexpected data, diff: %s", diff)
+	}
+}