Repository: kurator-dev/kurator
Pull request #541 (2024-01-08 06:58:22)

pipeline: add render custom task func
**What type of PR is this?**


/kind feature


**What this PR does / why we need it**:

part of #493


kurator-bot: [APPROVALNOTIFIER] This PR is **NOT APPROVED**

This pull-request has been approved by:
**Once this PR has been reviewed and has the lgtm label**, please assign kevin-wangzefeng for approval. For more information see [the Kubernetes Code Review Process](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process).

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=kurator-dev%2Fkurator).

<details open>
Needs approval from an approver in each of these files:

- **[OWNERS](https://github.com/kurator-dev/kurator/blob/main/OWNERS)**

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":["kevin-wangzefeng"]} -->
netlify[bot]: ### <span aria-hidden="true">👷</span> Deploy Preview for *kurator-dev* processing.


|  Name | Link |
|:-:|------------------------|
|<span aria-hidden="true">🔨</span> Latest commit | fc7aacd8eec6c9763593aac228605e50cc52e96c |
|<span aria-hidden="true">🔍</span> Latest deploy log | https://app.netlify.com/sites/kurator-dev/deploys/659b9d117444f40008fb6b38 |

Patch:
diff --git a/pkg/fleet-manager/pipeline/render/customTask.go b/pkg/fleet-manager/pipeline/render/customTask.go
new file mode 100644
index 0000000..89b8ffa
--- /dev/null
+++ b/pkg/fleet-manager/pipeline/render/customTask.go
@@ -0,0 +1,143 @@
+/*
+Copyright Kurator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package render
+
+import (
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	pipelineapi "kurator.dev/kurator/pkg/apis/pipeline/v1alpha1"
+)
+
+const (
+	CustomTaskTemplateName = "pipeline custom task template"
+)
+
+type CustomTaskConfig struct {
+	TaskName             string
+	PipelineName         string
+	PipelineNamespace    string
+	Image                string
+	Command              []string
+	Args                 []string
+	Env                  []corev1.EnvVar
+	ResourceRequirements *corev1.ResourceRequirements
+	Script               string
+	OwnerReference       *metav1.OwnerReference
+}
+
+// CustomTaskName is the name of custom task object, in case different pipeline have the same name task.
+func (cfg CustomTaskConfig) CustomTaskName() string {
+	return cfg.TaskName + "-" + cfg.PipelineName
+}
+
+// RenderCustomTaskWithPipeline takes a Pipeline object and generates YAML byte array configuration representing the CustomTask configuration.
+func RenderCustomTaskWithPipeline(pipeline *pipelineapi.Pipeline, taskName string, task *pipelineapi.CustomTask) ([]byte, error) {
+	cfg := CustomTaskConfig{
+		TaskName:             taskName,
+		PipelineName:         pipeline.Name,
+		PipelineNamespace:    pipeline.Namespace,
+		Image:                task.Image,
+		Command:              task.Command,
+		Args:                 task.Args,
+		Env:                  task.Env,
+		ResourceRequirements: &task.ResourceRequirements,
+		Script:               task.Script,
+		OwnerReference:       GeneratePipelineOwnerRef(pipeline),
+	}
+
+	return RenderCustomTask(cfg)
+}
+
+// RenderCustomTask takes a CustomTaskConfig object and generates YAML byte array configuration representing the CustomTask configuration.
+func RenderCustomTask(cfg CustomTaskConfig) ([]byte, error) {
+	if cfg.Image == "" || cfg.CustomTaskName() == "" {
+		return nil, fmt.Errorf("invalid RBACConfig: PipelineName and PipelineNamespace must not be empty")
+	}
+	return renderTemplate(CustomTaskTemplateContent, CustomTaskTemplateName, cfg)
+}
+
+const CustomTaskTemplateContent = `apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: {{ .CustomTaskName }}
+  namespace: {{ .PipelineNamespace }}
+{{- if .OwnerReference }}
+  ownerReferences:
+  - apiVersion: "{{ .OwnerReference.APIVersion }}"
+    kind: "{{ .OwnerReference.Kind }}"
+    name: "{{ .OwnerReference.Name }}"
+    uid: "{{ .OwnerReference.UID }}"
+{{- end }}
+spec:
+  description: >-
+    This task is a user-custom, single-step task.
+    The workspace is automatically and exclusively created named "source",
+    and assigned to the workspace of the pipeline in which this task is located.
+  workspaces:
+  - name: source
+    description: The workspace where user to run user-custom task.
+  steps:
+  - name: {{ .CustomTaskName }}
+    image: {{ .Image }}
+    {{- if .Env }}
+    env:
+    {{- range .Env }}
+    - name: {{ .Name }}
+      value: {{ .Value }}
+    {{- end }}
+    {{- end }}
+    {{- if .Command }}
+    command:
+    {{- range .Command }}
+    - {{ . }}
+    {{- end }}
+    {{- end }}
+    {{- if .Args }}
+    args:
+    {{- range .Args }}
+    - {{ . }}
+    {{- end }}
+    {{- end }}
+    {{- if .Script }}
+    script: |
+      {{ .Script }}
+    {{- end }}
+    {{- if .ResourceRequirements }}
+    resources:
+      {{- if .ResourceRequirements.Requests }}
+      requests:
+        {{- if .ResourceRequirements.Requests.Cpu }}
+        cpu: {{ .ResourceRequirements.Requests.Cpu }}
+        {{- end }}
+        {{- if .ResourceRequirements.Requests.Memory }}
+        memory: {{ .ResourceRequirements.Requests.Memory }}
+        {{- end }}
+      {{- end }}
+      {{- if .ResourceRequirements.Limits }}
+      limits:
+        {{- if .ResourceRequirements.Limits.Cpu }}
+        cpu: {{ .ResourceRequirements.Limits.Cpu }}
+        {{- end }}
+        {{- if .ResourceRequirements.Limits.Memory }}
+        memory: {{ .ResourceRequirements.Limits.Memory }}
+        {{- end }}
+      {{- end }}
+    {{- end }}
+`
diff --git a/pkg/fleet-manager/pipeline/render/util.go b/pkg/fleet-manager/pipeline/render/util.go
index 74d4b66..958b695 100644
--- a/pkg/fleet-manager/pipeline/render/util.go
+++ b/pkg/fleet-manager/pipeline/render/util.go
@@ -21,7 +21,10 @@ import (
 	"text/template"
 
 	"github.com/Masterminds/sprig/v3"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/yaml"
+
+	pipelineapi "kurator.dev/kurator/pkg/apis/pipeline/v1alpha1"
 )
 
 // renderTemplate reads, parses, and renders a template file using the provided configuration data.
@@ -56,3 +59,12 @@ func toYaml(value interface{}) string {
 
 	return string(y)
 }
+
+func GeneratePipelineOwnerRef(pipeline *pipelineapi.Pipeline) *metav1.OwnerReference {
+	return &metav1.OwnerReference{
+		APIVersion: pipeline.APIVersion,
+		Kind:       pipeline.Kind,
+		Name:       pipeline.Name,
+		UID:        pipeline.UID,
+	}
+}


Test patch:
diff --git a/pkg/fleet-manager/pipeline/render/customTask_test.go b/pkg/fleet-manager/pipeline/render/customTask_test.go
new file mode 100644
index 0000000..fc6af00
--- /dev/null
+++ b/pkg/fleet-manager/pipeline/render/customTask_test.go
@@ -0,0 +1,165 @@
+/*
+Copyright Kurator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package render
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+func TestRenderCustomTask(t *testing.T) {
+	expectedTaskFilePath := "testdata/custom-task/"
+	// Define test cases for various task templates and configurations.
+	cases := []struct {
+		name         string
+		cfg          CustomTaskConfig
+		expectError  bool
+		expectedFile string
+	}{
+		// ---- Case: Default Configuration for Go Test ----
+		// This case tests a simple configuration of to print the repo readme.
+		{
+			name: "cat-readme",
+			cfg: CustomTaskConfig{
+				TaskName:          "cat-readme",
+				PipelineName:      "test-pipeline",
+				PipelineNamespace: "default",
+				Image:             "zshusers/zsh:4.3.15",
+				Command: []string{
+					"/bin/sh",
+					"-c",
+				},
+				Args: []string{
+					"cat $(workspaces.source.path)/README.md",
+				},
+			},
+			expectError:  false,
+			expectedFile: "cat-readme.yaml",
+		},
+		{
+			name: "minimal-configuration",
+			cfg: CustomTaskConfig{
+				TaskName:          "minimal-task",
+				PipelineName:      "test-pipeline",
+				PipelineNamespace: "default",
+				Image:             "alpine:latest",
+			},
+			expectError:  false,
+			expectedFile: "minimal-task.yaml",
+		},
+		{
+			name: "complete-configuration",
+			cfg: CustomTaskConfig{
+				TaskName:          "complete-task",
+				PipelineName:      "test-pipeline",
+				PipelineNamespace: "default",
+				Image:             "python:3.8",
+				Command:           []string{"python", "-c"},
+				Args:              []string{"print('Hello World')"},
+				Env:               []corev1.EnvVar{{Name: "ENV_VAR", Value: "test"}},
+				ResourceRequirements: &corev1.ResourceRequirements{
+					Requests: corev1.ResourceList{
+						corev1.ResourceCPU:    resource.MustParse("100m"),
+						corev1.ResourceMemory: resource.MustParse("256Mi"),
+					},
+					Limits: corev1.ResourceList{
+						corev1.ResourceCPU:    resource.MustParse("200m"),
+						corev1.ResourceMemory: resource.MustParse("512Mi"),
+					},
+				},
+				Script: "print('This is a complete test')",
+			},
+			expectError:  false,
+			expectedFile: "complete-task.yaml",
+		},
+		{
+			name: "missing-required-fields-test-pipeline",
+			cfg: CustomTaskConfig{
+				PipelineNamespace: "default",
+			},
+			expectError:  true,
+			expectedFile: "",
+		},
+		{
+			name: "with-environment-variables-test-pipeline",
+			cfg: CustomTaskConfig{
+				TaskName:          "env-task",
+				PipelineName:      "test-pipeline",
+				PipelineNamespace: "default",
+				Image:             "node:14",
+				Env:               []corev1.EnvVar{{Name: "NODE_ENV", Value: "production"}},
+			},
+			expectError:  false,
+			expectedFile: "env-task.yaml",
+		},
+		{
+			name: "with-resource-requirements-test-pipeline",
+			cfg: CustomTaskConfig{
+				TaskName:          "resource-task",
+				PipelineName:      "test-pipeline",
+				PipelineNamespace: "default",
+				Image:             "golang:1.16",
+				ResourceRequirements: &corev1.ResourceRequirements{
+					Requests: corev1.ResourceList{
+						corev1.ResourceCPU:    resource.MustParse("500m"),
+						corev1.ResourceMemory: resource.MustParse("1Gi"),
+					},
+					Limits: corev1.ResourceList{
+						corev1.ResourceCPU:    resource.MustParse("1"),
+						corev1.ResourceMemory: resource.MustParse("2Gi"),
+					},
+				},
+			},
+			expectError:  false,
+			expectedFile: "resource-task.yaml",
+		},
+		{
+			name: "with-commands-and-arguments-test-pipeline",
+			cfg: CustomTaskConfig{
+				TaskName:          "cmd-args",
+				PipelineName:      "test-pipeline",
+				PipelineNamespace: "default",
+				Image:             "ubuntu:latest",
+				Command:           []string{"/bin/bash", "-c"},
+				Args:              []string{"echo 'Hello from command'"},
+			},
+			expectError:  false,
+			expectedFile: "cmd-args-task.yaml",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			result, err := RenderCustomTask(tc.cfg)
+
+			// Test assertions
+			if tc.expectError {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+
+				expected, err := os.ReadFile(expectedTaskFilePath + tc.expectedFile)
+				assert.NoError(t, err)
+				assert.Equal(t, string(expected), string(result))
+			}
+		})
+	}
+}
diff --git a/pkg/fleet-manager/pipeline/render/testdata/custom-task/cat-readme.yaml b/pkg/fleet-manager/pipeline/render/testdata/custom-task/cat-readme.yaml
new file mode 100644
index 0000000..8a5578c
--- /dev/null
+++ b/pkg/fleet-manager/pipeline/render/testdata/custom-task/cat-readme.yaml
@@ -0,0 +1,21 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: cat-readme-test-pipeline
+  namespace: default
+spec:
+  description: >-
+    This task is a user-custom, single-step task.
+    The workspace is automatically and exclusively created named "source",
+    and assigned to the workspace of the pipeline in which this task is located.
+  workspaces:
+  - name: source
+    description: The workspace where user to run user-custom task.
+  steps:
+  - name: cat-readme-test-pipeline
+    image: zshusers/zsh:4.3.15
+    command:
+    - /bin/sh
+    - -c
+    args:
+    - cat $(workspaces.source.path)/README.md
diff --git a/pkg/fleet-manager/pipeline/render/testdata/custom-task/cmd-args-task.yaml b/pkg/fleet-manager/pipeline/render/testdata/custom-task/cmd-args-task.yaml
new file mode 100644
index 0000000..f697e9f
--- /dev/null
+++ b/pkg/fleet-manager/pipeline/render/testdata/custom-task/cmd-args-task.yaml
@@ -0,0 +1,21 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: cmd-args-test-pipeline
+  namespace: default
+spec:
+  description: >-
+    This task is a user-custom, single-step task.
+    The workspace is automatically and exclusively created named "source",
+    and assigned to the workspace of the pipeline in which this task is located.
+  workspaces:
+  - name: source
+    description: The workspace where user to run user-custom task.
+  steps:
+  - name: cmd-args-test-pipeline
+    image: ubuntu:latest
+    command:
+    - /bin/bash
+    - -c
+    args:
+    - echo 'Hello from command'
diff --git a/pkg/fleet-manager/pipeline/render/testdata/custom-task/complete-task.yaml b/pkg/fleet-manager/pipeline/render/testdata/custom-task/complete-task.yaml
new file mode 100644
index 0000000..2c87073
--- /dev/null
+++ b/pkg/fleet-manager/pipeline/render/testdata/custom-task/complete-task.yaml
@@ -0,0 +1,33 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: complete-task-test-pipeline
+  namespace: default
+spec:
+  description: >-
+    This task is a user-custom, single-step task.
+    The workspace is automatically and exclusively created named "source",
+    and assigned to the workspace of the pipeline in which this task is located.
+  workspaces:
+  - name: source
+    description: The workspace where user to run user-custom task.
+  steps:
+  - name: complete-task-test-pipeline
+    image: python:3.8
+    env:
+    - name: ENV_VAR
+      value: test
+    command:
+    - python
+    - -c
+    args:
+    - print('Hello World')
+    script: |
+      print('This is a complete test')
+    resources:
+      requests:
+        cpu: 100m
+        memory: 256Mi
+      limits:
+        cpu: 200m
+        memory: 512Mi
diff --git a/pkg/fleet-manager/pipeline/render/testdata/custom-task/env-task.yaml b/pkg/fleet-manager/pipeline/render/testdata/custom-task/env-task.yaml
new file mode 100644
index 0000000..6ccdeaa
--- /dev/null
+++ b/pkg/fleet-manager/pipeline/render/testdata/custom-task/env-task.yaml
@@ -0,0 +1,19 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: env-task-test-pipeline
+  namespace: default
+spec:
+  description: >-
+    This task is a user-custom, single-step task.
+    The workspace is automatically and exclusively created named "source",
+    and assigned to the workspace of the pipeline in which this task is located.
+  workspaces:
+  - name: source
+    description: The workspace where user to run user-custom task.
+  steps:
+  - name: env-task-test-pipeline
+    image: node:14
+    env:
+    - name: NODE_ENV
+      value: production
diff --git a/pkg/fleet-manager/pipeline/render/testdata/custom-task/minimal-task.yaml b/pkg/fleet-manager/pipeline/render/testdata/custom-task/minimal-task.yaml
new file mode 100644
index 0000000..2dc1b48
--- /dev/null
+++ b/pkg/fleet-manager/pipeline/render/testdata/custom-task/minimal-task.yaml
@@ -0,0 +1,16 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: minimal-task-test-pipeline
+  namespace: default
+spec:
+  description: >-
+    This task is a user-custom, single-step task.
+    The workspace is automatically and exclusively created named "source",
+    and assigned to the workspace of the pipeline in which this task is located.
+  workspaces:
+  - name: source
+    description: The workspace where user to run user-custom task.
+  steps:
+  - name: minimal-task-test-pipeline
+    image: alpine:latest
diff --git a/pkg/fleet-manager/pipeline/render/testdata/custom-task/resource-task.yaml b/pkg/fleet-manager/pipeline/render/testdata/custom-task/resource-task.yaml
new file mode 100644
index 0000000..b279c73
--- /dev/null
+++ b/pkg/fleet-manager/pipeline/render/testdata/custom-task/resource-task.yaml
@@ -0,0 +1,23 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: resource-task-test-pipeline
+  namespace: default
+spec:
+  description: >-
+    This task is a user-custom, single-step task.
+    The workspace is automatically and exclusively created named "source",
+    and assigned to the workspace of the pipeline in which this task is located.
+  workspaces:
+  - name: source
+    description: The workspace where user to run user-custom task.
+  steps:
+  - name: resource-task-test-pipeline
+    image: golang:1.16
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1Gi
+      limits:
+        cpu: 1
+        memory: 2Gi