Repository: kedacore/keda

Issue:
Allow filtering labels out to avoid interactions with other stuff like tanka
### Proposal

KEDA should be offer a way of filtering out specific labels when it copies them from a `ScaledObject` to the `HPA`.

### Use-Case

When you create an `ScaledObject` using [tanka](https://tanka.dev/) it gets labeled with an environment hash (`tanka.dev/environment=<HASH>`) to get track of the objects and being able to prune removed objects from your code and so.

As KEDA copies all the labels applied to the `ScaledObject` to the `HPA` the tanka's environment label gets propagated so. This causes tanka to continuously show in the diff that the `HPA` needs to be removed, and, in fact, our deployment pipeline removes it (I have to thank KEDA being that quick on re-creating it, but is not the ideal scenario).

### Is this a feature you are interested in implementing yourself?

Maybe

### Anything else?

_No response_

Discussion:

Solution patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3d3063919..9564585b4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -71,6 +71,7 @@ To learn more about active deprecations, we recommend checking [GitHub Discussio
 
 ### Improvements
 
+- **General**: Allow excluding labels from being propagated from ScaledObject and ScaledJob to generated HPA and Job objects ([#6849](https://github.com/kedacore/keda/issues/6849))
 - **General**: Improve Events emitted from ScaledObject controller ([#6802](https://github.com/kedacore/keda/issues/6802))
 - **Datadog Scaler**: Fix bug with datadogNamespace config ([#6828](https://github.com/kedacore/keda/pull/6828))
 - **Metrics API**: Support multiple auth methods simultaneously in Metrics API scaler ([#6642](https://github.com/kedacore/keda/issues/6642))
diff --git a/apis/keda/v1alpha1/scaledjob_types.go b/apis/keda/v1alpha1/scaledjob_types.go
index deab161df..27cb09d2a 100644
--- a/apis/keda/v1alpha1/scaledjob_types.go
+++ b/apis/keda/v1alpha1/scaledjob_types.go
@@ -48,6 +48,8 @@ type ScaledJob struct {
 	Status ScaledJobStatus `json:"status,omitempty"`
 }
 
+const ScaledJobExcludedLabelsAnnotation = "scaledjob.keda.sh/job-excluded-labels"
+
 // ScaledJobSpec defines the desired state of ScaledJob
 type ScaledJobSpec struct {
 	JobTargetRef *batchv1.JobSpec `json:"jobTargetRef"`
diff --git a/apis/keda/v1alpha1/scaledobject_types.go b/apis/keda/v1alpha1/scaledobject_types.go
index 5a7cb3280..a23d6e5e2 100644
--- a/apis/keda/v1alpha1/scaledobject_types.go
+++ b/apis/keda/v1alpha1/scaledobject_types.go
@@ -53,6 +53,7 @@ type ScaledObject struct {
 
 const ScaledObjectOwnerAnnotation = "scaledobject.keda.sh/name"
 const ScaledObjectTransferHpaOwnershipAnnotation = "scaledobject.keda.sh/transfer-hpa-ownership"
+const ScaledObjectExcludedLabelsAnnotation = "scaledobject.keda.sh/hpa-excluded-labels"
 const ValidationsHpaOwnershipAnnotation = "validations.keda.sh/hpa-ownership"
 const PausedReplicasAnnotation = "autoscaling.keda.sh/paused-replicas"
 const PausedAnnotation = "autoscaling.keda.sh/paused"
diff --git a/controllers/keda/hpa.go b/controllers/keda/hpa.go
index 8ecebc775..31cbad6a2 100644
--- a/controllers/keda/hpa.go
+++ b/controllers/keda/hpa.go
@@ -98,7 +98,20 @@ func (r *ScaledObjectReconciler) newHPAForScaledObject(ctx context.Context, logg
 		"app.kubernetes.io/part-of":    scaledObject.Name,
 		"app.kubernetes.io/managed-by": "keda-operator",
 	}
+
+	excludedLabels := map[string]struct{}{}
+
+	if labels, ok := scaledObject.ObjectMeta.Annotations[kedav1alpha1.ScaledObjectExcludedLabelsAnnotation]; ok {
+		for _, excludedLabel := range strings.Split(labels, ",") {
+			excludedLabels[excludedLabel] = struct{}{}
+		}
+	}
+
 	for key, value := range scaledObject.ObjectMeta.Labels {
+		if _, ok := excludedLabels[key]; ok {
+			continue
+		}
+
 		labels[key] = value
 	}
 
diff --git a/pkg/scaling/executor/scale_jobs.go b/pkg/scaling/executor/scale_jobs.go
index 3f6b95bee..42d520c42 100644
--- a/pkg/scaling/executor/scale_jobs.go
+++ b/pkg/scaling/executor/scale_jobs.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	"sort"
 	"strconv"
+	"strings"
 
 	"github.com/go-logr/logr"
 	batchv1 "k8s.io/api/batch/v1"
@@ -159,7 +160,20 @@ func (e *scaleExecutor) generateJobs(logger logr.Logger, scaledJob *kedav1alpha1
 		"app.kubernetes.io/managed-by": "keda-operator",
 		"scaledjob.keda.sh/name":       scaledJob.GetName(),
 	}
+
+	excludedLabels := map[string]struct{}{}
+
+	if labels, ok := scaledJob.ObjectMeta.Annotations[kedav1alpha1.ScaledJobExcludedLabelsAnnotation]; ok {
+		for _, excludedLabel := range strings.Split(labels, ",") {
+			excludedLabels[excludedLabel] = struct{}{}
+		}
+	}
+
 	for key, value := range scaledJob.ObjectMeta.Labels {
+		if _, ok := excludedLabels[key]; ok {
+			continue
+		}
+
 		labels[key] = value
 	}
 



Test patch:
diff --git a/tests/helper/helper.go b/tests/helper/helper.go
index 0e5b61586..8c9ca3031 100644
--- a/tests/helper/helper.go
+++ b/tests/helper/helper.go
@@ -30,6 +30,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	autoscalingv2 "k8s.io/api/autoscaling/v2"
+	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -370,6 +371,27 @@ func WaitForJobCountUntilIteration(t *testing.T, kc *kubernetes.Clientset, names
 	return isTargetAchieved
 }
 
+func WaitForJobCreation(t *testing.T, kc *kubernetes.Clientset, scaledJobName, namespace string, iterations, intervalSeconds int) (*batchv1.Job, error) {
+	jobList := &batchv1.JobList{}
+	var err error
+
+	for i := 0; i < iterations; i++ {
+		jobList, err = kc.BatchV1().Jobs(namespace).List(context.Background(), metav1.ListOptions{
+			LabelSelector: fmt.Sprintf("scaledjob.keda.sh/name=%s", scaledJobName),
+		})
+
+		t.Log("Waiting for job creation")
+
+		if len(jobList.Items) > 0 {
+			return &jobList.Items[0], nil
+		}
+
+		time.Sleep(time.Duration(intervalSeconds) * time.Second)
+	}
+
+	return nil, err
+}
+
 // Waits until deployment count hits target or number of iterations are done.
 func WaitForPodCountInNamespace(t *testing.T, kc *kubernetes.Clientset, namespace string, target, iterations, intervalSeconds int) bool {
 	for i := 0; i < iterations; i++ {
diff --git a/tests/internals/scaled_job_validation/scaled_job_validation_test.go b/tests/internals/scaled_job_validation/scaled_job_validation_test.go
index 6074a0949..caa79afe7 100644
--- a/tests/internals/scaled_job_validation/scaled_job_validation_test.go
+++ b/tests/internals/scaled_job_validation/scaled_job_validation_test.go
@@ -17,13 +17,17 @@ const (
 )
 
 var (
-	testNamespace       = fmt.Sprintf("%s-ns", testName)
-	emptyTriggersSjName = fmt.Sprintf("%s-sj-empty-triggers", testName)
+	testNamespace           = fmt.Sprintf("%s-ns", testName)
+	emptyTriggersSjName     = fmt.Sprintf("%s-sj-empty-triggers", testName)
+	excludedLabelsSjName    = fmt.Sprintf("%s-sj-excluded-labels", testName)
+	monitoredDeploymentName = fmt.Sprintf("%s-monitored-deployment", testName)
 )
 
 type templateData struct {
-	TestNamespace       string
-	EmptyTriggersSjName string
+	TestNamespace           string
+	EmptyTriggersSjName     string
+	ExcludedLabelsSjName    string
+	MonitoredDeploymentName string
 }
 
 const (
@@ -48,6 +52,58 @@ spec:
         restartPolicy: Never
   triggers: []
 `
+
+	monitoredDeploymentTemplate = `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{.MonitoredDeploymentName}}
+  namespace: {{.TestNamespace}}
+  labels:
+    app: {{.MonitoredDeploymentName}}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{.MonitoredDeploymentName}}
+  template:
+    metadata:
+      labels:
+        app: {{.MonitoredDeploymentName}}
+    spec:
+      containers:
+        - name: nginx
+          image: 'ghcr.io/nginx/nginx-unprivileged:1.26'
+`
+
+	scaledJobTemplateWithExcludedLabels = `
+apiVersion: keda.sh/v1alpha1
+kind: ScaledJob
+metadata:
+  name: {{.ExcludedLabelsSjName}}
+  namespace: {{.TestNamespace}}
+  annotations:
+    scaledjob.keda.sh/job-excluded-labels: "foo.bar/environment,foo.bar/version"
+  labels:
+    team: backend
+    foo.bar/environment: bf5011472247b67cce3ee7b24c9a08c5
+    foo.bar/version: "1"
+spec:
+  jobTargetRef:
+    template:
+      spec:
+        containers:
+        - name: external-executor
+          image: busybox
+          command: ["sleep", "60"]
+          imagePullPolicy: IfNotPresent
+        restartPolicy: Never
+  triggers:
+    - type: kubernetes-workload
+      metadata:
+        podSelector: 'app={{.MonitoredDeploymentName}}'
+        value: '1'
+`
 )
 
 func TestScaledJobValidations(t *testing.T) {
@@ -61,6 +117,8 @@ func TestScaledJobValidations(t *testing.T) {
 
 	testTriggersWithEmptyArray(t, data)
 
+	testScaledJobWithExcludedLabels(t, data)
+
 	DeleteKubernetesResources(t, testNamespace, data, templates)
 }
 
@@ -72,9 +130,32 @@ func testTriggersWithEmptyArray(t *testing.T, data templateData) {
 	assert.Contains(t, err.Error(), "no triggers defined in the ScaledObject/ScaledJob")
 }
 
+func testScaledJobWithExcludedLabels(t *testing.T, data templateData) {
+	t.Log("--- scaled job with excluded labels ---")
+
+	err := KubectlApplyWithErrors(t, data, "monitoredDeploymentTemplate", monitoredDeploymentTemplate)
+	assert.NoError(t, err, "monitoredDeployment should be deployed")
+
+	err = KubectlApplyWithErrors(t, data, "scaledJobTemplateWithExcludedLabels", scaledJobTemplateWithExcludedLabels)
+	assert.NoError(t, err, "scaledJob should be deployed")
+
+	job, err := WaitForJobCreation(t, GetKubernetesClient(t), data.ExcludedLabelsSjName, data.TestNamespace, 10, 5)
+	assert.NoError(t, err, "job should be created")
+	assert.NotNil(t, job, "job should be created")
+
+	// Ensure that foo.bar/environment and foo.bar/version labels are not propagated
+	assert.Equal(t, "", job.Labels["foo.bar/environment"], "job should not have the 'foo.bar/environment' label")
+	assert.Equal(t, "", job.Labels["foo.bar/version"], "job should not have the 'foo.bar/version' label")
+
+	KubectlDeleteWithTemplate(t, data, "scaledJobTemplateWithExcludedLabels", scaledJobTemplateWithExcludedLabels)
+	KubectlDeleteWithTemplate(t, data, "monitoredDeploymentTemplate", monitoredDeploymentTemplate)
+}
+
 func getTemplateData() (templateData, []Template) {
 	return templateData{
-		TestNamespace:       testNamespace,
-		EmptyTriggersSjName: emptyTriggersSjName,
+		TestNamespace:           testNamespace,
+		EmptyTriggersSjName:     emptyTriggersSjName,
+		ExcludedLabelsSjName:    excludedLabelsSjName,
+		MonitoredDeploymentName: monitoredDeploymentName,
 	}, []Template{}
 }
diff --git a/tests/internals/scaled_object_validation/scaled_object_validation_test.go b/tests/internals/scaled_object_validation/scaled_object_validation_test.go
index 31bf5c55f..d20dd64e3 100644
--- a/tests/internals/scaled_object_validation/scaled_object_validation_test.go
+++ b/tests/internals/scaled_object_validation/scaled_object_validation_test.go
@@ -22,6 +22,7 @@ var (
 	scaledObject1Name                 = fmt.Sprintf("%s-so1", testName)
 	scaledObject2Name                 = fmt.Sprintf("%s-so2", testName)
 	emptyTriggersSoName               = fmt.Sprintf("%s-so-empty-triggers", testName)
+	excludedLabelsSoName              = fmt.Sprintf("%s-so-excluded-labels", testName)
 	hpaName                           = fmt.Sprintf("%s-hpa", testName)
 	ownershipTransferScaledObjectName = fmt.Sprintf("%s-ownership-transfer-so", testName)
 	ownershipTransferHpaName          = fmt.Sprintf("%s-ownership-transfer-hpa", testName)
@@ -185,6 +186,33 @@ spec:
     name: {{.DeploymentName}}
   triggers: []
 `
+
+	scaledObjectTemplateWithExcludedLabels = `
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: {{.ScaledObjectName}}
+  namespace: {{.TestNamespace}}
+  annotations:
+    scaledobject.keda.sh/hpa-excluded-labels: "foo.bar/environment,foo.bar/version"
+  labels:
+    team: backend
+    foo.bar/environment: bf5011472247b67cce3ee7b24c9a08c5
+    foo.bar/version: "1"
+spec:
+  scaleTargetRef:
+    name: {{.DeploymentName}}
+  advanced:
+    horizontalPodAutoscalerConfig:
+      name: {{.HpaName}}
+  triggers:
+  - type: cron
+    metadata:
+      timezone: Etc/UTC
+      start: 0 * * * *
+      end: 1 * * * *
+      desiredReplicas: '1'
+`
 )
 
 func TestScaledObjectValidations(t *testing.T) {
@@ -214,6 +242,8 @@ func TestScaledObjectValidations(t *testing.T) {
 
 	testTriggersWithEmptyArray(t, data)
 
+	testScaledObjectWithExcludedLabels(t, data)
+
 	DeleteKubernetesResources(t, testNamespace, data, templates)
 }
 
@@ -378,6 +408,25 @@ func testTriggersWithEmptyArray(t *testing.T, data templateData) {
 	assert.Contains(t, err.Error(), "no triggers defined in the ScaledObject/ScaledJob")
 }
 
+func testScaledObjectWithExcludedLabels(t *testing.T, data templateData) {
+	t.Log("--- scaled object with excluded labels ---")
+
+	data.ScaledObjectName = excludedLabelsSoName
+	data.HpaName = hpaName
+
+	err := KubectlApplyWithErrors(t, data, "scaledObjectTemplateWithExcludedLabels", scaledObjectTemplateWithExcludedLabels)
+	assert.NoError(t, err, "scaledObject should be deployed")
+
+	hpa, err := WaitForHpaCreation(t, GetKubernetesClient(t), data.HpaName, data.TestNamespace, 10, 5)
+	assert.NoError(t, err, "hpa should be created")
+
+	// Ensure that foo.bar/environment and foo.bar/version labels are not propagated
+	assert.Equal(t, "", hpa.Labels["foo.bar/environment"], "hpa should not have the 'foo.bar/environment' label")
+	assert.Equal(t, "", hpa.Labels["foo.bar/version"], "hpa should not have the 'foo.bar/version' label")
+
+	KubectlDeleteWithTemplate(t, data, "scaledObjectTemplateWithExcludedLabels", scaledObjectTemplateWithExcludedLabels)
+}
+
 func getTemplateData() (templateData, []Template) {
 	return templateData{
 			TestNamespace:       testNamespace,