Repository: SeldonIO/seldon-core (language: python)

Issue:
SeldonV2 - Ability to set component level labels/annotations in SeldonConfig/SeldonRuntime
I noticed that both SeldonConfig and SeldonRuntime Custom Resources use [ v1.PodSpec](https://github.com/kubernetes/api/blob/v0.24.4/core/v1/types.go#L3050) to define podSpec for the components, and re-use the Annotations/Metadata set in the CustomResource in the Pod spec within Deployment/Statefulset. 

References: 
a) https://github.com/SeldonIO/seldon-core/blob/v2.6.0/operator/controllers/reconcilers/seldon/deployment_reconciler.go#L54..L55 
b) https://github.com/SeldonIO/seldon-core/blob/60b7401ae5052ccf183c1453575611085bca4bb3/operator/controllers/reconcilers/seldon/statefulset_reconciler.go#L89..L90

**Looking at this I understand there is no provision to set component specific labels - For example, if I wanted to set some custom labels/annotations for the seldon-scheduler, and some different labels/annotations for Envoy, I could not find a way to do so.**

My goal is to be able to run all the Pods (except Envoy) defined in SeldonV2 Runtime definition within Istio Service Mesh (i.e. inject istio sidecar). One option I considered was to add the Istio labels/annotations to the CustomResource definition but then I would not be able to selectively prevent istio-proxy injection to the Envoy pod. 

Could you please provide guidance for this? 


Discussion / hints:

Fix patch:
diff --git a/k8s/Makefile b/k8s/Makefile
index 9e00bb0935..bb4d80b89a 100644
--- a/k8s/Makefile
+++ b/k8s/Makefile
@@ -18,8 +18,10 @@ create-helm-charts:
 	kustomize build kustomize/helm-servers/ > ${HELM_SERVERS_BASE}/seldon-v2-servers.yaml
 	sed -i 's/\(.*\)\(imagePullSecrets:\).*/{{- with .Values.imagePullSecrets }}\n\1\2\n\1{{- toYaml . | nindent 8 }}\n{{- end }}/g' ${HELM_COMPONENTS_BASE}/seldon-v2-components.yaml
 	sed -i 's/HACK_REMOVE_ME//' ${HELM_COMPONENTS_BASE}/seldon-v2-components.yaml
-	sed 's/HACK_REMOVE_ME//' ${HELM_SERVERS_BASE}/seldon-v2-servers.yaml \
-		> ${HELM_SERVERS_BASE}/.seldon-v2-servers.yaml
+	# kustomizse has builtin helper fuctions for meta.labels and metadata.annotations, so the field names are changed to escape kustomize's extra processing
+	sed 's/labelz:/labels:\n\ \ \ /' ${HELM_SERVERS_BASE}/seldon-v2-servers.yaml > ${HELM_SERVERS_BASE}/.seldon-v2-servers.yaml
+	sed -i 's/annotationz:/annotations:\n\ \ \ /' ${HELM_SERVERS_BASE}/.seldon-v2-servers.yaml
+	sed -i 's/HACK_REMOVE_ME//' ${HELM_SERVERS_BASE}/.seldon-v2-servers.yaml
 	sed -zi 's#\(apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole.*\- get\n\-\-\-\)#{{- if .Values.controller.clusterwide -}}\n\1\n{{- end }}#' ${HELM_COMPONENTS_BASE}/seldon-v2-components.yaml
 	sed -zi 's#\(apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding.*ServiceAccount\n  name: seldon-v2-controller-manager\n  namespace:[^-]\{27\}\n\-\-\-\)#{{- if .Values.controller.clusterwide -}}\n\1\n{{- end }}#' ${HELM_COMPONENTS_BASE}/seldon-v2-components.yaml
 	mv ${HELM_SERVERS_BASE}/.seldon-v2-servers.yaml ${HELM_SERVERS_BASE}/seldon-v2-servers.yaml
diff --git a/k8s/helm-charts/seldon-core-v2-crds/templates/seldon-v2-crds.yaml b/k8s/helm-charts/seldon-core-v2-crds/templates/seldon-v2-crds.yaml
index c6e2f2e443..fec62378ba 100644
--- a/k8s/helm-charts/seldon-core-v2-crds/templates/seldon-v2-crds.yaml
+++ b/k8s/helm-charts/seldon-core-v2-crds/templates/seldon-v2-crds.yaml
@@ -636,6 +636,14 @@ spec:
               components:
                 items:
                   properties:
+                    annotations:
+                      additionalProperties:
+                        type: string
+                      type: object
+                    labels:
+                      additionalProperties:
+                        type: string
+                      type: object
                     name:
                       type: string
                     podSpec:
diff --git a/k8s/helm-charts/seldon-core-v2-servers/templates/seldon-v2-servers.yaml b/k8s/helm-charts/seldon-core-v2-servers/templates/seldon-v2-servers.yaml
index bbe3954810..66441c058d 100644
--- a/k8s/helm-charts/seldon-core-v2-servers/templates/seldon-v2-servers.yaml
+++ b/k8s/helm-charts/seldon-core-v2-servers/templates/seldon-v2-servers.yaml
@@ -1,6 +1,10 @@
 apiVersion: mlops.seldon.io/v1alpha1
 kind: Server
 metadata:
+  annotations:
+    {{ .Values.mlserver.annotations | toYaml }}
+  labels:
+    {{ .Values.mlserver.labels | toYaml }}
   name: mlserver
   namespace: '{{ .Release.Namespace }}'
 spec:
@@ -11,6 +15,10 @@ spec:
 apiVersion: mlops.seldon.io/v1alpha1
 kind: Server
 metadata:
+  annotations:
+    {{ .Values.triton.annotations | toYaml }}
+  labels:
+    {{ .Values.triton.labels | toYaml }}
   name: triton
   namespace: '{{ .Release.Namespace }}'
 spec:
diff --git a/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml b/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml
index 7fdfb7fdbe..b5265f6894 100644
--- a/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml
+++ b/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml
@@ -894,7 +894,10 @@ metadata:
   namespace: '{{ .Release.Namespace }}'
 spec:
   components:
-  - name: seldon-scheduler
+  - annotations: {{- toYaml .Values.scheduler.annotations | nindent
+      8 }}
+    labels: {{- toYaml .Values.scheduler.labels | nindent 8 }}
+    name: seldon-scheduler
     podSpec:
       containers:
       - args:
@@ -1009,7 +1012,10 @@ spec:
         resources:
           requests:
             storage: '{{ .Values.scheduler.resources.storage }}'
-  - name: seldon-pipelinegateway
+  - annotations: {{- toYaml .Values.pipelinegateway.annotations | nindent
+      8 }}
+    labels: {{- toYaml .Values.pipelinegateway.labels | nindent 8 }}
+    name: seldon-pipelinegateway
     podSpec:
       containers:
       - args:
@@ -1145,7 +1151,10 @@ spec:
           name: seldon-tracing
         name: tracing-config-volume
     replicas: 1
-  - name: seldon-modelgateway
+  - annotations: {{- toYaml .Values.modelgateway.annotations | nindent
+      8 }}
+    labels: {{- toYaml .Values.modelgateway.labels | nindent 8 }}
+    name: seldon-modelgateway
     podSpec:
       containers:
       - args:
@@ -1263,7 +1272,10 @@ spec:
           name: seldon-tracing
         name: tracing-config-volume
     replicas: 1
-  - name: hodometer
+  - annotations: {{- toYaml .Values.hodometer.annotations | nindent
+      8 }}
+    labels: {{- toYaml .Values.hodometer.labels | nindent 8 }}
+    name: hodometer
     podSpec:
       containers:
       - env:
@@ -1319,7 +1331,9 @@ spec:
       serviceAccountName: hodometer
       terminationGracePeriodSeconds: 5
     replicas: 1
-  - name: seldon-envoy
+  - annotations: {{- toYaml .Values.envoy.annotations | nindent 8 }}
+    labels: {{- toYaml .Values.envoy.labels | nindent 8 }}
+    name: seldon-envoy
     podSpec:
       containers:
       - env:
@@ -1374,7 +1388,10 @@ spec:
         8 }}
       terminationGracePeriodSeconds: 5
     replicas: 1
-  - name: seldon-dataflow-engine
+  - annotations: {{- toYaml .Values.dataflow.annotations | nindent 8
+      }}
+    labels: {{- toYaml .Values.dataflow.labels | nindent 8 }}
+    name: seldon-dataflow-engine
     podSpec:
       containers:
       - env:
diff --git a/k8s/helm-charts/seldon-core-v2-setup/values.yaml b/k8s/helm-charts/seldon-core-v2-setup/values.yaml
index 0e861042d5..c8d7ae41e7 100644
--- a/k8s/helm-charts/seldon-core-v2-setup/values.yaml
+++ b/k8s/helm-charts/seldon-core-v2-setup/values.yaml
@@ -180,6 +180,10 @@ controller:
     runAsNonRoot: true
 
 envoy:
+  annotations:
+    "prometheus.io/path": "/stats/prometheus"
+    "prometheus.io/port": "9003"
+    "prometheus.io/scrape": "true"
   image:
     pullPolicy: IfNotPresent
     registry: docker.io
diff --git a/k8s/helm-charts/seldon-core-v2-setup/values.yaml.template b/k8s/helm-charts/seldon-core-v2-setup/values.yaml.template
index 655c3cb4f0..054ab5eb42 100644
--- a/k8s/helm-charts/seldon-core-v2-setup/values.yaml.template
+++ b/k8s/helm-charts/seldon-core-v2-setup/values.yaml.template
@@ -180,6 +180,10 @@ controller:
     runAsNonRoot: true
 
 envoy:
+  annotations:
+    "prometheus.io/path": "/stats/prometheus"
+    "prometheus.io/port": "9003"
+    "prometheus.io/scrape": "true"
   image:
     pullPolicy: IfNotPresent
     registry: docker.io
diff --git a/k8s/kustomize/helm-components-sc/patch_dataflow_json6902.yaml b/k8s/kustomize/helm-components-sc/patch_dataflow_json6902.yaml
index 71d0c5b78c..21713857f9 100644
--- a/k8s/kustomize/helm-components-sc/patch_dataflow_json6902.yaml
+++ b/k8s/kustomize/helm-components-sc/patch_dataflow_json6902.yaml
@@ -1,3 +1,9 @@
 - op: replace
   path: /spec/components/5/podSpec/securityContext
   value: HACK_REMOVE_ME{{- toYaml .Values.dataflow.securityContext | nindent 8 }}
+- op: replace
+  path: /spec/components/5/labels
+  value: HACK_REMOVE_ME{{- toYaml .Values.dataflow.labels | nindent 8 }}
+- op: replace
+  path: /spec/components/5/annotations
+  value: HACK_REMOVE_ME{{- toYaml .Values.dataflow.annotations | nindent 8 }}
diff --git a/k8s/kustomize/helm-components-sc/patch_envoy_json6902.yaml b/k8s/kustomize/helm-components-sc/patch_envoy_json6902.yaml
index ff1a02abdd..30283c0401 100644
--- a/k8s/kustomize/helm-components-sc/patch_envoy_json6902.yaml
+++ b/k8s/kustomize/helm-components-sc/patch_envoy_json6902.yaml
@@ -2,3 +2,9 @@
 - op: replace
   path: /spec/components/4/podSpec/securityContext
   value: HACK_REMOVE_ME{{- toYaml .Values.envoy.securityContext | nindent 8 }}
+- op: replace
+  path: /spec/components/4/labels
+  value: HACK_REMOVE_ME{{- toYaml .Values.envoy.labels | nindent 8 }}
+- op: replace
+  path: /spec/components/4/annotations
+  value: HACK_REMOVE_ME{{- toYaml .Values.envoy.annotations | nindent 8 }}
diff --git a/k8s/kustomize/helm-components-sc/patch_hodometer_json6902.yaml b/k8s/kustomize/helm-components-sc/patch_hodometer_json6902.yaml
index 8fe0fb4a87..fd4859d083 100644
--- a/k8s/kustomize/helm-components-sc/patch_hodometer_json6902.yaml
+++ b/k8s/kustomize/helm-components-sc/patch_hodometer_json6902.yaml
@@ -2,3 +2,9 @@
 - op: replace
   path: /spec/components/3/podSpec/securityContext  
   value: HACK_REMOVE_ME{{- toYaml .Values.hodometer.securityContext | nindent 8 }}
+- op: replace
+  path: /spec/components/3/labels
+  value: HACK_REMOVE_ME{{- toYaml .Values.hodometer.labels | nindent 8 }}
+- op: replace
+  path: /spec/components/3/annotations
+  value: HACK_REMOVE_ME{{- toYaml .Values.hodometer.annotations | nindent 8 }}
diff --git a/k8s/kustomize/helm-components-sc/patch_modelgateway_json6902.yaml b/k8s/kustomize/helm-components-sc/patch_modelgateway_json6902.yaml
index c8883500a1..63232a4f0e 100644
--- a/k8s/kustomize/helm-components-sc/patch_modelgateway_json6902.yaml
+++ b/k8s/kustomize/helm-components-sc/patch_modelgateway_json6902.yaml
@@ -1,3 +1,9 @@
 - op: replace
   path: /spec/components/2/podSpec/securityContext
   value: HACK_REMOVE_ME{{- toYaml .Values.modelgateway.securityContext | nindent 8 }}
+- op: replace
+  path: /spec/components/2/labels
+  value: HACK_REMOVE_ME{{- toYaml .Values.modelgateway.labels | nindent 8 }}
+- op: replace
+  path: /spec/components/2/annotations
+  value: HACK_REMOVE_ME{{- toYaml .Values.modelgateway.annotations | nindent 8 }}
diff --git a/k8s/kustomize/helm-components-sc/patch_pipelinegateway_json6902.yaml b/k8s/kustomize/helm-components-sc/patch_pipelinegateway_json6902.yaml
index c014d72687..f57f29f328 100644
--- a/k8s/kustomize/helm-components-sc/patch_pipelinegateway_json6902.yaml
+++ b/k8s/kustomize/helm-components-sc/patch_pipelinegateway_json6902.yaml
@@ -1,3 +1,9 @@
 - op: replace
   path: /spec/components/1/podSpec/securityContext  
   value: HACK_REMOVE_ME{{- toYaml .Values.pipelinegateway.securityContext | nindent 8 }}
+- op: replace
+  path: /spec/components/1/labels
+  value: HACK_REMOVE_ME{{- toYaml .Values.pipelinegateway.labels | nindent 8 }}
+- op: replace
+  path: /spec/components/1/annotations
+  value: HACK_REMOVE_ME{{- toYaml .Values.pipelinegateway.annotations | nindent 8 }}
diff --git a/k8s/kustomize/helm-components-sc/patch_scheduler_json6902.yaml b/k8s/kustomize/helm-components-sc/patch_scheduler_json6902.yaml
index 81f76824ff..df9a52f014 100644
--- a/k8s/kustomize/helm-components-sc/patch_scheduler_json6902.yaml
+++ b/k8s/kustomize/helm-components-sc/patch_scheduler_json6902.yaml
@@ -1,3 +1,9 @@
 - op: replace
   path: /spec/components/0/podSpec/securityContext
   value: HACK_REMOVE_ME{{- toYaml .Values.scheduler.securityContext | nindent 8 }}
+- op: replace
+  path: /spec/components/0/labels
+  value: HACK_REMOVE_ME{{- toYaml .Values.scheduler.labels | nindent 8 }}
+- op: replace
+  path: /spec/components/0/annotations
+  value: HACK_REMOVE_ME{{- toYaml .Values.scheduler.annotations | nindent 8 }}
diff --git a/k8s/kustomize/helm-servers/patch_mlserver.yaml b/k8s/kustomize/helm-servers/patch_mlserver.yaml
index 2dfe18f6a8..7729695312 100644
--- a/k8s/kustomize/helm-servers/patch_mlserver.yaml
+++ b/k8s/kustomize/helm-servers/patch_mlserver.yaml
@@ -2,6 +2,8 @@ apiVersion: mlops.seldon.io/v1alpha1
 kind: Server
 metadata:
   name: mlserver
+  labelz: HACK_REMOVE_ME{{ .Values.mlserver.labels | toYaml }}
+  annotationz: HACK_REMOVE_ME{{ .Values.mlserver.annotations | toYaml }}
 spec:
   replicas: HACK_REMOVE_ME{{ .Values.mlserver.replicas }}
   podSpec: HACK_REMOVE_ME{{ toJson .Values.mlserver.podSpec }}
diff --git a/k8s/kustomize/helm-servers/patch_triton.yaml b/k8s/kustomize/helm-servers/patch_triton.yaml
index 6b461c387b..f121adc785 100644
--- a/k8s/kustomize/helm-servers/patch_triton.yaml
+++ b/k8s/kustomize/helm-servers/patch_triton.yaml
@@ -2,6 +2,8 @@ apiVersion: mlops.seldon.io/v1alpha1
 kind: Server
 metadata:
   name: triton
+  labelz: HACK_REMOVE_ME{{ .Values.triton.labels | toYaml }}
+  annotationz: HACK_REMOVE_ME{{ .Values.triton.annotations | toYaml }}
 spec:
   replicas: HACK_REMOVE_ME{{ .Values.triton.replicas }}
   podSpec: HACK_REMOVE_ME{{ toJson .Values.triton.podSpec }}
diff --git a/k8s/yaml/components.yaml b/k8s/yaml/components.yaml
index 14d4755a81..b79aab90f0 100644
--- a/k8s/yaml/components.yaml
+++ b/k8s/yaml/components.yaml
@@ -538,7 +538,11 @@ metadata:
   name: default
 spec:
   components:
-  - name: seldon-scheduler
+  - annotations:
+        null
+    labels:
+        null
+    name: seldon-scheduler
     podSpec:
       containers:
       - args:
@@ -648,7 +652,11 @@ spec:
         resources:
           requests:
             storage: '1Gi'
-  - name: seldon-pipelinegateway
+  - annotations:
+        null
+    labels:
+        null
+    name: seldon-pipelinegateway
     podSpec:
       containers:
       - args:
@@ -778,7 +786,11 @@ spec:
           name: seldon-tracing
         name: tracing-config-volume
     replicas: 1
-  - name: seldon-modelgateway
+  - annotations:
+        null
+    labels:
+        null
+    name: seldon-modelgateway
     podSpec:
       containers:
       - args:
@@ -891,7 +903,11 @@ spec:
           name: seldon-tracing
         name: tracing-config-volume
     replicas: 1
-  - name: hodometer
+  - annotations:
+        null
+    labels:
+        null
+    name: hodometer
     podSpec:
       containers:
       - env:
@@ -944,7 +960,13 @@ spec:
       serviceAccountName: hodometer
       terminationGracePeriodSeconds: 5
     replicas: 1
-  - name: seldon-envoy
+  - annotations:
+        prometheus.io/path: /stats/prometheus
+        prometheus.io/port: "9003"
+        prometheus.io/scrape: "true"
+    labels:
+        null
+    name: seldon-envoy
     podSpec:
       containers:
       - env:
@@ -996,7 +1018,11 @@ spec:
         runAsUser: 1000
       terminationGracePeriodSeconds: 5
     replicas: 1
-  - name: seldon-dataflow-engine
+  - annotations:
+        null
+    labels:
+        null
+    name: seldon-dataflow-engine
     podSpec:
       containers:
       - env:
diff --git a/k8s/yaml/crds.yaml b/k8s/yaml/crds.yaml
index 99ac05ff19..d904d442eb 100644
--- a/k8s/yaml/crds.yaml
+++ b/k8s/yaml/crds.yaml
@@ -641,6 +641,14 @@ spec:
               components:
                 items:
                   properties:
+                    annotations:
+                      additionalProperties:
+                        type: string
+                      type: object
+                    labels:
+                      additionalProperties:
+                        type: string
+                      type: object
                     name:
                       type: string
                     podSpec:
diff --git a/k8s/yaml/runtime.yaml b/k8s/yaml/runtime.yaml
index 840617fed5..1951f26745 100644
--- a/k8s/yaml/runtime.yaml
+++ b/k8s/yaml/runtime.yaml
@@ -24,22 +24,28 @@ spec:
   - name: hodometer
     disable: false
     replicas: 1
+    podSpec: null
   - name: seldon-scheduler
     disable: false
     serviceType: LoadBalancer
+    podSpec: null
   - name: seldon-envoy
     disable: false
     replicas: 1
     serviceType: LoadBalancer
+    podSpec: null
   - name: seldon-dataflow-engine
     disable: false
     replicas: 1
+    podSpec: null
   - name: seldon-modelgateway
     disable: false
     replicas: 1
+    podSpec: null
   - name: seldon-pipelinegateway
     disable: false
     replicas: 1
+    podSpec: null
   config:
     agentConfig:
       rclone:
diff --git a/k8s/yaml/servers.yaml b/k8s/yaml/servers.yaml
index acc11ee2c9..6ca39bd0f5 100644
--- a/k8s/yaml/servers.yaml
+++ b/k8s/yaml/servers.yaml
@@ -3,8 +3,13 @@
 apiVersion: mlops.seldon.io/v1alpha1
 kind: Server
 metadata:
+  annotations:
+    null
+  labels:
+    null
   name: mlserver
 spec:
+  podSpec: null
   replicas: 1
   serverConfig: mlserver
 ---
@@ -12,7 +17,12 @@ spec:
 apiVersion: mlops.seldon.io/v1alpha1
 kind: Server
 metadata:
+  annotations:
+    null
+  labels:
+    null
   name: triton
 spec:
+  podSpec: null
   replicas: 1
   serverConfig: triton
diff --git a/operator/apis/mlops/v1alpha1/seldonconfig_types.go b/operator/apis/mlops/v1alpha1/seldonconfig_types.go
index 4ecec7b626..d2b77515a1 100644
--- a/operator/apis/mlops/v1alpha1/seldonconfig_types.go
+++ b/operator/apis/mlops/v1alpha1/seldonconfig_types.go
@@ -80,7 +80,10 @@ type TracingConfig struct {
 
 type ComponentDefn struct {
 	// +kubebuilder:validation:Required
+
 	Name                 string                  `json:"name"`
+	Labels               map[string]string       `json:"labels,omitempty"`
+	Annotations          map[string]string       `json:"annotations,omitempty"`
 	Replicas             *int32                  `json:"replicas,omitempty"`
 	PodSpec              *v1.PodSpec             `json:"podSpec,omitempty"`
 	VolumeClaimTemplates []PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"`
diff --git a/operator/apis/mlops/v1alpha1/zz_generated.deepcopy.go b/operator/apis/mlops/v1alpha1/zz_generated.deepcopy.go
index 70c5b1bdd6..a57f1f6cba 100644
--- a/operator/apis/mlops/v1alpha1/zz_generated.deepcopy.go
+++ b/operator/apis/mlops/v1alpha1/zz_generated.deepcopy.go
@@ -39,6 +39,20 @@ func (in *AgentConfiguration) DeepCopy() *AgentConfiguration {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ComponentDefn) DeepCopyInto(out *ComponentDefn) {
 	*out = *in
+	if in.Labels != nil {
+		in, out := &in.Labels, &out.Labels
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.Annotations != nil {
+		in, out := &in.Annotations, &out.Annotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	if in.Replicas != nil {
 		in, out := &in.Replicas, &out.Replicas
 		*out = new(int32)
diff --git a/operator/config/crd/bases/mlops.seldon.io_seldonconfigs.yaml b/operator/config/crd/bases/mlops.seldon.io_seldonconfigs.yaml
index d03cc9257c..bb06101931 100644
--- a/operator/config/crd/bases/mlops.seldon.io_seldonconfigs.yaml
+++ b/operator/config/crd/bases/mlops.seldon.io_seldonconfigs.yaml
@@ -38,6 +38,14 @@ spec:
               components:
                 items:
                   properties:
+                    annotations:
+                      additionalProperties:
+                        type: string
+                      type: object
+                    labels:
+                      additionalProperties:
+                        type: string
+                      type: object
                     name:
                       type: string
                     podSpec:
diff --git a/operator/config/seldonconfigs/default.yaml b/operator/config/seldonconfigs/default.yaml
index c135bd7bf1..1a6be91dff 100644
--- a/operator/config/seldonconfigs/default.yaml
+++ b/operator/config/seldonconfigs/default.yaml
@@ -45,6 +45,10 @@ spec:
       terminationGracePeriodSeconds: 5
   - name: seldon-envoy
     replicas: 1
+    annotations:
+        "prometheus.io/path": "/stats/prometheus"
+        "prometheus.io/port": "9003"
+        "prometheus.io/scrape": "true"
     podSpec:
       containers:
       - image: seldonio/seldon-envoy:latest
diff --git a/operator/controllers/reconcilers/seldon/deployment_reconciler.go b/operator/controllers/reconcilers/seldon/deployment_reconciler.go
index 5cce767ec3..3fe708a2dc 100644
--- a/operator/controllers/reconcilers/seldon/deployment_reconciler.go
+++ b/operator/controllers/reconcilers/seldon/deployment_reconciler.go
@@ -40,12 +40,16 @@ func NewComponentDeploymentReconciler(
 	common common.ReconcilerConfig,
 	meta metav1.ObjectMeta,
 	podSpec *v1.PodSpec,
+	componentLabels map[string]string,
+	componentAnnotations map[string]string,
 	override *mlopsv1alpha1.OverrideSpec,
 	seldonConfigMeta metav1.ObjectMeta,
 	annotator *patch.Annotator,
 ) (*ComponentDeploymentReconciler, error) {
 	labels := utils.MergeMaps(meta.Labels, seldonConfigMeta.Labels)
+	labels = utils.MergeMaps(componentLabels, labels)
 	annotations := utils.MergeMaps(meta.Annotations, seldonConfigMeta.Annotations)
+	annotations = utils.MergeMaps(componentAnnotations, annotations)
 	deployment, err := toDeployment(name, meta, podSpec, override, labels, annotations)
 	if err != nil {
 		return nil, err
@@ -62,20 +66,14 @@ func (s *ComponentDeploymentReconciler) GetResources() []client.Object {
 	return []client.Object{s.Deployment}
 }
 
-func addEnvoyAnnotations(annotations map[string]string) map[string]string {
-	annotations["prometheus.io/path"] = "/stats/prometheus"
-	annotations["prometheus.io/scrape"] = "true"
-	annotations["prometheus.io/port"] = "9003"
-	return annotations
-}
-
 func toDeployment(
 	name string,
 	meta metav1.ObjectMeta,
 	podSpec *v1.PodSpec,
 	override *mlopsv1alpha1.OverrideSpec,
 	labels map[string]string,
-	annotations map[string]string) (*appsv1.Deployment, error) {
+	annotations map[string]string,
+) (*appsv1.Deployment, error) {
 	var replicas int32
 	if override != nil && override.Replicas != nil {
 		replicas = *override.Replicas
@@ -90,10 +88,6 @@ func toDeployment(
 			return nil, err
 		}
 	}
-	// Envoy annotations
-	if name == mlopsv1alpha1.EnvoyName {
-		annotations = addEnvoyAnnotations(annotations)
-	}
 	metaLabels := utils.MergeMaps(map[string]string{constants.KubernetesNameLabelKey: name}, labels)
 	templateLabels := utils.MergeMaps(map[string]string{constants.KubernetesNameLabelKey: name}, labels)
 	d := &appsv1.Deployment{
diff --git a/operator/controllers/reconcilers/seldon/runtime_reconciler.go b/operator/controllers/reconcilers/seldon/runtime_reconciler.go
index f64b8a6871..89cf2acf1f 100644
--- a/operator/controllers/reconcilers/seldon/runtime_reconciler.go
+++ b/operator/controllers/reconcilers/seldon/runtime_reconciler.go
@@ -29,8 +29,8 @@ type SeldonRuntimeReconciler struct {
 
 func NewSeldonRuntimeReconciler(
 	runtime *mlopsv1alpha1.SeldonRuntime,
-	commonConfig common.ReconcilerConfig) (common.Reconciler, error) {
-
+	commonConfig common.ReconcilerConfig,
+) (common.Reconciler, error) {
 	var err error
 
 	seldonConfig, err := mlopsv1alpha1.GetSeldonConfigForSeldonRuntime(runtime.Spec.SeldonConfig, commonConfig.Client)
@@ -58,6 +58,8 @@ func NewSeldonRuntimeReconciler(
 					runtime.ObjectMeta,
 					c.PodSpec,
 					c.VolumeClaimTemplates,
+					c.Labels,
+					c.Annotations,
 					overrides[c.Name],
 					seldonConfig.ObjectMeta,
 					annotator,
@@ -73,6 +75,8 @@ func NewSeldonRuntimeReconciler(
 					commonConfig,
 					runtime.ObjectMeta,
 					c.PodSpec,
+					c.Labels,
+					c.Annotations,
 					overrides[c.Name],
 					seldonConfig.ObjectMeta,
 					annotator,
diff --git a/operator/controllers/reconcilers/seldon/statefulset_reconciler.go b/operator/controllers/reconcilers/seldon/statefulset_reconciler.go
index a4b0af56b8..20722f14ea 100644
--- a/operator/controllers/reconcilers/seldon/statefulset_reconciler.go
+++ b/operator/controllers/reconcilers/seldon/statefulset_reconciler.go
@@ -46,12 +46,16 @@ func NewComponentStatefulSetReconciler(
 	meta metav1.ObjectMeta,
 	podSpec *v1.PodSpec,
 	volumeClaimTemplates []mlopsv1alpha1.PersistentVolumeClaim,
+	componentLabels map[string]string,
+	componentAnnotations map[string]string,
 	override *mlopsv1alpha1.OverrideSpec,
 	seldonConfigMeta metav1.ObjectMeta,
 	annotator *patch.Annotator,
 ) (*ComponentStatefulSetReconciler, error) {
 	labels := utils.MergeMaps(meta.Labels, seldonConfigMeta.Labels)
+	labels = utils.MergeMaps(componentLabels, labels)
 	annotations := utils.MergeMaps(meta.Annotations, seldonConfigMeta.Annotations)
+	annotations = utils.MergeMaps(componentAnnotations, annotations)
 	statefulSet, err := toStatefulSet(name, meta, podSpec, volumeClaimTemplates, override, labels, annotations)
 	if err != nil {
 		return nil, err


Test patch:
diff --git a/operator/controllers/reconcilers/seldon/deployment_reconciler_test.go b/operator/controllers/reconcilers/seldon/deployment_reconciler_test.go
index 7311b62c9c..2631a40da4 100644
--- a/operator/controllers/reconcilers/seldon/deployment_reconciler_test.go
+++ b/operator/controllers/reconcilers/seldon/deployment_reconciler_test.go
@@ -124,6 +124,8 @@ func TestDeploymentReconcile(t *testing.T) {
 				common.ReconcilerConfig{Ctx: context.TODO(), Logger: logger, Client: client},
 				meta,
 				test.podSpec,
+				make(map[string]string),
+				make(map[string]string),
 				test.override,
 				test.seldonConfigMeta,
 				annotator)
@@ -225,6 +227,8 @@ func TestDeploymentOverride(t *testing.T) {
 				common.ReconcilerConfig{Ctx: context.TODO(), Logger: logger, Client: client},
 				meta,
 				test.podSpec,
+				make(map[string]string),
+				make(map[string]string),
 				test.override,
 				metav1.ObjectMeta{},
 				annotator)
diff --git a/operator/controllers/reconcilers/seldon/runtime_reconciler_test.go b/operator/controllers/reconcilers/seldon/runtime_reconciler_test.go
index d89bf8fced..55d246bfb1 100644
--- a/operator/controllers/reconcilers/seldon/runtime_reconciler_test.go
+++ b/operator/controllers/reconcilers/seldon/runtime_reconciler_test.go
@@ -53,7 +53,9 @@ func TestRuntimeReconcile(t *testing.T) {
 				Spec: mlopsv1alpha1.SeldonConfigSpec{
 					Components: []*mlopsv1alpha1.ComponentDefn{
 						{
-							Name: mlopsv1alpha1.SchedulerName,
+							Name:        mlopsv1alpha1.SchedulerName,
+							Labels:      map[string]string{"test-label": "123"},
+							Annotations: map[string]string{"test-annotation": "456"},
 							PodSpec: &v1.PodSpec{
 								Containers: []v1.Container{
 									{
@@ -94,7 +96,9 @@ func TestRuntimeReconcile(t *testing.T) {
 				Spec: mlopsv1alpha1.SeldonConfigSpec{
 					Components: []*mlopsv1alpha1.ComponentDefn{
 						{
-							Name: mlopsv1alpha1.PipelineGatewayName,
+							Name:        mlopsv1alpha1.PipelineGatewayName,
+							Labels:      map[string]string{"test-label": "123"},
+							Annotations: map[string]string{"test-annotation": "456"},
 							PodSpec: &v1.PodSpec{
 								Containers: []v1.Container{
 									{
@@ -105,7 +109,9 @@ func TestRuntimeReconcile(t *testing.T) {
 							},
 						},
 						{
-							Name: mlopsv1alpha1.ModelGatewayName,
+							Name:        mlopsv1alpha1.ModelGatewayName,
+							Labels:      map[string]string{"test-label": "123"},
+							Annotations: map[string]string{"test-annotation": "456"},
 							PodSpec: &v1.PodSpec{
 								Containers: []v1.Container{
 									{
@@ -172,6 +178,8 @@ func TestRuntimeReconcile(t *testing.T) {
 						Namespace: test.runtime.GetNamespace(),
 					}, ss)
 					g.Expect(err).To(BeNil())
+					g.Expect(ss.Labels["test-label"]).To(Equal("123"))
+					g.Expect(ss.Annotations["test-annotation"]).To(Equal("456"))
 				}
 				for _, depName := range test.expectedDeployments {
 					dep := &appsv1.Deployment{}
@@ -180,6 +188,8 @@ func TestRuntimeReconcile(t *testing.T) {
 						Namespace: test.runtime.GetNamespace(),
 					}, dep)
 					g.Expect(err).To(BeNil())
+					g.Expect(dep.Labels["test-label"]).To(Equal("123"))
+					g.Expect(dep.Annotations["test-annotation"]).To(Equal("456"))
 				}
 			}
 		})
diff --git a/operator/controllers/reconcilers/seldon/statefulset_reconciler_test.go b/operator/controllers/reconcilers/seldon/statefulset_reconciler_test.go
index f1ab052df8..d821eec722 100644
--- a/operator/controllers/reconcilers/seldon/statefulset_reconciler_test.go
+++ b/operator/controllers/reconcilers/seldon/statefulset_reconciler_test.go
@@ -34,13 +34,15 @@ func TestStatefulSetReconcile(t *testing.T) {
 	g := NewGomegaWithT(t)
 
 	type test struct {
-		name             string
-		statefulSetName  string
-		podSpec          *v1.PodSpec
-		volumeClaims     []mlopsv1alpha1.PersistentVolumeClaim
-		override         *mlopsv1alpha1.OverrideSpec
-		seldonConfigMeta metav1.ObjectMeta
-		error            bool
+		name                 string
+		statefulSetName      string
+		podSpec              *v1.PodSpec
+		volumeClaims         []mlopsv1alpha1.PersistentVolumeClaim
+		componentLabels      map[string]string
+		componentAnnotations map[string]string
+		override             *mlopsv1alpha1.OverrideSpec
+		seldonConfigMeta     metav1.ObjectMeta
+		error                bool
 	}
 
 	tests := []test{
@@ -78,6 +80,8 @@ func TestStatefulSetReconcile(t *testing.T) {
 				meta,
 				test.podSpec,
 				test.volumeClaims,
+				test.componentLabels,
+				test.componentAnnotations,
 				test.override,
 				test.seldonConfigMeta,
 				annotator)