Repository: odahu/odahu-flow (language: python)

Issue:
Model deployment not work with two node pools
**Describe the bug**
In some situations, Knative start creating revisions abnormally

**To Reproduce**
1. Create a cluster with next node pools for deployment:
```  model_deployment_gpu:
    machine_type: n1-standard-8
    disk_size_gb: 100
    preemptible: true
    max_node_count: 1
    gpu:
      - type: nvidia-tesla-p100
        count: 1
    labels:
      mode: odahu-flow-deployment
      gpu: enabled
    taints:
      - key: dedicated
        value: deployment
        effect: NO_SCHEDULE
  model_deployment:
    max_node_count: 3
    preemptible: true
    labels:
      mode: odahu-flow-deployment
    taints:
      - key: dedicated
        value: deployment
        effect: NO_SCHEDULE```


2. Apply model  with similar configuration

```id: counter
spec:
 resources:
    limits:
      gpu: ''
      cpu: '2'
      memory: 2Gi
    requests:
      gpu: ''
      cpu: 125m
      memory: 128Mi
  minReplicas: 1
  maxReplicas: 1
  livenessProbeInitialDelay: 10
  readinessProbeInitialDelay: 10
  roleName: role-counter
  imagePullConnID: docker-ci
```

**Expected behavior**
Model has deployed only one revision and ModelDeployment is created by Knative

**Actual behavior**
Knative starts to create a huge count of revisions and model deployments. Model status is blinking




Discussion / hints:

Fix patch:
diff --git a/packages/operator/controllers/modeldeployment_controller.go b/packages/operator/controllers/modeldeployment_controller.go
index 089ef2004..c4aa5c61d 100644
--- a/packages/operator/controllers/modeldeployment_controller.go
+++ b/packages/operator/controllers/modeldeployment_controller.go
@@ -36,6 +36,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"knative.dev/serving/pkg/apis/serving"
 	knservingv1 "knative.dev/serving/pkg/apis/serving/v1"
+	"reflect"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -71,14 +72,15 @@ const (
 	KnativeAutoscalingMetric             = "autoscaling.knative.dev/metric"
 	DefaultKnativeAutoscalingMetric      = "concurrency"
 	DefaultKnativeAutoscalingClass       = "kpa.autoscaling.knative.dev"
-	DodelNameAnnotationKey               = "modelName"
+	ModelNameAnnotationKey               = "modelName"
+	ModelDeploymentVersionKey            = "modelDeploymentVersion"
 
-	IstioRewriteHTTPProbesAnnotation     = "sidecar.istio.io/rewriteAppHTTPProbers"
-	OdahuAuthorizationLabel          	 = "odahu-flow-authorization"
+	IstioRewriteHTTPProbesAnnotation = "sidecar.istio.io/rewriteAppHTTPProbers"
+	OdahuAuthorizationLabel          = "odahu-flow-authorization"
 
-	deploymentIDLabel 					 = "odahu.org/deploymentID"
-	cmPolicySuffix    					 = "opa-policy"
-	podPolicyLabel   					 = "opa-policy-config-map-name"
+	deploymentIDLabel = "odahu.org/deploymentID"
+	cmPolicySuffix    = "opa-policy"
+	podPolicyLabel    = "opa-policy-config-map-name"
 )
 
 var (
@@ -147,28 +149,31 @@ func (r *ModelDeploymentReconciler) ReconcileKnativeConfiguration(
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      KnativeConfigurationName(modelDeploymentCR),
 			Namespace: modelDeploymentCR.Namespace,
+			Annotations: map[string]string{
+				ModelDeploymentVersionKey: modelDeploymentCR.ResourceVersion,
+			},
 		},
 		Spec: knservingv1.ConfigurationSpec{
 			Template: knservingv1.RevisionTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels: map[string]string{
-						DodelNameAnnotationKey: modelDeploymentCR.Name,
+						ModelNameAnnotationKey: modelDeploymentCR.Name,
 						deploymentIDLabel:      modelDeploymentCR.Name,
 						// We must provide it because we don't use knative built-in routing
 						// And w/o this label revision is considered as "Unreachable" so then minScale is ignored and
 						// replicas are scaled to 0 even if user set minScale > 0
 						// TODO in future: move from creating knservingv1.Configuration -> knservingv1.Service
 						// https://github.com/knative/serving/blob/a333742324081d769d1b234622f3fc4cfd181ca4/pkg/apis/autoscaling/v1alpha1/pa_lifecycle.go#L85
-						serving.RouteLabelKey: modelDeploymentCR.Name,
+						serving.RouteLabelKey:   modelDeploymentCR.Name,
 						OdahuAuthorizationLabel: "enabled",
-						podPolicyLabel: getCMPolicyName(modelDeploymentCR),
+						podPolicyLabel:          getCMPolicyName(modelDeploymentCR),
 					},
 					Annotations: map[string]string{
-						KnativeAutoscalingClass:     DefaultKnativeAutoscalingClass,
-						KnativeAutoscalingMetric:    DefaultKnativeAutoscalingMetric,
-						KnativeMinReplicasKey:       strconv.Itoa(int(*modelDeploymentCR.Spec.MinReplicas)),
-						KnativeMaxReplicasKey:       strconv.Itoa(int(*modelDeploymentCR.Spec.MaxReplicas)),
-						KnativeAutoscalingTargetKey: KnativeAutoscalingTargetDefaultValue,
+						KnativeAutoscalingClass:          DefaultKnativeAutoscalingClass,
+						KnativeAutoscalingMetric:         DefaultKnativeAutoscalingMetric,
+						KnativeMinReplicasKey:            strconv.Itoa(int(*modelDeploymentCR.Spec.MinReplicas)),
+						KnativeMaxReplicasKey:            strconv.Itoa(int(*modelDeploymentCR.Spec.MaxReplicas)),
+						KnativeAutoscalingTargetKey:      KnativeAutoscalingTargetDefaultValue,
 						IstioRewriteHTTPProbesAnnotation: "true",
 					},
 				},
@@ -192,11 +197,6 @@ func (r *ModelDeploymentReconciler) ReconcileKnativeConfiguration(
 		return err
 	}
 
-	if err := odahuflow.StoreHashKnative(knativeConfiguration); err != nil {
-		log.Error(err, "Cannot apply obj hash")
-		return err
-	}
-
 	found := &knservingv1.Configuration{}
 	err = r.Get(context.TODO(), types.NamespacedName{
 		Name: knativeConfiguration.Name, Namespace: knativeConfiguration.Namespace,
@@ -209,26 +209,29 @@ func (r *ModelDeploymentReconciler) ReconcileKnativeConfiguration(
 		return err
 	}
 
-	if !odahuflow.ObjsEqualByHash(knativeConfiguration, found) {
-		log.Info(fmt.Sprintf(
-			"Knative Configuration hashes aren't equal. Update the %s Knative Configuration",
-			knativeConfiguration.Name,
-		))
+	modelDeploymentVersion := found.Annotations[ModelDeploymentVersionKey]
+	if modelDeploymentVersion == modelDeploymentCR.ResourceVersion {
+		log.Info("Model Deployment version is up to date, skipping")
+		return nil
+	}
 
-		found.Spec = knativeConfiguration.Spec
-		found.ObjectMeta.Annotations = knativeConfiguration.ObjectMeta.Annotations
-		found.ObjectMeta.Labels = knativeConfiguration.ObjectMeta.Labels
+	log.Info(fmt.Sprintf(
+		"Knative Configuration bases on version %s, update to reflect version %s",
+		modelDeploymentVersion, modelDeploymentCR.ResourceVersion,
+	))
 
-		log.Info(fmt.Sprintf("Updating %s Knative Configuration", knativeConfiguration.ObjectMeta.Name))
-		err = r.Update(context.TODO(), found)
-		if err != nil {
-			return err
-		}
-	} else {
-		log.Info(fmt.Sprintf(
-			"Knative Configuration hashes equal. Skip updating of the %s Knative Configuration",
-			knativeConfiguration.Name,
-		))
+	found.Spec = knativeConfiguration.Spec
+	for k, v := range knativeConfiguration.Labels {
+		found.Labels[k] = v
+	}
+	for k, v := range knativeConfiguration.Annotations {
+		found.Annotations[k] = v
+	}
+
+	log.Info(fmt.Sprintf("Updating %s Knative Configuration", knativeConfiguration.ObjectMeta.Name))
+	err = r.Update(context.TODO(), found)
+	if err != nil {
+		return err
 	}
 
 	return nil
@@ -330,64 +333,78 @@ func (r *ModelDeploymentReconciler) reconcileService(
 	log logr.Logger,
 	modelDeploymentCR *odahuflowv1alpha1.ModelDeployment,
 ) error {
-	service := &corev1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      modelDeploymentCR.Name,
-			Namespace: modelDeploymentCR.Namespace,
-		},
-		Spec: corev1.ServiceSpec{
-			Ports: []corev1.ServicePort{
-				{
-					Name:       "http",
-					Protocol:   corev1.ProtocolTCP,
-					Port:       80,
-					TargetPort: intstr.FromInt(8012),
-				},
-			},
-			Type: corev1.ServiceTypeClusterIP,
-		},
-	}
 
-	if err := controllerutil.SetControllerReference(modelDeploymentCR, service, r.scheme); err != nil {
-		return err
+	// Ensures that Service has a required port
+	fulfilServiceObject := func(service *corev1.Service) (updated bool) {
+		requiredPort := corev1.ServicePort{
+			Name:       "http",
+			Protocol:   corev1.ProtocolTCP,
+			Port:       80,
+			TargetPort: intstr.FromInt(8012),
+		}
+
+		found := false
+		for i, port := range service.Spec.Ports {
+			if port.Name != requiredPort.Name {
+				continue
+			}
+			found = true
+			if port != requiredPort {
+				service.Spec.Ports[i] = requiredPort
+				updated = true
+			}
+		}
+
+		if !found {
+			service.Spec.Ports = append(service.Spec.Ports, requiredPort)
+			updated = true
+		}
+
+		return updated
 	}
 
-	if err := odahuflow.StoreHash(service); err != nil {
-		log.Error(err, "Cannot apply obj hash")
-		return err
+	serviceNamespacedName := types.NamespacedName{
+		Name: modelDeploymentCR.Name, Namespace: modelDeploymentCR.Namespace,
 	}
 
-	found := &corev1.Service{}
-	err := r.Get(context.TODO(), types.NamespacedName{
-		Name: service.Name, Namespace: service.Namespace,
-	}, found)
+	foundService := &corev1.Service{}
+	err := r.Get(context.TODO(), serviceNamespacedName, foundService)
 	if err != nil && errors.IsNotFound(err) {
-		log.Info(fmt.Sprintf("Creating %s k8s service", service.ObjectMeta.Name))
-		err = r.Create(context.TODO(), service)
+		log.Info(fmt.Sprintf("Creating %s k8s service", serviceNamespacedName.Name))
+
+		newService := &corev1.Service{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      serviceNamespacedName.Name,
+				Namespace: serviceNamespacedName.Namespace,
+			},
+			Spec: corev1.ServiceSpec{
+				Type: corev1.ServiceTypeClusterIP,
+			},
+		}
+		fulfilServiceObject(newService)
+
+		if err := controllerutil.SetControllerReference(modelDeploymentCR, newService, r.scheme); err != nil {
+			return err
+		}
+
+		err = r.Create(context.TODO(), newService)
 		return err
 	} else if err != nil {
 		return err
 	}
 
-	if !odahuflow.ObjsEqualByHash(service, found) {
-		log.Info(fmt.Sprintf("Service hashes aren't equal. Update the %s service", service.Name))
+	needToUpdate := fulfilServiceObject(foundService)
 
-		// ClusterIP must not change
-		clusterIP := found.Spec.ClusterIP
-		found.Spec = service.Spec
-		found.Spec.ClusterIP = clusterIP
-
-		log.Info(fmt.Sprintf("Updating %s k8s service", service.ObjectMeta.Name))
-		err = r.Update(context.TODO(), found)
-		if err != nil {
-			return err
-		}
-	} else {
-		log.Info(fmt.Sprintf("k8s service hashes equal. Skip updating of the %s service", service.Name))
+	if !needToUpdate {
+		log.Info("K8s Service already has required port, skipping")
+		return nil
 	}
 
-	if err := r.reconcileEndpoints(log, modelDeploymentCR); err != nil {
-		log.Error(err, "Can not reconcile endpoints")
+	log.Info(fmt.Sprintf("Update ports of %s K8s Service...", foundService.Name))
+
+	err = r.Update(context.TODO(), foundService)
+	if err != nil {
+		return err
 	}
 
 	return nil
@@ -401,63 +418,60 @@ func (r *ModelDeploymentReconciler) reconcileEndpoints(
 	lastRevisionName := modelDeploymentCR.Status.LastRevisionName
 	if len(lastRevisionName) == 0 {
 		log.Info("Last revision name is empty")
-
 		return nil
 	}
 
 	knativeEndpoints := &corev1.Endpoints{}
-
 	if err := r.Get(context.TODO(), types.NamespacedName{
 		Namespace: modelDeploymentCR.Namespace,
 		Name:      lastRevisionName,
 	}, knativeEndpoints); err != nil {
 		log.Error(err, "Cannot get the knative endpoints endpoints",
 			"last revision name", lastRevisionName)
-
 		return err
 	}
 
-	endpoints := &corev1.Endpoints{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      modelDeploymentCR.Name,
-			Namespace: modelDeploymentCR.Namespace,
-		},
-		Subsets: knativeEndpoints.Subsets,
-	}
-
-	if err := controllerutil.SetControllerReference(modelDeploymentCR, endpoints, r.scheme); err != nil {
-		return err
-	}
-
-	if err := odahuflow.StoreHash(endpoints); err != nil {
-		log.Error(err, "Cannot apply obj hash")
-		return err
+	endpointsNamespacedName := types.NamespacedName{
+		Name:      modelDeploymentCR.Name,
+		Namespace: modelDeploymentCR.Namespace,
 	}
 
 	found := &corev1.Endpoints{}
-	err := r.Get(context.TODO(), types.NamespacedName{
-		Name: endpoints.Name, Namespace: endpoints.Namespace,
-	}, found)
+	err := r.Get(context.TODO(), endpointsNamespacedName, found)
 	if err != nil && errors.IsNotFound(err) {
-		log.Info(fmt.Sprintf("Creating %s k8s endpoints", endpoints.ObjectMeta.Name))
+		log.Info(fmt.Sprintf("Creating %s k8s endpoints", endpointsNamespacedName.Name))
+
+		endpoints := &corev1.Endpoints{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      endpointsNamespacedName.Name,
+				Namespace: endpointsNamespacedName.Namespace,
+			},
+			Subsets: knativeEndpoints.Subsets,
+		}
+
+		if err := controllerutil.SetControllerReference(modelDeploymentCR, endpoints, r.scheme); err != nil {
+			return err
+		}
+
 		err = r.Create(context.TODO(), endpoints)
 		return err
 	} else if err != nil {
 		return err
 	}
 
-	if !odahuflow.ObjsEqualByHash(endpoints, found) {
-		log.Info(fmt.Sprintf("Endpoints hashes aren't equal. Update the %s endpoints", endpoints.Name))
+	if reflect.DeepEqual(found.Subsets, knativeEndpoints.Subsets) {
+		log.Info("Endpoints are associated with up-to-date Model Deployment version, skipping")
+		return nil
+	}
 
-		found.Subsets = endpoints.Subsets
+	log.Info(fmt.Sprintf("Endpoints are not up-to-date with latest Knative revision."+
+		"Update the %s endpoints", found.Name))
 
-		log.Info(fmt.Sprintf("Updating %s k8s endpoints", endpoints.ObjectMeta.Name))
-		err = r.Update(context.TODO(), found)
-		if err != nil {
-			return err
-		}
-	} else {
-		log.Info(fmt.Sprintf("k8s endpoints hashes equal. Skip updating of the %s endpoints", endpoints.Name))
+	found.Subsets = knativeEndpoints.Subsets
+
+	err = r.Update(context.TODO(), found)
+	if err != nil {
+		return err
 	}
 
 	return nil
@@ -502,7 +516,7 @@ func (r *ModelDeploymentReconciler) cleanupOldRevisions(
 	knativeRevisions := &knservingv1.RevisionList{}
 
 	labelSelectorReq, err := labels.NewRequirement(
-		DodelNameAnnotationKey,
+		ModelNameAnnotationKey,
 		selection.DoubleEquals,
 		[]string{modelDeploymentCR.Name},
 	)
@@ -530,7 +544,7 @@ func (r *ModelDeploymentReconciler) cleanupOldRevisions(
 		// pin variable
 		kr := kr
 
-		modelDeploymentName, ok := kr.Labels[DodelNameAnnotationKey]
+		modelDeploymentName, ok := kr.Labels[ModelNameAnnotationKey]
 		if !ok || modelDeploymentName != modelDeploymentCR.Name {
 			continue
 		}
@@ -662,8 +676,6 @@ func (r *ModelDeploymentReconciler) Reconcile(request ctrl.Request) (ctrl.Result
 
 	log.Info("Run reconciling of model deployment")
 
-
-
 	if err := r.reconcilePolicyCM(log, modelDeploymentCR); err != nil {
 		log.Error(err, "Reconcile policy config map")
 		return reconcile.Result{}, err
@@ -687,20 +699,25 @@ func (r *ModelDeploymentReconciler) Reconcile(request ctrl.Request) (ctrl.Result
 	}
 
 	if !configurationReady {
-		log.Info("Configuration was not updated yet. Put the Model Deployment back in the queue")
+		log.Info("Configuration was not updated yet. Update Status and Put the Model Deployment back in the queue")
 
 		_ = r.reconcileStatus(log, modelDeploymentCR, odahuflowv1alpha1.ModelDeploymentStateProcessing, "")
 
 		return reconcile.Result{RequeueAfter: DefaultRequeueDelay}, nil
 	}
 
-	log.Info("Reconcile default Model Route")
+	log.Info("Reconcile K8s Service")
 
 	if err := r.reconcileService(log, modelDeploymentCR); err != nil {
 		log.Error(err, "Reconcile the k8s service")
 		return reconcile.Result{}, err
 	}
 
+	if err := r.reconcileEndpoints(log, modelDeploymentCR); err != nil {
+		log.Error(err, "Can not reconcile endpoints")
+		return reconcile.Result{}, err
+	}
+
 	modelDeployment := &appsv1.Deployment{}
 	modelDeploymentKey := types.NamespacedName{
 		Name:      knativeDeploymentName(latestReadyRevision),
@@ -708,6 +725,7 @@ func (r *ModelDeploymentReconciler) Reconcile(request ctrl.Request) (ctrl.Result
 	}
 
 	if err := r.Client.Get(context.TODO(), modelDeploymentKey, modelDeployment); errors.IsNotFound(err) {
+		log.Info("Knative Revision Deployment not found. Re-queueing...")
 		_ = r.reconcileStatus(log, modelDeploymentCR, odahuflowv1alpha1.ModelDeploymentStateProcessing, latestReadyRevision)
 
 		return reconcile.Result{RequeueAfter: DefaultRequeueDelay}, nil
@@ -722,6 +740,7 @@ func (r *ModelDeploymentReconciler) Reconcile(request ctrl.Request) (ctrl.Result
 	modelDeploymentCR.Status.Deployment = modelDeployment.Name
 
 	if modelDeploymentCR.Status.Replicas != modelDeploymentCR.Status.AvailableReplicas {
+		log.Info(fmt.Sprintf("Not enough replicas running. Requeue after %s", DefaultRequeueDelay))
 		_ = r.reconcileStatus(log, modelDeploymentCR, odahuflowv1alpha1.ModelDeploymentStateProcessing, latestReadyRevision)
 
 		return reconcile.Result{RequeueAfter: DefaultRequeueDelay}, nil
diff --git a/packages/operator/controllers/modeldeployment_event_hadler.go b/packages/operator/controllers/modeldeployment_event_hadler.go
index b2402f42b..13136cd17 100644
--- a/packages/operator/controllers/modeldeployment_event_hadler.go
+++ b/packages/operator/controllers/modeldeployment_event_hadler.go
@@ -53,7 +53,7 @@ func (e *EnqueueRequestForImplicitOwner) Generic(evt event.GenericEvent, q workq
 
 func (e *EnqueueRequestForImplicitOwner) addEvent(meta metav1.Object, q workqueue.RateLimitingInterface) {
 	labels := meta.GetLabels()
-	name, ok := labels[DodelNameAnnotationKey]
+	name, ok := labels[ModelNameAnnotationKey]
 	if !ok {
 		return
 	}
diff --git a/packages/operator/controllers/modelpackaging_controller.go b/packages/operator/controllers/modelpackaging_controller.go
index 1339e9922..33f6451f4 100644
--- a/packages/operator/controllers/modelpackaging_controller.go
+++ b/packages/operator/controllers/modelpackaging_controller.go
@@ -249,11 +249,6 @@ func (r *ModelPackagingReconciler) reconcileTaskRun(
 		return nil, err
 	}
 
-	if err := odahuflow.StoreHash(taskRun); err != nil {
-		log.Error(err, "Cannot apply obj hash")
-		return nil, err
-	}
-
 	found := &tektonv1beta1.TaskRun{}
 	err = r.Get(context.TODO(), types.NamespacedName{
 		Name: taskRun.Name, Namespace: r.packagingConfig.Namespace,
@@ -286,11 +281,6 @@ func (r *ModelPackagingReconciler) createResultConfigMap(packagingCR *odahuflowv
 		return err
 	}
 
-	if err := odahuflow.StoreHash(resultCM); err != nil {
-		log.Error(err, "Cannot apply obj hash")
-		return err
-	}
-
 	found := &corev1.ConfigMap{}
 	err := r.Get(context.TODO(), types.NamespacedName{
 		Name: resultCM.Name, Namespace: r.packagingConfig.Namespace,
diff --git a/packages/operator/controllers/modelroute_controller.go b/packages/operator/controllers/modelroute_controller.go
index 97ae748be..408811f84 100644
--- a/packages/operator/controllers/modelroute_controller.go
+++ b/packages/operator/controllers/modelroute_controller.go
@@ -22,7 +22,6 @@ import (
 	v1alpha3_istio_api "github.com/aspenmesh/istio-client-go/pkg/apis/networking/v1alpha3"
 	gogotypes "github.com/gogo/protobuf/types"
 	"github.com/odahu/odahu-flow/packages/operator/pkg/config"
-	"github.com/odahu/odahu-flow/packages/operator/pkg/odahuflow"
 	v1alpha3_istio "istio.io/api/networking/v1alpha3"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -47,11 +46,12 @@ import (
 )
 
 const (
-	knativeRevisionHeader                   = "knative-serving-revision"
-	knativeNamespaceHeader                  = "knative-serving-namespace"
-	defaultRetryAttempts                    = 30
-	defaultListOfRetryCauses                = "5xx,connect-failure,refused-stream"
-	routeForLabelPrefix						= "odahu-route-for-"
+	knativeRevisionHeader    = "knative-serving-revision"
+	knativeNamespaceHeader   = "knative-serving-namespace"
+	defaultRetryAttempts     = 30
+	defaultListOfRetryCauses = "5xx,connect-failure,refused-stream"
+	routeForLabelPrefix      = "odahu-route-for-"
+	ModelRouteVersionKey     = "modelRouteVersion"
 )
 
 var (
@@ -177,6 +177,9 @@ func (r *ModelRouteReconciler) reconcileVirtualService(modelRouteCR *odahuflowv1
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      VirtualServiceName(modelRouteCR),
 			Namespace: modelRouteCR.Namespace,
+			Annotations: map[string]string{
+				ModelRouteVersionKey: modelRouteCR.ResourceVersion,
+			},
 		},
 		Spec: v1alpha3_istio_api.VirtualServiceSpec{
 			VirtualService: v1alpha3_istio.VirtualService{
@@ -234,11 +237,6 @@ func (r *ModelRouteReconciler) reconcileVirtualService(modelRouteCR *odahuflowv1
 		return reconileAgain, err
 	}
 
-	if err := odahuflow.StoreHash(vservice); err != nil {
-		log.Error(err, "Cannot apply obj hash")
-		return reconileAgain, err
-	}
-
 	found := &v1alpha3_istio_api.VirtualService{}
 	err := r.Get(context.TODO(), types.NamespacedName{
 		Name: vservice.Name, Namespace: vservice.Namespace,
@@ -251,23 +249,26 @@ func (r *ModelRouteReconciler) reconcileVirtualService(modelRouteCR *odahuflowv1
 		return reconileAgain, err
 	}
 
-	if !odahuflow.ObjsEqualByHash(vservice, found) {
-		log.Info(fmt.Sprintf("Istio Virtual Service hashes don't equal. Update the %s Model route", vservice.Name))
+	modelDeploymentVersion := found.Annotations[ModelRouteVersionKey]
+	if modelDeploymentVersion == modelRouteCR.ResourceVersion {
+		log.Info("Istio VService is associated with up-to-date Model Route version, skipping")
+		return reconileAgain, err
+	}
+
+	log.Info(fmt.Sprintf("Istio Virtual Service hashes don't equal. Update the %s Model route", vservice.Name))
 
-		found.Spec = vservice.Spec
-		found.ObjectMeta.Annotations = vservice.ObjectMeta.Annotations
-		found.ObjectMeta.Labels = vservice.ObjectMeta.Labels
+	found.Spec = vservice.Spec
+	for k, v := range vservice.Labels {
+		found.Labels[k] = v
+	}
+	for k, v := range vservice.Annotations {
+		found.Annotations[k] = v
+	}
 
-		log.Info(fmt.Sprintf("Updating %s k8s Istio Virtual Service", vservice.ObjectMeta.Name))
-		err = r.Update(context.TODO(), found)
-		if err != nil {
-			return reconileAgain, err
-		}
-	} else {
-		log.Info(fmt.Sprintf(
-			"Istio Virtual Service hashes equal. Skip updating of the %s Istio Virtual Service",
-			vservice.Name),
-		)
+	log.Info(fmt.Sprintf("Updating %s k8s Istio Virtual Service", vservice.ObjectMeta.Name))
+	err = r.Update(context.TODO(), found)
+	if err != nil {
+		return reconileAgain, err
 	}
 
 	return reconileAgain, err
@@ -338,7 +339,7 @@ func (r *ModelRouteReconciler) reconcileRouteForLabels(route *odahuflowv1alpha1.
 		}
 	}
 
-	var newRouteForLabels []string  //nolint
+	var newRouteForLabels []string //nolint
 	for _, md := range route.Spec.ModelDeploymentTargets {
 		newRouteForLabels = append(newRouteForLabels, routeForLabelKey(md.Name))
 	}
@@ -375,42 +376,42 @@ func (r *ModelRouteReconciler) SetupBuilder(mgr ctrl.Manager) *ctrl.Builder {
 		Owns(&v1alpha3_istio_api.VirtualService{}).
 		// Route should reacts on changes in any ModelDeployment from its targets
 		Watches(&source.Kind{Type: &odahuflowv1alpha1.ModelDeployment{}}, &handler.EnqueueRequestsFromMapFunc{
-		ToRequests: handler.ToRequestsFunc(func(object handler.MapObject) []reconcile.Request {
-
-			mdName := object.Meta.GetName()
-
-			var result []reconcile.Request
-			ctx, cancel := context.WithTimeout(context.Background(), time.Second * 5)
-			defer cancel()
-
-			selector := labels.NewSelector()
-			req, err := labels.NewRequirement(routeForLabelKey(mdName), selection.Exists, nil)
-			if err != nil {
-				log.Error(err, "Unable to create label requirement")
-				return result
-			}
-			selector.Add(*req)
+			ToRequests: handler.ToRequestsFunc(func(object handler.MapObject) []reconcile.Request {
+
+				mdName := object.Meta.GetName()
+
+				var result []reconcile.Request
+				ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+				defer cancel()
+
+				selector := labels.NewSelector()
+				req, err := labels.NewRequirement(routeForLabelKey(mdName), selection.Exists, nil)
+				if err != nil {
+					log.Error(err, "Unable to create label requirement")
+					return result
+				}
+				selector.Add(*req)
+
+				modelRoutes := &odahuflowv1alpha1.ModelRouteList{}
+				if err := r.List(ctx, modelRoutes, &client.ListOptions{
+					LabelSelector: selector,
+				}); err != nil {
+					log.Error(err, "Unable to fetch ModelRouteList for ModelDeployment")
+					return result
+				}
+
+				for _, mr := range modelRoutes.Items {
+					result = append(result, reconcile.Request{
+						NamespacedName: types.NamespacedName{
+							Namespace: mr.Namespace,
+							Name:      mr.Name,
+						},
+					})
+				}
 
-			modelRoutes := &odahuflowv1alpha1.ModelRouteList{}
-			if err := r.List(ctx, modelRoutes, &client.ListOptions{
-				LabelSelector: selector,
-			}); err != nil {
-				log.Error(err, "Unable to fetch ModelRouteList for ModelDeployment")
 				return result
-			}
-
-			for _, mr := range modelRoutes.Items {
-				result = append(result, reconcile.Request{
-					NamespacedName: types.NamespacedName{
-						Namespace: mr.Namespace,
-						Name: mr.Name,
-					},
-				})
-			}
-
-			return result
-		}),
-	})
+			}),
+		})
 }
 
 func (r *ModelRouteReconciler) SetupWithManager(mgr ctrl.Manager) error {
diff --git a/packages/operator/controllers/modeltraining_controller.go b/packages/operator/controllers/modeltraining_controller.go
index b5dbe7b27..4a9bd1c71 100644
--- a/packages/operator/controllers/modeltraining_controller.go
+++ b/packages/operator/controllers/modeltraining_controller.go
@@ -278,11 +278,6 @@ func (r *ModelTrainingReconciler) reconcileTaskRun(
 		return nil, err
 	}
 
-	if err := odahuflow.StoreHash(taskRun); err != nil {
-		log.Error(err, "Cannot apply obj hash")
-		return nil, err
-	}
-
 	found := &tektonv1beta1.TaskRun{}
 	err = r.Get(context.TODO(), types.NamespacedName{
 		Name: taskRun.Name, Namespace: r.trainingConfig.Namespace,
@@ -314,11 +309,6 @@ func (r *ModelTrainingReconciler) createResultConfigMap(trainingCR *odahuflowv1a
 		return err
 	}
 
-	if err := odahuflow.StoreHash(resultCM); err != nil {
-		log.Error(err, "Cannot apply obj hash")
-		return err
-	}
-
 	found := &corev1.ConfigMap{}
 	err := r.Get(context.TODO(), types.NamespacedName{
 		Name: resultCM.Name, Namespace: r.trainingConfig.Namespace,
diff --git a/packages/operator/pkg/odahuflow/k8s.go b/packages/operator/pkg/odahuflow/k8s.go
index a8d1f6d8a..fbab9594f 100644
--- a/packages/operator/pkg/odahuflow/k8s.go
+++ b/packages/operator/pkg/odahuflow/k8s.go
@@ -17,13 +17,7 @@
 package odahuflow
 
 import (
-	"crypto/sha512"
-	"encoding/base64"
-	"encoding/json"
 	"fmt"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	knservingv1 "knative.dev/serving/pkg/apis/serving/v1"
 )
 
 const (
@@ -48,58 +42,3 @@ func GenerateTrainingResultCMName(mtID string) string {
 func GenerateDeploymentConnectionSecretName(connName string) string {
 	return fmt.Sprintf("%s-regsecret", connName)
 }
-
-// Compute hash and store it in the annotations
-func StoreHash(obj metav1.Object) error {
-	h := sha512.New()
-	jsonData, err := json.Marshal(obj)
-	if err != nil {
-		return err
-	}
-	_, err = h.Write(jsonData)
-	if err != nil {
-		return err
-	}
-
-	annotations := map[string]string{}
-	if obj.GetAnnotations() != nil {
-		for k, v := range obj.GetAnnotations() {
-			annotations[k] = v
-		}
-	}
-	annotations[LastAppliedHashAnnotation] = base64.StdEncoding.EncodeToString(h.Sum(nil))
-
-	obj.SetAnnotations(annotations)
-
-	return nil
-}
-
-// Compute hash and store it in the annotations
-func StoreHashKnative(obj *knservingv1.Configuration) error {
-	h := sha512.New()
-	jsonData, err := json.Marshal(obj)
-	if err != nil {
-		return err
-	}
-
-	_, err = h.Write(jsonData)
-	if err != nil {
-		return err
-	}
-
-	annotations := map[string]string{}
-	if obj.GetAnnotations() != nil {
-		for k, v := range obj.GetAnnotations() {
-			annotations[k] = v
-		}
-	}
-	annotations[LastAppliedHashAnnotation] = base64.StdEncoding.EncodeToString(h.Sum(nil))
-
-	obj.SetAnnotations(annotations)
-
-	return nil
-}
-
-func ObjsEqualByHash(firstObj, secondObj metav1.Object) bool {
-	return firstObj.GetAnnotations()[LastAppliedHashAnnotation] == secondObj.GetAnnotations()[LastAppliedHashAnnotation]
-}


Test patch:
diff --git a/packages/operator/controllers/modeldeployment_controller_test.go b/packages/operator/controllers/modeldeployment_controller_test.go
index dbe853d34..8352f5862 100644
--- a/packages/operator/controllers/modeldeployment_controller_test.go
+++ b/packages/operator/controllers/modeldeployment_controller_test.go
@@ -161,8 +161,8 @@ func (s *ModelDeploymentControllerSuite) TestReconcile() {
 
 	configurationLabels := knativeConfiguration.Spec.Template.ObjectMeta.Labels
 	s.Assertions.Len(configurationLabels, 5)
-	s.Assertions.Contains(configurationLabels, DodelNameAnnotationKey)
-	s.Assertions.Equal(md.Name, configurationLabels[DodelNameAnnotationKey])
+	s.Assertions.Contains(configurationLabels, ModelNameAnnotationKey)
+	s.Assertions.Equal(md.Name, configurationLabels[ModelNameAnnotationKey])
 
 	podSpec := knativeConfiguration.Spec.Template.Spec
 	s.Assertions.Len(podSpec.Containers, 1)
@@ -281,9 +281,8 @@ func (s *ModelDeploymentControllerSuite) createDeployment(md *odahuflowv1alpha1.
 	return func() { s.k8sClient.Delete(context.TODO(), md) }
 }
 
-func (s *ModelDeploymentControllerSuite) getKnativeConfiguration(md *odahuflowv1alpha1.ModelDeployment) (
-	*knservingv1.Configuration,
-) {
+func (s *ModelDeploymentControllerSuite) getKnativeConfiguration(md *odahuflowv1alpha1.ModelDeployment,
+) *knservingv1.Configuration {
 	configuration := &knservingv1.Configuration{}
 	configurationKey := types.NamespacedName{Name: KnativeConfigurationName(md), Namespace: md.Namespace}
 	s.Assertions.Eventually(
diff --git a/packages/tests/e2e/robot/tests/api/resources/training_packaging/valid/training.mlflow-gpu.not_default.yaml b/packages/tests/e2e/robot/tests/api/resources/training_packaging/valid/training.mlflow-gpu.not_default.yaml
index 09ae4d41e..31409de06 100644
--- a/packages/tests/e2e/robot/tests/api/resources/training_packaging/valid/training.mlflow-gpu.not_default.yaml
+++ b/packages/tests/e2e/robot/tests/api/resources/training_packaging/valid/training.mlflow-gpu.not_default.yaml
@@ -18,9 +18,9 @@ spec:
     limits:
       cpu: 3
       memory: 2Gi
-      gpu: 2
+      gpu: 1
     requests:
       cpu: 3
       memory: 2Gi
-      gpu: 2
+      gpu: 1
   vcsName: odahu-flow-examples
\ No newline at end of file