Repository: SeldonIO/seldon-core (language: python)

Issue:
Seldon Manager reconciliation logs are not controlled via MANAGER_LOG_LEVEL
<!-- Welcome and thank you for helping us make Seldon Core better!

To help us address your issue, please provide us as much of the information requested below as possible. Thanks! -->

## Describe the bug

We are using Seldon v1 **version 1.16.0**. In our Azure managed Kubernetes test environment I have been seeing a lot of reconciliation logs. I understand that these logs are emitted whenever their are differences detected that need to be reconciled. We have set our MANAGER_LOG_LEVEL to ERROR and we still see them, but we don't see any other error logs saying reconcile failed or issues with reconcile.

![Screenshot 2023-06-14 at 10 04 27 AM](https://github.com/SeldonIO/seldon-core/assets/25337979/190f1cf5-d6b2-4979-baa9-7af3c9db36b0)

![Screenshot 2023-06-14 at 10 02 05 AM](https://github.com/SeldonIO/seldon-core/assets/25337979/4d6bb8a2-6ded-4fd8-b58e-c625a568ebd6)

I think these logs are not logged via logger. Please take a look at how we can control these logs. 

## To reproduce

- Bring up seldon manager and resources managed by the manager.
- Whenever there is some reconcile request, you should see these logs.

## Expected behaviour
<!-- A clear and concise description of what you expected to happen. -->
MANAGER_LOG_LEVEL should also control these logs and these logs should be DEBUG logs.

## Environment
 K8s: v1.26.3  

<!-- You Can fill it manually or paste the output of the command below:

* Cloud Provider: Azure
* Kubernetes Cluster Version:  v1.26.3
* Deployed Seldon System Images: seldon-core-operator:1.16.0

Alternatively run `echo "#### Kubernetes version:\n $(kubectl version) \n\n#### Seldon Images:\n$(kubectl get --namespace seldon-system deploy seldon-controller-manager -o yaml  | grep seldonio)"`
-->




Discussion / hints:
Hi @KashyapAdarsh,

From a quick look at the code, I can see that although the Helm charts use `MANAGER_LOG_LEVEL`, the operator's `main.go` uses `SELDON_LOG_LEVEL`.  See [here](https://github.com/SeldonIO/seldon-core/blob/f9d39508c1957c7aacd09b0a46e73e7171dcbee4/operator/main.go#L149) and [here](https://github.com/SeldonIO/seldon-core/blob/f9d39508c1957c7aacd09b0a46e73e7171dcbee4/operator/main.go#L54).

Could you try setting `SELDON_LOG_LEVEL` and confirm it that behaves the way you'd expect?  If so, we can look at updating the Helm chart.
Hey @agrski Thank you for the response. Did you mean set `SELDON_LOG_LEVEL` for seldon-controller-manager or the SeldonDeployment resource? The logs are seen in Manager right? 
@agrski I don't think that helps. I tried setting that env variable for `seldon-controller-manager` deployment and I still see those logs.
![Screenshot 2023-06-15 at 11 22 35 AM](https://github.com/SeldonIO/seldon-core/assets/132311616/87a02967-a2a3-409e-a8b0-974eaf69e094)

These logs do not have any log level as well. Its like print statements that are sent to STDOUT.

![Screenshot 2023-06-15 at 11 24 22 AM](https://github.com/SeldonIO/seldon-core/assets/132311616/b759799c-e4f8-4c32-b00e-7700227283ad)



Fix patch:
diff --git a/operator/apis/machinelearning.seldon.io/v1/prepack.go b/operator/apis/machinelearning.seldon.io/v1/prepack.go
index a60224e5fe..75038a4590 100644
--- a/operator/apis/machinelearning.seldon.io/v1/prepack.go
+++ b/operator/apis/machinelearning.seldon.io/v1/prepack.go
@@ -154,8 +154,6 @@ func getPredictorServerConfigs() (map[string]PredictorServerConfig, error) {
 	err := C.Get(context.TODO(), k8types.NamespacedName{Name: ControllerConfigMapName, Namespace: ControllerNamespace}, configMap)
 
 	if err != nil {
-		fmt.Println("Failed to find config map " + ControllerConfigMapName)
-		fmt.Println(err)
 		return map[string]PredictorServerConfig{}, err
 	}
 	return getPredictorServerConfigsFromMap(configMap)
diff --git a/operator/controllers/seldondeployment_controller.go b/operator/controllers/seldondeployment_controller.go
index 5b6a55800f..a1bf4f1b15 100644
--- a/operator/controllers/seldondeployment_controller.go
+++ b/operator/controllers/seldondeployment_controller.go
@@ -120,6 +120,24 @@ type components struct {
 	addressable           *machinelearningv1.SeldonAddressable
 }
 
+func (c *components) deploymentHasHpa(depname string) bool {
+	for _, hpa := range c.hpas {
+		if hpa.Name == depname {
+			return true
+		}
+	}
+	return false
+}
+
+func (c *components) deploymentHasKeda(depname string) bool {
+	for _, scaledObject := range c.kedaScaledObjects {
+		if scaledObject.Name == depname {
+			return true
+		}
+	}
+	return false
+}
+
 type httpGrpcPorts struct {
 	httpPort int
 	grpcPort int
@@ -494,6 +512,7 @@ func (r *SeldonDeploymentReconciler) createComponents(ctx context.Context, mlDep
 			deploy := createDeploymentWithoutEngine(depName, seldonId, cSpec, &p, mlDep, securityContext, true)
 
 			if cSpec.KedaSpec != nil { // Add KEDA if needed
+				r.Log.Info("Creating keda scaled object", "deployment", depName)
 				c.kedaScaledObjects = append(c.kedaScaledObjects, createKeda(cSpec, depName, seldonId, namespace))
 			} else if cSpec.HpaSpec != nil { // Add HPA if needed
 				c.hpas = append(c.hpas, createHpa(cSpec, depName, seldonId, namespace))
@@ -1700,6 +1719,10 @@ func (r *SeldonDeploymentReconciler) createDeployments(components *components, i
 				patch.IgnoreField("apiVersion"),
 				patch.IgnoreField("metadata"),
 			}
+			// Ignore replica count disparities as that is controlled by HPA/KEDA
+			if components.deploymentHasHpa(deploy.Name) || components.deploymentHasKeda(deploy.Name) {
+				opts = append(opts, utils2.IgnoreReplicas())
+			}
 			patcherMaker := patch.NewPatchMaker(annotator, &patch.K8sStrategicMergePatcher{}, &patch.BaseJSONMergePatcher{})
 			patchResult, err := patcherMaker.Calculate(found, deploy, opts...)
 			if err != nil {
@@ -1707,14 +1730,14 @@ func (r *SeldonDeploymentReconciler) createDeployments(components *components, i
 			}
 			if !patchResult.IsEmpty() {
 				log.Info("Updating Deployment", "namespace", deploy.Namespace, "name", deploy.Name)
-				fmt.Printf("\n%s\n", patchResult.String())
+				log.V(5).Info("Deployment differs", "patch result", patchResult.String())
 				b, err := json.Marshal(deploy.Spec.Template.Spec)
 				if err == nil {
-					fmt.Printf("\n%s\n", string(b))
+					log.V(5).Info("Deployment differs", "existing", string(b))
 				}
 				b2, err := json.Marshal(found.Spec.Template.Spec)
 				if err == nil {
-					fmt.Printf("\n%s\n", string(b2))
+					log.V(5).Info("Deployment differs", "found", string(b2))
 				}
 
 				desiredDeployment := found.DeepCopy()
diff --git a/operator/controllers/utils/patch_utils.go b/operator/controllers/utils/patch_utils.go
new file mode 100644
index 0000000000..be76e48bf2
--- /dev/null
+++ b/operator/controllers/utils/patch_utils.go
@@ -0,0 +1,46 @@
+package utils
+
+import (
+	"emperror.dev/errors"
+	"github.com/banzaicloud/k8s-objectmatcher/patch"
+	json "github.com/json-iterator/go"
+)
+
+func IgnoreReplicas() patch.CalculateOption {
+	return func(current, modified []byte) ([]byte, []byte, error) {
+		current, err := deleteReplicas(current)
+		if err != nil {
+			return []byte{}, []byte{}, errors.Wrap(err, "could not delete replicas field from current byte sequence")
+		}
+
+		modified, err = deleteReplicas(modified)
+		if err != nil {
+			return []byte{}, []byte{}, errors.Wrap(err, "could not delete replicas field from modified byte sequence")
+		}
+
+		return current, modified, nil
+	}
+}
+
+func deleteReplicas(obj []byte) ([]byte, error) {
+	resource := map[string]interface{}{}
+	err := json.Unmarshal(obj, &resource)
+	if err != nil {
+		return []byte{}, errors.Wrap(err, "could not unmarshal byte sequence")
+	}
+
+	if spec, ok := resource["spec"]; ok {
+		if spec, ok := spec.(map[string]interface{}); ok {
+			if _, ok := spec["replicas"]; ok {
+				spec["replicas"] = nil
+			}
+		}
+	}
+
+	obj, err = json.ConfigCompatibleWithStandardLibrary.Marshal(resource)
+	if err != nil {
+		return []byte{}, errors.Wrap(err, "could not marshal byte sequence")
+	}
+
+	return obj, nil
+}


Test patch:
diff --git a/operator/controllers/utils/patch_utils_test.go b/operator/controllers/utils/patch_utils_test.go
new file mode 100644
index 0000000000..cd033222ff
--- /dev/null
+++ b/operator/controllers/utils/patch_utils_test.go
@@ -0,0 +1,94 @@
+package utils
+
+import (
+	"encoding/json"
+	. "github.com/onsi/gomega"
+	v1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"testing"
+)
+
+func getTestDeployment(replicas *int32) *v1.Deployment {
+	d := &v1.Deployment{
+		Spec: v1.DeploymentSpec{
+			Template: corev1.PodTemplateSpec{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name: "mycontainer",
+						},
+					},
+				},
+			},
+		},
+	}
+	if replicas != nil {
+		d.Spec.Replicas = replicas
+	}
+	return d
+}
+
+func TestReplica(t *testing.T) {
+	g := NewGomegaWithT(t)
+	type test struct {
+		name         string
+		dep1         *v1.Deployment
+		dep2         *v1.Deployment
+		expectedDep1 *v1.Deployment
+		expectedDep2 *v1.Deployment
+	}
+	getInt32Ptr := func(v int32) *int32 { return &v }
+	tests := []test{
+		{
+			name:         "no replicas",
+			dep1:         getTestDeployment(nil),
+			dep2:         getTestDeployment(nil),
+			expectedDep1: getTestDeployment(nil),
+			expectedDep2: getTestDeployment(nil),
+		},
+		{
+			name:         "first dep with replicas",
+			dep1:         getTestDeployment(getInt32Ptr(1)),
+			dep2:         getTestDeployment(nil),
+			expectedDep1: getTestDeployment(nil),
+			expectedDep2: getTestDeployment(nil),
+		},
+		{
+			name:         "second dep with replicas",
+			dep1:         getTestDeployment(nil),
+			dep2:         getTestDeployment(getInt32Ptr(1)),
+			expectedDep1: getTestDeployment(nil),
+			expectedDep2: getTestDeployment(nil),
+		},
+		{
+			name:         "both deps with replicas",
+			dep1:         getTestDeployment(getInt32Ptr(2)),
+			dep2:         getTestDeployment(getInt32Ptr(4)),
+			expectedDep1: getTestDeployment(nil),
+			expectedDep2: getTestDeployment(nil),
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			f := IgnoreReplicas()
+			b1, err := json.Marshal(test.dep1)
+			g.Expect(err).To(BeNil())
+			b2, err := json.Marshal(test.dep2)
+			g.Expect(err).To(BeNil())
+
+			b3, b4, err := f(b1, b2)
+			g.Expect(err).To(BeNil())
+
+			d1 := &v1.Deployment{}
+			err = json.Unmarshal(b3, d1)
+			g.Expect(err).To(BeNil())
+
+			d2 := &v1.Deployment{}
+			err = json.Unmarshal(b4, d2)
+			g.Expect(err).To(BeNil())
+
+			g.Expect(d1).To(Equal(test.expectedDep1))
+			g.Expect(d2).To(Equal(test.expectedDep2))
+		})
+	}
+}