Repository: codeready-toolchain/member-operator
Pull request #633: fix requeue in idler controller
Author: login: MatousJobanek
Labels: name: approved
Created: 2025-03-03T15:35:28Z | Merged: 2025-03-07T07:03:11Z

[KUBESAW-266](https://issues.redhat.com/browse/KUBESAW-266)

Added watcher on pods with a predicate that should trigger the reconcile only if:
* pod exists in users namespace
* either the startTime was set in the pod's status, or it reached the restart threshold.

two fixes of scheduling the next requeue of the idler controller:
1. drops the `RequeueTimeThreshold` - the original value caused that the controller kept reconciling as crazy in instances with a high number of users 
2. take into account that VMs should live only one twelfth of the idler timeout - the next requeue should be scheduled accordingly.


Diff:
diff --git a/controllers/idler/idler_controller.go b/controllers/idler/idler_controller.go
index 963d2778..62415fe6 100644
--- a/controllers/idler/idler_controller.go
+++ b/controllers/idler/idler_controller.go
@@ -15,8 +15,11 @@ import (
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/scale"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
+	runtimeCluster "sigs.k8s.io/controller-runtime/pkg/cluster"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	openshiftappsv1 "github.com/openshift/api/apps/v1"
 	errs "github.com/pkg/errors"
@@ -34,8 +37,7 @@ import (
 )
 
 const (
-	RestartThreshold     = 50
-	RequeueTimeThreshold = 300 * time.Second
+	restartThreshold = 50
 )
 
 var SupportedScaleResources = map[schema.GroupVersionKind]schema.GroupVersionResource{
@@ -47,9 +49,11 @@ var vmGVR = schema.GroupVersionResource{Group: "kubevirt.io", Version: "v1", Res
 var vmInstanceGVR = schema.GroupVersionResource{Group: "kubevirt.io", Version: "v1", Resource: "virtualmachineinstances"}
 
 // SetupWithManager sets up the controller with the Manager.
-func (r *Reconciler) SetupWithManager(mgr manager.Manager) error {
+func (r *Reconciler) SetupWithManager(mgr manager.Manager, allNamespaceCluster runtimeCluster.Cluster) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&toolchainv1alpha1.Idler{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
+		WatchesRawSource(source.Kind(allNamespaceCluster.GetCache(), &corev1.Pod{}),
+			handler.EnqueueRequestsFromMapFunc(MapPodToIdler), builder.WithPredicates(PodIdlerPredicate{})).
 		Complete(r)
 }
 
@@ -107,59 +111,68 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.
 		logger.Error(err, "failed to ensure idling")
 		return reconcile.Result{}, r.setStatusFailed(ctx, idler, err.Error())
 	}
-	if err := r.ensureIdling(ctx, idler); err != nil {
+	requeueAfter, err := r.ensureIdling(ctx, idler)
+	if err != nil {
 		return reconcile.Result{}, r.wrapErrorWithStatusUpdate(ctx, idler, r.setStatusFailed, err,
 			"failed to ensure idling '%s'", idler.Name)
 	}
-	// Requeue in shortest of the following values idler.Spec.TimeoutSeconds or RequeueTimeThreshold or nextPodToBeKilledAfter
-	after := findShortestRequeueDuration(idler)
-	logger.Info("requeueing for next pod to check", "after_seconds", after.Seconds())
-	return reconcile.Result{
-		Requeue:      true,
-		RequeueAfter: after,
-	}, r.setStatusReady(ctx, idler)
+	result := reconcile.Result{}
+	if len(idler.Status.Pods) > 0 { // schedule the next requeue only if something is being tracked
+		logger.Info("requeueing for next pod to check", "after_seconds", requeueAfter.Seconds())
+		result = reconcile.Result{
+			Requeue:      true,
+			RequeueAfter: requeueAfter,
+		}
+	} else {
+		// if nothing is being tracked then only wait for the next reconcile triggered
+		// by a pod starting
+		logger.Info("no pods being tracked")
+	}
+
+	return result, r.setStatusReady(ctx, idler)
 }
 
-func (r *Reconciler) ensureIdling(ctx context.Context, idler *toolchainv1alpha1.Idler) error {
+func (r *Reconciler) ensureIdling(ctx context.Context, idler *toolchainv1alpha1.Idler) (time.Duration, error) {
 	// Get all pods running in the namespace
 	podList := &corev1.PodList{}
 	if err := r.AllNamespacesClient.List(ctx, podList, client.InNamespace(idler.Name)); err != nil {
-		return err
+		return 0, err
 	}
+	requeueAfter := time.Duration(idler.Spec.TimeoutSeconds) * time.Second
 	newStatusPods := make([]toolchainv1alpha1.Pod, 0, 10)
 	for _, pod := range podList.Items {
 		pod := pod // TODO We won't need it after upgrading to go 1.22: https://go.dev/blog/loopvar-preview
 		podLogger := log.FromContext(ctx).WithValues("pod_name", pod.Name, "pod_phase", pod.Status.Phase)
 		podCtx := log.IntoContext(ctx, podLogger)
+		timeoutSeconds := idler.Spec.TimeoutSeconds
+		if isOwnedByVM(pod.ObjectMeta) {
+			// use 1/12th of the timeout for VMs to have more aggressive idling to decrease
+			// the infra costs because VMs consume much more resources
+			timeoutSeconds = timeoutSeconds / 12
+		}
 		if trackedPod := findPodByName(idler, pod.Name); trackedPod != nil {
 			// check the restart count for the trackedPod
 			restartCount := getHighestRestartCount(pod.Status)
-			if restartCount > RestartThreshold {
+			if restartCount > restartThreshold {
 				podLogger.Info("Pod is restarting too often. Killing the pod", "restart_count", restartCount)
 				// Check if it belongs to a controller (Deployment, DeploymentConfig, etc) and scale it down to zero.
 				err := deletePodsAndCreateNotification(podCtx, pod, r, idler)
 				if err != nil {
-					return err
+					return 0, err
 				}
 				continue
 			}
-			timeoutSeconds := idler.Spec.TimeoutSeconds
-			if isOwnedByVM(pod.ObjectMeta) {
-				// use 1/12th of the timeout for VMs
-				timeoutSeconds = timeoutSeconds / 12
-			}
 			// Already tracking this pod. Check the timeout.
 			if time.Now().After(trackedPod.StartTime.Add(time.Duration(timeoutSeconds) * time.Second)) {
 				podLogger.Info("Pod running for too long. Killing the pod.", "start_time", trackedPod.StartTime.Format("2006-01-02T15:04:05Z"), "timeout_seconds", timeoutSeconds)
 				// Check if it belongs to a controller (Deployment, DeploymentConfig, etc) and scale it down to zero.
 				err := deletePodsAndCreateNotification(podCtx, pod, r, idler)
 				if err != nil {
-					return err
+					return 0, err
 				}
-
-			} else {
-				newStatusPods = append(newStatusPods, *trackedPod) // keep tracking
+				continue
 			}
+			newStatusPods = append(newStatusPods, *trackedPod) // keep tracking
 
 		} else if pod.Status.StartTime != nil { // Ignore pods without StartTime
 			podLogger.Info("New pod detected. Start tracking.")
@@ -168,9 +181,23 @@ func (r *Reconciler) ensureIdling(ctx context.Context, idler *toolchainv1alpha1.
 				StartTime: *pod.Status.StartTime,
 			})
 		}
+		killAfter := time.Until(pod.Status.StartTime.Add(time.Duration(timeoutSeconds+1) * time.Second))
+		requeueAfter = shorterDuration(requeueAfter, killAfter)
 	}
 
-	return r.updateStatusPods(ctx, idler, newStatusPods)
+	return requeueAfter, r.updateStatusPods(ctx, idler, newStatusPods)
+}
+
+func shorterDuration(first, second time.Duration) time.Duration {
+	shorter := first
+	if first > second {
+		shorter = second
+	}
+	// do not allow negative durations: if a pod has timed out, then it should be killed immediately
+	if shorter < 0 {
+		shorter = 0
+	}
+	return shorter
 }
 
 // Check if the pod belongs to a controller (Deployment, DeploymentConfig, etc) and scale it down to zero.
@@ -602,47 +629,6 @@ func findPodByName(idler *toolchainv1alpha1.Idler, name string) *toolchainv1alph
 	return nil
 }
 
-// nextPodToBeKilledAfter checks the start times of all the tracked pods in the Idler and the timeout left
-// for the next pod to be killed.
-// If there is no pod to kill, the func returns `nil`
-func nextPodToBeKilledAfter(idler *toolchainv1alpha1.Idler) *time.Duration {
-	if len(idler.Status.Pods) == 0 {
-		// no pod tracked, so nothing to kill
-		return nil
-	}
-	var d time.Duration
-	for _, pod := range idler.Status.Pods {
-		killAfter := time.Until(pod.StartTime.Add(time.Duration(idler.Spec.TimeoutSeconds+1) * time.Second))
-		if d == 0 || killAfter < d {
-			d = killAfter
-		}
-	}
-	// do not allow negative durations: if a pod has timed out, then it should be killed immediately
-	if d < 0 {
-		d = 0
-	}
-	return &d
-}
-
-// findShortestRequeueDuration finds the shortest duration the given durations
-// returns the shortest duration to requeue after for idler
-func findShortestRequeueDuration(idler *toolchainv1alpha1.Idler) time.Duration {
-	durations := make([]*time.Duration, 0, 3)
-	nextPodToKillAfter := nextPodToBeKilledAfter(idler)
-	maxRequeueDuration := RequeueTimeThreshold
-	idlerTimeoutDuration := time.Duration(idler.Spec.TimeoutSeconds) * time.Second
-	durations = append(durations, nextPodToKillAfter, &maxRequeueDuration, &idlerTimeoutDuration)
-	var shortest *time.Duration
-	for _, d := range durations {
-		if d != nil {
-			if shortest == nil || *d < *shortest {
-				shortest = d
-			}
-		}
-	}
-	return *shortest
-}
-
 // updateStatusPods updates the status pods to the new ones but only if something changed. Order is ignored.
 func (r *Reconciler) updateStatusPods(ctx context.Context, idler *toolchainv1alpha1.Idler, newPods []toolchainv1alpha1.Pod) error {
 	nothingChanged := len(idler.Status.Pods) == len(newPods)
diff --git a/controllers/idler/idler_controller_test.go b/controllers/idler/idler_controller_test.go
index 1072985d..6a2b21f9 100644
--- a/controllers/idler/idler_controller_test.go
+++ b/controllers/idler/idler_controller_test.go
@@ -107,13 +107,13 @@ func TestEnsureIdling(t *testing.T) {
 
 	logf.SetLogger(zap.New(zap.UseDevMode(true)))
 
-	t.Run("No pods in namespace managed by idler", func(t *testing.T) {
+	t.Run("No pods in namespace managed by idler, no requeue time", func(t *testing.T) {
 		// given
 		idler := &toolchainv1alpha1.Idler{
 			ObjectMeta: metav1.ObjectMeta{
 				Name: "john-dev",
 			},
-			Spec: toolchainv1alpha1.IdlerSpec{TimeoutSeconds: TestIdlerTimeOutSeconds},
+			Spec: toolchainv1alpha1.IdlerSpec{TimeoutSeconds: TestIdlerTimeOutSeconds * 100},
 		}
 
 		reconciler, req, cl, _, _ := prepareReconcile(t, idler.Name, getHostCluster, idler)
@@ -124,11 +124,8 @@ func TestEnsureIdling(t *testing.T) {
 
 		// then
 		require.NoError(t, err)
-		// no pods found - the controller will requeue after 5 mins
-		assert.Equal(t, reconcile.Result{
-			Requeue:      true,
-			RequeueAfter: RequeueTimeThreshold,
-		}, res)
+		// no pods found - the controller won't schedule requeue
+		assert.Equal(t, reconcile.Result{}, res)
 		memberoperatortest.AssertThatIdler(t, idler.Name, cl).HasConditions(memberoperatortest.Running())
 	})
 
@@ -263,7 +260,7 @@ func TestEnsureIdling(t *testing.T) {
 					HasConditions(memberoperatortest.Running(), memberoperatortest.IdlerNotificationCreated())
 
 				assert.True(t, res.Requeue)
-				assert.Less(t, int64(res.RequeueAfter), int64(time.Duration(idler.Spec.TimeoutSeconds)*time.Second))
+				assert.Less(t, int64(res.RequeueAfter), int64(time.Duration(idler.Spec.TimeoutSeconds)*time.Second)/12)
 
 				t.Run("Third Reconcile. Stop tracking deleted pods.", func(t *testing.T) {
 					//when
@@ -298,15 +295,60 @@ func TestEnsureIdling(t *testing.T) {
 							TracksPods([]*corev1.Pod{}).
 							HasConditions(memberoperatortest.Running(), memberoperatortest.IdlerNotificationCreated())
 
-						// requeue after the idler timeout
-						assert.Equal(t, reconcile.Result{
-							Requeue:      true,
-							RequeueAfter: RequeueTimeThreshold,
-						}, res)
+						// no pods being tracked -> no scheduled requeue
+						assert.Equal(t, reconcile.Result{}, res)
 					})
 				})
 			})
 		})
+
+		t.Run("requeue time with workloads", func(t *testing.T) {
+			// given
+			startTimes := payloadStartTimes{
+				defaultStartTime: time.Now(),
+				vmStartTime:      time.Now(),
+			}
+			t.Run("with VM", func(t *testing.T) {
+				// given
+				reconciler, req, _, _, _ := prepareReconcile(t, idler.Name, getHostCluster, idler, nsTmplSet, mur)
+				payloads := preparePayloads(t, reconciler, idler.Name, "", startTimes)
+				preparePayloadCrashloopingPodsWithinThreshold(t, reconciler, idler.Name, "inThreshRestarts-", startTimes)
+				preparePayloadCrashloopingAboveThreshold(t, reconciler, idler.Name, "restartCount-")
+				// start tracking
+				_, err := reconciler.Reconcile(context.TODO(), req)
+				require.NoError(t, err)
+
+				//when
+				res, err := reconciler.Reconcile(context.TODO(), req)
+
+				// then
+				require.NoError(t, err)
+				assert.True(t, res.Requeue)
+				// with VMs, it needs to be approx one twelfth of the idler timeout plus-minus one second
+				assert.Greater(t, int64(res.RequeueAfter), int64((TestIdlerTimeOutSeconds/12-1)*time.Second))
+				assert.Less(t, int64(res.RequeueAfter), int64((TestIdlerTimeOutSeconds/12+1)*time.Second))
+
+				t.Run("without VM", func(t *testing.T) {
+					// given
+					// delete all VM pods
+					for _, pod := range payloads.controlledPods {
+						if len(pod.OwnerReferences) > 0 && pod.OwnerReferences[0].Name == payloads.virtualmachineinstance.GetName() {
+							require.NoError(t, reconciler.AllNamespacesClient.Delete(context.TODO(), pod))
+						}
+					}
+
+					//when
+					res, err := reconciler.Reconcile(context.TODO(), req)
+
+					// then
+					require.NoError(t, err)
+					assert.True(t, res.Requeue)
+					// without VMs, it needs to be approx the idler timeout plus-minus one second
+					assert.Greater(t, int64(res.RequeueAfter), int64((TestIdlerTimeOutSeconds-1)*time.Second))
+					assert.Less(t, int64(res.RequeueAfter), int64((TestIdlerTimeOutSeconds+1)*time.Second))
+				})
+			})
+		})
 	})
 
 	t.Run("Create notification the first time resources Idled", func(t *testing.T) {
@@ -536,11 +578,8 @@ func TestEnsureIdlingFailed(t *testing.T) {
 				res, err := reconciler.Reconcile(context.TODO(), req)
 
 				// then
-				require.NoError(t, err) // 'NotFound' errors are ignored!
-				assert.Equal(t, reconcile.Result{
-					Requeue:      true,
-					RequeueAfter: RequeueTimeThreshold,
-				}, res)
+				require.NoError(t, err)                  // 'NotFound' errors are ignored!
+				assert.Equal(t, reconcile.Result{}, res) // no other pods being tracked
 				memberoperatortest.AssertThatIdler(t, idler.Name, cl).ContainsCondition(memberoperatortest.Running())
 			}
 
diff --git a/controllers/idler/mapper.go b/controllers/idler/mapper.go
new file mode 100644
index 00000000..fb3647d0
--- /dev/null
+++ b/controllers/idler/mapper.go
@@ -0,0 +1,19 @@
+package idler
+
+import (
+	"context"
+
+	"k8s.io/apimachinery/pkg/types"
+	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+// MapPodToIdler maps the pod to the idler
+func MapPodToIdler(_ context.Context, obj runtimeclient.Object) []reconcile.Request {
+	return []reconcile.Request{{
+		// the idler should have the same name as the user's namespace
+		NamespacedName: types.NamespacedName{
+			Name: obj.GetNamespace(),
+		},
+	}}
+}
diff --git a/controllers/idler/mapper_test.go b/controllers/idler/mapper_test.go
new file mode 100644
index 00000000..2d529546
--- /dev/null
+++ b/controllers/idler/mapper_test.go
@@ -0,0 +1,24 @@
+package idler
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestMapper(t *testing.T) {
+	// given
+	pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "jane-dev", Name: "my-pod"}}
+
+	// when
+	requests := MapPodToIdler(context.TODO(), pod)
+
+	// then
+	require.Len(t, requests, 1)
+	assert.Equal(t, "jane-dev", requests[0].Name)
+	assert.Empty(t, requests[0].Namespace)
+}
diff --git a/controllers/idler/predicate.go b/controllers/idler/predicate.go
new file mode 100644
index 00000000..5f4a4b55
--- /dev/null
+++ b/controllers/idler/predicate.go
@@ -0,0 +1,46 @@
+package idler
+
+import (
+	"github.com/codeready-toolchain/member-operator/pkg/webhook/mutatingwebhook"
+	corev1 "k8s.io/api/core/v1"
+	runtimeevent "sigs.k8s.io/controller-runtime/pkg/event"
+)
+
+type PodIdlerPredicate struct {
+}
+
+// Update triggers reconcile if the pod runs in users namespace
+// and if either the highest restart count is higher than the threshold
+// or the startTime was newly set in the new version of the pod
+func (p PodIdlerPredicate) Update(event runtimeevent.UpdateEvent) bool {
+	newPod, ok := event.ObjectNew.(*corev1.Pod) // this can be replaced by the typed-predicate in the next version of k8s
+	if !ok {
+		return false
+	}
+	// all pods running in users' namespaces have the priorityClassName set, so trigger reconcile only
+	// if the pod contains the same class name to ensure that the pod runs in a user's namespace
+	// (we don't care about other pods)
+	if newPod.Spec.PriorityClassName != mutatingwebhook.PriorityClassName {
+		return false
+	}
+	if oldPod, ok := event.ObjectOld.(*corev1.Pod); ok {
+		startTimeNewlySet := oldPod.Status.StartTime == nil && newPod.Status.StartTime != nil
+		return startTimeNewlySet || getHighestRestartCount(newPod.Status) > restartThreshold
+	}
+	return false
+}
+
+// Create doesn't trigger reconcile
+func (PodIdlerPredicate) Create(_ runtimeevent.CreateEvent) bool {
+	return false
+}
+
+// Delete doesn't trigger reconcile
+func (PodIdlerPredicate) Delete(_ runtimeevent.DeleteEvent) bool {
+	return false
+}
+
+// Generic doesn't trigger reconcile
+func (p PodIdlerPredicate) Generic(_ runtimeevent.GenericEvent) bool {
+	return false
+}
diff --git a/controllers/idler/predicate_test.go b/controllers/idler/predicate_test.go
new file mode 100644
index 00000000..7cc75ee2
--- /dev/null
+++ b/controllers/idler/predicate_test.go
@@ -0,0 +1,113 @@
+package idler
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+)
+
+func TestPredicate(t *testing.T) {
+	// given
+	predicate := PodIdlerPredicate{}
+	testPriorityClassData := map[string]struct {
+		pod            *corev1.Pod
+		expectedResult bool
+	}{
+		"pod with sandbox class": {
+			pod: &corev1.Pod{Spec: corev1.PodSpec{
+				PriorityClassName: "sandbox-users-pods",
+			}},
+			expectedResult: true,
+		},
+		"pod without class": {
+			pod:            &corev1.Pod{Spec: corev1.PodSpec{}},
+			expectedResult: false,
+		},
+		"pod with other class": {
+			pod: &corev1.Pod{Spec: corev1.PodSpec{
+				PriorityClassName: "some-class",
+			}},
+			expectedResult: false,
+		},
+	}
+
+	testRestartData := map[string]struct {
+		newPodStatus corev1.PodStatus
+		expected     bool
+	}{
+		"with container above threshold": {
+			newPodStatus: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{
+				{RestartCount: 51},
+				{RestartCount: 49},
+			}},
+			expected: true,
+		},
+		"with containers under threshold": {
+			newPodStatus: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{
+				{RestartCount: 0},
+				{RestartCount: 49},
+			}},
+			expected: false,
+		},
+		"without container statuses": {
+			newPodStatus: corev1.PodStatus{},
+			expected:     false,
+		},
+	}
+
+	startTime := metav1.Now()
+	testStartTimeData := map[string]struct {
+		oldPod    *corev1.Pod
+		startTime *metav1.Time
+		expected  bool
+	}{
+		"without startTime": {
+			oldPod:    &corev1.Pod{},
+			startTime: nil,
+			expected:  false,
+		},
+		"with added startTime": {
+			oldPod:    &corev1.Pod{},
+			startTime: &startTime,
+			expected:  true,
+		},
+		"with startTime already present": {
+			oldPod: &corev1.Pod{Status: corev1.PodStatus{
+				StartTime: &startTime,
+			}},
+			startTime: &startTime,
+			expected:  false,
+		},
+	}
+
+	for classTestName, classTestData := range testPriorityClassData {
+		t.Run(classTestName, func(t *testing.T) {
+			for subTestName, restartData := range testRestartData {
+				t.Run(subTestName, func(t *testing.T) {
+					for subSubTestName, startTimeData := range testStartTimeData {
+						t.Run(subSubTestName, func(t *testing.T) {
+							// given
+							pod := classTestData.pod.DeepCopy()
+							pod.Status = restartData.newPodStatus
+							pod.Status.StartTime = startTimeData.startTime
+							expectedResult := classTestData.expectedResult && (restartData.expected || startTimeData.expected)
+
+							// when & then
+							assert.Equal(t, expectedResult, predicate.Update(event.UpdateEvent{
+								ObjectOld: startTimeData.oldPod,
+								ObjectNew: pod,
+							}))
+
+							assert.False(t, predicate.Create(event.CreateEvent{Object: classTestData.pod}))
+							assert.False(t, predicate.Generic(event.GenericEvent{Object: classTestData.pod}))
+							assert.False(t, predicate.Delete(event.DeleteEvent{Object: classTestData.pod}))
+						})
+					}
+				})
+			}
+		})
+	}
+}
diff --git a/main.go b/main.go
index f5a055e6..9f4baf7f 100644
--- a/main.go
+++ b/main.go
@@ -5,9 +5,10 @@ import (
 	"fmt"
 	"os"
 	goruntime "runtime"
-	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"time"
 
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+
 	"github.com/codeready-toolchain/member-operator/controllers/idler"
 	membercfgctrl "github.com/codeready-toolchain/member-operator/controllers/memberoperatorconfig"
 	"github.com/codeready-toolchain/member-operator/controllers/memberstatus"
@@ -248,7 +249,7 @@ func main() {
 		DynamicClient:       dynamicClient,
 		GetHostCluster:      cluster.GetHostCluster,
 		Namespace:           namespace,
-	}).SetupWithManager(mgr); err != nil {
+	}).SetupWithManager(mgr, allNamespacesCluster); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Idler")
 		os.Exit(1)
 	}
diff --git a/pkg/webhook/mutatingwebhook/userpods_mutate.go b/pkg/webhook/mutatingwebhook/userpods_mutate.go
index 55872fa8..144e2ae2 100644
--- a/pkg/webhook/mutatingwebhook/userpods_mutate.go
+++ b/pkg/webhook/mutatingwebhook/userpods_mutate.go
@@ -18,7 +18,7 @@ var (
 		{
 			"op":    "replace",
 			"path":  "/spec/priorityClassName",
-			"value": priorityClassName,
+			"value": PriorityClassName,
 		},
 		{
 			"op":    "replace",
@@ -32,7 +32,7 @@ var (
 
 const (
 	priority          = int32(-3)
-	priorityClassName = "sandbox-users-pods"
+	PriorityClassName = "sandbox-users-pods"
 )
 
 func init() {


Reviews:
user: login: metlos
body: Nice simplification of the code 👍🏼 
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-03T17:15:35Z
user: login: alexeykazakov
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-03T18:49:30Z
user: login: ranakan19
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T03:37:34Z
user: login: ranakan19
body: Thanks for the changes!

Couple of thought/comments/questions:
1. Sorry, could you please clarify the issue with shorter VM idling mentioned in point 2 of PR description? Was it not functioning as expected before, aside from the 5m requeue time being too aggressive causing issues?
2. The current implementation checks restartCount against the threshold but doesn’t tightly couple requeueTime to it. Previously also the logic for restartCount wasn't tied to timeout, but shorter requeue time would almost instantly idle the pod once it reaches restartThreshold. For instance, reaching a restartCount of 50 takes about 4 hours, yet idling a crash-looping pod with requeueTimeThreshold set to 6 hours can take anywhere between 6 to 12 hours. As Alexey pointed out its still better than not idling crashlooping pods at all, but a requeueTimethreshold shorter than 6 hr should help too.
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T05:47:27Z
user: login: MatousJobanek
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T09:12:51Z
user: login: MatousJobanek
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T09:17:00Z
user: login: metlos
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T11:50:52Z
user: login: MatousJobanek
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T12:07:32Z
user: login: MatousJobanek
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T12:11:24Z
user: login: metlos
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-05T15:33:03Z
user: login: metlos
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-05T15:35:26Z
user: login: rajivnathan
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-05T17:20:16Z
user: login: alexeykazakov
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-05T19:17:38Z
user: login: ranakan19
body: Great improvements! predicate looks real nice.
minor - the testIdlertimeoutseconds const in idler_controller_test can be a lower value now 
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-06T06:06:17Z
user: login: MatousJobanek
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-06T12:54:25Z
user: login: MatousJobanek
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-06T12:57:51Z
user: login: MatousJobanek
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-06T15:04:43Z
user: login: rajivnathan
body: Nice! 👍 
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-06T15:29:38Z

Review comments:
diff_hunk: @@ -107,30 +107,36 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.
 		logger.Error(err, "failed to ensure idling")
 		return reconcile.Result{}, r.setStatusFailed(ctx, idler, err.Error())
 	}
-	if err := r.ensureIdling(ctx, idler); err != nil {
+	requeueAfter, err := r.ensureIdling(ctx, idler)
+	if err != nil {
 		return reconcile.Result{}, r.wrapErrorWithStatusUpdate(ctx, idler, r.setStatusFailed, err,
 			"failed to ensure idling '%s'", idler.Name)
 	}
 	// Requeue in shortest of the following values idler.Spec.TimeoutSeconds or RequeueTimeThreshold or nextPodToBeKilledAfter
-	after := findShortestRequeueDuration(idler)
-	logger.Info("requeueing for next pod to check", "after_seconds", after.Seconds())
+	logger.Info("requeueing for next pod to check", "after_seconds", requeueAfter.Seconds())
 	return reconcile.Result{
 		Requeue:      true,
-		RequeueAfter: after,
+		RequeueAfter: requeueAfter,
 	}, r.setStatusReady(ctx, idler)
 }
 
-func (r *Reconciler) ensureIdling(ctx context.Context, idler *toolchainv1alpha1.Idler) error {
+func (r *Reconciler) ensureIdling(ctx context.Context, idler *toolchainv1alpha1.Idler) (time.Duration, error) {
 	// Get all pods running in the namespace
 	podList := &corev1.PodList{}
 	if err := r.AllNamespacesClient.List(ctx, podList, client.InNamespace(idler.Name)); err != nil {
-		return err
+		return 0, err
 	}
+	requeueAfter := shorterDuration(RequeueTimeThreshold, time.Duration(idler.Spec.TimeoutSeconds)*time.Second)
 	newStatusPods := make([]toolchainv1alpha1.Pod, 0, 10)
 	for _, pod := range podList.Items {
 		pod := pod // TODO We won't need it after upgrading to go 1.22: https://go.dev/blog/loopvar-preview
 		podLogger := log.FromContext(ctx).WithValues("pod_name", pod.Name, "pod_phase", pod.Status.Phase)
 		podCtx := log.IntoContext(ctx, podLogger)
+		timeoutSeconds := idler.Spec.TimeoutSeconds
+		if isOwnedByVM(pod.ObjectMeta) {
+			// use 1/12th of the timeout for VMs
path: controllers/idler/idler_controller.go
user: login: metlos
body: I'd appreciate the comment to explain why this needs to happen for the VMs, not merely stating that it does :) That is unless the knowledge is lost in the depths of time, seeing that you copied this code from its prior location.
created_at: 2025-03-03T17:00:19Z
author_association: CONTRIBUTOR
diff_hunk: @@ -35,7 +35,7 @@ import (
 
 const (
 	RestartThreshold     = 50
-	RequeueTimeThreshold = 300 * time.Second
+	RequeueTimeThreshold = 6 * time.Hour
path: controllers/idler/idler_controller.go
user: login: alexeykazakov
body: I wonder if we can make it a bit shorter like 3h? Basically crash-lopping pods will be killed after the threshold is reached (50 restarts) plus up to `RequeueTimeThreshold`. With this change it's up to 6h instead of 5m extra time after the threshold. Which is fine. But potentially could be shorter (3h???) if it's not too often for the idler with many namespaces.
But again, it's minor. We can keep it as 6h for now and shorten it after we switch to a pod watcher.
created_at: 2025-03-03T18:15:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -107,30 +107,36 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.
 		logger.Error(err, "failed to ensure idling")
 		return reconcile.Result{}, r.setStatusFailed(ctx, idler, err.Error())
 	}
-	if err := r.ensureIdling(ctx, idler); err != nil {
+	requeueAfter, err := r.ensureIdling(ctx, idler)
+	if err != nil {
 		return reconcile.Result{}, r.wrapErrorWithStatusUpdate(ctx, idler, r.setStatusFailed, err,
 			"failed to ensure idling '%s'", idler.Name)
 	}
 	// Requeue in shortest of the following values idler.Spec.TimeoutSeconds or RequeueTimeThreshold or nextPodToBeKilledAfter
-	after := findShortestRequeueDuration(idler)
-	logger.Info("requeueing for next pod to check", "after_seconds", after.Seconds())
+	logger.Info("requeueing for next pod to check", "after_seconds", requeueAfter.Seconds())
 	return reconcile.Result{
 		Requeue:      true,
-		RequeueAfter: after,
+		RequeueAfter: requeueAfter,
 	}, r.setStatusReady(ctx, idler)
 }
 
-func (r *Reconciler) ensureIdling(ctx context.Context, idler *toolchainv1alpha1.Idler) error {
+func (r *Reconciler) ensureIdling(ctx context.Context, idler *toolchainv1alpha1.Idler) (time.Duration, error) {
 	// Get all pods running in the namespace
 	podList := &corev1.PodList{}
 	if err := r.AllNamespacesClient.List(ctx, podList, client.InNamespace(idler.Name)); err != nil {
-		return err
+		return 0, err
 	}
+	requeueAfter := shorterDuration(RequeueTimeThreshold, time.Duration(idler.Spec.TimeoutSeconds)*time.Second)
 	newStatusPods := make([]toolchainv1alpha1.Pod, 0, 10)
 	for _, pod := range podList.Items {
 		pod := pod // TODO We won't need it after upgrading to go 1.22: https://go.dev/blog/loopvar-preview
 		podLogger := log.FromContext(ctx).WithValues("pod_name", pod.Name, "pod_phase", pod.Status.Phase)
 		podCtx := log.IntoContext(ctx, podLogger)
+		timeoutSeconds := idler.Spec.TimeoutSeconds
+		if isOwnedByVM(pod.ObjectMeta) {
+			// use 1/12th of the timeout for VMs
path: controllers/idler/idler_controller.go
user: login: alexeykazakov
body: The reason is that we want to be more aggressive with VM idling. Ideally we should extend our Idlier API and introduce a new field there. Spec.VMTimeoutSeconds or something like this. Having 1/12 of the Spec.TimeoutSecond for VM was just a quick way for us (for having more aggressive VM Idler).
created_at: 2025-03-03T18:18:56Z
author_association: CONTRIBUTOR
diff_hunk: @@ -35,7 +35,7 @@ import (
 
 const (
 	RestartThreshold     = 50
-	RequeueTimeThreshold = 300 * time.Second
+	RequeueTimeThreshold = 6 * time.Hour
path: controllers/idler/idler_controller.go
user: login: ranakan19
body: +1
created_at: 2025-03-04T03:37:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -107,30 +107,36 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.
 		logger.Error(err, "failed to ensure idling")
 		return reconcile.Result{}, r.setStatusFailed(ctx, idler, err.Error())
 	}
-	if err := r.ensureIdling(ctx, idler); err != nil {
+	requeueAfter, err := r.ensureIdling(ctx, idler)
+	if err != nil {
 		return reconcile.Result{}, r.wrapErrorWithStatusUpdate(ctx, idler, r.setStatusFailed, err,
 			"failed to ensure idling '%s'", idler.Name)
 	}
 	// Requeue in shortest of the following values idler.Spec.TimeoutSeconds or RequeueTimeThreshold or nextPodToBeKilledAfter
-	after := findShortestRequeueDuration(idler)
-	logger.Info("requeueing for next pod to check", "after_seconds", after.Seconds())
+	logger.Info("requeueing for next pod to check", "after_seconds", requeueAfter.Seconds())
 	return reconcile.Result{
 		Requeue:      true,
-		RequeueAfter: after,
+		RequeueAfter: requeueAfter,
 	}, r.setStatusReady(ctx, idler)
 }
 
-func (r *Reconciler) ensureIdling(ctx context.Context, idler *toolchainv1alpha1.Idler) error {
+func (r *Reconciler) ensureIdling(ctx context.Context, idler *toolchainv1alpha1.Idler) (time.Duration, error) {
 	// Get all pods running in the namespace
 	podList := &corev1.PodList{}
 	if err := r.AllNamespacesClient.List(ctx, podList, client.InNamespace(idler.Name)); err != nil {
-		return err
+		return 0, err
 	}
+	requeueAfter := shorterDuration(RequeueTimeThreshold, time.Duration(idler.Spec.TimeoutSeconds)*time.Second)
 	newStatusPods := make([]toolchainv1alpha1.Pod, 0, 10)
 	for _, pod := range podList.Items {
 		pod := pod // TODO We won't need it after upgrading to go 1.22: https://go.dev/blog/loopvar-preview
 		podLogger := log.FromContext(ctx).WithValues("pod_name", pod.Name, "pod_phase", pod.Status.Phase)
 		podCtx := log.IntoContext(ctx, podLogger)
+		timeoutSeconds := idler.Spec.TimeoutSeconds
+		if isOwnedByVM(pod.ObjectMeta) {
+			// use 1/12th of the timeout for VMs
path: controllers/idler/idler_controller.go
user: login: MatousJobanek
body: Yeah, I updated the comment a bit
created_at: 2025-03-04T09:12:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -35,7 +35,7 @@ import (
 
 const (
 	RestartThreshold     = 50
-	RequeueTimeThreshold = 300 * time.Second
+	RequeueTimeThreshold = 6 * time.Hour
path: controllers/idler/idler_controller.go
user: login: MatousJobanek
body: well, I'm not really a fan of the short requeue time. having 3h would result in a new reconcile of the controller every 5-6 seconds in an environment with 2k namespaces. Do we really need that?

Anyway, I implemented the watcher with a predicate that should take care of this without triggering unnecessary reconcile loops. 
created_at: 2025-03-04T09:17:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,19 @@
+package idler
+
+import (
+	"context"
+
+	"k8s.io/apimachinery/pkg/types"
+	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+// MapPodToIdler maps the pod to the idler
+func MapPodToIdler(_ context.Context, obj runtimeclient.Object) []reconcile.Request {
+	return []reconcile.Request{{
+		// the idler should have the same name as the user's namespace
+		NamespacedName: types.NamespacedName{
+			Name: obj.GetNamespace(),
path: controllers/idler/mapper.go
user: login: metlos
body: Does not specifying the namespace of the idler resource work correctly with the controller manager? 
created_at: 2025-03-04T11:43:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,52 @@
+package idler
+
+import (
+	"github.com/codeready-toolchain/member-operator/pkg/webhook/mutatingwebhook"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	runtimeevent "sigs.k8s.io/controller-runtime/pkg/event"
+)
+
+type PodIdlerPredicate struct {
+}
+
+// Update triggers reconcile if the pod runs in users namespace
+// and if either the highest restart count is higher than the threshold
+// or the startTime was newly set in the new version of the pod
+func (p PodIdlerPredicate) Update(event runtimeevent.UpdateEvent) bool {
+	isUserPod, newPod := isUserPod(event.ObjectNew)
+	if !isUserPod {
+		return false
+	}
+	if oldPod, ok := event.ObjectOld.(*corev1.Pod); ok {
+		startTimeNewlySet := oldPod.Status.StartTime == nil && newPod.Status.StartTime != nil
+		return startTimeNewlySet || getHighestRestartCount(newPod.Status) > RestartThreshold
path: controllers/idler/predicate.go
user: login: metlos
body: Maybe have the `RestartThreshold` as a field of the predicate instead of using the global variable? That'll a) make the code slightly more local and therefore IMHO more readable and b) open it up to be user-configurable.
created_at: 2025-03-04T11:47:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,19 @@
+package idler
+
+import (
+	"context"
+
+	"k8s.io/apimachinery/pkg/types"
+	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+// MapPodToIdler maps the pod to the idler
+func MapPodToIdler(_ context.Context, obj runtimeclient.Object) []reconcile.Request {
+	return []reconcile.Request{{
+		// the idler should have the same name as the user's namespace
+		NamespacedName: types.NamespacedName{
+			Name: obj.GetNamespace(),
path: controllers/idler/mapper.go
user: login: MatousJobanek
body: the idler is a cluster-scoped resource, there is no namespace to be created :-)
We decided to define it as a cluster-scoped resource in order to prevent users from modifying it.
created_at: 2025-03-04T12:07:32Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,52 @@
+package idler
+
+import (
+	"github.com/codeready-toolchain/member-operator/pkg/webhook/mutatingwebhook"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	runtimeevent "sigs.k8s.io/controller-runtime/pkg/event"
+)
+
+type PodIdlerPredicate struct {
+}
+
+// Update triggers reconcile if the pod runs in users namespace
+// and if either the highest restart count is higher than the threshold
+// or the startTime was newly set in the new version of the pod
+func (p PodIdlerPredicate) Update(event runtimeevent.UpdateEvent) bool {
+	isUserPod, newPod := isUserPod(event.ObjectNew)
+	if !isUserPod {
+		return false
+	}
+	if oldPod, ok := event.ObjectOld.(*corev1.Pod); ok {
+		startTimeNewlySet := oldPod.Status.StartTime == nil && newPod.Status.StartTime != nil
+		return startTimeNewlySet || getHighestRestartCount(newPod.Status) > RestartThreshold
path: controllers/idler/predicate.go
user: login: MatousJobanek
body: it should use the same value as is used in idler_controller.go. There is nothing to be configurable for now, if we wanted to have it configurable, then it should go into the ToolchainConfig, but this is not really needed.
I unexported the constant to make it clear that it's not a global variable, but specific to the controller. 2001124d
created_at: 2025-03-04T12:11:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -248,7 +249,7 @@ func main() {
 		DynamicClient:       dynamicClient,
 		GetHostCluster:      cluster.GetHostCluster,
 		Namespace:           namespace,
-	}).SetupWithManager(mgr); err != nil {
+	}).SetupWithManager(mgr, allNamespacesCluster); err != nil {
path: main.go
user: login: rajivnathan
body: Just curious, do you know the difference between `allNamespacesCluster.GetCache()` and `allNamespacesCache`? I'm wondering if we have overlap between these clients and caches.
created_at: 2025-03-05T14:35:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -107,59 +111,68 @@ func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.
 		logger.Error(err, "failed to ensure idling")
 		return reconcile.Result{}, r.setStatusFailed(ctx, idler, err.Error())
 	}
-	if err := r.ensureIdling(ctx, idler); err != nil {
+	requeueAfter, err := r.ensureIdling(ctx, idler)
+	if err != nil {
 		return reconcile.Result{}, r.wrapErrorWithStatusUpdate(ctx, idler, r.setStatusFailed, err,
 			"failed to ensure idling '%s'", idler.Name)
 	}
-	// Requeue in shortest of the following values idler.Spec.TimeoutSeconds or RequeueTimeThreshold or nextPodToBeKilledAfter
-	after := findShortestRequeueDuration(idler)
-	logger.Info("requeueing for next pod to check", "after_seconds", after.Seconds())
-	return reconcile.Result{
-		Requeue:      true,
-		RequeueAfter: after,
-	}, r.setStatusReady(ctx, idler)
+	result := reconcile.Result{}
+	if len(idler.Status.Pods) > 0 { // schedule the next requeue only if something is being tracked
+		logger.Info("requeueing for next pod to check", "after_seconds", requeueAfter.Seconds())
+		result = reconcile.Result{
+			Requeue:      true,
+			RequeueAfter: requeueAfter,
+		}
+	} else {
+		// if nothing is being tracked then only wait for the next reconcile triggered
+		// by starting the pod
path: controllers/idler/idler_controller.go
user: login: rajivnathan
body: ```suggestion
		// if nothing is being tracked then only wait for the next reconcile triggered
		// by a pod starting
```
created_at: 2025-03-05T15:15:50Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,19 @@
+package idler
+
+import (
+	"context"
+
+	"k8s.io/apimachinery/pkg/types"
+	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+// MapPodToIdler maps the pod to the idler
+func MapPodToIdler(_ context.Context, obj runtimeclient.Object) []reconcile.Request {
+	return []reconcile.Request{{
+		// the idler should have the same name as the user's namespace
+		NamespacedName: types.NamespacedName{
+			Name: obj.GetNamespace(),
path: controllers/idler/mapper.go
user: login: metlos
body: 🤦🏼 👍🏼 
created_at: 2025-03-05T15:33:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -107,13 +107,13 @@ func TestEnsureIdling(t *testing.T) {
 
 	logf.SetLogger(zap.New(zap.UseDevMode(true)))
 
-	t.Run("No pods in namespace managed by idler", func(t *testing.T) {
+	t.Run("No pods in namespace managed by idler, idler timeout is longer than requeue-time-threshold", func(t *testing.T) {
path: controllers/idler/idler_controller_test.go
user: login: rajivnathan
body: Isn't `RequeueTimeThreshold` removed?
created_at: 2025-03-05T15:58:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -248,7 +249,7 @@ func main() {
 		DynamicClient:       dynamicClient,
 		GetHostCluster:      cluster.GetHostCluster,
 		Namespace:           namespace,
-	}).SetupWithManager(mgr); err != nil {
+	}).SetupWithManager(mgr, allNamespacesCluster); err != nil {
path: main.go
user: login: MatousJobanek
body: it's actually the same. 
https://github.com/codeready-toolchain/member-operator/blob/fc363451b7190fd597f62cb968f9d6bf74be5a35/main.go#L199
https://github.com/codeready-toolchain/member-operator/blob/fc363451b7190fd597f62cb968f9d6bf74be5a35/main.go#L338-L345
The code in the main.go could be a bit refactored to make it clear, but that's for different PR
created_at: 2025-03-06T12:54:25Z
author_association: CONTRIBUTOR
diff_hunk: @@ -107,13 +107,13 @@ func TestEnsureIdling(t *testing.T) {
 
 	logf.SetLogger(zap.New(zap.UseDevMode(true)))
 
-	t.Run("No pods in namespace managed by idler", func(t *testing.T) {
+	t.Run("No pods in namespace managed by idler, idler timeout is longer than requeue-time-threshold", func(t *testing.T) {
path: controllers/idler/idler_controller_test.go
user: login: MatousJobanek
body: of course :-) 
good catch, thanks,  fixed in 4e807fb0
created_at: 2025-03-06T12:57:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -248,7 +249,7 @@ func main() {
 		DynamicClient:       dynamicClient,
 		GetHostCluster:      cluster.GetHostCluster,
 		Namespace:           namespace,
-	}).SetupWithManager(mgr); err != nil {
+	}).SetupWithManager(mgr, allNamespacesCluster); err != nil {
path: main.go
user: login: MatousJobanek
body: As discussed in the call, I misunderstood the variables you were referring to. So yes, you are right, there is a duplication of the clients - addressed here: https://github.com/codeready-toolchain/member-operator/pull/635
Thanks for pointing it out :+1: 
created_at: 2025-03-06T15:04:43Z
author_association: CONTRIBUTOR

PR comments:
user: login: codecov[bot]
created_at: 2025-03-03T17:36:10Z
body: ## [Codecov](https://app.codecov.io/gh/codeready-toolchain/member-operator/pull/633?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=codeready-toolchain) Report
Attention: Patch coverage is `88.57143%` with `8 lines` in your changes missing coverage. Please review.
> Project coverage is 81.79%. Comparing base [(`fc36345`)](https://app.codecov.io/gh/codeready-toolchain/member-operator/commit/fc363451b7190fd597f62cb968f9d6bf74be5a35?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=codeready-toolchain) to head [(`4e807fb`)](https://app.codecov.io/gh/codeready-toolchain/member-operator/commit/4e807fb05530961fbc3851071e4241d769012b40?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=codeready-toolchain).
> Report is 1 commits behind head on master.

| [Files with missing lines](https://app.codecov.io/gh/codeready-toolchain/member-operator/pull/633?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=codeready-toolchain) | Patch % | Lines |
|---|---|---|
| [controllers/idler/idler\_controller.go](https://app.codecov.io/gh/codeready-toolchain/member-operator/pull/633?src=pr&el=tree&filepath=controllers%2Fidler%2Fidler_controller.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=codeready-toolchain#diff-Y29udHJvbGxlcnMvaWRsZXIvaWRsZXJfY29udHJvbGxlci5nbw==) | 90.90% | [4 Missing :warning: ](https://app.codecov.io/gh/codeready-toolchain/member-operator/pull/633?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=codeready-toolchain) |
| [controllers/idler/predicate.go](https://app.codecov.io/gh/codeready-toolchain/member-operator/pull/633?src=pr&el=tree&filepath=controllers%2Fidler%2Fpredicate.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=codeready-toolchain#diff-Y29udHJvbGxlcnMvaWRsZXIvcHJlZGljYXRlLmdv) | 78.94% | [3 Missing and 1 partial :warning: ](https://app.codecov.io/gh/codeready-toolchain/member-operator/pull/633?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=codeready-toolchain) |

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master     #633      +/-   ##
==========================================
+ Coverage   81.71%   81.79%   +0.07%     
==========================================
  Files          29       31       +2     
  Lines        3330     3350      +20     
==========================================
+ Hits         2721     2740      +19     
- Misses        459      460       +1     
  Partials      150      150              
```

| [Files with missing lines](https://app.codecov.io/gh/codeready-toolchain/member-operator/pull/633?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=codeready-toolchain) | Coverage Δ | |
|---|---|---|
| [controllers/idler/mapper.go](https://app.codecov.io/gh/codeready-toolchain/member-operator/pull/633?src=pr&el=tree&filepath=controllers%2Fidler%2Fmapper.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=codeready-toolchain#diff-Y29udHJvbGxlcnMvaWRsZXIvbWFwcGVyLmdv) | `100.00% <100.00%> (ø)` | |
| [pkg/webhook/mutatingwebhook/userpods\_mutate.go](https://app.codecov.io/gh/codeready-toolchain/member-operator/pull/633?src=pr&el=tree&filepath=pkg%2Fwebhook%2Fmutatingwebhook%2Fuserpods_mutate.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=codeready-toolchain#diff-cGtnL3dlYmhvb2svbXV0YXRpbmd3ZWJob29rL3VzZXJwb2RzX211dGF0ZS5nbw==) | `87.09% <ø> (ø)` | |
| [controllers/idler/idler\_controller.go](https://app.codecov.io/gh/codeready-toolchain/member-operator/pull/633?src=pr&el=tree&filepath=controllers%2Fidler%2Fidler_controller.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=codeready-toolchain#diff-Y29udHJvbGxlcnMvaWRsZXIvaWRsZXJfY29udHJvbGxlci5nbw==) | `93.76% <90.90%> (+0.49%)` | :arrow_up: |
| [controllers/idler/predicate.go](https://app.codecov.io/gh/codeready-toolchain/member-operator/pull/633?src=pr&el=tree&filepath=controllers%2Fidler%2Fpredicate.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=codeready-toolchain#diff-Y29udHJvbGxlcnMvaWRsZXIvcHJlZGljYXRlLmdv) | `78.94% <78.94%> (ø)` | |

</details>
<details><summary>🚀 New features to boost your workflow: </summary>

- ❄ [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE
user: login: MatousJobanek
created_at: 2025-03-04T09:28:37Z
body: > Sorry, could you please clarify the issue with shorter VM idling mentioned in point 2 of PR description? Was it not functioning as expected before, aside from the 5m requeue time being too aggressive causing issues?

yeah, you are actually right, with the extreme short requeue time it wasn't actually a big problem - the VM would eventually get killed within 5 minutes since the scheduled idle time. I was mainly looking at the logic where it calculated the requeue time from the list of tracked pods, and this logic didn't take into account the pods owned by VM. As a result, modifying the requeue thredhold time had indirectly a negative impact on the VM idling. 

> The current implementation checks restartCount against the threshold but doesn’t tightly couple requeueTime to it. Previously also the logic for restartCount wasn't tied to timeout, but shorter requeue time would almost instantly idle the pod once it reaches restartThreshold. For instance, reaching a restartCount of 50 takes about 4 hours, yet idling a crash-looping pod with requeueTimeThreshold set to 6 hours can take anywhere between 6 to 12 hours. As Alexey pointed out its still better than not idling crashlooping pods at all, but a requeueTimethreshold shorter than 6 hr should help too.

We need to find a balance between overloading the controller and our logs vs being able to proactively act on the changes in the cluster. The best way to act on the changes is using a watcher, so I replaced the approach of using the requeue threshold time with a watcher and a predicate.
author_association: CONTRIBUTOR
user: login: ranakan19
created_at: 2025-03-06T05:56:47Z
body: > I was mainly looking at the logic where it calculated the requeue time from the list of tracked pods, and this logic didn't take into account the pods owned by VM

Ah because only the timeoutsecond variable was updated within the tracledPods loop, while the requeuetime calculation relied on idler spec! I see now, thank you so much :) 
author_association: CONTRIBUTOR
user: login: sonarqubecloud[bot]
created_at: 2025-03-06T12:57:51Z
body: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=codeready-toolchain_member-operator&pullRequest=633) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [2 New issues](https://sonarcloud.io/project/issues?id=codeready-toolchain_member-operator&pullRequest=633&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [0 Accepted issues](https://sonarcloud.io/project/issues?id=codeready-toolchain_member-operator&pullRequest=633&issueStatuses=ACCEPTED)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=codeready-toolchain_member-operator&pullRequest=633&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Coverage on New Code](https://sonarcloud.io/component_measures?id=codeready-toolchain_member-operator&pullRequest=633&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=codeready-toolchain_member-operator&pullRequest=633&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarQube Cloud](https://sonarcloud.io/dashboard?id=codeready-toolchain_member-operator&pullRequest=633)


author_association: NONE
user: login: MatousJobanek
created_at: 2025-03-06T13:22:33Z
body: >  minor - the testIdlertimeoutseconds const in idler_controller_test can be a lower value now

it's a unit-test, it's just a number. TBH, it's sometimes even better to keep it higher to have a room for playing with the age of the workload and other numbers, so keeping it as it is
author_association: CONTRIBUTOR
user: login: openshift-ci[bot]
created_at: 2025-03-06T15:29:42Z
body: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/codeready-toolchain/member-operator/pull/633#pullrequestreview-2662254779" title="Approved">alexeykazakov</a>*, *<a href="https://github.com/codeready-toolchain/member-operator/pull/633#" title="Author self-approved">MatousJobanek</a>*, *<a href="https://github.com/codeready-toolchain/member-operator/pull/633#pullrequestreview-2661548557" title="Approved">metlos</a>*, *<a href="https://github.com/codeready-toolchain/member-operator/pull/633#pullrequestreview-2664812823" title="Approved">rajivnathan</a>*, *<a href="https://github.com/codeready-toolchain/member-operator/pull/633#pullrequestreview-2663418915" title="Approved">ranakan19</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=codeready-toolchain%2Fmember-operator).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/codeready-toolchain/member-operator/blob/master/OWNERS)~~ [MatousJobanek,alexeykazakov,metlos,rajivnathan,ranakan19]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
author_association: NONE