Repository: openshift-kni/numaresources-operator
Pull request #1216: OCPBUGS-51261:controller: track rte configmap state
Author: login: Tal-or
Labels: name: approved
description: Indicates a PR has been approved by an approver from all required OWNERS files.
name: lgtm
description: Indicates that a PR is ready to be merged.
Created: 2025-03-10T07:54:28Z | Merged: 2025-03-12T10:46:05Z

Kubelet controller is responsible to track `KubeletConfig` objects and
generate a `ConfigMap` data from them that will be served to RTE.

NROP controller is responsible to track the `ConfigMap` creation,
feed it to RTE and restart RTE in case the `ConfigMap` data has changed.

The latter is broken, and this PR is fixing it.

In addition the `ConfigMap` should be per `DaemonSet`/`NodeGroup`, while
previously it was common at the CR level which is wrong.

Because cluster might have multiple node groups which are configured
slightly different.

Signed-off-by: Talor Itzhak <titzhak@redhat.com>


Diff:
diff --git a/cmd/main.go b/cmd/main.go
index 025efc0b7b..e07ddf4e11 100644
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -458,9 +458,6 @@ func renderRTEManifests(rteManifests rtemanifests.Manifests, namespace string, i
 	if err != nil {
 		return mf, err
 	}
-	if mf.ConfigMap != nil {
-		rteupdate.DaemonSetHashAnnotation(mf.DaemonSet, hash.ConfigMapData(mf.ConfigMap))
-	}
 	return mf, err
 }
 
diff --git a/internal/controller/numaresourcesoperator_controller.go b/internal/controller/numaresourcesoperator_controller.go
index 16c75da0fc..4e0df5feec 100644
--- a/internal/controller/numaresourcesoperator_controller.go
+++ b/internal/controller/numaresourcesoperator_controller.go
@@ -58,7 +58,6 @@ import (
 	"github.com/openshift-kni/numaresources-operator/internal/dangling"
 	"github.com/openshift-kni/numaresources-operator/internal/relatedobjects"
 	"github.com/openshift-kni/numaresources-operator/pkg/apply"
-	"github.com/openshift-kni/numaresources-operator/pkg/hash"
 	"github.com/openshift-kni/numaresources-operator/pkg/images"
 	"github.com/openshift-kni/numaresources-operator/pkg/loglevel"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectnames"
@@ -535,15 +534,6 @@ func (r *NUMAResourcesOperatorReconciler) syncNUMAResourcesOperatorResources(ctx
 		return dsPoolPairs, err
 	}
 
-	// ConfigMap should be provided by the kubeletconfig reconciliation loop
-	if r.RTEManifests.Core.ConfigMap != nil {
-		cmHash, err := hash.ComputeCurrentConfigMap(ctx, r.Client, r.RTEManifests.Core.ConfigMap)
-		if err != nil {
-			return dsPoolPairs, err
-		}
-		rteupdate.DaemonSetHashAnnotation(r.RTEManifests.Core.DaemonSet, cmHash)
-	}
-
 	rteupdate.SecurityContextConstraint(r.RTEManifests.Core.SecurityContextConstraint, true) // force to legacy context
 	// SCC v2 needs no updates
 
@@ -753,6 +743,8 @@ func daemonsetUpdater(poolName string, gdm *rtestate.GeneratedDesiredManifest) e
 			klog.V(5).InfoS("DaemonSet update: selinux options", "container", manifests.ContainerNameRTE, "context", selinux.RTEContextType)
 		}
 	}
+	// it's possible that the hash will be empty if kubelet controller hasn't created a configmap
+	rteupdate.DaemonSetHashAnnotation(gdm.DaemonSet, gdm.RTEConfigHash)
 	return nil
 }
 
diff --git a/internal/controller/numaresourcesoperator_controller_test.go b/internal/controller/numaresourcesoperator_controller_test.go
index bdd8a517fd..4b7bd33186 100644
--- a/internal/controller/numaresourcesoperator_controller_test.go
+++ b/internal/controller/numaresourcesoperator_controller_test.go
@@ -54,6 +54,7 @@ import (
 	"github.com/openshift-kni/numaresources-operator/internal/api/annotations"
 	inthelper "github.com/openshift-kni/numaresources-operator/internal/api/annotations/helper"
 	testobjs "github.com/openshift-kni/numaresources-operator/internal/objects"
+	"github.com/openshift-kni/numaresources-operator/pkg/hash"
 	"github.com/openshift-kni/numaresources-operator/pkg/images"
 	rtemetricsmanifests "github.com/openshift-kni/numaresources-operator/pkg/metrics/manifests/monitor"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectnames"
@@ -191,6 +192,7 @@ var _ = Describe("Test NUMAResourcesOperator Reconcile", func() {
 				var mcp2 *machineconfigv1.MachineConfigPool
 				var mcp1Selector, mcp2Selector *metav1.LabelSelector
 				var nroKey client.ObjectKey
+				var cm1, cm2 *corev1.ConfigMap
 
 				var reconciler *NUMAResourcesOperatorReconciler
 				var ng1, ng2 nropv1.NodeGroup
@@ -224,8 +226,11 @@ var _ = Describe("Test NUMAResourcesOperator Reconcile", func() {
 					mcp1 = testobjs.NewMachineConfigPool(pn1, mcp1Selector.MatchLabels, mcp1Selector, mcp1Selector)
 					mcp2 = testobjs.NewMachineConfigPool(pn2, mcp2Selector.MatchLabels, mcp2Selector, mcp2Selector)
 
+					cm1 = testobjs.NewRTEConfigMap(objectnames.GetComponentName(nro.Name, mcp1.Name), testNamespace, "single-numa-node", "pod")
+					cm2 = testobjs.NewRTEConfigMap(objectnames.GetComponentName(nro.Name, mcp2.Name), testNamespace, "single-numa-node", "container")
+
 					var err error
-					reconciler, err = NewFakeNUMAResourcesOperatorReconciler(platform.OpenShift, defaultOCPVersion, nro, mcp1, mcp2)
+					reconciler, err = NewFakeNUMAResourcesOperatorReconciler(platform.OpenShift, defaultOCPVersion, nro, mcp1, mcp2, cm1, cm2)
 					Expect(err).ToNot(HaveOccurred())
 
 					// on the first iteration with the default RTE SELinux policy we expect immediate update, thus the reconciliation result is empty
@@ -239,19 +244,28 @@ var _ = Describe("Test NUMAResourcesOperator Reconcile", func() {
 				})
 
 				It("should create all CRDs and objects and operator status are updated from the first reconcile iteration", func() {
-					By("Check DaemonSets are created")
+					By("Check DaemonSet1 is created")
 					dsKey := client.ObjectKey{
 						Name:      objectnames.GetComponentName(nro.Name, pn1),
 						Namespace: testNamespace,
 					}
 					ds := &appsv1.DaemonSet{}
 					Expect(reconciler.Client.Get(context.TODO(), dsKey, ds)).ToNot(HaveOccurred())
+					By("Check DaemonSet1 has configmap hash")
+					annot, ok := ds.Spec.Template.Annotations[hash.ConfigMapAnnotation]
+					Expect(ok).To(BeTrue())
+					Expect(annot).To(BeEquivalentTo(hash.ConfigMapData(cm1)), "plain text data: %s", cm1.Data)
 
+					By("Check DaemonSet2 is created")
 					dsKey = client.ObjectKey{
 						Name:      objectnames.GetComponentName(nro.Name, pn2),
 						Namespace: testNamespace,
 					}
 					Expect(reconciler.Client.Get(context.TODO(), dsKey, ds)).To(Succeed())
+					By("Check DaemonSet2 has configmap hash")
+					annot, ok = ds.Spec.Template.Annotations[hash.ConfigMapAnnotation]
+					Expect(ok).To(BeTrue())
+					Expect(annot).To(BeEquivalentTo(hash.ConfigMapData(cm2)), "plain text data: %s", cm2.Data)
 
 					By("Check status is updated")
 					Expect(reconciler.Client.Get(context.TODO(), nroKey, nro)).ToNot(HaveOccurred())
diff --git a/internal/objects/objects.go b/internal/objects/objects.go
index ee8dbd0518..c3be9ed771 100644
--- a/internal/objects/objects.go
+++ b/internal/objects/objects.go
@@ -217,3 +217,21 @@ func GetDaemonSetListFromNodeGroupStatuses(groups []nropv1.NodeGroupStatus) []nr
 	}
 	return dss
 }
+
+// NewRTEConfigMap create a configmap similar to one created by KubeletController
+func NewRTEConfigMap(name, ns, policy, scope string) *corev1.ConfigMap {
+	data := fmt.Sprintf("kubelet:\n\t\ttopologyManagerPolicy: %s\ntopologyManagerScope: %s", policy, scope)
+	return &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "ConfigMap",
+			APIVersion: corev1.SchemeGroupVersion.String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: ns,
+		},
+		Data: map[string]string{
+			"config.yaml": data,
+		},
+	}
+}
diff --git a/pkg/hash/configmap.go b/pkg/hash/configmap.go
index 3484bd616f..053858af01 100644
--- a/pkg/hash/configmap.go
+++ b/pkg/hash/configmap.go
@@ -17,40 +17,19 @@
 package hash
 
 import (
-	"context"
 	"crypto/sha256"
 	"fmt"
 
 	corev1 "k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/klog/v2"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 // hash package purpose is to compute a ConfigMap hash
 // that will be attached as an annotation to workload resources (DaemonSet, Deployment, etc.)
-// in order to allow them to track ConfigMap changes
+//  to allow them to track ConfigMap changes
 // more about this technique here: https://blog.questionable.services/article/kubernetes-deployments-configmap-change/
 
 const ConfigMapAnnotation = "configmap.hash"
 
-func ComputeCurrentConfigMap(ctx context.Context, cli client.Reader, cm *corev1.ConfigMap) (string, error) {
-	updatedConfigMap := &corev1.ConfigMap{}
-	key := client.ObjectKeyFromObject(cm)
-
-	if err := cli.Get(ctx, key, updatedConfigMap); err != nil {
-		// ConfigMap not created yet, use the data from the manifests
-		if apierrors.IsNotFound(err) {
-			updatedConfigMap = cm
-		} else {
-			return "", fmt.Errorf("could not calculate ConfigMap %q hash: %w", key.String(), err)
-		}
-	}
-	cmHash := ConfigMapData(updatedConfigMap)
-	klog.InfoS("configmap hash calculated", "hash", cmHash)
-	return cmHash, nil
-}
-
 func ConfigMapData(cm *corev1.ConfigMap) string {
 	var dataAsString string
 
diff --git a/pkg/hash/configmap_test.go b/pkg/hash/configmap_test.go
index c719d049a6..0f49418169 100644
--- a/pkg/hash/configmap_test.go
+++ b/pkg/hash/configmap_test.go
@@ -17,50 +17,14 @@
 package hash
 
 import (
-	"context"
 	"regexp"
 	"strings"
 	"testing"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
-
-	"github.com/openshift-kni/numaresources-operator/internal/objects"
 )
 
-func TestComputeCurrentConfigMap(t *testing.T) {
-	testCases := []struct {
-		name        string
-		cli         client.Client
-		cm          *corev1.ConfigMap
-		expectedOut string
-		expectedErr bool
-	}{
-		{
-			name: "map not created",
-			cli:  fake.NewFakeClient(),
-			cm:   objects.NewKubeletConfigConfigMap("test", map[string]string{}, objects.NewKubeletConfigAutoresizeControlPlane()),
-			// verified manually
-			expectedOut: "SHA256:93909e569a15b6e4a5eefdcac4153f2c8179bf155143d10dac589f62ddcdf742",
-			expectedErr: false,
-		},
-	}
-
-	for _, tc := range testCases {
-		out, err := ComputeCurrentConfigMap(context.TODO(), tc.cli, tc.cm.DeepCopy())
-		gotErr := (err != nil)
-		if gotErr != tc.expectedErr {
-			t.Fatalf("got error %v expected error=%v", err, tc.expectedErr)
-		}
-		if out != tc.expectedOut {
-			t.Fatalf("got output %q expected %q", out, tc.expectedOut)
-		}
-	}
-}
-
 func TestConfigMapData(t *testing.T) {
 	cm := &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
diff --git a/pkg/objectstate/rte/machineconfigpool.go b/pkg/objectstate/rte/machineconfigpool.go
index 618319dbbd..40b4c95969 100644
--- a/pkg/objectstate/rte/machineconfigpool.go
+++ b/pkg/objectstate/rte/machineconfigpool.go
@@ -21,6 +21,7 @@ import (
 	"fmt"
 
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	machineconfigv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
@@ -28,6 +29,7 @@ import (
 	nropv1 "github.com/openshift-kni/numaresources-operator/api/v1"
 	nodegroupv1 "github.com/openshift-kni/numaresources-operator/api/v1/helper/nodegroup"
 	"github.com/openshift-kni/numaresources-operator/internal/api/annotations"
+	"github.com/openshift-kni/numaresources-operator/pkg/hash"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectnames"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectstate"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectstate/compare"
@@ -56,6 +58,12 @@ func (obj machineConfigPoolFinder) UpdateFromClient(ctx context.Context, cli cli
 		if dsm.daemonSetError = cli.Get(ctx, key, ds); dsm.daemonSetError == nil {
 			dsm.daemonSet = ds
 		}
+
+		cm := &corev1.ConfigMap{}
+		if err := cli.Get(ctx, key, cm); err == nil {
+			// we're storing the updated hash only in the case that kubelet controller created a configmap
+			dsm.rteConfigHash = hash.ConfigMapData(cm)
+		}
 		obj.em.daemonSets[generatedName] = dsm
 
 		mcName := objectnames.GetMachineConfigName(obj.instance.Name, mcp.Name)
@@ -76,12 +84,14 @@ func (obj machineConfigPoolFinder) FindState(mf Manifests, tree nodegroupv1.Tree
 	for _, mcp := range tree.MachineConfigPools {
 		var existingDs client.Object
 		var loadError error
+		var rteComputedConfigHash string
 
 		generatedName := objectnames.GetComponentName(obj.instance.Name, mcp.Name)
 		existingDaemonSet, ok := obj.em.daemonSets[generatedName]
 		if ok {
 			existingDs = existingDaemonSet.daemonSet
 			loadError = existingDaemonSet.daemonSetError
+			rteComputedConfigHash = existingDaemonSet.rteConfigHash
 		} else {
 			loadError = fmt.Errorf("failed to find daemon set %s/%s", mf.Core.DaemonSet.Namespace, mf.Core.DaemonSet.Name)
 		}
@@ -101,6 +111,7 @@ func (obj machineConfigPoolFinder) FindState(mf Manifests, tree nodegroupv1.Tree
 			MachineConfigPool:     mcp.DeepCopy(),
 			NodeGroup:             tree.NodeGroup.DeepCopy(),
 			DaemonSet:             desiredDaemonSet,
+			RTEConfigHash:         rteComputedConfigHash,
 			IsCustomPolicyEnabled: annotations.IsCustomPolicyEnabled(tree.NodeGroup.Annotations),
 		}
 
diff --git a/pkg/objectstate/rte/nodegroup.go b/pkg/objectstate/rte/nodegroup.go
index 902c8e6dff..01ff877d92 100644
--- a/pkg/objectstate/rte/nodegroup.go
+++ b/pkg/objectstate/rte/nodegroup.go
@@ -21,11 +21,13 @@ import (
 	"fmt"
 
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	nropv1 "github.com/openshift-kni/numaresources-operator/api/v1"
 	nodegroupv1 "github.com/openshift-kni/numaresources-operator/api/v1/helper/nodegroup"
 	"github.com/openshift-kni/numaresources-operator/internal/api/annotations"
+	"github.com/openshift-kni/numaresources-operator/pkg/hash"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectnames"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectstate"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectstate/compare"
@@ -53,6 +55,11 @@ func (obj nodeGroupFinder) UpdateFromClient(ctx context.Context, cli client.Clie
 	if dsm.daemonSetError = cli.Get(ctx, key, ds); dsm.daemonSetError == nil {
 		dsm.daemonSet = ds
 	}
+	cm := &corev1.ConfigMap{}
+	if err := cli.Get(ctx, key, cm); err == nil {
+		// we're storing the updated hash only in the case that kubelet controller created a configmap
+		dsm.rteConfigHash = hash.ConfigMapData(cm)
+	}
 	obj.em.daemonSets[generatedName] = dsm
 }
 
@@ -60,6 +67,7 @@ func (obj nodeGroupFinder) FindState(mf Manifests, tree nodegroupv1.Tree) []obje
 	var ret []objectstate.ObjectState
 	var existingDs client.Object
 	var loadError error
+	var rteConfigHash string
 
 	poolName := *tree.NodeGroup.PoolName
 
@@ -68,6 +76,7 @@ func (obj nodeGroupFinder) FindState(mf Manifests, tree nodegroupv1.Tree) []obje
 	if ok {
 		existingDs = existingDaemonSet.daemonSet
 		loadError = existingDaemonSet.daemonSetError
+		rteConfigHash = existingDaemonSet.rteConfigHash
 	} else {
 		loadError = fmt.Errorf("failed to find daemon set %s/%s", mf.Core.DaemonSet.Namespace, mf.Core.DaemonSet.Name)
 	}
@@ -85,6 +94,7 @@ func (obj nodeGroupFinder) FindState(mf Manifests, tree nodegroupv1.Tree) []obje
 		MachineConfigPool:     nil,
 		NodeGroup:             tree.NodeGroup.DeepCopy(),
 		DaemonSet:             desiredDaemonSet,
+		RTEConfigHash:         rteConfigHash,
 		IsCustomPolicyEnabled: annotations.IsCustomPolicyEnabled(tree.NodeGroup.Annotations),
 	}
 
diff --git a/pkg/objectstate/rte/rte.go b/pkg/objectstate/rte/rte.go
index c13a4f5fbe..b4bfb5b285 100644
--- a/pkg/objectstate/rte/rte.go
+++ b/pkg/objectstate/rte/rte.go
@@ -56,6 +56,7 @@ type rteHelper interface {
 type daemonSetManifest struct {
 	daemonSet      *appsv1.DaemonSet
 	daemonSetError error
+	rteConfigHash  string
 }
 
 type machineConfigManifest struct {
@@ -227,6 +228,8 @@ type GeneratedDesiredManifest struct {
 	NodeGroup         *nropv1.NodeGroup
 	// generated manifests
 	DaemonSet             *appsv1.DaemonSet
+	RTEConfigHash         string
+	ConfigMap             *corev1.ConfigMap
 	IsCustomPolicyEnabled bool
 }
 
diff --git a/pkg/objectupdate/rte/rte.go b/pkg/objectupdate/rte/rte.go
index 69cb12cecc..6a8fada768 100644
--- a/pkg/objectupdate/rte/rte.go
+++ b/pkg/objectupdate/rte/rte.go
@@ -127,6 +127,7 @@ func DaemonSetHashAnnotation(ds *appsv1.DaemonSet, cmHash string) {
 		template.Annotations = map[string]string{}
 	}
 	template.Annotations[hash.ConfigMapAnnotation] = cmHash
+	klog.V(4).InfoS("DaemonSet RTE ConfigMap hash annotation updated", "namespace", ds.Namespace, "DaemonSetName", ds.Name, "hash", cmHash)
 }
 
 const _MiB = 1024 * 1024


Reviews:
user: login: ffromani
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-10T08:38:01Z
user: login: ffromani
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-10T08:56:31Z
user: login: Tal-or
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-10T09:02:08Z
user: login: ffromani
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-10T09:12:30Z
user: login: ffromani
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-10T09:19:02Z
user: login: Tal-or
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-10T09:21:03Z
user: login: ffromani
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-10T13:13:25Z
user: login: ffromani
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-10T13:14:47Z
user: login: Tal-or
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-10T13:23:44Z
user: login: ffromani
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-10T13:25:07Z
user: login: ffromani
body: ok, last comment inline, but is worthy. If you add the logs (V>=4) we were talking about it's a nice bonus.
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-10T14:46:20Z
user: login: Tal-or
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-10T15:45:50Z
user: login: ffromani
body: the main problem here is we push updates in the state extraction code which was a bad idea in the first place back in time `daemonsetUpdater`. It should have been cleaner since the beginning.

Now's probably late to clean the board - we should wait 4.20 for it, if ever, so let's try to have a more targeted fix.
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-11T08:46:28Z
user: login: ffromani
body: something like this should fit more the existing (very debatable, see #1217) layout:
```
diff --git a/internal/controller/numaresourcesoperator_controller.go b/internal/controller/numaresourcesoperator_controller.go
index a10433bc..c44e8c21 100644
--- a/internal/controller/numaresourcesoperator_controller.go
+++ b/internal/controller/numaresourcesoperator_controller.go
@@ -758,6 +758,9 @@ func daemonsetUpdater(poolName string, gdm *rtestate.GeneratedDesiredManifest) e
 			klog.V(5).InfoS("DaemonSet update: selinux options", "container", manifests.ContainerNameRTE, "context", selinux.RTEContextType)
 		}
 	}
+
+	// set hash here
+
 	return nil
 }
 
diff --git a/pkg/objectstate/rte/machineconfigpool.go b/pkg/objectstate/rte/machineconfigpool.go
index 618319db..fa0a9140 100644
--- a/pkg/objectstate/rte/machineconfigpool.go
+++ b/pkg/objectstate/rte/machineconfigpool.go
@@ -75,12 +75,14 @@ func (obj machineConfigPoolFinder) FindState(mf Manifests, tree nodegroupv1.Tree
 	var ret []objectstate.ObjectState
 	for _, mcp := range tree.MachineConfigPools {
 		var existingDs client.Object
+		var rteConfigHash string
 		var loadError error
 
 		generatedName := objectnames.GetComponentName(obj.instance.Name, mcp.Name)
 		existingDaemonSet, ok := obj.em.daemonSets[generatedName]
 		if ok {
 			existingDs = existingDaemonSet.daemonSet
+			rteConfigHash = existingDaemonSet.rteComputedConfigHash
 			loadError = existingDaemonSet.daemonSetError
 		} else {
 			loadError = fmt.Errorf("failed to find daemon set %s/%s", mf.Core.DaemonSet.Namespace, mf.Core.DaemonSet.Name)
@@ -101,6 +103,7 @@ func (obj machineConfigPoolFinder) FindState(mf Manifests, tree nodegroupv1.Tree
 			MachineConfigPool:     mcp.DeepCopy(),
 			NodeGroup:             tree.NodeGroup.DeepCopy(),
 			DaemonSet:             desiredDaemonSet,
+			RTEComputedConfigHash: rteConfigHash,
 			IsCustomPolicyEnabled: annotations.IsCustomPolicyEnabled(tree.NodeGroup.Annotations),
 		}
 
diff --git a/pkg/objectstate/rte/nodegroup.go b/pkg/objectstate/rte/nodegroup.go
index 902c8e6d..c4adfff6 100644
--- a/pkg/objectstate/rte/nodegroup.go
+++ b/pkg/objectstate/rte/nodegroup.go
@@ -59,6 +59,7 @@ func (obj nodeGroupFinder) UpdateFromClient(ctx context.Context, cli client.Clie
 func (obj nodeGroupFinder) FindState(mf Manifests, tree nodegroupv1.Tree) []objectstate.ObjectState {
 	var ret []objectstate.ObjectState
 	var existingDs client.Object
+	var rteConfigHash string
 	var loadError error
 
 	poolName := *tree.NodeGroup.PoolName
@@ -67,6 +68,7 @@ func (obj nodeGroupFinder) FindState(mf Manifests, tree nodegroupv1.Tree) []obje
 	existingDaemonSet, ok := obj.em.daemonSets[generatedName]
 	if ok {
 		existingDs = existingDaemonSet.daemonSet
+		rteConfigHash = existingDaemonSet.rteComputedConfigHash
 		loadError = existingDaemonSet.daemonSetError
 	} else {
 		loadError = fmt.Errorf("failed to find daemon set %s/%s", mf.Core.DaemonSet.Namespace, mf.Core.DaemonSet.Name)
@@ -85,6 +87,7 @@ func (obj nodeGroupFinder) FindState(mf Manifests, tree nodegroupv1.Tree) []obje
 		MachineConfigPool:     nil,
 		NodeGroup:             tree.NodeGroup.DeepCopy(),
 		DaemonSet:             desiredDaemonSet,
+		RTEComputedConfigHash: rteConfigHash,
 		IsCustomPolicyEnabled: annotations.IsCustomPolicyEnabled(tree.NodeGroup.Annotations),
 	}
 
diff --git a/pkg/objectstate/rte/rte.go b/pkg/objectstate/rte/rte.go
index c13a4f5f..43df352e 100644
--- a/pkg/objectstate/rte/rte.go
+++ b/pkg/objectstate/rte/rte.go
@@ -54,8 +54,9 @@ type rteHelper interface {
 }
 
 type daemonSetManifest struct {
-	daemonSet      *appsv1.DaemonSet
-	daemonSetError error
+	daemonSet             *appsv1.DaemonSet
+	daemonSetError        error
+	rteComputedConfigHash string
 }
 
 type machineConfigManifest struct {
@@ -227,6 +228,7 @@ type GeneratedDesiredManifest struct {
 	NodeGroup         *nropv1.NodeGroup
 	// generated manifests
 	DaemonSet             *appsv1.DaemonSet
+	RTEComputedConfigHash string
 	IsCustomPolicyEnabled bool
 }
 ```
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-11T09:18:11Z
user: login: ffromani
body: /approve
/lgtm

thanks!
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-12T10:28:07Z

Review comments:
diff_hunk: @@ -407,3 +416,20 @@ func getObject(ctx context.Context, cli client.Client, key client.ObjectKey, obj
 	*err = cli.Get(ctx, key, obj)
 	return *err == nil
 }
+
+func findConfigMapState(name, namespace string, configMaps map[string]configMapManifest) *corev1.ConfigMap {
+	var desiredConfigMap *corev1.ConfigMap
+	existingConfigMap, ok := configMaps[name]
+	errFormat := fmt.Sprintf("failed to find config map %s/%s", namespace, name)
path: pkg/objectstate/rte/rte.go
user: login: ffromani
body: it seems we can log this in few flows, but all the flows should be mutually exclusive. So I'd avoid this and inline the formatting in the logging sites.
created_at: 2025-03-10T08:38:01Z
author_association: MEMBER
diff_hunk: @@ -753,6 +744,11 @@ func daemonsetUpdater(poolName string, gdm *rtestate.GeneratedDesiredManifest) e
 			klog.V(5).InfoS("DaemonSet update: selinux options", "container", manifests.ContainerNameRTE, "context", selinux.RTEContextType)
 		}
 	}
+	if gdm.ConfigMap != nil {
path: internal/controller/numaresourcesoperator_controller.go
user: login: ffromani
body: on which cases we do expect `gdm.ConfigMap` to be nil?
created_at: 2025-03-10T08:56:31Z
author_association: MEMBER
diff_hunk: @@ -753,6 +744,11 @@ func daemonsetUpdater(poolName string, gdm *rtestate.GeneratedDesiredManifest) e
 			klog.V(5).InfoS("DaemonSet update: selinux options", "container", manifests.ContainerNameRTE, "context", selinux.RTEContextType)
 		}
 	}
+	if gdm.ConfigMap != nil {
path: internal/controller/numaresourcesoperator_controller.go
user: login: Tal-or
body: In case that NROP installed on the cluster, but kubelet object has not been created just yet.
In the case kubelet controller will not create the configmap needed for RTE
created_at: 2025-03-10T09:02:08Z
author_association: COLLABORATOR
diff_hunk: @@ -753,6 +744,11 @@ func daemonsetUpdater(poolName string, gdm *rtestate.GeneratedDesiredManifest) e
 			klog.V(5).InfoS("DaemonSet update: selinux options", "container", manifests.ContainerNameRTE, "context", selinux.RTEContextType)
 		}
 	}
+	if gdm.ConfigMap != nil {
path: internal/controller/numaresourcesoperator_controller.go
user: login: ffromani
body: ok, makes sense
created_at: 2025-03-10T09:12:30Z
author_association: MEMBER
diff_hunk: @@ -80,11 +89,27 @@ func (obj nodeGroupFinder) FindState(mf Manifests, tree nodegroupv1.Tree) []obje
 		HyperShiftNodePoolLabel: poolName,
 	}
 
+	var desiredConfigMap *corev1.ConfigMap
+	existingConfigMap, ok := obj.em.configMaps[generatedName]
+	errFormat := fmt.Errorf("failed to find config map %s/%s", mf.Core.DaemonSet.Namespace, generatedName)
+	// a case where kubelet controller is not creating the ConfigMap, is undesired but yet possible.
+	// this is why the operator should log here
+	// and not bubble up the error that would interfere with its normal operation.
+	switch {
+	case !ok:
+		klog.ErrorS(errFormat, "")
+	case existingConfigMap.configMapError != nil:
+		klog.ErrorS(existingConfigMap.configMapError, errFormat.Error())
+	default:
+		desiredConfigMap = existingConfigMap.configMap
path: pkg/objectstate/rte/nodegroup.go
user: login: ffromani
body: do we need a DeepCopy here?
created_at: 2025-03-10T09:17:31Z
author_association: MEMBER
diff_hunk: @@ -96,11 +105,27 @@ func (obj machineConfigPoolFinder) FindState(mf Manifests, tree nodegroupv1.Tree
 			updateError = fmt.Errorf("the machine config pool %q does not have node selector", mcp.Name)
 		}
 
+		var desiredConfigMap *corev1.ConfigMap
+		existingConfigMap, ok := obj.em.configMaps[generatedName]
+		errFormat := fmt.Errorf("failed to find config map %s/%s", mf.Core.DaemonSet.Namespace, generatedName)
path: pkg/objectstate/rte/machineconfigpool.go
user: login: ffromani
body: please inline in the calling site
created_at: 2025-03-10T09:18:19Z
author_association: MEMBER
diff_hunk: @@ -80,11 +89,27 @@ func (obj nodeGroupFinder) FindState(mf Manifests, tree nodegroupv1.Tree) []obje
 		HyperShiftNodePoolLabel: poolName,
 	}
 
+	var desiredConfigMap *corev1.ConfigMap
+	existingConfigMap, ok := obj.em.configMaps[generatedName]
+	errFormat := fmt.Errorf("failed to find config map %s/%s", mf.Core.DaemonSet.Namespace, generatedName)
path: pkg/objectstate/rte/nodegroup.go
user: login: ffromani
body: please inline in the calling site
created_at: 2025-03-10T09:18:54Z
author_association: MEMBER
diff_hunk: @@ -80,11 +89,27 @@ func (obj nodeGroupFinder) FindState(mf Manifests, tree nodegroupv1.Tree) []obje
 		HyperShiftNodePoolLabel: poolName,
 	}
 
+	var desiredConfigMap *corev1.ConfigMap
+	existingConfigMap, ok := obj.em.configMaps[generatedName]
+	errFormat := fmt.Errorf("failed to find config map %s/%s", mf.Core.DaemonSet.Namespace, generatedName)
+	// a case where kubelet controller is not creating the ConfigMap, is undesired but yet possible.
+	// this is why the operator should log here
+	// and not bubble up the error that would interfere with its normal operation.
+	switch {
+	case !ok:
+		klog.ErrorS(errFormat, "")
+	case existingConfigMap.configMapError != nil:
+		klog.ErrorS(existingConfigMap.configMapError, errFormat.Error())
+	default:
+		desiredConfigMap = existingConfigMap.configMap
path: pkg/objectstate/rte/nodegroup.go
user: login: Tal-or
body: No. We're not mutating the config map.
All the controller is doing is exercise a hash function on the data
created_at: 2025-03-10T09:21:03Z
author_association: COLLABORATOR
diff_hunk: @@ -74,6 +82,8 @@ func (obj nodeGroupFinder) FindState(mf Manifests, tree nodegroupv1.Tree) []obje
 
 	desiredDaemonSet := mf.Core.DaemonSet.DeepCopy()
 	desiredDaemonSet.Name = generatedName
+	// it's possible that the hash will be empty if kubelet controller hasn't created a configmap
+	rteupdate.DaemonSetHashAnnotation(desiredDaemonSet, existingDaemonSet.rteComputedConfigHash)
path: pkg/objectstate/rte/nodegroup.go
user: login: ffromani
body: I'm thinking aloud. Should we set it to "" (empty string) or should we set it only if `!= ""`? What are the pros and cons in both scenarios?
created_at: 2025-03-10T13:13:25Z
author_association: MEMBER
diff_hunk: @@ -74,6 +82,8 @@ func (obj nodeGroupFinder) FindState(mf Manifests, tree nodegroupv1.Tree) []obje
 
 	desiredDaemonSet := mf.Core.DaemonSet.DeepCopy()
 	desiredDaemonSet.Name = generatedName
+	// it's possible that the hash will be empty if kubelet controller hasn't created a configmap
+	rteupdate.DaemonSetHashAnnotation(desiredDaemonSet, existingDaemonSet.rteComputedConfigHash)
path: pkg/objectstate/rte/nodegroup.go
user: login: ffromani
body: do we log when we change annotation? we probably should (V>=4)
created_at: 2025-03-10T13:14:47Z
author_association: MEMBER
diff_hunk: @@ -74,6 +82,8 @@ func (obj nodeGroupFinder) FindState(mf Manifests, tree nodegroupv1.Tree) []obje
 
 	desiredDaemonSet := mf.Core.DaemonSet.DeepCopy()
 	desiredDaemonSet.Name = generatedName
+	// it's possible that the hash will be empty if kubelet controller hasn't created a configmap
+	rteupdate.DaemonSetHashAnnotation(desiredDaemonSet, existingDaemonSet.rteComputedConfigHash)
path: pkg/objectstate/rte/nodegroup.go
user: login: Tal-or
body: If kubeletconfig has been completely deleted from the cluster, we should be able to signal that correctly.
setting the annotation value to `""` gives us what we need.
It will cause RTE pods to restart and they'll reflect the right behavior as in the case of no kubeletconfig configuration at all.
created_at: 2025-03-10T13:23:44Z
author_association: COLLABORATOR
diff_hunk: @@ -74,6 +82,8 @@ func (obj nodeGroupFinder) FindState(mf Manifests, tree nodegroupv1.Tree) []obje
 
 	desiredDaemonSet := mf.Core.DaemonSet.DeepCopy()
 	desiredDaemonSet.Name = generatedName
+	// it's possible that the hash will be empty if kubelet controller hasn't created a configmap
+	rteupdate.DaemonSetHashAnnotation(desiredDaemonSet, existingDaemonSet.rteComputedConfigHash)
path: pkg/objectstate/rte/nodegroup.go
user: login: ffromani
body: good point, it's worthy to keep it. Please do *NOT* resubmit just to add logs
created_at: 2025-03-10T13:25:07Z
author_association: MEMBER
diff_hunk: @@ -21,17 +21,20 @@ import (
 	"fmt"
 
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	machineconfigv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
 
 	nropv1 "github.com/openshift-kni/numaresources-operator/api/v1"
 	nodegroupv1 "github.com/openshift-kni/numaresources-operator/api/v1/helper/nodegroup"
 	"github.com/openshift-kni/numaresources-operator/internal/api/annotations"
+	"github.com/openshift-kni/numaresources-operator/pkg/hash"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectnames"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectstate"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectstate/compare"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectstate/merge"
+	rteupdate "github.com/openshift-kni/numaresources-operator/pkg/objectupdate/rte"
path: pkg/objectstate/rte/machineconfigpool.go
user: login: ffromani
body: ok, this is the only remaining item (extra logging is a nice bonus but not required).
Conceptually, `objectstate` sits below in the stack with respect to `objectupdate`:

```
controller
objectupdate
objectstate
client
```
IOW, objectstate can't import objectupdate, but objectupdate can call objectstate (controller can and do import them directly).
We can bypass layers from up to down, but we should never break layering from the bottom to the up
created_at: 2025-03-10T14:45:38Z
author_association: MEMBER
diff_hunk: @@ -21,17 +21,20 @@ import (
 	"fmt"
 
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	machineconfigv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
 
 	nropv1 "github.com/openshift-kni/numaresources-operator/api/v1"
 	nodegroupv1 "github.com/openshift-kni/numaresources-operator/api/v1/helper/nodegroup"
 	"github.com/openshift-kni/numaresources-operator/internal/api/annotations"
+	"github.com/openshift-kni/numaresources-operator/pkg/hash"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectnames"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectstate"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectstate/compare"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectstate/merge"
+	rteupdate "github.com/openshift-kni/numaresources-operator/pkg/objectupdate/rte"
path: pkg/objectstate/rte/machineconfigpool.go
user: login: Tal-or
body: I completely see your point and that's why I preferred passing the configmap, because it is more align with current flow. Anyway I'll fix that, I'll find a way to update the ds later in the code 
created_at: 2025-03-10T15:45:50Z
author_association: COLLABORATOR
diff_hunk: @@ -21,17 +21,20 @@ import (
 	"fmt"
 
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	machineconfigv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
 
 	nropv1 "github.com/openshift-kni/numaresources-operator/api/v1"
 	nodegroupv1 "github.com/openshift-kni/numaresources-operator/api/v1/helper/nodegroup"
 	"github.com/openshift-kni/numaresources-operator/internal/api/annotations"
+	"github.com/openshift-kni/numaresources-operator/pkg/hash"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectnames"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectstate"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectstate/compare"
 	"github.com/openshift-kni/numaresources-operator/pkg/objectstate/merge"
+	rteupdate "github.com/openshift-kni/numaresources-operator/pkg/objectupdate/rte"
path: pkg/objectstate/rte/machineconfigpool.go
user: login: ffromani
body: I may be wrong but  I think there should be a reasonnably easy way to not break the layering and yet only carry to hash around
created_at: 2025-03-11T08:32:16Z
author_association: MEMBER

PR comments:
user: login: ffromani
created_at: 2025-03-10T08:22:57Z
body: conceptually LGTM. Please fix CI errors
author_association: MEMBER
user: login: openshift-ci-robot
created_at: 2025-03-10T09:13:06Z
body: @Tal-or: This pull request references [Jira Issue OCPBUGS-51261](https://issues.redhat.com//browse/OCPBUGS-51261), which is invalid:
 - expected the bug to target the "4.19.0" version, but no target version was set

Comment <code>/jira refresh</code> to re-evaluate validity if changes to the Jira bug are made, or edit the title of this pull request to link to a different bug.

The bug has been updated to refer to the pull request using the external bug tracker.

<details>

In response to [this](https://github.com/openshift-kni/numaresources-operator/pull/1216):

>Kubelet controller is responsible to track `KubeletConfig` objects and
>generate a `ConfigMap` data from them that will be served to RTE.
>
>NROP controller is responsible to track the `ConfigMap` creation,
>feed it to RTE and restart RTE in case the `ConfigMap` data has changed.
>
>The latter is broken, and this PR is fixing it.
>
>In addition the `ConfigMap` should be per `DaemonSet`/`NodeGroup`, while
>previously it was common at the CR level which is wrong.
>
>Because cluster might have multiple node groups which are configured
>slightly different.
>
>Signed-off-by: Talor Itzhak <titzhak@redhat.com>


Instructions for interacting with me using PR comments are available [here](https://prow.ci.openshift.org/command-help?repo=openshift-kni%2Fnumaresources-operator).  If you have questions or suggestions related to my behavior, please file an issue against the [openshift-eng/jira-lifecycle-plugin](https://github.com/openshift-eng/jira-lifecycle-plugin/issues/new) repository.
</details>
author_association: NONE
user: login: ffromani
created_at: 2025-03-10T09:19:34Z
body: this work should probably be backported down to 4.16
author_association: MEMBER
user: login: Tal-or
created_at: 2025-03-10T10:55:11Z
body: /cherry-pick release-4.18
author_association: COLLABORATOR
user: login: openshift-cherrypick-robot
created_at: 2025-03-10T10:55:14Z
body: @Tal-or: once the present PR merges, I will cherry-pick it on top of `release-4.18` in a new PR and assign it to you.

<details>

In response to [this](https://github.com/openshift-kni/numaresources-operator/pull/1216#issuecomment-2710179067):

>/cherry-pick release-4.18


Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>
author_association: NONE
user: login: ffromani
created_at: 2025-03-10T11:14:27Z
body: so `ConfigMap` is already an exception in the context of this controller. We don't own it, we don't reconcile it, we "just" need to read it. So, idea: why don't we read it _and_ store its hash in the internal data structures vs storing the object and computing when needed? this
1. will further reinforce the exceptional treatment
2. will remove any ambiguity about ownerhip, mutation or not
author_association: MEMBER
user: login: Tal-or
created_at: 2025-03-10T12:00:54Z
body: > so `ConfigMap` is already an exception in the context of this controller. We don't own it, we don't reconcile it, we "just" need to read it. So, idea: why don't we read it _and_ store its hash in the internal data structures vs storing the object and computing when needed? this
> 
> 1. will further reinforce the exceptional treatment
> 2. will remove any ambiguity about ownerhip, mutation or not

Sure this is an option. I used the above approach simply because it was aligned with the existing architecture
author_association: COLLABORATOR
user: login: ffromani
created_at: 2025-03-10T12:43:35Z
body: > > so `ConfigMap` is already an exception in the context of this controller. We don't own it, we don't reconcile it, we "just" need to read it. So, idea: why don't we read it _and_ store its hash in the internal data structures vs storing the object and computing when needed? this
> > 
> > 1. will further reinforce the exceptional treatment
> > 2. will remove any ambiguity about ownerhip, mutation or not
> 
> Sure this is an option. I used the above approach simply because it was aligned with the existing architecture

thanks, looks better IMO and almost ready to go (gonna review shortly).
My general rule is: code which behaves the same (or kinda the same) should look the same (or kinda the same). Code which behaves different can and should look different. So in this case looking different is a feature IMO.
author_association: MEMBER
user: login: ffromani
created_at: 2025-03-12T07:49:25Z
body: /retest
author_association: MEMBER
user: login: openshift-ci[bot]
created_at: 2025-03-12T10:28:14Z
body: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/openshift-kni/numaresources-operator/pull/1216#pullrequestreview-2677886000" title="LGTM">ffromani</a>*, *<a href="https://github.com/openshift-kni/numaresources-operator/pull/1216#" title="Author self-approved">Tal-or</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=openshift-kni%2Fnumaresources-operator).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/openshift-kni/numaresources-operator/blob/main/OWNERS)~~ [Tal-or,ffromani]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
author_association: CONTRIBUTOR
user: login: openshift-ci-robot
created_at: 2025-03-12T10:46:10Z
body: @Tal-or: [Jira Issue OCPBUGS-51261](https://issues.redhat.com//browse/OCPBUGS-51261): All pull requests linked via external trackers have merged:
 * [openshift-kni/numaresources-operator#1216](https://github.com/openshift-kni/numaresources-operator/pull/1216)

[Jira Issue OCPBUGS-51261](https://issues.redhat.com//browse/OCPBUGS-51261) has been moved to the MODIFIED state.

<details>

In response to [this](https://github.com/openshift-kni/numaresources-operator/pull/1216):

>Kubelet controller is responsible to track `KubeletConfig` objects and
>generate a `ConfigMap` data from them that will be served to RTE.
>
>NROP controller is responsible to track the `ConfigMap` creation,
>feed it to RTE and restart RTE in case the `ConfigMap` data has changed.
>
>The latter is broken, and this PR is fixing it.
>
>In addition the `ConfigMap` should be per `DaemonSet`/`NodeGroup`, while
>previously it was common at the CR level which is wrong.
>
>Because cluster might have multiple node groups which are configured
>slightly different.
>
>Signed-off-by: Talor Itzhak <titzhak@redhat.com>


Instructions for interacting with me using PR comments are available [here](https://prow.ci.openshift.org/command-help?repo=openshift-kni%2Fnumaresources-operator).  If you have questions or suggestions related to my behavior, please file an issue against the [openshift-eng/jira-lifecycle-plugin](https://github.com/openshift-eng/jira-lifecycle-plugin/issues/new) repository.
</details>
author_association: NONE
user: login: openshift-cherrypick-robot
created_at: 2025-03-12T10:46:48Z
body: @Tal-or: new pull request created: #1225

<details>

In response to [this](https://github.com/openshift-kni/numaresources-operator/pull/1216#issuecomment-2710179067):

>/cherry-pick release-4.18


Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes-sigs/prow](https://github.com/kubernetes-sigs/prow/issues/new?title=Prow%20issue:) repository.
</details>
author_association: NONE