Repository: koordinator-sh/koordinator

Issue:
[proposal] support init containers in runtime hooks
<!-- Please only use this template for submitting proposal -->

**What is your proposal**:
- [ ] batch-resource
- [ ] coresched
- [ ] cpu-normalization
- [x] cpusetAllocator

- [ ] exclude init containers which resorucePolicy!=Always

**Why is this needed**:
init containers has not supported in these plugins.

**Is there a suggested solution, if so, please add it**:
implementation ref of cpusetAllocator: https://github.com/koordinator-sh/koordinator/pull/2349/files

Hints from issue comments:
@stale: This issue has been automatically marked as stale because it has not had recent activity.
This bot triages issues and PRs according to the following rules:
- After 90d of inactivity, `lifecycle/stale` is applied
- After 30d of inactivity since `lifecycle/stale` was applied, the issue is closed
You can:
- Mark this issue or PR as fresh with `/remove-lifecycle stale`
- Close this issue or PR with `/close`
Thank you for your contributions.
@stale: This issue has been automatically closed because it has not had recent activity.
This bot triages issues and PRs according to the following rules:
- After 90d of inactivity, `lifecycle/stale` is applied
- After 30d of inactivity since `lifecycle/stale` was applied, the issue is closed
You can:
- Reopen this PR with `/reopen`
Thank you for your contributions.


Solution patch:
diff --git a/pkg/koordlet/runtimehooks/hooks/cpuset/rule.go b/pkg/koordlet/runtimehooks/hooks/cpuset/rule.go
index 9bdf2176..5ad4a1c7 100644
--- a/pkg/koordlet/runtimehooks/hooks/cpuset/rule.go
+++ b/pkg/koordlet/runtimehooks/hooks/cpuset/rule.go
@@ -208,7 +208,11 @@ func (p *cpusetPlugin) ruleUpdateCb(target *statesinformer.CallbackTarget) error
 		return nil
 	}
 	for _, podMeta := range target.Pods {
-		for _, containerStat := range podMeta.Pod.Status.ContainerStatuses {
+		allContainerStatus := make([]corev1.ContainerStatus, 0, len(podMeta.Pod.Status.ContainerStatuses)+len(podMeta.Pod.Status.InitContainerStatuses))
+		allContainerStatus = append(allContainerStatus, podMeta.Pod.Status.ContainerStatuses...)
+		allContainerStatus = append(allContainerStatus, podMeta.Pod.Status.InitContainerStatuses...)
+		for _, containerStat := range allContainerStatus {
+			// TODO exclude some init containers, e.g. restartPolicy != Always
 			containerCtx := &protocol.ContainerContext{}
 			containerCtx.FromReconciler(podMeta, containerStat.Name, false)
 			if err := p.SetContainerCPUSet(containerCtx); err != nil {
diff --git a/pkg/koordlet/runtimehooks/protocol/container_context.go b/pkg/koordlet/runtimehooks/protocol/container_context.go
index e9715c2d..91e082b3 100644
--- a/pkg/koordlet/runtimehooks/protocol/container_context.go
+++ b/pkg/koordlet/runtimehooks/protocol/container_context.go
@@ -128,27 +128,55 @@ func (c *ContainerRequest) FromReconciler(podMeta *statesinformer.PodMeta, conta
 	c.PodMeta.FromReconciler(podMeta.Pod.ObjectMeta)
 	c.ContainerMeta.Name = containerName
 	c.ContainerMeta.Sandbox = sandbox
+	c.PodLabels = podMeta.Pod.Labels
+	c.PodAnnotations = podMeta.Pod.Annotations
+
 	if sandbox {
 		var err error
 		if c.ContainerMeta.ID, err = koordletutil.GetPodSandboxContainerID(podMeta.Pod); err != nil {
 			klog.V(4).Infof("failed to get sandbox container ID for pod %s, err: %s",
 				podMeta.Key(), err)
-			return
 		} else if c.ContainerMeta.ID == "" {
 			klog.V(4).Infof("container ID is empty for pod %s, pod may not start, skip", podMeta.Key())
-			return
+		} else {
+			c.CgroupParent, _ = koordletutil.GetContainerCgroupParentDirByID(podMeta.CgroupDir, c.ContainerMeta.ID)
+			klog.V(5).Infof("got sandbox %s cgroup parent %s for pod %s", c.ContainerMeta.ID, c.CgroupParent, podMeta.Key())
 		}
-	} else {
-		for _, containerStat := range podMeta.Pod.Status.ContainerStatuses {
-			if containerStat.Name == containerName {
-				c.ContainerMeta.ID = containerStat.ContainerID
+		return
+	}
+
+	foundInContainer := false
+	for _, containerStat := range podMeta.Pod.Status.ContainerStatuses {
+		if containerStat.Name == containerName {
+			c.ContainerMeta.ID = containerStat.ContainerID
+			foundInContainer = true
+			break
+		}
+	}
+	if !foundInContainer {
+		for _, initContainerStat := range podMeta.Pod.Status.InitContainerStatuses {
+			if initContainerStat.Name == containerName {
+				c.ContainerMeta.ID = initContainerStat.ContainerID
 				break
 			}
 		}
 	}
+	if c.ContainerMeta.ID == "" {
+		klog.V(4).Infof("container ID not found in container and init container status for pod %s, pod may not start, skip", podMeta.Key())
+		return
+	}
+	c.CgroupParent, _ = koordletutil.GetContainerCgroupParentDirByID(podMeta.CgroupDir, c.ContainerMeta.ID)
+
+	var containerCandidates []corev1.Container
+	if foundInContainer {
+		containerCandidates = podMeta.Pod.Spec.Containers
+	} else {
+		containerCandidates = podMeta.Pod.Spec.InitContainers
+	}
+
 	var specFromContainer *apiext.ExtendedResourceContainerSpec
-	for i := range podMeta.Pod.Spec.Containers {
-		containerSpec := podMeta.Pod.Spec.Containers[i]
+	for i := range containerCandidates {
+		containerSpec := containerCandidates[i]
 		if containerSpec.Name == containerName {
 			if c.ContainerEnvs == nil {
 				c.ContainerEnvs = map[string]string{}
@@ -162,9 +190,7 @@ func (c *ContainerRequest) FromReconciler(podMeta *statesinformer.PodMeta, conta
 			break
 		}
 	}
-	c.PodLabels = podMeta.Pod.Labels
-	c.PodAnnotations = podMeta.Pod.Annotations
-	c.CgroupParent, _ = koordletutil.GetContainerCgroupParentDirByID(podMeta.CgroupDir, c.ContainerMeta.ID)
+
 	// retrieve ExtendedResources from container spec and pod annotations (prefer container spec)
 	specFromAnnotations, err := apiext.GetExtendedResourceSpec(podMeta.Pod.Annotations)
 	if err != nil {
diff --git a/pkg/koordlet/runtimehooks/reconciler/reconciler.go b/pkg/koordlet/runtimehooks/reconciler/reconciler.go
index e7fc1a26..46fc562f 100644
--- a/pkg/koordlet/runtimehooks/reconciler/reconciler.go
+++ b/pkg/koordlet/runtimehooks/reconciler/reconciler.go
@@ -484,6 +484,32 @@ func (c *reconciler) reconcilePodCgroup(stopCh <-chan struct{}) {
 					}
 				}
 
+				for _, initContainerStat := range podMeta.Pod.Status.InitContainerStatuses {
+					// TODO exclude some init containers, e.g. restartPolicy != Always
+					for resourceType, r := range globalCgroupReconcilers.containerLevel {
+						condition := r.filter.Filter(podMeta)
+						reconcileFn, ok := r.fn[condition]
+						if !ok {
+							klog.V(5).Infof("calling reconcile function %v aborted for init container %v/%v, condition %s not registered",
+								r.description[condition], podMeta.Key(), initContainerStat.Name, condition)
+							continue
+						}
+
+						containerCtx := protocol.HooksProtocolBuilder.Container(podMeta, initContainerStat.Name)
+						start := time.Now()
+						if err := reconcileFn(containerCtx); err != nil {
+							metrics.RecordRuntimeHookReconcilerInvokedDurationMilliSeconds(string(ContainerLevel), resourceType, err, metrics.SinceInSeconds(start))
+							klog.Warningf("calling reconcile function %v for init container %v/%v failed, error %v",
+								r.description[condition], podMeta.Key(), initContainerStat.Name, err)
+						} else {
+							containerCtx.ReconcilerDone(c.executor)
+							metrics.RecordRuntimeHookReconcilerInvokedDurationMilliSeconds(string(ContainerLevel), resourceType, nil, metrics.SinceInSeconds(start))
+							klog.V(5).Infof("calling reconcile function %v for init container %v/%v finish",
+								r.description[condition], podMeta.Key(), initContainerStat.Name)
+						}
+					}
+				}
+
 				for _, containerStat := range podMeta.Pod.Status.ContainerStatuses {
 					for resourceType, r := range globalCgroupReconcilers.containerLevel {
 						condition := r.filter.Filter(podMeta)
diff --git a/pkg/koordlet/util/pod.go b/pkg/koordlet/util/pod.go
index 10d53d7a..eff14ce6 100644
--- a/pkg/koordlet/util/pod.go
+++ b/pkg/koordlet/util/pod.go
@@ -61,13 +61,25 @@ func GetPodSandboxContainerID(pod *corev1.Pod) (string, error) {
 	// get runtime type and dir names of known containers
 	containerSubDirNames := make(map[string]struct{}, len(pod.Status.ContainerStatuses))
 	containerRuntime := system.RuntimeTypeUnknown
+	for _, containerStat := range pod.Status.InitContainerStatuses {
+		runtimeType, containerDirName, err := system.CgroupPathFormatter.ContainerDirFn(containerStat.ContainerID)
+		if err != nil {
+			return "", err
+		}
+		containerSubDirNames[containerDirName] = struct{}{}
+		if containerRuntime == system.RuntimeTypeUnknown {
+			containerRuntime = runtimeType
+		}
+	}
 	for _, containerStat := range pod.Status.ContainerStatuses {
 		runtimeType, containerDirName, err := system.CgroupPathFormatter.ContainerDirFn(containerStat.ContainerID)
 		if err != nil {
 			return "", err
 		}
 		containerSubDirNames[containerDirName] = struct{}{}
-		containerRuntime = runtimeType
+		if containerRuntime == system.RuntimeTypeUnknown {
+			containerRuntime = runtimeType
+		}
 	}
 
 	sandboxCandidates := make([]string, 0)



Test patch:
diff --git a/pkg/koordlet/runtimehooks/hooks/cpuset/rule_test.go b/pkg/koordlet/runtimehooks/hooks/cpuset/rule_test.go
index f1347876..e05022f0 100644
--- a/pkg/koordlet/runtimehooks/hooks/cpuset/rule_test.go
+++ b/pkg/koordlet/runtimehooks/hooks/cpuset/rule_test.go
@@ -794,6 +794,7 @@ func Test_cpusetPlugin_ruleUpdateCbForPods(t *testing.T) {
 		sandboxID string
 	}
 	type args struct {
+		rule      *cpusetRule
 		pods      []*testPod
 		podAllocs map[string]ext.ResourceStatus
 	}
@@ -810,6 +811,15 @@ func Test_cpusetPlugin_ruleUpdateCbForPods(t *testing.T) {
 		{
 			name: "set container cpuset",
 			args: args{
+				rule: &cpusetRule{
+					sharePools: []ext.CPUSharedPool{
+						{
+							Socket: 0,
+							Node:   0,
+							CPUSet: "0-1,5-7",
+						},
+					},
+				},
 				pods: []*testPod{
 					{
 						pod: &corev1.Pod{
@@ -834,6 +844,43 @@ func Test_cpusetPlugin_ruleUpdateCbForPods(t *testing.T) {
 						},
 						sandboxID: "containerd://pod-with-cpuset-alloc-sandbox-id",
 					},
+					{
+						pod: &corev1.Pod{
+							ObjectMeta: metav1.ObjectMeta{
+								UID: "pod-cpu-share-uid",
+								Labels: map[string]string{
+									ext.LabelPodQoS: string(ext.QoSLS),
+								},
+							},
+							Spec: corev1.PodSpec{
+								InitContainers: []corev1.Container{
+									{
+										Name: "init-container-with-cpu-share-name",
+									},
+								},
+								Containers: []corev1.Container{
+									{
+										Name: "container-with-cpu-share-name",
+									},
+								},
+							},
+							Status: corev1.PodStatus{
+								InitContainerStatuses: []corev1.ContainerStatus{
+									{
+										Name:        "init-container-with-cpu-share-name",
+										ContainerID: "containerd://init-container-with-cpu-share-uid",
+									},
+								},
+								ContainerStatuses: []corev1.ContainerStatus{
+									{
+										Name:        "container-with-cpu-share-name",
+										ContainerID: "containerd://container-with-cpu-share-uid",
+									},
+								},
+							},
+						},
+						sandboxID: "containerd://pod-cpu-share-sandbox-id",
+					},
 					{
 						pod: &corev1.Pod{
 							ObjectMeta: metav1.ObjectMeta{
@@ -870,10 +917,13 @@ func Test_cpusetPlugin_ruleUpdateCbForPods(t *testing.T) {
 			wants: wants{
 				containersCPUSet: map[string]string{
 					"container-with-cpuset-alloc-name":     "2-4",
+					"init-container-with-cpu-share-name":   "0-1,5-7",
+					"container-with-cpu-share-name":        "0-1,5-7",
 					"container-with-bad-cpuset-alloc-name": "",
 				},
 				sandboxCPUSet: map[string]string{
 					"pod-with-cpuset-alloc-uid":     "2-4",
+					"pod-cpu-share-uid":             "0-1,5-7",
 					"pod-with-bad-cpuset-alloc-uid": "",
 				},
 			},
@@ -897,6 +947,11 @@ func Test_cpusetPlugin_ruleUpdateCbForPods(t *testing.T) {
 			// init cgroups cpuset file
 			for _, testPod := range tt.args.pods {
 				podMeta := podUIDMetas[string(testPod.pod.UID)]
+				for _, initContainerStat := range podMeta.Pod.Status.InitContainerStatuses {
+					containerPath, err := koordletutil.GetContainerCgroupParentDirByID(podMeta.CgroupDir, initContainerStat.ContainerID)
+					assert.NoError(t, err, "get init container cgroup path during init container cpuset")
+					initCPUSet(containerPath, "", testHelper)
+				}
 				for _, containerStat := range podMeta.Pod.Status.ContainerStatuses {
 					containerPath, err := koordletutil.GetContainerCgroupParentDirByID(podMeta.CgroupDir, containerStat.ContainerID)
 					assert.NoError(t, err, "get container cgroup path during init container cpuset")
@@ -921,7 +976,7 @@ func Test_cpusetPlugin_ruleUpdateCbForPods(t *testing.T) {
 				}
 			}
 
-			p := &cpusetPlugin{executor: resourceexecutor.NewResourceUpdateExecutor()}
+			p := &cpusetPlugin{executor: resourceexecutor.NewResourceUpdateExecutor(), rule: tt.args.rule}
 			stop := make(chan struct{})
 			defer func() { close(stop) }()
 			p.executor.Run(stop)
@@ -940,6 +995,14 @@ func Test_cpusetPlugin_ruleUpdateCbForPods(t *testing.T) {
 
 			for _, testPod := range tt.args.pods {
 				podMeta := podUIDMetas[string(testPod.pod.UID)]
+				for _, initContainerStat := range podMeta.Pod.Status.InitContainerStatuses {
+					containerPath, err := koordletutil.GetContainerCgroupParentDirByID(podMeta.CgroupDir, initContainerStat.ContainerID)
+					assert.NoError(t, err, "get init contaienr cgorup path during check container cpuset")
+					gotCPUSet := getCPUSet(containerPath, testHelper)
+					assert.Equal(t, tt.wants.containersCPUSet[initContainerStat.Name], gotCPUSet,
+						"container cpuset after callback should be equal")
+				}
+
 				for _, containerStat := range podMeta.Pod.Status.ContainerStatuses {
 					containerPath, err := koordletutil.GetContainerCgroupParentDirByID(podMeta.CgroupDir, containerStat.ContainerID)
 					assert.NoError(t, err, "get contaienr cgorup path during check container cpuset")
diff --git a/pkg/koordlet/runtimehooks/reconciler/reconciler_test.go b/pkg/koordlet/runtimehooks/reconciler/reconciler_test.go
index 73e6df6c..68656674 100755
--- a/pkg/koordlet/runtimehooks/reconciler/reconciler_test.go
+++ b/pkg/koordlet/runtimehooks/reconciler/reconciler_test.go
@@ -195,6 +195,11 @@ func Test_reconciler_reconcilePodCgroup(t *testing.T) {
 							UID:       "test1-pod-uid",
 						},
 						Spec: corev1.PodSpec{
+							InitContainers: []corev1.Container{
+								{
+									Name: "test1-init-container-name",
+								},
+							},
 							Containers: []corev1.Container{
 								{
 									Name: "test1-container-name",
@@ -202,6 +207,12 @@ func Test_reconciler_reconcilePodCgroup(t *testing.T) {
 							},
 						},
 						Status: corev1.PodStatus{
+							InitContainerStatuses: []corev1.ContainerStatus{
+								{
+									Name:        "test1-init-container-name",
+									ContainerID: "test1-init-container-id",
+								},
+							},
 							ContainerStatuses: []corev1.ContainerStatus{
 								{
 									Name:        "test1-container-name",
@@ -219,6 +230,11 @@ func Test_reconciler_reconcilePodCgroup(t *testing.T) {
 							UID:       "test2-pod-uid",
 						},
 						Spec: corev1.PodSpec{
+							InitContainers: []corev1.Container{
+								{
+									Name: "test2-init-container-name",
+								},
+							},
 							Containers: []corev1.Container{
 								{
 									Name: "test2-container-name",
@@ -226,6 +242,12 @@ func Test_reconciler_reconcilePodCgroup(t *testing.T) {
 							},
 						},
 						Status: corev1.PodStatus{
+							InitContainerStatuses: []corev1.ContainerStatus{
+								{
+									Name:        "test2-init-container-name",
+									ContainerID: "test2-init-container-id",
+								},
+							},
 							ContainerStatuses: []corev1.ContainerStatus{
 								{
 									Name:        "test2-container-name",
@@ -243,6 +265,11 @@ func Test_reconciler_reconcilePodCgroup(t *testing.T) {
 							UID:       "test3-pod-uid",
 						},
 						Spec: corev1.PodSpec{
+							InitContainers: []corev1.Container{
+								{
+									Name: "test3-init-container-name",
+								},
+							},
 							Containers: []corev1.Container{
 								{
 									Name: "test3-container-name",
@@ -250,6 +277,12 @@ func Test_reconciler_reconcilePodCgroup(t *testing.T) {
 							},
 						},
 						Status: corev1.PodStatus{
+							InitContainerStatuses: []corev1.ContainerStatus{
+								{
+									Name:        "test3-init-container-name",
+									ContainerID: "test3-init-container-id",
+								},
+							},
 							ContainerStatuses: []corev1.ContainerStatus{
 								{
 									Name:        "test3-container-name",
@@ -268,9 +301,12 @@ func Test_reconciler_reconcilePodCgroup(t *testing.T) {
 				genPodKey("test-ns", "test3-pod-name"): "test3-pod-uid",
 			},
 			wantContainers: map[string]string{
-				genContainerKey("test-ns", "test1-pod-name", "test1-container-name"): "test1-container-id",
-				genContainerKey("test-ns", "test2-pod-name", "test2-container-name"): "test2-container-id",
-				genContainerKey("test-ns", "test3-pod-name", "test3-container-name"): "test3-container-id",
+				genContainerKey("test-ns", "test1-pod-name", "test1-container-name"):      "test1-container-id",
+				genContainerKey("test-ns", "test1-pod-name", "test1-init-container-name"): "test1-init-container-id",
+				genContainerKey("test-ns", "test2-pod-name", "test2-container-name"):      "test2-container-id",
+				genContainerKey("test-ns", "test2-pod-name", "test2-init-container-name"): "test2-init-container-id",
+				genContainerKey("test-ns", "test3-pod-name", "test3-container-name"):      "test3-container-id",
+				genContainerKey("test-ns", "test3-pod-name", "test3-init-container-name"): "test3-init-container-id",
 			},
 			wantPods4AllPods: map[string]string{
 				genPodKey("test-ns", "test1-pod-name"): "test1-pod-uid",
diff --git a/pkg/koordlet/util/pod_test.go b/pkg/koordlet/util/pod_test.go
index df33055a..60a8bdd8 100644
--- a/pkg/koordlet/util/pod_test.go
+++ b/pkg/koordlet/util/pod_test.go
@@ -233,6 +233,7 @@ func TestGetPodSandboxContainerID(t *testing.T) {
 			fields: fields{
 				otherContaienrIDs: []string{
 					"containerd://testPodSandboxHashID",
+					"containerd://testInitContainerHashID",
 					"containerd://testContainerHashID",
 				},
 			},
@@ -243,6 +244,13 @@ func TestGetPodSandboxContainerID(t *testing.T) {
 						UID:  "test-pod-uid",
 					},
 					Status: corev1.PodStatus{
+						InitContainerStatuses: []corev1.ContainerStatus{
+							{
+								Name:        "test-init-container-name",
+								ContainerID: "containerd://testInitContainerHashID",
+								Started:     nil,
+							},
+						},
 						ContainerStatuses: []corev1.ContainerStatus{
 							{
 								Name:        "test-container-name",
@@ -261,6 +269,7 @@ func TestGetPodSandboxContainerID(t *testing.T) {
 			fields: fields{
 				otherContaienrIDs: []string{
 					"docker://testPodSandboxHashID",
+					"docker://testInitContainerHashID",
 					"docker://testContainerHashID",
 				},
 			},
@@ -271,6 +280,13 @@ func TestGetPodSandboxContainerID(t *testing.T) {
 						UID:  "test-pod-uid",
 					},
 					Status: corev1.PodStatus{
+						InitContainerStatuses: []corev1.ContainerStatus{
+							{
+								Name:        "test-container-name",
+								ContainerID: "docker://testInitContainerHashID",
+								Started:     nil,
+							},
+						},
 						ContainerStatuses: []corev1.ContainerStatus{
 							{
 								Name:        "test-container-name",