Change:
We should add a unit test to Eviction manager that accounts for PID Pressure. The test should cover similar cases as DiskPressure or MemoryPressure. 1) PIDPressure doesn't exist on node. 2) Force PidPressure on node We should include test cases for reclaming resources in terms of pressure and verifying differences between soft and hard eviction.
/sig node /kind feature /help
This request has been marked as needing help from a contributor. Please ensure that the issue body includes answers to the following questions: Why are we solving this issue? To address this issue, are there any code changes? If there are code changes, what needs to be done in the code and what places can the assignee treat as reference points? Does this issue have zero to low barrier of entry? How can the assignee reach out to you for help? For more details on the requirements of such an issue, please see and ensure that they are met. If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : /sig node /kind feature /help </details
/triage accepted /priority important-longterm
/good-first-issue
This request has been marked as suitable for new contributors. Please ensure that the issue body includes answers to the following questions: Why are we solving this issue? To address this issue, are there any code changes? If there are code changes, what needs to be done in the code and what places can the assignee treat as reference points? Does this issue have zero to low barrier of entry? How can the assignee reach out to you for help? For more details on the requirements of such an issue, please see and ensure that they are met. If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : /good-first-issue Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign
it looks like has requested to work on this.
/assign
Hi What do you mean by, "verifying differences between soft and hard eviction."?
It’s the same idea as what we do for disk, memory or inode pressure. Hard eviction means we trigger eviction with no grace period while soft eviction means that we obey a grace period. Test should check for that.
I have opened a PR based on what I could understand using a debugger and the other tests. This is my first time contributing to the codebase. Happy to take brutal feedback.
Hi, i would like to support this issue. Let me know if any help is required.
/assign

Discussion:
return pod, podStats } <ins> func makePodWithPIDStats(name string, priority int32, processCount uint64) (*v1.Pod, statsapi.PodStats) { pod := newPod(name, priority, []v1.Container{ newContainer(name, nil, nil), }, nil) podStats := newPodProcessStats(pod, processCount) return pod, podStats } </ins> func makePodWithDiskStats(name string, priority int32, requests v1.ResourceList, limits v1.ResourceList, rootFsUsed, logsUsed, perLocalVolumeUsed string) (*v1.Pod, statsapi.PodStats) { pod := newPod(name, priority, []v1.Container{ newContainer(name, requests, limits),

return pod, podStats } <ins> func makePIDStats(nodeAvailablePIDs string, numberOfRunningProcesses string, podStats map[*v1.Pod]statsapi.PodStats) *statsapi.Summary { val := resource.MustParse(nodeAvailablePIDs) availablePIDs := int64(val.Value()) parsed := resource.MustParse(numberOfRunningProcesses) NumberOfRunningProcesses := int64(parsed.Value()) result := &statsapi.Summary{ Node: statsapi.NodeStats{ Rlimit: &statsapi.RlimitStats{ MaxPID:                &availablePIDs, NumOfRunningProcesses: &NumberOfRunningProcesses, }, }, Pods: []statsapi.PodStats{}, } for _, podStat := range podStats { result.Pods = append(result.Pods, podStat) } return result } </ins> func makeMemoryStats(nodeAvailableBytes string, podStats map[*v1.Pod]statsapi.PodStats) *statsapi.Summary { val := resource.MustParse(nodeAvailableBytes) availableBytes := uint64(val.Value())

requests                 v1.ResourceList limits                   v1.ResourceList memoryWorkingSet         string <ins> pidUsage                 uint64 </ins> rootFsUsed               string logsFsUsed               string logsFsInodesUsed         string

} } <ins> func TestPIDPressure_VerifyPodStatus(t *testing.T) { testCases := map[string]struct { wantPodStatus v1.PodStatus }{ "eviction due to pid pressure": { wantPodStatus: v1.PodStatus{ Phase:   v1.PodFailed, Reason:  "Evicted", Message: "The node was low on resource: pids. Threshold quantity: 1200, available: 500. ", }, }, } for name, tc := range testCases { for _, enablePodDisruptionConditions := range []bool{true, false} { t.Run(fmt.Sprintf("%s;PodDisruptionConditions=%v", name, enablePodDisruptionConditions), func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodDisruptionConditions, enablePodDisruptionConditions)() podMaker := makePodWithPIDStats summaryStatsMaker := makePIDStats podsToMake := []podToMake{ {name: "pod1", priority: lowPriority, pidUsage: 500}, {name: "pod2", priority: defaultPriority, pidUsage: 500}, } pods := []*v1.Pod{} podStats := map[*v1.Pod]statsapi.PodStats{} for _, podToMake := range podsToMake { pod, podStat := podMaker(podToMake.name, podToMake.priority, 2) pods = append(pods, pod) podStats[pod] = podStat } activePodsFunc := func() []*v1.Pod { return pods } fakeClock := testingclock.NewFakeClock(time.Now()) podKiller := &mockPodKiller{} diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: ptr.To(false)} diskGC := &mockDiskGC{err: nil} nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""} config := Config{ PressureTransitionPeriod: time.Minute * 5, Thresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalPIDAvailable, Operator: evictionapi.OpLessThan, Value: evictionapi.ThresholdValue{ Quantity: quantityMustParse("1200"), }, }, }, } summaryProvider := &fakeSummaryProvider{result: summaryStatsMaker("1500", "1000", podStats)} manager := &managerImpl{ clock:                        fakeClock, killPodFunc:                  podKiller.killPodNow, imageGC:                      diskGC, containerGC:                  diskGC, config:                       config, recorder:                     &record.FakeRecorder{}, summaryProvider:              summaryProvider, nodeRef:                      nodeRef, nodeConditionsLastObservedAt: nodeConditionsObservedAt{}, thresholdsFirstObservedAt:    thresholdsObservedAt{}, } // synchronize to detect the PID pressure _, err := manager.synchronize(diskInfoProvider, activePodsFunc) if err != nil { t.Fatalf("Manager expects no error but got %v", err) } // verify PID pressure is detected if !manager.IsUnderPIDPressure() { t.Fatalf("Manager should have detected PID pressure") } // verify a pod is selected for eviction if podKiller.pod == nil { t.Fatalf("Manager should have selected a pod for eviction") } wantPodStatus := tc.wantPodStatus.DeepCopy() if enablePodDisruptionConditions { wantPodStatus.Conditions = append(wantPodStatus.Conditions, v1.PodCondition{ Type:    "DisruptionTarget", Status:  "True", Reason:  "TerminationByKubelet", Message: "The node was low on resource: pids. Threshold quantity: 1200, available: 500. ", }) } // verify the pod status after applying the status update function podKiller.statusFn(&podKiller.pod.Status) if diff := cmp.Diff(*wantPodStatus, podKiller.pod.Status, cmpopts.IgnoreFields(v1.PodCondition{}, "LastProbeTime", "LastTransitionTime")); diff != "" { t.Errorf("Unexpected pod status of the evicted pod (-want,+got):n%s", diff) } }) } } } </ins> func TestDiskPressureNodeFs_VerifyPodStatus(t *testing.T) { testCases := map[string]struct { nodeFsStats                   string

} } <ins> func TestPIDPressure(t *testing.T) { testCases := []struct { name                               string podsToMake                         []podToMake evictPodIndex                      int noPressurePIDUsage                 string pressurePIDUsageWithGracePeriod    string pressurePIDUsageWithoutGracePeriod string totalPID                           string }{ { name: "eviction due to pid pressure", podsToMake: []podToMake{ {name: "high-priority-high-usage", priority: highPriority, pidUsage: 900}, {name: "default-priority-low-usage", priority: defaultPriority, pidUsage: 100}, {name: "default-priority-medium-usage", priority: defaultPriority, pidUsage: 400}, {name: "low-priority-high-usage", priority: lowPriority, pidUsage: 600}, {name: "low-priority-low-usage", priority: lowPriority, pidUsage: 50}, }, evictPodIndex:                      3, // we expect the low-priority-high-usage pod to be evicted noPressurePIDUsage:                 "300", pressurePIDUsageWithGracePeriod:    "700", pressurePIDUsageWithoutGracePeriod: "1200", totalPID:                           "2000", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { podMaker := makePodWithPIDStats summaryStatsMaker := makePIDStats pods := []*v1.Pod{} podStats := map[*v1.Pod]statsapi.PodStats{} for _, podToMake := range tc.podsToMake { pod, podStat := podMaker(podToMake.name, podToMake.priority, podToMake.pidUsage) pods = append(pods, pod) podStats[pod] = podStat } podToEvict := pods[tc.evictPodIndex] activePodsFunc := func() []*v1.Pod { return pods } fakeClock := testingclock.NewFakeClock(time.Now()) podKiller := &mockPodKiller{} diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: ptr.To(false)} diskGC := &mockDiskGC{err: nil} nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""} config := Config{ MaxPodGracePeriodSeconds: 5, PressureTransitionPeriod: time.Minute * 5, Thresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalPIDAvailable, Operator: evictionapi.OpLessThan, Value: evictionapi.ThresholdValue{ Quantity: quantityMustParse("1200"), }, }, { Signal:   evictionapi.SignalPIDAvailable, Operator: evictionapi.OpLessThan, Value: evictionapi.ThresholdValue{ Quantity: quantityMustParse("1500"), }, GracePeriod: time.Minute * 2, }, }, } summaryProvider := &fakeSummaryProvider{result: summaryStatsMaker(tc.totalPID, tc.noPressurePIDUsage, podStats)} manager := &managerImpl{ clock:                        fakeClock, killPodFunc:                  podKiller.killPodNow, imageGC:                      diskGC, containerGC:                  diskGC, config:                       config, recorder:                     &record.FakeRecorder{}, summaryProvider:              summaryProvider, nodeRef:                      nodeRef, nodeConditionsLastObservedAt: nodeConditionsObservedAt{}, thresholdsFirstObservedAt:    thresholdsObservedAt{}, } // create a pod to test admission podToAdmit, _ := podMaker("pod-to-admit", defaultPriority, 50) // synchronize _, err := manager.synchronize(diskInfoProvider, activePodsFunc) if err != nil { t.Fatalf("Manager expects no error but got %v", err) } // we should not have PID pressure if manager.IsUnderPIDPressure() { t.Fatalf("Manager should not report PID pressure") } // try to admit our pod (should succeed) if result := manager.Admit(&lifecycle.PodAdmitAttributes{Pod: podToAdmit}); !result.Admit { t.Fatalf("Admit pod: %v, expected: %v, actual: %v", podToAdmit, true, result.Admit) } // induce soft threshold for PID pressure fakeClock.Step(1 * time.Minute) summaryProvider.result = summaryStatsMaker(tc.totalPID, tc.pressurePIDUsageWithGracePeriod, podStats) _, err = manager.synchronize(diskInfoProvider, activePodsFunc) if err != nil { t.Fatalf("Manager expects no error but got %v", err) } // now, we should have PID pressure if !manager.IsUnderPIDPressure() { t.Errorf("Manager should report PID pressure since soft threshold was met") } // verify no pod was yet killed because there has not yet been enough time passed if podKiller.pod != nil { t.Errorf("Manager should not have killed a pod yet, but killed: %v", podKiller.pod.Name) } // step forward in time past the grace period fakeClock.Step(3 * time.Minute) // no change in PID stats to simulate continued pressure _, err = manager.synchronize(diskInfoProvider, activePodsFunc) if err != nil { t.Fatalf("Manager expects no error but got %v", err) } // verify PID pressure is still reported if !manager.IsUnderPIDPressure() { t.Errorf("Manager should still report PID pressure") } // verify the right pod was killed with the right grace period. if podKiller.pod != podToEvict { t.Errorf("Manager chose to kill pod: %v, but should have chosen %v", podKiller.pod.Name, podToEvict.Name) } if podKiller.gracePeriodOverride == nil { t.Errorf("Manager chose to kill pod but should have had a grace period override.") } observedGracePeriod := *podKiller.gracePeriodOverride if observedGracePeriod != manager.config.MaxPodGracePeriodSeconds { t.Errorf("Manager chose to kill pod with incorrect grace period.  Expected: %d, actual: %d", manager.config.MaxPodGracePeriodSeconds, observedGracePeriod) } // reset state podKiller.pod = nil podKiller.gracePeriodOverride = nil // remove PID pressure by simulating increased PID availability fakeClock.Step(20 * time.Minute) summaryProvider.result = summaryStatsMaker(tc.totalPID, tc.noPressurePIDUsage, podStats) // Simulate increased PID availability _, err = manager.synchronize(diskInfoProvider, activePodsFunc) if err != nil { t.Fatalf("Manager expects no error but got %v", err) } // verify PID pressure is resolved if manager.IsUnderPIDPressure() { t.Errorf("Manager should not report PID pressure") } // re-induce PID pressure fakeClock.Step(1 * time.Minute) summaryProvider.result = summaryStatsMaker(tc.totalPID, tc.pressurePIDUsageWithoutGracePeriod, podStats) _, err = manager.synchronize(diskInfoProvider, activePodsFunc) if err != nil { t.Fatalf("Manager expects no error but got %v", err) } // verify PID pressure is reported again if !manager.IsUnderPIDPressure() { t.Errorf("Manager should report PID pressure") } // verify the right pod was killed with the right grace period. if podKiller.pod != podToEvict { t.Errorf("Manager chose to kill pod: %v, but should have chosen %v", podKiller.pod.Name, podToEvict.Name) } if podKiller.gracePeriodOverride == nil { t.Errorf("Manager chose to kill pod but should have had a grace period override.") } observedGracePeriod = *podKiller.gracePeriodOverride if observedGracePeriod != int64(0) { t.Errorf("Manager chose to kill pod with incorrect grace period.  Expected: %d, actual: %d", 0, observedGracePeriod) } // try to admit our pod (should fail) if result := manager.Admit(&lifecycle.PodAdmitAttributes{Pod: podToAdmit}); result.Admit { t.Fatalf("Admit pod: %v, expected: %v, actual: %v", podToAdmit, false, result.Admit) } // reduce PID pressure fakeClock.Step(1 * time.Minute) summaryProvider.result = summaryStatsMaker(tc.totalPID, tc.noPressurePIDUsage, podStats) podKiller.pod = nil // reset state _, err = manager.synchronize(diskInfoProvider, activePodsFunc) if err != nil { t.Fatalf("Manager expects no error but got %v", err) } // we should have PID pressure (because transition period not yet met) if !manager.IsUnderPIDPressure() { t.Errorf("Manager should report PID pressure") } // no pod should have been killed if podKiller.pod != nil { t.Errorf("Manager chose to kill pod: %v when no pod should have been killed", podKiller.pod.Name) } // try to admit our pod (should fail) if result := manager.Admit(&lifecycle.PodAdmitAttributes{Pod: podToAdmit}); result.Admit { t.Fatalf("Admit pod: %v, expected: %v, actual: %v", podToAdmit, false, result.Admit) } // move the clock past the transition period fakeClock.Step(5 * time.Minute) summaryProvider.result = summaryStatsMaker(tc.totalPID, tc.noPressurePIDUsage, podStats) _, err = manager.synchronize(diskInfoProvider, activePodsFunc) if err != nil { t.Fatalf("Manager expects no error but got %v", err) } // we should not have PID pressure (because transition period met) if manager.IsUnderPIDPressure() { t.Errorf("Manager should not report PID pressure") } // no pod should have been killed if podKiller.pod != nil { t.Errorf("Manager chose to kill pod: %v when no pod should have been killed", podKiller.pod.Name) } // try to admit our pod (should succeed) if result := manager.Admit(&lifecycle.PodAdmitAttributes{Pod: podToAdmit}); !result.Admit { t.Fatalf("Admit pod: %v, expected: %v, actual: %v", podToAdmit, true, result.Admit) } }) } } </ins> func TestAdmitUnderNodeConditions(t *testing.T) { manager := &managerImpl{} pods := []*v1.Pod{