Change:
After enabling In-Place Pod Vertical Scaling, if deploy a pod without setting memory (or cpu) request, kubelet will be failed at its second restart kubelet restart successfully kubelet with In-Place Pod Vertical Scaling enabled a pod without setting memory request, e.g: kubelet for the first time kubelet for the second time will fail with error: After deploying the pod, the following information is saved in the file : Then restart kubelet for the first time, the relevant content in the file becomes: When deploying the pod, as there are no records in the checkpoint (), kubelet will get resource config from directly and save it to the checkpoint (without memory request) When restarting kubelet for the first time, it will restore the previously saved data from the checkpoint. Since the memory request was not previously set, it will be set to an empty value and resaved to the checkpoint When restarting kubelet for the second time, it will process as a value of 0 (not empty), which resulting in the inconsistency between the calculated checksum during data recovery and the previous one <details</details<details</details<details</details<details</details<details</details<details</details
/sig node
cc
/triage accepted
/assign
xlink: /cc
thanks for finding this issue and the fix. It looks good with suggestion. For beta, I wonder if we can do away with local checkpoint and rely on PodStatus as persistent store and source of truth for allocated resources. It will simplify the implementation.
Using resource.Quantity in checkoutPoint is a bad usage. I found except the case in this issue, whenever adding a static pod to the kubelet, then restart the kubelet, restore the resize status checkpoint will fail, casing kubelet panic. The reason is , before restart kubelet, the checkpoint struct in kubelet will looks like: and the the podstatusmanager_state file is looks like: and after restart kubelet and recover the checkoutpoint state, the checkpoint struct in kubelet will looks like: the difference is between "kube-scheduler:map[cpu:{i:{value:6 scale:-1} d:{Dec:<nil} s: Format:DecimalSI}" and "kube-scheduler:map[cpu:{i:{value:600 scale:-3} d:{Dec:<nil} s:600m Format:DecimalSI} ". It's caused by json Marshal and Unmarshal of resource.Quantity. So, I think Using resource.Quantity in checkoutPoint is a bad case. We should consider using PodStatus in apiserver instead of checkoutPoint.
Is this problem solved? Now I can still reproduce this problem in the official version of 1.28. Thanks~
Could anyone please let me know what is the state of the art to solve this issue?
we have seen the similar problem after enabling InPlaceUpdate pod resources and could we reopen this ?
can we just do marshal&unmarshal before storing state checkpoint to avoid the issue of resource.Quantity?
I am +1 on doing away with the checkpointing code altogether and leveraging PodStatus. We discussed this in the Kubernetes Contributor Summit in Chicago. I just have not been able to find the cycles to work on it :(

Discussion:
otherPods := make([]*v1.Pod, 0, len(pods)) for _, p := range pods { op := p.DeepCopy() <del> for _, c := range op.Spec.Containers { allocatedResources, found := kl.statusManager.GetContainerResourceAllocation(string(p.UID), c.Name) if c.Resources.Requests != nil && found { c.Resources.Requests[v1.ResourceCPU] = allocatedResources[v1.ResourceCPU] c.Resources.Requests[v1.ResourceMemory] = allocatedResources[v1.ResourceMemory] } } </del> <ins> kl.updateContainerResourceAllocation(op) </ins> otherPods = append(otherPods, op) } attrs.OtherPods = otherPods

// To handle kubelet restarts, test pod admissibility using AllocatedResources values // (for cpu & memory) from checkpoint store. If found, that is the source of truth. podCopy := pod.DeepCopy() <del> for _, c := range podCopy.Spec.Containers { allocatedResources, found := kl.statusManager.GetContainerResourceAllocation(string(pod.UID), c.Name) if c.Resources.Requests != nil && found { c.Resources.Requests[v1.ResourceCPU] = allocatedResources[v1.ResourceCPU] c.Resources.Requests[v1.ResourceMemory] = allocatedResources[v1.ResourceMemory] } } </del> <ins> kl.updateContainerResourceAllocation(podCopy) </ins> // Check if we can admit the pod; if not, reject it. if ok, reason, message := kl.canAdmitPod(activePods, podCopy); !ok { kl.rejectPod(pod, reason, message)

} } <ins> // updateContainerResourceAllocation updates AllocatedResources values // (for cpu & memory) from checkpoint store func (kl *Kubelet) updateContainerResourceAllocation(pod *v1.Pod) { for _, c := range pod.Spec.Containers { allocatedResources, found := kl.statusManager.GetContainerResourceAllocation(string(pod.UID), c.Name) if c.Resources.Requests != nil && found { if _, ok := allocatedResources[v1.ResourceCPU]; ok { c.Resources.Requests[v1.ResourceCPU] = allocatedResources[v1.ResourceCPU] } if _, ok := allocatedResources[v1.ResourceMemory]; ok { c.Resources.Requests[v1.ResourceMemory] = allocatedResources[v1.ResourceMemory] } } } } </ins> // HandlePodUpdates is the callback in the SyncHandler interface for pods // being updated from a config source. func (kl *Kubelet) HandlePodUpdates(pods []*v1.Pod) {

serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats" "k8s.io/kubernetes/pkg/kubelet/stats" "k8s.io/kubernetes/pkg/kubelet/status" <ins> "k8s.io/kubernetes/pkg/kubelet/status/state" </ins> statustest "k8s.io/kubernetes/pkg/kubelet/status/testing" "k8s.io/kubernetes/pkg/kubelet/token" kubetypes "k8s.io/kubernetes/pkg/kubelet/types"

checkPodStatus(t, kl, podToAdmit, v1.PodPending) } <ins> func TestPodResourceAllocationReset(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.InPlacePodVerticalScaling, true)() testKubelet := newTestKubelet(t, false) defer testKubelet.Cleanup() kubelet := testKubelet.kubelet kubelet.statusManager = status.NewFakeManager() nodes := []*v1.Node{ { ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}, Status: v1.NodeStatus{ Capacity: v1.ResourceList{ v1.ResourceCPU:    resource.MustParse("8"), v1.ResourceMemory: resource.MustParse("8Gi"), }, Allocatable: v1.ResourceList{ v1.ResourceCPU:    resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("4Gi"), v1.ResourcePods:   *resource.NewQuantity(40, resource.DecimalSI), }, }, }, } kubelet.nodeLister = testNodeLister{nodes: nodes} cpu500m := resource.MustParse("500m") cpu800m := resource.MustParse("800m") mem500M := resource.MustParse("500Mi") mem800M := resource.MustParse("800Mi") cpu500mMem500MPodSpec := &v1.PodSpec{ Containers: []v1.Container{ { Name: "c1", Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{v1.ResourceCPU: cpu500m, v1.ResourceMemory: mem500M}, }, }, }, } cpu800mMem800MPodSpec := cpu500mMem500MPodSpec.DeepCopy() cpu800mMem800MPodSpec.Containers[0].Resources.Requests = v1.ResourceList{v1.ResourceCPU: cpu800m, v1.ResourceMemory: mem800M} cpu800mPodSpec := cpu500mMem500MPodSpec.DeepCopy() cpu800mPodSpec.Containers[0].Resources.Requests = v1.ResourceList{v1.ResourceCPU: cpu800m} mem800MPodSpec := cpu500mMem500MPodSpec.DeepCopy() mem800MPodSpec.Containers[0].Resources.Requests = v1.ResourceList{v1.ResourceMemory: mem800M} cpu500mPodSpec := cpu500mMem500MPodSpec.DeepCopy() cpu500mPodSpec.Containers[0].Resources.Requests = v1.ResourceList{v1.ResourceCPU: cpu500m} mem500MPodSpec := cpu500mMem500MPodSpec.DeepCopy() mem500MPodSpec.Containers[0].Resources.Requests = v1.ResourceList{v1.ResourceMemory: mem500M} emptyPodSpec := cpu500mMem500MPodSpec.DeepCopy() emptyPodSpec.Containers[0].Resources.Requests = v1.ResourceList{} tests := []struct { name                          string pod                           *v1.Pod existingPodAllocation         *v1.Pod expectedPodResourceAllocation state.PodResourceAllocation }{ { name: "Having both memory and cpu, resource allocation not exists", pod:  podWithUIDNameNsSpec("1", "pod1", "foo", *cpu500mMem500MPodSpec), expectedPodResourceAllocation: state.PodResourceAllocation{ "1": map[string]v1.ResourceList{ cpu500mMem500MPodSpec.Containers[0].Name: cpu500mMem500MPodSpec.Containers[0].Resources.Requests, }, }, }, { name:                  "Having both memory and cpu, resource allocation exists", pod:                   podWithUIDNameNsSpec("2", "pod2", "foo", *cpu500mMem500MPodSpec), existingPodAllocation: podWithUIDNameNsSpec("2", "pod2", "foo", *cpu500mMem500MPodSpec), expectedPodResourceAllocation: state.PodResourceAllocation{ "2": map[string]v1.ResourceList{ cpu500mMem500MPodSpec.Containers[0].Name: cpu500mMem500MPodSpec.Containers[0].Resources.Requests, }, }, }, { name:                  "Having both memory and cpu, resource allocation exists (with different value)", pod:                   podWithUIDNameNsSpec("3", "pod3", "foo", *cpu500mMem500MPodSpec), existingPodAllocation: podWithUIDNameNsSpec("3", "pod3", "foo", *cpu800mMem800MPodSpec), expectedPodResourceAllocation: state.PodResourceAllocation{ "3": map[string]v1.ResourceList{ cpu800mMem800MPodSpec.Containers[0].Name: cpu800mMem800MPodSpec.Containers[0].Resources.Requests, }, }, }, { name: "Only has cpu, resource allocation not exists", pod:  podWithUIDNameNsSpec("4", "pod5", "foo", *cpu500mPodSpec), expectedPodResourceAllocation: state.PodResourceAllocation{ "4": map[string]v1.ResourceList{ cpu500mPodSpec.Containers[0].Name: cpu500mPodSpec.Containers[0].Resources.Requests, }, }, }, { name:                  "Only has cpu, resource allocation exists", pod:                   podWithUIDNameNsSpec("5", "pod5", "foo", *cpu500mPodSpec), existingPodAllocation: podWithUIDNameNsSpec("5", "pod5", "foo", *cpu500mPodSpec), expectedPodResourceAllocation: state.PodResourceAllocation{ "5": map[string]v1.ResourceList{ cpu500mPodSpec.Containers[0].Name: cpu500mPodSpec.Containers[0].Resources.Requests, }, }, }, { name:                  "Only has cpu, resource allocation exists (with different value)", pod:                   podWithUIDNameNsSpec("6", "pod6", "foo", *cpu500mPodSpec), existingPodAllocation: podWithUIDNameNsSpec("6", "pod6", "foo", *cpu800mPodSpec), expectedPodResourceAllocation: state.PodResourceAllocation{ "6": map[string]v1.ResourceList{ cpu800mPodSpec.Containers[0].Name: cpu800mPodSpec.Containers[0].Resources.Requests, }, }, }, { name: "Only has memory, resource allocation not exists", pod:  podWithUIDNameNsSpec("7", "pod7", "foo", *mem500MPodSpec), expectedPodResourceAllocation: state.PodResourceAllocation{ "7": map[string]v1.ResourceList{ mem500MPodSpec.Containers[0].Name: mem500MPodSpec.Containers[0].Resources.Requests, }, }, }, { name:                  "Only has memory, resource allocation exists", pod:                   podWithUIDNameNsSpec("8", "pod8", "foo", *mem500MPodSpec), existingPodAllocation: podWithUIDNameNsSpec("8", "pod8", "foo", *mem500MPodSpec), expectedPodResourceAllocation: state.PodResourceAllocation{ "8": map[string]v1.ResourceList{ mem500MPodSpec.Containers[0].Name: mem500MPodSpec.Containers[0].Resources.Requests, }, }, }, { name:                  "Only has memory, resource allocation exists (with different value)", pod:                   podWithUIDNameNsSpec("9", "pod9", "foo", *mem500MPodSpec), existingPodAllocation: podWithUIDNameNsSpec("9", "pod9", "foo", *mem800MPodSpec), expectedPodResourceAllocation: state.PodResourceAllocation{ "9": map[string]v1.ResourceList{ mem800MPodSpec.Containers[0].Name: mem800MPodSpec.Containers[0].Resources.Requests, }, }, }, { name: "No CPU and memory, resource allocation not exists", pod:  podWithUIDNameNsSpec("10", "pod10", "foo", *emptyPodSpec), expectedPodResourceAllocation: state.PodResourceAllocation{ "10": map[string]v1.ResourceList{ emptyPodSpec.Containers[0].Name: emptyPodSpec.Containers[0].Resources.Requests, }, }, }, { name:                  "No CPU and memory, resource allocation exists", pod:                   podWithUIDNameNsSpec("11", "pod11", "foo", *emptyPodSpec), existingPodAllocation: podWithUIDNameNsSpec("11", "pod11", "foo", *emptyPodSpec), expectedPodResourceAllocation: state.PodResourceAllocation{ "11": map[string]v1.ResourceList{ emptyPodSpec.Containers[0].Name: emptyPodSpec.Containers[0].Resources.Requests, }, }, }, } for _, tc := range tests { if tc.existingPodAllocation != nil { // when kubelet restarts, AllocatedResources has already existed before adding pod err := kubelet.statusManager.SetPodAllocation(tc.existingPodAllocation) if err != nil { t.Fatalf("failed to set pod allocation: %v", err) } } kubelet.HandlePodAdditions([]*v1.Pod{tc.pod}) allocatedResources, found := kubelet.statusManager.GetContainerResourceAllocation(string(tc.pod.UID), tc.pod.Spec.Containers[0].Name) if !found { t.Fatalf("resource allocation should exist: (pod: %#v, container: %s)", tc.pod, tc.pod.Spec.Containers[0].Name) } assert.Equal(t, tc.expectedPodResourceAllocation[string(tc.pod.UID)][tc.pod.Spec.Containers[0].Name], allocatedResources, tc.name) } } </ins> func TestHandlePodResourcesResize(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.InPlacePodVerticalScaling, true)() testKubelet := newTestKubelet(t, false)