Change:
<ins> /* Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package state import ( "os" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" testutil "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/testing" ) const testingCheckpoint = "dramanager_checkpoint_test" // TODO (https://github.com/kubernetes/kubernetes/issues/123552): reconsider what data gets stored in checkpoints and whether that is really necessary. func TestCheckpointGetOrCreate(t *testing.T) { testCases := []struct { description                string checkpointContent          string expectedError              string expectedClaimInfoStateList ClaimInfoStateList }{ { description:                "new-checkpoint", expectedClaimInfoStateList: nil, }, { description:       "single-claim-info-state", checkpointContent: `{"Data":"{"kind":"DRACheckpoint","apiVersion":"checkpoint.dra.kubelet.k8s.io/v1","ClaimInfoStateList":[{"ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"DriverState":{"test-driver.cdi.k8s.io":{"Devices":[{"PoolName":"worker-1","DeviceName":"dev-1","RequestNames":["test request"],"CDIDeviceIDs":["example.com/example=cdi-example"]}]}}}]}","Checksum":1656016162}`, expectedClaimInfoStateList: ClaimInfoStateList{ { DriverState: map[string]DriverState{ "test-driver.cdi.k8s.io": { Devices: []Device{ { PoolName:     "worker-1", DeviceName:   "dev-1", RequestNames: []string{"test request"}, CDIDeviceIDs: []string{"example.com/example=cdi-example"}, }, }, }, }, ClaimUID:  "067798be-454e-4be4-9047-1aa06aea63f7", ClaimName: "example", Namespace: "default", PodUIDs:   sets.New("139cdb46-f989-4f17-9561-ca10cfb509a6"), }, }, }, { description:       "claim-info-state-with-multiple-devices", checkpointContent: `{"Data":"{"kind":"DRACheckpoint","apiVersion":"checkpoint.dra.kubelet.k8s.io/v1","ClaimInfoStateList":[{"ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"DriverState":{"test-driver.cdi.k8s.io":{"Devices":[{"PoolName":"worker-1","DeviceName":"dev-1","RequestNames":["test request"],"CDIDeviceIDs":["example.com/example=cdi-example"]},{"PoolName":"worker-1","DeviceName":"dev-2","RequestNames":["test request"],"CDIDeviceIDs":["example.com/example=cdi-example"]}]}}}]}","Checksum":3369508096}`, expectedClaimInfoStateList: ClaimInfoStateList{ { DriverState: map[string]DriverState{ "test-driver.cdi.k8s.io": { Devices: []Device{ { PoolName:     "worker-1", DeviceName:   "dev-1", RequestNames: []string{"test request"}, CDIDeviceIDs: []string{"example.com/example=cdi-example"}, }, { PoolName:     "worker-1", DeviceName:   "dev-2", RequestNames: []string{"test request"}, CDIDeviceIDs: []string{"example.com/example=cdi-example"}, }, }, }, }, ClaimUID:  "067798be-454e-4be4-9047-1aa06aea63f7", ClaimName: "example", Namespace: "default", PodUIDs:   sets.New("139cdb46-f989-4f17-9561-ca10cfb509a6"), }, }, }, { description:       "two-claim-info-states", checkpointContent: `{"Data":"{"kind":"DRACheckpoint","apiVersion":"checkpoint.dra.kubelet.k8s.io/v1","ClaimInfoStateList":[{"ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example-1","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"DriverState":{"test-driver.cdi.k8s.io":{"Devices":[{"PoolName":"worker-1","DeviceName":"dev-1","RequestNames":["test request"],"CDIDeviceIDs":["example.com/example=cdi-example"]}]}}},{"ClaimUID":"4cf8db2d-06c0-7d70-1a51-e59b25b2c16c","ClaimName":"example-2","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"DriverState":{"test-driver.cdi.k8s.io":{"Devices":[{"PoolName":"worker-1","DeviceName":"dev-2","RequestNames":null,"CDIDeviceIDs":null}]}}}]}","Checksum":1582256999}`, expectedClaimInfoStateList: ClaimInfoStateList{ { DriverState: map[string]DriverState{ "test-driver.cdi.k8s.io": { Devices: []Device{ { PoolName:     "worker-1", DeviceName:   "dev-1", RequestNames: []string{"test request"}, CDIDeviceIDs: []string{"example.com/example=cdi-example"}, }, }, }, }, ClaimUID:  "067798be-454e-4be4-9047-1aa06aea63f7", ClaimName: "example-1", Namespace: "default", PodUIDs:   sets.New("139cdb46-f989-4f17-9561-ca10cfb509a6"), }, { DriverState: map[string]DriverState{ "test-driver.cdi.k8s.io": { Devices: []Device{ { PoolName:   "worker-1", DeviceName: "dev-2", }, }, }, }, ClaimUID:  "4cf8db2d-06c0-7d70-1a51-e59b25b2c16c", ClaimName: "example-2", Namespace: "default", PodUIDs:   sets.New("139cdb46-f989-4f17-9561-ca10cfb509a6"), }, }, }, { description:       "incorrect-checksum", checkpointContent: `{"Data":"{"kind":"DRACheckpoint","apiVersion":"checkpoint.dra.kubelet.k8s.io/v1","Entries":[{"ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example-1","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"DriverState":{"test-driver.cdi.k8s.io":{"Devices":[{"PoolName":"worker-1","DeviceName":"dev-1","RequestNames":["test request"],"CDIDeviceIDs":["example.com/example=cdi-example"]}]}}},{"ClaimUID":"4cf8db2d-06c0-7d70-1a51-e59b25b2c16c","ClaimName":"example-2","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"DriverState":{"test-driver.cdi.k8s.io":{"Devices":[{"PoolName":"worker-1","DeviceName":"dev-2","RequestNames":null,"CDIDeviceIDs":null}]}}}]}","Checksum":2930258365}`, expectedError:     "checkpoint is corrupted", }, { description:       "invalid-JSON", checkpointContent: `{`, expectedError:     "unexpected end of JSON input", }, } // create temp dir testingDir, err := os.MkdirTemp("", "dramanager_state_test") if err != nil { t.Fatal(err) } defer func() { if err := os.RemoveAll(testingDir); err != nil { t.Fatal(err) } }() // create checkpoint manager for testing cpm, err := checkpointmanager.NewCheckpointManager(testingDir) require.NoError(t, err, "could not create testing checkpoint manager") for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { // ensure there is no previous checkpoint require.NoError(t, cpm.RemoveCheckpoint(testingCheckpoint), "could not remove testing checkpoint") // prepare checkpoint for testing if strings.TrimSpace(tc.checkpointContent) != "" { mock := &testutil.MockCheckpoint{Content: tc.checkpointContent} require.NoError(t, cpm.CreateCheckpoint(testingCheckpoint, mock), "could not create testing checkpoint") } checkpointer, err := NewCheckpointer(testingDir, testingCheckpoint) require.NoError(t, err, "could not create testing checkpointer") checkpoint, err := checkpointer.GetOrCreate() if strings.TrimSpace(tc.expectedError) != "" { assert.ErrorContains(t, err, tc.expectedError) } else { require.NoError(t, err, "unexpected error") stateList, err := checkpoint.GetClaimInfoStateList() require.NoError(t, err, "could not get data entries from checkpoint") require.NoError(t, err) assert.Equal(t, tc.expectedClaimInfoStateList, stateList) } }) } } func TestCheckpointStateStore(t *testing.T) { testCases := []struct { description               string claimInfoStateList        ClaimInfoStateList expectedCheckpointContent string }{ { description: "single-claim-info-state", claimInfoStateList: ClaimInfoStateList{ { DriverState: map[string]DriverState{ "test-driver.cdi.k8s.io": { Devices: []Device{ { PoolName:     "worker-1", DeviceName:   "dev-1", RequestNames: []string{"test request"}, CDIDeviceIDs: []string{"example.com/example=cdi-example"}, }, }, }, }, ClaimUID:  "067798be-454e-4be4-9047-1aa06aea63f7", ClaimName: "example", Namespace: "default", PodUIDs:   sets.New("139cdb46-f989-4f17-9561-ca10cfb509a6"), }, }, expectedCheckpointContent: `{"Data":"{"kind":"DRACheckpoint","apiVersion":"checkpoint.dra.kubelet.k8s.io/v1","ClaimInfoStateList":[{"ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"DriverState":{"test-driver.cdi.k8s.io":{"Devices":[{"PoolName":"worker-1","DeviceName":"dev-1","RequestNames":["test request"],"CDIDeviceIDs":["example.com/example=cdi-example"]}]}}}]}","Checksum":1656016162}`, }, { description: "claim-info-state-with-multiple-devices", claimInfoStateList: ClaimInfoStateList{ { DriverState: map[string]DriverState{ "test-driver.cdi.k8s.io": { Devices: []Device{ { PoolName:     "worker-1", DeviceName:   "dev-1", RequestNames: []string{"test request"}, CDIDeviceIDs: []string{"example.com/example=cdi-example"}, }, { PoolName:     "worker-1", DeviceName:   "dev-2", RequestNames: []string{"test request"}, CDIDeviceIDs: []string{"example.com/example=cdi-example"}, }, }, }, }, ClaimUID:  "067798be-454e-4be4-9047-1aa06aea63f7", ClaimName: "example", Namespace: "default", PodUIDs:   sets.New("139cdb46-f989-4f17-9561-ca10cfb509a6"), }, }, expectedCheckpointContent: `{"Data":"{"kind":"DRACheckpoint","apiVersion":"checkpoint.dra.kubelet.k8s.io/v1","ClaimInfoStateList":[{"ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"DriverState":{"test-driver.cdi.k8s.io":{"Devices":[{"PoolName":"worker-1","DeviceName":"dev-1","RequestNames":["test request"],"CDIDeviceIDs":["example.com/example=cdi-example"]},{"PoolName":"worker-1","DeviceName":"dev-2","RequestNames":["test request"],"CDIDeviceIDs":["example.com/example=cdi-example"]}]}}}]}","Checksum":3369508096}`, }, { description: "two-claim-info-states", claimInfoStateList: ClaimInfoStateList{ { DriverState: map[string]DriverState{ "test-driver.cdi.k8s.io": { Devices: []Device{ { PoolName:     "worker-1", DeviceName:   "dev-1", RequestNames: []string{"test request"}, CDIDeviceIDs: []string{"example.com/example=cdi-example"}, }, }, }, }, ClaimUID:  "067798be-454e-4be4-9047-1aa06aea63f7", ClaimName: "example-1", Namespace: "default", PodUIDs:   sets.New("139cdb46-f989-4f17-9561-ca10cfb509a6"), }, { DriverState: map[string]DriverState{ "test-driver.cdi.k8s.io": { Devices: []Device{ { PoolName:   "worker-1", DeviceName: "dev-2", }, }, }, }, ClaimUID:  "4cf8db2d-06c0-7d70-1a51-e59b25b2c16c", ClaimName: "example-2", Namespace: "default", PodUIDs:   sets.New("139cdb46-f989-4f17-9561-ca10cfb509a6"), }, }, expectedCheckpointContent: `{"Data":"{"kind":"DRACheckpoint","apiVersion":"checkpoint.dra.kubelet.k8s.io/v1","ClaimInfoStateList":[{"ClaimUID":"067798be-454e-4be4-9047-1aa06aea63f7","ClaimName":"example-1","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"DriverState":{"test-driver.cdi.k8s.io":{"Devices":[{"PoolName":"worker-1","DeviceName":"dev-1","RequestNames":["test request"],"CDIDeviceIDs":["example.com/example=cdi-example"]}]}}},{"ClaimUID":"4cf8db2d-06c0-7d70-1a51-e59b25b2c16c","ClaimName":"example-2","Namespace":"default","PodUIDs":{"139cdb46-f989-4f17-9561-ca10cfb509a6":{}},"DriverState":{"test-driver.cdi.k8s.io":{"Devices":[{"PoolName":"worker-1","DeviceName":"dev-2","RequestNames":null,"CDIDeviceIDs":null}]}}}]}","Checksum":1582256999}`, }, } // Should return an error, stateDir cannot be an empty string if _, err := NewCheckpointer("", testingCheckpoint); err == nil { t.Fatal("expected error but got nil") } // create temp dir testingDir, err := os.MkdirTemp("", "dramanager_state_test") if err != nil { t.Fatal(err) } defer func() { if err := os.RemoveAll(testingDir); err != nil { t.Fatal(err) } }() // NewCheckpointState with an empty checkpointName should return an error if _, err = NewCheckpointer(testingDir, ""); err == nil { t.Fatal("expected error but got nil") } cpm, err := checkpointmanager.NewCheckpointManager(testingDir) require.NoError(t, err, "could not create testing checkpoint manager") require.NoError(t, cpm.RemoveCheckpoint(testingCheckpoint), "could not remove testing checkpoint") cs, err := NewCheckpointer(testingDir, testingCheckpoint) require.NoError(t, err, "could not create testing checkpointState instance") for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { checkpoint, err := NewCheckpoint(tc.claimInfoStateList) require.NoError(t, err, "could not create Checkpoint") err = cs.Store(checkpoint) require.NoError(t, err, "could not store checkpoint") err = cpm.GetCheckpoint(testingCheckpoint, checkpoint) require.NoError(t, err, "could not get checkpoint") checkpointContent, err := checkpoint.MarshalCheckpoint() require.NoError(t, err, "could not Marshal Checkpoint") assert.Equal(t, tc.expectedCheckpointContent, string(checkpointContent)) }) } } </ins>

Discussion:
When adding the field to , old checkpoint files became invalid because the checksum that gets calculated by hashing the output in no longer matches. This caused kubelet to fail to start with: Here's what the checksum in a unit test gets calculcated for: Note the new . Not sure what the right behavior is. Perhaps kubelet should have ignored the invalid checkpoint? Revert the update in The state.DRAManagerCheckpoint contains a version field, which is set to v1.I think that's where things fall apart: that version must exactly match the type that gets serialized, which is not the case here. The type changes over time, but the version is fixed. Let's reconsider what should get checkpointed and how we can make that stable. Also, is checkpointing really worth it? What bad things happen if we remove it? master n/a No response No response No response /sig node /cc
/cc
We originally didn't have checkpointing and it was by in to fix
The reasons for checkpoint were: (what Kevin mentioned) of the pod resource API DAR extensions concerns raised on what happen if kubelet restart and we get GetPodResourcesRequest. The request should return the DRA claimed devices but if we didn't checkpoint the DRA cache is empty and we would return empty resources force delete see [2] My original PR didn't checkpoint the ResourceHandle is was in PR [2]. Maybe we should exclude ResourceHandle from the checksum hash calculation [1] [2] -
So the cache makes sense. What doesn't make sense to me is how the consistency of that on-disk file is checked and how it is versioned. Shouldn't the checksum be calculated for the bytes on disk (version-independent) instead of the in-memory representation (version-dependent)? Or is the goal to protect against "I can't read this data anymore" instead of "bit has flipped on disk"? The assumption is that the content can always be decoded from what was written in the past into the type which is used now. This works with JSON if the newer type is a true superset of the old one, but that's not necessarily guaranteed. The proper way of doing this is to define a scheme which includes conversion between different versions.

This cannot be done when embedding API types from because conversion between current v1alpha2 and a future v1beta1 can only be done in the apiserver, not client-side. I don't see a quick solution here. In the short-term, how do we deal with version upgrades? Require that users drain the node (thus clearing the cache) before upgrading or downgrading kubelet?
I am still trying to understand the problem. the problem that we checksum the version? the problem that we don't have conversion mechanism to move from version X -version Y on upgrade? the issue related only for upgrade or there are other cases which checksum changes? In general we the version to checkpoint file so that in the future when the DRA checkpoint struct changed we can have mechanism to convert old checkpoint version to new checkpoint version (in the initial PR we didn't implement conversion mechanism as I thought it will be on DRA checkpoint struct change) . If you take a look on the CPU Manager checkpoint [1] you will see that it detecting if the scheme matches to V1 first and if not it assume V2. We wanted that it will part of the checkpoint file so we won't need such detection and in the file we will know that correct version of the DRA checkpoint struct. Now if someone change the DRA checkpoint scheme and didn't implement migration flow (similar to CPU manager [2]) or change the version that indeed an issue, [1] - [2] -
The problem is that after an upgrade of kubelet from <1.30 to 1.30, it won't be able to read an existing, non-empty checkpoint file. Worse, it will refuse to start because of that. The root cause is the lack of proper versioning and lack of conversion code. Any version change potentially has this problem (including downgrades). I'm not aware of other failure scenarios.
Since it is not a GA feature, we cannot fix this in 1.30. But marking it as important long-term. /priority important-longterm
This has to be solved before graduating from alpha
I intend to move all version-dependent code into drivers (). This will simplify the checkpoint content a bit, but the overall questions around its design remain.

/kind feature /remove-kind bug /triage accepted I think a good forum to discuss this could be the KEP for DRA, though we can keep this tracker here
Isn't it because checkpoint structure was changed without incrementing version and adding conversion code? Anyway, I'd propose to remove version from the checkpoint structure and comment it to prevent changing it without changing its version. the implementation should be similar to the cpu maanager checkpointing. If it's been working for CPU (and device?) managers for some time, it should hopefully work for DRA as well. Any other ideas?
Yes. I haven't looked into this again. Can you implement your proposal on top of and submit a PR for review, or do you want to discuss this first?
if you're ok with my proposal I'm ok to implement it. Otherwise it needs to be discussed further.
I don't fully understand the implications without looking into this further, so I can say neither yes nor no to it at this point. do you have an opinion?