Change:
assert.IsType(t, core.GetActionImpl{}, actions[9]) } <ins> func TestUpdateNodeStatusAndVolumesInUseWithoutNodeLease(t *testing.T) { defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.NodeLease, false)() cases := []struct { desc                  string existingVolumes       []v1.UniqueVolumeName // volumes to initially populate volumeManager existingNode          *v1.Node              // existing node object expectedNode          *v1.Node              // new node object after patch expectedReportedInUse []v1.UniqueVolumeName // expected volumes reported in use in volumeManager }{ { desc:         "no volumes and no update", existingNode: &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}}, expectedNode: &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}}, }, { desc:            "volumes inuse on node and volumeManager", existingVolumes: []v1.UniqueVolumeName{"vol1"}, existingNode: &v1.Node{ ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}, Status: v1.NodeStatus{ VolumesInUse: []v1.UniqueVolumeName{"vol1"}, }, }, expectedNode: &v1.Node{ ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}, Status: v1.NodeStatus{ VolumesInUse: []v1.UniqueVolumeName{"vol1"}, }, }, expectedReportedInUse: []v1.UniqueVolumeName{"vol1"}, }, { desc: "volumes inuse on node but not in volumeManager", existingNode: &v1.Node{ ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}, Status: v1.NodeStatus{ VolumesInUse: []v1.UniqueVolumeName{"vol1"}, }, }, expectedNode: &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}}, }, { desc:            "volumes inuse in volumeManager but not on node", existingVolumes: []v1.UniqueVolumeName{"vol1"}, existingNode:    &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}}, expectedNode: &v1.Node{ ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}, Status: v1.NodeStatus{ VolumesInUse: []v1.UniqueVolumeName{"vol1"}, }, }, expectedReportedInUse: []v1.UniqueVolumeName{"vol1"}, }, } for _, tc := range cases { t.Run(tc.desc, func(t *testing.T) { // Setup testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() kubelet := testKubelet.kubelet kubelet.kubeClient = nil // ensure only the heartbeat client is used kubelet.containerManager = &localCM{ContainerManager: cm.NewStubContainerManager()} kubelet.lastStatusReportTime = kubelet.clock.Now() kubelet.nodeStatusReportFrequency = time.Hour kubelet.machineInfo = &cadvisorapi.MachineInfo{} // override test volumeManager fakeVolumeManager := kubeletvolume.NewFakeVolumeManager(tc.existingVolumes) kubelet.volumeManager = fakeVolumeManager // Only test VolumesInUse setter kubelet.setNodeStatusFuncs = []func(*v1.Node) error{ nodestatus.VolumesInUse(kubelet.volumeManager.ReconcilerStatesHasBeenSynced, kubelet.volumeManager.GetVolumesInUse), } kubeClient := testKubelet.fakeKubeClient kubeClient.ReactionChain = fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{*tc.existingNode}}).ReactionChain // Execute assert.NoError(t, kubelet.updateNodeStatus()) // Validate actions := kubeClient.Actions() if tc.expectedNode != nil { assert.Len(t, actions, 2) assert.IsType(t, core.GetActionImpl{}, actions[0]) assert.IsType(t, core.PatchActionImpl{}, actions[1]) patchAction := actions[1].(core.PatchActionImpl) updatedNode, err := applyNodeStatusPatch(tc.existingNode, patchAction.GetPatch()) require.NoError(t, err) assert.True(t, apiequality.Semantic.DeepEqual(tc.expectedNode, updatedNode), "%s", diff.ObjectDiff(tc.expectedNode, updatedNode)) } else { assert.Len(t, actions, 1) assert.IsType(t, core.GetActionImpl{}, actions[0]) } reportedInUse := fakeVolumeManager.GetVolumesReportedInUse() assert.True(t, apiequality.Semantic.DeepEqual(tc.expectedReportedInUse, reportedInUse), "%s", diff.ObjectDiff(tc.expectedReportedInUse, reportedInUse)) }) } } func TestUpdateNodeStatusAndVolumesInUseWithNodeLease(t *testing.T) { defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.NodeLease, true)() cases := []struct { desc                  string existingVolumes       []v1.UniqueVolumeName // volumes to initially populate volumeManager existingNode          *v1.Node              // existing node object expectedNode          *v1.Node              // new node object after patch expectedReportedInUse []v1.UniqueVolumeName // expected volumes reported in use in volumeManager }{ { desc:         "no volumes and no update", existingNode: &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}}, }, { desc:            "volumes inuse on node and volumeManager", existingVolumes: []v1.UniqueVolumeName{"vol1"}, existingNode: &v1.Node{ ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}, Status: v1.NodeStatus{ VolumesInUse: []v1.UniqueVolumeName{"vol1"}, }, }, expectedReportedInUse: []v1.UniqueVolumeName{"vol1"}, }, { desc: "volumes inuse on node but not in volumeManager", existingNode: &v1.Node{ ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}, Status: v1.NodeStatus{ VolumesInUse: []v1.UniqueVolumeName{"vol1"}, }, }, expectedNode: &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}}, }, { desc:            "volumes inuse in volumeManager but not on node", existingVolumes: []v1.UniqueVolumeName{"vol1"}, existingNode:    &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}}, expectedNode: &v1.Node{ ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}, Status: v1.NodeStatus{ VolumesInUse: []v1.UniqueVolumeName{"vol1"}, }, }, expectedReportedInUse: []v1.UniqueVolumeName{"vol1"}, }, } for _, tc := range cases { t.Run(tc.desc, func(t *testing.T) { // Setup testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() kubelet := testKubelet.kubelet kubelet.kubeClient = nil // ensure only the heartbeat client is used kubelet.containerManager = &localCM{ContainerManager: cm.NewStubContainerManager()} kubelet.lastStatusReportTime = kubelet.clock.Now() kubelet.nodeStatusReportFrequency = time.Hour kubelet.machineInfo = &cadvisorapi.MachineInfo{} // override test volumeManager fakeVolumeManager := kubeletvolume.NewFakeVolumeManager(tc.existingVolumes) kubelet.volumeManager = fakeVolumeManager // Only test VolumesInUse setter kubelet.setNodeStatusFuncs = []func(*v1.Node) error{ nodestatus.VolumesInUse(kubelet.volumeManager.ReconcilerStatesHasBeenSynced, kubelet.volumeManager.GetVolumesInUse), } kubeClient := testKubelet.fakeKubeClient kubeClient.ReactionChain = fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{*tc.existingNode}}).ReactionChain // Execute assert.NoError(t, kubelet.updateNodeStatus()) // Validate actions := kubeClient.Actions() if tc.expectedNode != nil { assert.Len(t, actions, 2) assert.IsType(t, core.GetActionImpl{}, actions[0]) assert.IsType(t, core.PatchActionImpl{}, actions[1]) patchAction := actions[1].(core.PatchActionImpl) updatedNode, err := applyNodeStatusPatch(tc.existingNode, patchAction.GetPatch()) require.NoError(t, err) assert.True(t, apiequality.Semantic.DeepEqual(tc.expectedNode, updatedNode), "%s", diff.ObjectDiff(tc.expectedNode, updatedNode)) } else { assert.Len(t, actions, 1) assert.IsType(t, core.GetActionImpl{}, actions[0]) } reportedInUse := fakeVolumeManager.GetVolumesReportedInUse() assert.True(t, apiequality.Semantic.DeepEqual(tc.expectedReportedInUse, reportedInUse), "%s", diff.ObjectDiff(tc.expectedReportedInUse, reportedInUse)) }) } } </ins> func TestRegisterWithApiServer(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup()

Discussion:
<!-- Please only use this template for submitting reports about failing tests in Kubernetes CI jobs --Which jobs are failing: Which test(s) are failing: [sig-storage] CSI Volumes [Driver: csi-hostpath-v0] [Testpattern: Dynamic PV (default fs)] subPath should unmount if pod is gracefully deleted while kubelet is down [Disruptive][Slow] [sig-storage] CSI Volumes [Driver: csi-hostpath] [Testpattern: Dynamic PV (default fs)] subPath should unmount if pod is gracefully deleted while kubelet is down [Disruptive][Slow] [sig-storage] CSI Volumes [Driver: ][Serial] [Testpattern: Dynamic PV (default fs)] subPath should unmount if pod is gracefully deleted while kubelet is down [Disruptive][Slow] Since when has it been failing: 2019-01-15 (when they were ) Testgrid link: Reason for failure: They all seem to be failing with: The failures seem to start with .
/sig storage
Summoning and This seems to have been failing since it was in master-blocking.
This is the same as
Thanks closing this one. /close
Closing this issue. <detailsIn response to : Thanks closing this one. Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/reopen I took a look at the latest test failures and I think it is a different issue than
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Looking at one test failure here: Sequence of events: stopped, pod deleted: started again: volume to dsw started and failed because it's not in the node's volumesinuse list. This repeats for a while. also started, but we skipped cleaning up because the volume is in dsw so we want to let volume manager gracefully cleanup don't remove volume from dsw because actual state has not been updated? So it seems like there's a problem with populating the node's volumesinuse after a kubelet restart.
cc
/assign will continue investigating.

i will try repro-ing at without different commits to try to figure out which pr introduced the regression
/priority important-soon
I ran these tests against a cluster with the NodeLease feature disabled, and the tests pass. So I think the issue is related to the new NodeLease feature enabled a month ago: cc
I discussed with VolumesInUse is updated periodically in the Node.Status. So we need to make sure this flow is working well with the NodeLease feature.
and I walked through the code. The NodeLease feature introduced an early exit to the node status update function, so our theory is we need to add a call to . I will make the change and test it out