Change:
<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: bug /kind bug What happened: Deployment running with read-only GCE PD. A node containing one of the deployment pods is deleted and recreated soon after. New pod is scheduled to the same node, but is stuck in ContainerCreating state. The node's kubelet logs show continuous "Volume is not yet attached according to node" errors. What you expected to happen: Pod recreated successfully. Anything else we need to know?: Cause by PR The issue happens as follows: When the node is deleted, GCE PD is detached from the cloud provider. AttachDetachController discovers the PD has been detached, and marks it as detached. The original pod still exists in the API server (i.e. before garbage collection), so AttachDetachController reattaches the PD, successfully. Before the node recovers, ADC tries to update the node status but finds that the node doesn't exist. Following the logic introduced in it never attempts to update the node status again. The node recovers, a new pod is scheduled to it, waits for a successful attach signal in the node status but never gets it. /cc /sig storage
/assign

Discussion:
// GetNodesToUpdateStatusFor returns the map of nodeNames to nodeToUpdateStatusFor GetNodesToUpdateStatusFor() map[types.NodeName]nodeToUpdateStatusFor <del> // Removes the given node from the record of attach updates. The node's entire // volumesToReportAsAttached list is removed. RemoveNodeFromAttachUpdates(nodeName types.NodeName) error </del> } // AttachedVolume represents a volume that is attached to a node.

asw.addVolumeToReportAsAttached(volumeName, nodeName) } <del> func (asw *actualStateOfWorld) RemoveNodeFromAttachUpdates(nodeName types.NodeName) error { asw.Lock() defer asw.Unlock() _, nodeToUpdateExists := asw.nodesToUpdateStatusFor[nodeName] if nodeToUpdateExists { delete(asw.nodesToUpdateStatusFor, nodeName) return nil } return fmt.Errorf("node %q does not exist in volumesToReportAsAttached list", nodeName) } </del> func (asw *actualStateOfWorld) AddVolumeNode( uniqueName v1.UniqueVolumeName, volumeSpec *volume.Spec, nodeName types.NodeName, devicePath string) (v1.UniqueVolumeName, error) { asw.Lock()

} } <del> // Test_RemoveNodeFromAttachUpdates_Positive expects an entire node entry to be removed // from nodesToUpdateStatusFor func Test_RemoveNodeFromAttachUpdates_Positive(t *testing.T) { // Arrange volumePluginMgr, _ := volumetesting.GetTestVolumePluginMgr(t) asw := &actualStateOfWorld{ attachedVolumes:        make(map[v1.UniqueVolumeName]attachedVolume), nodesToUpdateStatusFor: make(map[types.NodeName]nodeToUpdateStatusFor), volumePluginMgr:        volumePluginMgr, } nodeName := types.NodeName("node-1") nodeToUpdate := nodeToUpdateStatusFor{ nodeName:                  nodeName, statusUpdateNeeded:        true, volumesToReportAsAttached: make(map[v1.UniqueVolumeName]v1.UniqueVolumeName), } asw.nodesToUpdateStatusFor[nodeName] = nodeToUpdate // Act err := asw.RemoveNodeFromAttachUpdates(nodeName) // Assert if err != nil { t.Fatalf("RemoveNodeFromAttachUpdates should not return error, but got: %v", err) } if len(asw.nodesToUpdateStatusFor) > 0 { t.Fatal("nodesToUpdateStatusFor should be empty as its only entry has been deleted.") } } // Test_RemoveNodeFromAttachUpdates_Negative_NodeDoesNotExist expects an error to be thrown // when nodeName is not in nodesToUpdateStatusFor. func Test_RemoveNodeFromAttachUpdates_Negative_NodeDoesNotExist(t *testing.T) { // Arrange volumePluginMgr, _ := volumetesting.GetTestVolumePluginMgr(t) asw := &actualStateOfWorld{ attachedVolumes:        make(map[v1.UniqueVolumeName]attachedVolume), nodesToUpdateStatusFor: make(map[types.NodeName]nodeToUpdateStatusFor), volumePluginMgr:        volumePluginMgr, } nodeName := types.NodeName("node-1") nodeToUpdate := nodeToUpdateStatusFor{ nodeName:                  nodeName, statusUpdateNeeded:        true, volumesToReportAsAttached: make(map[v1.UniqueVolumeName]v1.UniqueVolumeName), } asw.nodesToUpdateStatusFor[nodeName] = nodeToUpdate // Act err := asw.RemoveNodeFromAttachUpdates("node-2") // Assert if err == nil { t.Fatal("RemoveNodeFromAttachUpdates should return an error as the nodeName doesn't exist.") } if len(asw.nodesToUpdateStatusFor) != 1 { t.Fatal("The length of nodesToUpdateStatusFor should not change because no operation was performed.") } } // Test_RemoveNodeFromAttachUpdates_Negative_Empty expects an error to be thrown // when a nodesToUpdateStatusFor is empty. func Test_RemoveNodeFromAttachUpdates_Negative_Empty(t *testing.T) { // Arrange volumePluginMgr, _ := volumetesting.GetTestVolumePluginMgr(t) asw := &actualStateOfWorld{ attachedVolumes:        make(map[v1.UniqueVolumeName]attachedVolume), nodesToUpdateStatusFor: make(map[types.NodeName]nodeToUpdateStatusFor), volumePluginMgr:        volumePluginMgr, } // Act err := asw.RemoveNodeFromAttachUpdates("node-1") // Assert if err == nil { t.Fatal("RemoveNodeFromAttachUpdates should return an error as nodeToUpdateStatusFor is empty.") } if len(asw.nodesToUpdateStatusFor) != 0 { t.Fatal("The length of nodesToUpdateStatusFor should be 0.") } } </del> func verifyAttachedVolume( t *testing.T, attachedVolumes []AttachedVolume,

nodeObj, err := nsu.nodeLister.Get(string(nodeName)) if errors.IsNotFound(err) { // If node does not exist, its status cannot be updated. <del> // Remove the node entry from the collection of attach updates, preventing the // status updater from unnecessarily updating the node. </del> <ins> // Do nothing so that there is no retry until node is created. </ins> glog.V(2).Infof( "Could not update node status. Failed to find node %q in NodeInformer cache. Error: '%v'", nodeName, err) <del> nsu.actualStateOfWorld.RemoveNodeFromAttachUpdates(nodeName) </del> continue } else if err != nil { // For all other errors, log error and reset flag statusUpdateNeeded