Change:
introduced logic to make kubelet volume manager wait for attach detach controller to finish attach before it starts mount. The controller should also not start a detach operation until the node status has been updated removing the volume from the list of attached volumes. Otherwise if a pod is rapidly created and deleted on the same node, it could result in a mount starting immediately before a detach starts. Props to and his pet set tests for helping catch this.
Hmm, this just happened , just a lot less frequently. Haven't debugged, but it looks like the failed pod did reland on the same node: pet-0, jenkins-e2e-minion-group-4mn2 (). Reopening, will try to confirm when i have time, and close if it's not the same thing.
Just following up with an update, Saad and i are tag teaming this. He's uploading a pr that I'm going to reivew. We're tracking a couple of issues: need to update the "volumes in use" list before polling on the "volumes attached" list. Best option is to write a operation executor, so we don't hammer the apiserver. needs to get up to date node object from apiserver to determine volumes in use. The update that removes volumes from the "attached list" shoud return a fresh node object, we can just iterate the volumes its using (this must happen after popping from attached, though). leaky volumes in the node object (this is not about the physical volumes but about the size of the json blob that's the node object growing). The proposed fix is to use the desired state of the world to update the list of volumes in use. There's a last point I need to follow up on. We might leave the node object in an inconsistent state with the proposed fix, but it should be ok since nothing is checking the inconsistent fields. This will happen if we pop a volume from the attached list, but then find its in use in the updated node object. ping me when you're ready to go, of if you need help

Discussion:
if !rc.desiredStateOfWorld.VolumeExists( attachedVolume.VolumeName, attachedVolume.NodeName) { // Volume exists in actual state of world but not desired <ins> // Mark desire to detach timeElapsed, err := rc.actualStateOfWorld.MarkDesireToDetach(attachedVolume.VolumeName, attachedVolume.NodeName) if err != nil { glog.Errorf("Unexpected error actualStateOfWorld.MarkDesireToDetach(): %v", err) } // Update Node Status to indicate volume is no longer safe to mount. err = rc.nodeStatusUpdater.UpdateNodeStatuses() if err != nil { // Skip detaching this volume if unable to update node status glog.Infof("UpdateNodeStatuses failed with: %v", err) continue } </ins> if !attachedVolume.MountedByNode { glog.V(5).Infof("Attempting to start DetachVolume for volume %q from node %q", attachedVolume.VolumeName, attachedVolume.NodeName) err := rc.attacherDetacher.DetachVolume(attachedVolume.AttachedVolume, rc.actualStateOfWorld)

} } else { // If volume is not safe to detach (is mounted) wait a max amount of time before detaching any way. <del> timeElapsed, err := rc.actualStateOfWorld.MarkDesireToDetach(attachedVolume.VolumeName, attachedVolume.NodeName) if err != nil { glog.Errorf("Unexpected error actualStateOfWorld.MarkDesireToDetach(): %v", err) } </del> if timeElapsed > rc.maxWaitForUnmountDuration { glog.V(5).Infof("Attempting to start DetachVolume for volume %q from node %q. Volume is not safe to detach, but maxWaitForUnmountDuration expired.", attachedVolume.VolumeName, attachedVolume.NodeName) err := rc.attacherDetacher.DetachVolume(attachedVolume.AttachedVolume, rc.actualStateOfWorld)

asw := cache.NewActualStateOfWorld(volumePluginMgr) fakeKubeClient := controllervolumetesting.CreateTestClient() ad := operationexecutor.NewOperationExecutor(fakeKubeClient, volumePluginMgr) <del> nodeInformer := informers.CreateSharedNodeIndexInformer( fakeKubeClient, resyncPeriod) nsu := statusupdater.NewNodeStatusUpdater( fakeKubeClient, nodeInformer, asw) </del> <ins> nsu := statusupdater.NewFakeNodeStatusUpdater(false /* returnError */) </ins> reconciler := NewReconciler( reconcilerLoopPeriod, maxWaitForUnmountDuration, dsw, asw, ad, nsu) podName := "pod-uid"

waitForDetachCallCount(t, 1 /* expectedDetachCallCount */, fakePlugin) } <ins> // Populates desiredStateOfWorld cache with one node/volume/pod tuple. // Has node update fail // Calls Run() // Verifies there is one attach call and no detach calls. // Marks the node/volume as unmounted. // Deletes the node/volume/pod tuple from desiredStateOfWorld cache. // Verifies there are NO detach call and no (new) attach calls. func Test_Run_Negative_OneDesiredVolumeAttachThenDetachWithUnmountedVolumeUpdateStatusFail(t *testing.T) { // Arrange volumePluginMgr, fakePlugin := volumetesting.GetTestVolumePluginMgr(t) dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) asw := cache.NewActualStateOfWorld(volumePluginMgr) fakeKubeClient := controllervolumetesting.CreateTestClient() ad := operationexecutor.NewOperationExecutor(fakeKubeClient, volumePluginMgr) nsu := statusupdater.NewFakeNodeStatusUpdater(true /* returnError */) reconciler := NewReconciler( reconcilerLoopPeriod, maxWaitForUnmountDuration, dsw, asw, ad, nsu) podName := "pod-uid" volumeName := api.UniqueVolumeName("volume-name") volumeSpec := controllervolumetesting.GetTestVolumeSpec(string(volumeName), volumeName) nodeName := "node-name" dsw.AddNode(nodeName) volumeExists := dsw.VolumeExists(volumeName, nodeName) if volumeExists { t.Fatalf( "Volume %q/node %q should not exist, but it does.", volumeName, nodeName) } generatedVolumeName, podAddErr := dsw.AddPod(types.UniquePodName(podName), controllervolumetesting.NewPod(podName, podName), volumeSpec, nodeName) if podAddErr != nil { t.Fatalf("AddPod failed. Expected: <no error> Actual: <%v>", podAddErr) } // Act go reconciler.Run(wait.NeverStop) // Assert waitForNewAttacherCallCount(t, 1 /* expectedCallCount */, fakePlugin) verifyNewAttacherCallCount(t, false /* expectZeroNewAttacherCallCount */, fakePlugin) waitForAttachCallCount(t, 1 /* expectedAttachCallCount */, fakePlugin) verifyNewDetacherCallCount(t, true /* expectZeroNewDetacherCallCount */, fakePlugin) waitForDetachCallCount(t, 0 /* expectedDetachCallCount */, fakePlugin) // Act dsw.DeletePod(types.UniquePodName(podName), generatedVolumeName, nodeName) volumeExists = dsw.VolumeExists(generatedVolumeName, nodeName) if volumeExists { t.Fatalf( "Deleted pod %q from volume %q/node %q. Volume should also be deleted but it still exists.", podName, generatedVolumeName, nodeName) } asw.SetVolumeMountedByNode(generatedVolumeName, nodeName, true /* mounted */) asw.SetVolumeMountedByNode(generatedVolumeName, nodeName, false /* mounted */) // Assert verifyNewDetacherCallCount(t, true /* expectZeroNewDetacherCallCount */, fakePlugin) verifyNewAttacherCallCount(t, false /* expectZeroNewAttacherCallCount */, fakePlugin) waitForAttachCallCount(t, 1 /* expectedAttachCallCount */, fakePlugin) verifyNewDetacherCallCount(t, false /* expectZeroNewDetacherCallCount */, fakePlugin) waitForDetachCallCount(t, 0 /* expectedDetachCallCount */, fakePlugin) } </ins> func waitForNewAttacherCallCount( t *testing.T, expectedCallCount int,

<ins> /* Copyright 2016 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package statusupdater import ( "fmt" ) func NewFakeNodeStatusUpdater(returnError bool) NodeStatusUpdater { return &fakeNodeStatusUpdater{ returnError: returnError, } } type fakeNodeStatusUpdater struct { returnError bool } func (fnsu *fakeNodeStatusUpdater) UpdateNodeStatuses() error { if fnsu.returnError { return fmt.Errorf("fake error on update node status") } return nil } </ins>