Code change:
The "" E2E test will sometimes fail because it fails to delete the PD used in the test after the test is finished. The error: Detailed test log: The disk that fails to delete still shows up in : My attempts to manually try to delete the PD fail with: (the work around for ) doesn't change anything here:
is the disk still mounted ? It looks like the detach call did not succeed for some reason
I'll have to get another repro and verify.
okay cool. If you get a repro check to see if the disk is still mounted. If so then the cleanup code is not being called. If not then the clean up started, detach was called but failed for some reason; look in the logs for something like "googleapi: some error about detaching". It can also just be a race condition. That is, detach was called but is taking too long; if the disk does not detach then you cannot delete it.

Related discussion:
text: package gce_pd import ( <del> "errors" </del> "fmt" "os" "path" <ins> "path/filepath" "strings" </ins> "time" "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider" "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/gce" <ins> "github.com/GoogleCloudPlatform/kubernetes/pkg/util" </ins> "github.com/GoogleCloudPlatform/kubernetes/pkg/util/exec" "github.com/GoogleCloudPlatform/kubernetes/pkg/util/mount" <ins> "github.com/GoogleCloudPlatform/kubernetes/pkg/util/operationmanager" </ins> "github.com/golang/glog" ) <ins> const ( diskByIdPath         = "/dev/disk/by-id/" diskGooglePrefix     = "google-" diskScsiGooglePrefix = "scsi-0Google_PersistentDisk_" diskPartitionSuffix  = "-part" diskSDPath           = "/dev/sd" diskSDPattern        = "/dev/sd*" maxChecks            = 10 maxRetries           = 10 checkSleepDuration   = time.Second ) // Singleton operation manager for managing detach clean up go routines var detachCleanupManager = operationmanager.NewOperationManager() </ins> type GCEDiskUtil struct{} // Attaches a disk specified by a volume.GCEPersistentDisk to the current kubelet. // Mounts the disk to it's global path. <del> func (util *GCEDiskUtil) AttachAndMountDisk(pd *gcePersistentDisk, globalPDPath string) error { </del> <ins> func (diskUtil *GCEDiskUtil) AttachAndMountDisk(pd *gcePersistentDisk, globalPDPath string) error { glog.V(5).Infof("AttachAndMountDisk(pd, %q) where pd is %#vrn", globalPDPath, pd) // Terminate any in progress verify detach go routines, this will block until the goroutine is ready to exit because the channel is unbuffered detachCleanupManager.Send(pd.pdName, true) sdBefore, err := filepath.Glob(diskSDPattern) if err != nil { glog.Errorf("Error filepath.Glob("%s"): %vrn", diskSDPattern, err) } sdBeforeSet := util.NewStringSet(sdBefore...) </ins> gce, err := cloudprovider.GetCloudProvider("gce", nil) if err != nil { return err } if err := gce.(*gce_cloud.GCECloud).AttachDisk(pd.pdName, pd.readOnly); err != nil { return err } <del> devicePaths := []string{ path.Join("/dev/disk/by-id/", "google-"+pd.pdName), path.Join("/dev/disk/by-id/", "scsi-0Google_PersistentDisk_"+pd.pdName), } if pd.partition != "" { for i, path := range devicePaths { devicePaths[i] = path + "-part" + pd.partition } } //TODO(jonesdl) There should probably be better method than busy-waiting here. numTries := 0 devicePath := "" // Wait for the disk device to be created for { for _, path := range devicePaths { _, err := os.Stat(path) if err == nil { devicePath = path break } if err != nil && !os.IsNotExist(err) { return err } } if devicePath != "" { break } numTries++ if numTries == 10 { return errors.New("Could not attach disk: Timeout after 10s") } time.Sleep(time.Second) </del> <ins> devicePath, err := verifyAttached(pd, sdBeforeSet, gce) if err != nil { return err </ins> } // Only mount the PD globally once.