Change:
if err := gce.(*gce_cloud.GCECloud).DetachDisk(pd.pdName); err != nil { return err } <ins> // Verify disk detached, retry if needed. go verifyDetached(pd, gce) return nil } // Verifys the disk device to be created has been succesffully attached, and retries if it fails. func verifyAttached(pd *gcePersistentDisk, sdBeforeSet util.StringSet, gce cloudprovider.Interface) (string, error) { devicePaths := getDiskByIdPaths(pd) for numRetries := 0; numRetries < maxRetries; numRetries++ { for numChecks := 0; numChecks < maxChecks; numChecks++ { if err := udevadmChangeToNewDrives(sdBeforeSet); err != nil { // udevadm errors should not block disk attachment, log and continue glog.Errorf("%v", err) } for _, path := range devicePaths { if pathExists, err := pathExists(path); err != nil { return "", err } else if pathExists { // A device path has succesfully been created for the PD glog.V(5).Infof("Succesfully attached GCE PD %q.", pd.pdName) return path, nil } } // Sleep then check again glog.V(5).Infof("Waiting for GCE PD %q to attach.", pd.pdName) time.Sleep(checkSleepDuration) } // Try attaching the disk again glog.Warningf("Timed out waiting for GCE PD %q to attach. Retrying attach.", pd.pdName) if err := gce.(*gce_cloud.GCECloud).AttachDisk(pd.pdName, pd.readOnly); err != nil { return "", err } } return "", fmt.Errorf("Could not attach GCE PD %q. Timeout waiting for mount paths to be created.", pd.pdName) } // Veify the specified persistent disk device has been succesfully detached, and retries if it fails. // This function is intended to be called asynchronously as a go routine. func verifyDetached(pd *gcePersistentDisk, gce cloudprovider.Interface) { defer util.HandleCrash() // Setting bufferSize to 0 so that when senders send, they are blocked until we recieve. This avoids the need to have a separate exit check. ch, err := detachCleanupManager.Start(pd.pdName, 0 /* bufferSize */) if err != nil { glog.Errorf("Error adding %q to detachCleanupManager: %v", pd.pdName, err) return } defer detachCleanupManager.Close(pd.pdName) devicePaths := getDiskByIdPaths(pd) for numRetries := 0; numRetries < maxRetries; numRetries++ { for numChecks := 0; numChecks < maxChecks; numChecks++ { select { case <-ch: glog.Warningf("Terminating GCE PD %q detach verification. Another attach/detach call was made for this PD.", pd.pdName) return default: allPathsRemoved := true for _, path := range devicePaths { if err := udevadmChangeToDrive(path); err != nil { // udevadm errors should not block disk detachment, log and continue glog.Errorf("%v", err) } if exists, err := pathExists(path); err != nil { glog.Errorf("Error check path: %v", err) return } else { allPathsRemoved = allPathsRemoved && !exists } } if allPathsRemoved { // All paths to the PD have been succefully removed glog.V(5).Infof("Succesfully detached GCE PD %q.", pd.pdName) return } // Sleep then check again glog.V(5).Infof("Waiting for GCE PD %q to detach.", pd.pdName) time.Sleep(checkSleepDuration) } } // Try detaching disk again glog.Warningf("Timed out waiting for GCE PD %q to detach. Retrying detach.", pd.pdName) if err := gce.(*gce_cloud.GCECloud).DetachDisk(pd.pdName); err != nil { glog.Errorf("Error on retry detach PD %q: %v", pd.pdName, err) return } } glog.Errorf("Could not detach GCE PD %q. One or more mount paths was not removed.", pd.pdName) } // Returns list of all /dev/disk/by-id/* paths for given PD. func getDiskByIdPaths(pd *gcePersistentDisk) []string { devicePaths := []string{ path.Join(diskByIdPath, diskGooglePrefix+pd.pdName), path.Join(diskByIdPath, diskScsiGooglePrefix+pd.pdName), } if pd.partition != "" { for i, path := range devicePaths { devicePaths[i] = path + diskPartitionSuffix + pd.partition } } return devicePaths } // Checks if the specified path exists func pathExists(path string) (bool, error) { _, err := os.Stat(path) if err == nil { return true, nil } else if os.IsNotExist(err) { return false, nil } else { return false, err } } // Calls "udevadm trigger --action=change" for newly created "/dev/sd*" drives (exist only in after set). // This is workaround for Issue #7972. Once the underlying issue has been resolved, this may be removed. func udevadmChangeToNewDrives(sdBeforeSet util.StringSet) error { sdAfter, err := filepath.Glob(diskSDPattern) if err != nil { return fmt.Errorf("Error filepath.Glob("%s"): %vrn", diskSDPattern, err) } for _, sd := range sdAfter { if !sdBeforeSet.Has(sd) { return udevadmChangeToDrive(sd) } } return nil } // Calls "udevadm trigger --action=change" on the specified drive. // drivePath must be the the block device path to trigger on, in the format "/dev/sd*", or a symlink to it. // This is workaround for Issue #7972. Once the underlying issue has been resolved, this may be removed. func udevadmChangeToDrive(drivePath string) error { glog.V(5).Infof("udevadmChangeToDrive: drive=%q", drivePath) // Evaluate symlink, if any drive, err := filepath.EvalSymlinks(drivePath) if err != nil { return fmt.Errorf("udevadmChangeToDrive: filepath.EvalSymlinks(%q) failed with %v.", drivePath, err) } glog.V(5).Infof("udevadmChangeToDrive: symlink path is %q", drive) // Check to make sure input is "/dev/sd*" if !strings.Contains(drive, diskSDPath) { return fmt.Errorf("udevadmChangeToDrive: expected input in the form "%s" but drive is %q.", diskSDPattern, drive) } // Call "udevadm trigger --action=change --property-match=DEVNAME=/dev/sd..." _, err = exec.New().Command( "udevadm", "trigger", "--action=change", fmt.Sprintf("--property-match=DEVNAME=%s", drive)).CombinedOutput() if err != nil { return fmt.Errorf("udevadmChangeToDrive: udevadm trigger failed for drive %q with %v.", drive, err) } </ins> return nil }

Discussion:
The "" E2E test will sometimes fail because it fails to delete the PD used in the test after the test is finished. The error: Detailed test log: The disk that fails to delete still shows up in : My attempts to manually try to delete the PD fail with: (the work around for ) doesn't change anything here:
is the disk still mounted ? It looks like the detach call did not succeed for some reason
I'll have to get another repro and verify.
okay cool. If you get a repro check to see if the disk is still mounted. If so then the cleanup code is not being called. If not then the clean up started, detach was called but failed for some reason; look in the logs for something like "googleapi: some error about detaching". It can also just be a race condition. That is, detach was called but is taking too long; if the disk does not detach then you cannot delete it.