Change:
return nil } <ins> func (l *persistentVolumeLabel) findVolumeLabels(volume *api.PersistentVolume) (map[string]string, error) { existingLabels := volume.Labels // All cloud providers set only these two labels. domain, domainOK := existingLabels[v1.LabelZoneFailureDomain] region, regionOK := existingLabels[v1.LabelZoneRegion] isDynamicallyProvisioned := metav1.HasAnnotation(volume.ObjectMeta, persistentvolume.AnnDynamicallyProvisioned) if isDynamicallyProvisioned && domainOK && regionOK { // PV already has all the labels and we can trust the dynamic provisioning that it provided correct values. return map[string]string{ v1.LabelZoneFailureDomain: domain, v1.LabelZoneRegion:        region, }, nil } // Either missing labels or we don't trust the user provided correct values. switch { case volume.Spec.AWSElasticBlockStore != nil: labels, err := l.findAWSEBSLabels(volume) if err != nil { return nil, fmt.Errorf("error querying AWS EBS volume %s: %v", volume.Spec.AWSElasticBlockStore.VolumeID, err) } return labels, nil case volume.Spec.GCEPersistentDisk != nil: labels, err := l.findGCEPDLabels(volume) if err != nil { return nil, fmt.Errorf("error querying GCE PD volume %s: %v", volume.Spec.GCEPersistentDisk.PDName, err) } return labels, nil case volume.Spec.AzureDisk != nil: labels, err := l.findAzureDiskLabels(volume) if err != nil { return nil, fmt.Errorf("error querying AzureDisk volume %s: %v", volume.Spec.AzureDisk.DiskName, err) } return labels, nil case volume.Spec.Cinder != nil: labels, err := l.findCinderDiskLabels(volume) if err != nil { return nil, fmt.Errorf("error querying Cinder volume %s: %v", volume.Spec.Cinder.VolumeID, err) } return labels, nil case volume.Spec.VsphereVolume != nil: labels, err := l.findVsphereVolumeLabels(volume) if err != nil { return nil, fmt.Errorf("error querying vSphere Volume %s: %v", volume.Spec.VsphereVolume.VolumePath, err) } return labels, nil } // Unrecognized volume, do not add any labels return nil, nil } </ins> func (l *persistentVolumeLabel) findAWSEBSLabels(volume *api.PersistentVolume) (map[string]string, error) { // Ignore any volumes that are being provisioned if volume.Spec.AWSElasticBlockStore.VolumeID == cloudvolume.ProvisionedVolumeName {

Discussion:
API server latency for PV creation depends on underlying cloud API latency. We have admission plugin that examines incoming PV and adds topology labels and node selector: When the cluster (and cloud) are under pressure, e.g. lot of PVCs are created and dynamically provisioned at one time and attached to nodes, API server latency increases. We've seen cases when we ran out of cloud API request quota and the admission plugin needed to sleep for couple of seconds (on AWS). There should be an easy fix: check, if the created PV already has all necessary labels and topology information and ask the cloud for them only when they're missing. Dynamically provisioned volumes already have these labels. The admission plugin will then ask the cloud only for manually provisioned PVs, which we believe is just a fraction of all PVs in the cluster. There is one glitch: previously, when user created a PV with wrong topology labels, the admission plugin corrected them. With the fix suggested above, this PV won't be corrected. /sig storage cc
Implementation of the above: