Change:
API server latency for PV creation depends on underlying cloud API latency. We have admission plugin that examines incoming PV and adds topology labels and node selector: When the cluster (and cloud) are under pressure, e.g. lot of PVCs are created and dynamically provisioned at one time and attached to nodes, API server latency increases. We've seen cases when we ran out of cloud API request quota and the admission plugin needed to sleep for couple of seconds (on AWS). There should be an easy fix: check, if the created PV already has all necessary labels and topology information and ask the cloud for them only when they're missing. Dynamically provisioned volumes already have these labels. The admission plugin will then ask the cloud only for manually provisioned PVs, which we believe is just a fraction of all PVs in the cluster. There is one glitch: previously, when user created a PV with wrong topology labels, the admission plugin corrected them. With the fix suggested above, this PV won't be corrected. /sig storage cc
Implementation of the above:

Discussion:
deps = [ "//pkg/apis/core:go_default_library", "//pkg/apis/core/v1:go_default_library", <ins> "//pkg/controller/volume/persistentvolume/util:go_default_library", </ins> "//pkg/kubeapiserver/admission:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", <ins> "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", </ins> "//staging/src/k8s.io/apiserver/pkg/admission:go_default_library", "//staging/src/k8s.io/cloud-provider:go_default_library", "//staging/src/k8s.io/cloud-provider/volume:go_default_library",

embed = [":go_default_library"], deps = [ "//pkg/apis/core:go_default_library", <ins> "//pkg/controller/volume/persistentvolume/util:go_default_library", </ins> "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

"sync" v1 "k8s.io/api/core/v1" <ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" </ins> "k8s.io/apiserver/pkg/admission" cloudprovider "k8s.io/cloud-provider" cloudvolume "k8s.io/cloud-provider/volume"

"k8s.io/klog" api "k8s.io/kubernetes/pkg/apis/core" k8s_api_v1 "k8s.io/kubernetes/pkg/apis/core/v1" <ins> persistentvolume "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" </ins> kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" )

return nil } <del> var volumeLabels map[string]string if volume.Spec.AWSElasticBlockStore != nil { labels, err := l.findAWSEBSLabels(volume) if err != nil { return admission.NewForbidden(a, fmt.Errorf("error querying AWS EBS volume %s: %v", volume.Spec.AWSElasticBlockStore.VolumeID, err)) } volumeLabels = labels } if volume.Spec.GCEPersistentDisk != nil { labels, err := l.findGCEPDLabels(volume) if err != nil { return admission.NewForbidden(a, fmt.Errorf("error querying GCE PD volume %s: %v", volume.Spec.GCEPersistentDisk.PDName, err)) } volumeLabels = labels } if volume.Spec.AzureDisk != nil { labels, err := l.findAzureDiskLabels(volume) if err != nil { return admission.NewForbidden(a, fmt.Errorf("error querying AzureDisk volume %s: %v", volume.Spec.AzureDisk.DiskName, err)) } volumeLabels = labels } if volume.Spec.Cinder != nil { labels, err := l.findCinderDiskLabels(volume) if err != nil { return admission.NewForbidden(a, fmt.Errorf("error querying Cinder volume %s: %v", volume.Spec.Cinder.VolumeID, err)) } volumeLabels = labels } if volume.Spec.VsphereVolume != nil { labels, err := l.findVsphereVolumeLabels(volume) if err != nil { return admission.NewForbidden(a, fmt.Errorf("error querying vSphere Volume %s: %v", volume.Spec.VsphereVolume.VolumePath, err)) } volumeLabels = labels </del> <ins> volumeLabels, err := l.findVolumeLabels(volume) if err != nil { return admission.NewForbidden(a, err) </ins> } requirements := make([]api.NodeSelectorRequirement, 0) if len(volumeLabels) != 0 { if volume.Labels == nil {

return nil } <ins> func (l *persistentVolumeLabel) findVolumeLabels(volume *api.PersistentVolume) (map[string]string, error) { existingLabels := volume.Labels // All cloud providers set only these two labels. domain, domainOK := existingLabels[v1.LabelZoneFailureDomain] region, regionOK := existingLabels[v1.LabelZoneRegion] isDynamicallyProvisioned := metav1.HasAnnotation(volume.ObjectMeta, persistentvolume.AnnDynamicallyProvisioned) if isDynamicallyProvisioned && domainOK && regionOK { // PV already has all the labels and we can trust the dynamic provisioning that it provided correct values. return map[string]string{ v1.LabelZoneFailureDomain: domain, v1.LabelZoneRegion:        region, }, nil } // Either missing labels or we don't trust the user provided correct values. switch { case volume.Spec.AWSElasticBlockStore != nil: labels, err := l.findAWSEBSLabels(volume) if err != nil { return nil, fmt.Errorf("error querying AWS EBS volume %s: %v", volume.Spec.AWSElasticBlockStore.VolumeID, err) } return labels, nil case volume.Spec.GCEPersistentDisk != nil: labels, err := l.findGCEPDLabels(volume) if err != nil { return nil, fmt.Errorf("error querying GCE PD volume %s: %v", volume.Spec.GCEPersistentDisk.PDName, err) } return labels, nil case volume.Spec.AzureDisk != nil: labels, err := l.findAzureDiskLabels(volume) if err != nil { return nil, fmt.Errorf("error querying AzureDisk volume %s: %v", volume.Spec.AzureDisk.DiskName, err) } return labels, nil case volume.Spec.Cinder != nil: labels, err := l.findCinderDiskLabels(volume) if err != nil { return nil, fmt.Errorf("error querying Cinder volume %s: %v", volume.Spec.Cinder.VolumeID, err) } return labels, nil case volume.Spec.VsphereVolume != nil: labels, err := l.findVsphereVolumeLabels(volume) if err != nil { return nil, fmt.Errorf("error querying vSphere Volume %s: %v", volume.Spec.VsphereVolume.VolumePath, err) } return labels, nil } // Unrecognized volume, do not add any labels return nil, nil } </ins> func (l *persistentVolumeLabel) findAWSEBSLabels(volume *api.PersistentVolume) (map[string]string, error) { // Ignore any volumes that are being provisioned if volume.Spec.AWSElasticBlockStore.VolumeID == cloudvolume.ProvisionedVolumeName {

admissiontesting "k8s.io/apiserver/pkg/admission/testing" cloudprovider "k8s.io/cloud-provider" api "k8s.io/kubernetes/pkg/apis/core" <ins> persistentvolume "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util" </ins> ) type mockVolumes struct {

err: nil, }, { <ins> name:    "existing labels from dynamic provisioning are not changed", handler: newPersistentVolumeLabel(), pvlabeler: mockVolumeLabels(map[string]string{ v1.LabelZoneFailureDomain: "domain1", v1.LabelZoneRegion:        "region1", }), preAdmissionPV: &api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "awsebs", Namespace: "myns", Labels: map[string]string{ v1.LabelZoneFailureDomain: "existingDomain", v1.LabelZoneRegion:        "existingRegion", }, Annotations: map[string]string{ persistentvolume.AnnDynamicallyProvisioned: "kubernetes.io/aws-ebs", }, }, Spec: api.PersistentVolumeSpec{ PersistentVolumeSource: api.PersistentVolumeSource{ AWSElasticBlockStore: &api.AWSElasticBlockStoreVolumeSource{ VolumeID: "123", }, }, }, }, postAdmissionPV: &api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name:      "awsebs", Namespace: "myns", Labels: map[string]string{ v1.LabelZoneFailureDomain: "existingDomain", v1.LabelZoneRegion:        "existingRegion", }, Annotations: map[string]string{ persistentvolume.AnnDynamicallyProvisioned: "kubernetes.io/aws-ebs", }, }, Spec: api.PersistentVolumeSpec{ PersistentVolumeSource: api.PersistentVolumeSource{ AWSElasticBlockStore: &api.AWSElasticBlockStoreVolumeSource{ VolumeID: "123", }, }, NodeAffinity: &api.VolumeNodeAffinity{ Required: &api.NodeSelector{ NodeSelectorTerms: []api.NodeSelectorTerm{ { MatchExpressions: []api.NodeSelectorRequirement{ { Key:      v1.LabelZoneRegion, Operator: api.NodeSelectorOpIn, Values:   []string{"existingRegion"}, }, { Key:      v1.LabelZoneFailureDomain, Operator: api.NodeSelectorOpIn, Values:   []string{"existingDomain"}, }, }, }, }, }, }, }, }, err: nil, }, { name:    "existing labels from user are changed", handler: newPersistentVolumeLabel(), pvlabeler: mockVolumeLabels(map[string]string{ v1.LabelZoneFailureDomain: "domain1", v1.LabelZoneRegion:        "region1", }), preAdmissionPV: &api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name: "awsebs", Namespace: "myns", Labels: map[string]string{ v1.LabelZoneFailureDomain: "existingDomain", v1.LabelZoneRegion:        "existingRegion", }, }, Spec: api.PersistentVolumeSpec{ PersistentVolumeSource: api.PersistentVolumeSource{ AWSElasticBlockStore: &api.AWSElasticBlockStoreVolumeSource{ VolumeID: "123", }, }, }, }, postAdmissionPV: &api.PersistentVolume{ ObjectMeta: metav1.ObjectMeta{ Name:      "awsebs", Namespace: "myns", Labels: map[string]string{ v1.LabelZoneFailureDomain: "domain1", v1.LabelZoneRegion:        "region1", }, }, Spec: api.PersistentVolumeSpec{ PersistentVolumeSource: api.PersistentVolumeSource{ AWSElasticBlockStore: &api.AWSElasticBlockStoreVolumeSource{ VolumeID: "123", }, }, NodeAffinity: &api.VolumeNodeAffinity{ Required: &api.NodeSelector{ NodeSelectorTerms: []api.NodeSelectorTerm{ { MatchExpressions: []api.NodeSelectorRequirement{ { Key:      v1.LabelZoneRegion, Operator: api.NodeSelectorOpIn, Values:   []string{"region1"}, }, { Key:      v1.LabelZoneFailureDomain, Operator: api.NodeSelectorOpIn, Values:   []string{"domain1"}, }, }, }, }, }, }, }, }, err: nil, }, { </ins> name:    "GCE PD PV labeled correctly", handler: newPersistentVolumeLabel(), pvlabeler: mockVolumeLabels(map[string]string{