Change:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: We have pretty large clusters in aws and in production, we found an inefficiency in provisioning dynamic EBS volumes, where aws-ebs volume provisioner tries to look for available zone from aws directly. With scale, this can trigger massive from ec2, and affect cluster stability. This is also related to What you expected to happen: To simply get zones with available instances, we can get information from API server for efficiency. There might be a delay between node gets shutting down and such change is reflected in api server, but in case node is not ready or gets terminated, there will be higher level reconciliation triggered so we are safe. How to reproduce it (as minimally and precisely as possible): We only expect this to be hit when the cluster is huge (or aws account is huge) so not easy to reproduce. /assign /sig aws /sig cloud-provider
/assign

Discussion:
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", <ins> "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", </ins> "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/client-go/util/testing:go_default_library", "//staging/src/k8s.io/legacy-cloud-providers/aws:go_default_library", <ins> "//vendor/github.com/stretchr/testify/assert:go_default_library", </ins> "//vendor/k8s.io/klog:go_default_library", ], )

"reflect" "testing" <ins> "github.com/stretchr/testify/assert" </ins> "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" <ins> "k8s.io/apimachinery/pkg/util/sets" </ins> "k8s.io/client-go/kubernetes/fake" utiltesting "k8s.io/client-go/util/testing" "k8s.io/kubernetes/pkg/util/mount"

t.Errorf("Expected mount options to be %v got %v", expectedMountOptions, mountOptions) } } <ins> func TestGetCandidateZone(t *testing.T) { const testZone = "my-zone-1a" // TODO: add test case for immediate bind volume when we have a good way to mock Cloud outside cloudprovider tests := []struct { cloud         *aws.Cloud node          *v1.Node expectedZones sets.String }{ { cloud: nil, node: &v1.Node{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ v1.LabelZoneFailureDomain: testZone, }, }, }, expectedZones: sets.NewString(), }, { cloud:         nil, node:          &v1.Node{}, expectedZones: sets.NewString(), }, } for _, test := range tests { zones, err := getCandidateZones(test.cloud, test.node) assert.Nil(t, err) assert.Equal(t, test.expectedZones, zones) } } </ins>

"k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/sets" <del> cloudprovider "k8s.io/cloud-provider" </del> <ins> "k8s.io/cloud-provider" </ins> volumehelpers "k8s.io/cloud-provider/volume/helpers" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/volume"

capacity := c.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)] <del> zonesWithNodes, err := cloud.GetCandidateZonesForDynamicVolume() </del> <ins> zonesWithNodes, err := getCandidateZones(cloud, node) </ins> if err != nil { <del> return "", 0, nil, "", fmt.Errorf("error querying for all zones: %v", err) </del> <ins> return "", 0, nil, "", fmt.Errorf("error finding candidate zone for pvc: %v", err) </ins> } volumeOptions, err := populateVolumeOptions(c.plugin.GetPluginName(), c.options.PVC.Name, capacity, tags, c.options.Parameters, node, allowedTopologies, zonesWithNodes)

return name, volumeOptions.CapacityGB, labels, fstype, nil } <ins> // getCandidateZones finds possible zones that a volume can be created in func getCandidateZones(cloud *aws.Cloud, selectedNode *v1.Node) (sets.String, error) { if selectedNode != nil { // For topology aware volume provisioning, node is already selected so we use the zone from // selected node directly instead of candidate zones. // We can assume the information is always available as node controller shall maintain it. return sets.NewString(), nil } // For non-topology-aware volumes (those that binds immediately), we fall back to original logic to query // cloud provider for possible zones return cloud.GetCandidateZonesForDynamicVolume() } </ins> // returns volumeOptions for EBS based on storageclass parameters and node configuration func populateVolumeOptions(pluginName, pvcName string, capacityGB resource.Quantity, tags map[string]string, storageParams map[string]string, node *v1.Node, allowedTopologies []v1.TopologySelectorTerm, zonesWithNodes sets.String) (*aws.VolumeOptions, error) { requestGiB, err := volumehelpers.RoundUpToGiBInt(capacityGB)