Change:
// GetCandidateZonesForDynamicVolume retrieves  a list of all the zones in which nodes are running // It currently involves querying all instances func (c *Cloud) GetCandidateZonesForDynamicVolume() (sets.String, error) { <del> // We don't currently cache this; it is currently used only in volume // creation which is expected to be a comparatively rare occurrence. // TODO: Caching / expose v1.Nodes to the cloud provider? // TODO: We could also query for subnets, I think // Note: It is more efficient to call the EC2 API twice with different tag // filters than to call it once with a tag filter that results in a logical // OR. For really large clusters the logical OR will result in EC2 API rate // limiting. instances := []*ec2.Instance{} baseFilters := []*ec2.Filter{newEc2Filter("instance-state-name", "running")} </del> <ins> zones := sets.NewString() </ins> <del> filters := c.tagging.addFilters(baseFilters) di, err := c.describeInstances(filters) </del> <ins> // TODO: list from cache? nodes, err := c.kubeClient.CoreV1().Nodes().List(metav1.ListOptions{}) </ins> if err != nil { <ins> klog.Errorf("Failed to get nodes from api server: %#v", err) </ins> return nil, err } <del> instances = append(instances, di...) if c.tagging.usesLegacyTags { filters = c.tagging.addLegacyFilters(baseFilters) di, err = c.describeInstances(filters) if err != nil { return nil, err </del> <ins> for _, n := range nodes.Items { if !c.isNodeReady(&n) { klog.V(4).Infof("Ignoring not ready node %q in zone discovery", n.Name) continue </ins> } <del> instances = append(instances, di...) } if len(instances) == 0 { return nil, fmt.Errorf("no instances returned") } zones := sets.NewString() for _, instance := range instances { // We skip over master nodes, if the installation tool labels them with one of the well-known master labels // This avoids creating a volume in a zone where only the master is running - e.g. #34583 // This is a short-term workaround until the scheduler takes care of zone selection master := false for _, tag := range instance.Tags { tagKey := aws.StringValue(tag.Key) if awsTagNameMasterRoles.Has(tagKey) { master = true </del> <ins> // In some cluster provisioning software, a node can be both a minion and a master. Therefore we white-list // here, and only filter out node that is not minion AND is labeled as master explicitly if c.isMinionNode(&n) || !c.isMasterNode(&n) { if zone, ok := n.Labels[v1.LabelZoneFailureDomain]; ok { zones.Insert(zone) } else { klog.Warningf("Node %s does not have zone label, ignore for zone discovery.", n.Name) </ins> } <ins> } else { klog.V(4).Infof("Ignoring master node %q in zone discovery", n.Name) </ins> } <ins> } </ins> <del> if master { klog.V(4).Infof("Ignoring master instance %q in zone discovery", aws.StringValue(instance.InstanceId)) continue } </del> <ins> klog.V(2).Infof("Found instances in zones %s", zones) return zones, nil } </ins> <del> if instance.Placement != nil { zone := aws.StringValue(instance.Placement.AvailabilityZone) zones.Insert(zone) </del> <ins> // isNodeReady checks node condition and return true if NodeReady is marked as true func (c *Cloud) isNodeReady(node *v1.Node) bool { for _, c := range node.Status.Conditions { if c.Type == v1.NodeReady { return c.Status == v1.ConditionTrue </ins> } } <ins> return false } // isMasterNode checks if the node is labeled as master func (c *Cloud) isMasterNode(node *v1.Node) bool { // Master node has one or more of the following labels: // // 	kubernetes.io/role: master //	node-role.kubernetes.io/master: "" //	node-role.kubernetes.io/master: "true" if val, ok := node.Labels[labelKeyNodeMaster]; ok && val != "false" { return true } else if role, ok := node.Labels[labelKeyNodeRole]; ok && role == nodeMasterRole { return true } return false } </ins> <del> klog.V(2).Infof("Found instances in zones %s", zones) return zones, nil </del> <ins> // isMinionNode checks if the node is labeled as minion func (c *Cloud) isMinionNode(node *v1.Node) bool { // Minion node has one or more oof the following labels: // // 	kubernetes.io/role: "node" //	node-role.kubernetes.io/compute: "true" //	node-role.kubernetes.io/node: "" if val, ok := node.Labels[labelKeyNodeMinion]; ok && val != "false" { return true } else if val, ok := node.Labels[labelKeyNodeCompute]; ok && val != "false" { return true } else if role, ok := node.Labels[labelKeyNodeRole]; ok && role == nodeMinionRole { return true } return false </ins> } // GetZone implements Zones.GetZone

Discussion:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: We have pretty large clusters in aws and in production, we found an inefficiency in provisioning dynamic EBS volumes, where aws-ebs volume provisioner tries to look for available zone from aws directly. With scale, this can trigger massive from ec2, and affect cluster stability. This is also related to What you expected to happen: To simply get zones with available instances, we can get information from API server for efficiency. There might be a delay between node gets shutting down and such change is reflected in api server, but in case node is not ready or gets terminated, there will be higher level reconciliation triggered so we are safe. How to reproduce it (as minimally and precisely as possible): We only expect this to be hit when the cluster is huge (or aws account is huge) so not easy to reproduce. /assign /sig aws /sig cloud-provider
/assign