Change:
On AWS when we call "attach" of a volume then we periodically we check (via API call) if volume has indeed been attached. We run the check in a loop with timeout of 21 minutes. AWS documents that - if a volume is stuck in "attaching" state for too long, the node where volume is being attached is in "dirty" state and must be rebooted to fix the problem. Until node is rebooted, more pods with volumes to same node is only going be stuck in same state (i.e "attaching"). We propose that - if attach operation times out waiting for volume in "attaching" state, we should simply taint the node so as it can not affect other pods and a admin can manually fix the node. /sig aws /sig storage cc

Discussion:
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", "//vendor/gopkg.in/gcfg.v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", <ins> "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", </ins> "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", <ins> "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/kubernetes/scheme:go_default_library", "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//vendor/k8s.io/client-go/tools/record:go_default_library", </ins> ], )

"github.com/aws/aws-sdk-go/service/kms" "github.com/golang/glog" "github.com/prometheus/client_golang/prometheus" <ins> clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/record" </ins> "path" "k8s.io/api/core/v1" <ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" </ins> "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" <ins> "k8s.io/client-go/kubernetes/scheme" v1core "k8s.io/client-go/kubernetes/typed/core/v1" </ins> "k8s.io/kubernetes/pkg/api/v1/service" "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/controller"

// For example: "Key1=Val1,Key2=Val2,KeyNoVal1=,KeyNoVal2" const ServiceAnnotationLoadBalancerAdditionalTags = "service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags" <ins> // Event key when a volume is stuck on attaching state when being attached to a volume const volumeAttachmentStuck = "VolumeAttachmentStuck" // Indicates that a node has volumes stuck in attaching state and hence it is not fit for scheduling more pods const nodeWithImpairedVolumes = "NodeWithImpairedVolumes" </ins> const ( // volumeAttachmentConsecutiveErrorLimit is the number of consecutive errors we will ignore when waiting for a volume to attach/detach volumeAttachmentStatusConsecutiveErrorLimit = 10

instanceCache instanceCache <ins> clientBuilder    controller.ControllerClientBuilder kubeClient       clientset.Interface eventBroadcaster record.EventBroadcaster eventRecorder    record.EventRecorder </ins> // We keep an active list of devices we have assigned but not yet // attached, to avoid a race condition where we assign a device mapping // and then get a second request before we attach the volume

} // Initialize passes a Kubernetes clientBuilder interface to the cloud provider <del> func (c *Cloud) Initialize(clientBuilder controller.ControllerClientBuilder) {} </del> <ins> func (c *Cloud) Initialize(clientBuilder controller.ControllerClientBuilder) { c.clientBuilder = clientBuilder c.kubeClient = clientBuilder.ClientOrDie("cloud-provider") c.eventBroadcaster = record.NewBroadcaster() c.eventBroadcaster.StartLogging(glog.Infof) c.eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(c.kubeClient.CoreV1().RESTClient()).Events("")}) c.eventRecorder = c.eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "aws-cloudprovider"}) } </ins> // Clusters returns the list of clusters. func (c *Cloud) Clusters() (cloudprovider.Clusters, bool) {

return volumes[0], nil } <ins> // applyUnSchedulableTaint applies a unschedulable taint to a node after verifying // if node has become unusable because of volumes getting stuck in attaching state. func (c *Cloud) applyUnSchedulableTaint(nodeName types.NodeName, reason string) { node, fetchErr := c.kubeClient.CoreV1().Nodes().Get(string(nodeName), metav1.GetOptions{}) if fetchErr != nil { glog.Errorf("Error fetching node %s with %v", nodeName, fetchErr) return } taint := &v1.Taint{ Key:    nodeWithImpairedVolumes, Value:  "true", Effect: v1.TaintEffectNoSchedule, } err := controller.AddOrUpdateTaintOnNode(c.kubeClient, string(nodeName), taint) if err != nil { glog.Errorf("Error applying taint to node %s with error %v", nodeName, err) return } c.eventRecorder.Eventf(node, v1.EventTypeWarning, volumeAttachmentStuck, reason) } </ins> // waitForAttachmentStatus polls until the attachment status is the expected value // On success, it returns the last attachment state. func (d *awsDisk) waitForAttachmentStatus(status string) (*ec2.VolumeAttachment, error) {

} attachment, err := disk.waitForAttachmentStatus("attached") if err != nil { <ins> if err == wait.ErrWaitTimeout { c.applyUnSchedulableTaint(nodeName, "Volume stuck in attaching state - node needs reboot to fix impaired state.") } </ins> return "", err }