Change:
What happened: Upgraded from 1.16.6 to 1.17.2 and will no longer start on control-plane hosts. It fails with: What you expected to happen: to start, static pods to launch, to connect to in the static pod How to reproduce it (as minimally and precisely as possible): Have the , , , etc in the static pod manifest folder. Try to start the with the container images not preloaded on the host. Just in case it is needed: is used as the backend is run using Anything else we need to know?: This looks to be two things interacting poorly: 1- It takes a long time for static pods to be created for the first time when the can't talk to yet. 2- The new CSI migration code force terminates causing these log lines to appear in : It looks like introduced this new fatal error. Environment: Kubernetes version (use ): 1.17.2 Cloud provider or hardware configuration: Bare-metal OS (e.g: ): Fedora 31..1 (CoreOS preview)" Kernel (e.g. ): 5.3.12-300.fc31.x86_64 Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig cluster-lifecycle
Actually, probably: /sig node
The referenced PR was in 1.14, so that interaction is not new. What is likely new is the size increase of hyperkube in 1.17 (xref ). When not pre-pulled, this can increase the time it takes to start hyperkube for the first time. Does the image pull make progress as the kubelet restarts/retries?
It might make sense for the initializeCSINode() method to: first wait to successfully talk to the kube apiserver at all, perhaps indefinitely after successfully talking to the kube apiserver, then try to register the CSINode object with backoff, and fail if it cannot do that successfully
/sig storage /remove-sig cluster-lifecycle
It doesn't seem to. Once the connection to containerd is closed, it cancels the context. Otherwise I would expect this to succeed eventually if the pull happened in the background. That sounds reasonable to me assuming it isn't blocking the static pod creation code. Do you happen to know why it takes a very long time for the static pod creation code to be called? I would expect the pods to be created very soon after the kubelet is started.
Pulling the image and starting the processes for those pods happens immediately (from the containerd logs, you can see the image pull happening... it just doesn't complete in time before the independent kubelet logic to register the CSI node info times out).
Ah, yes. I just upgraded a couple of days ago so I didn't check whether I still saw the 2 minute delay in creating the "pod sandboxes" (this is before the pull even happens). It now is only a 23 second delay: containerd: kubelet:
ack, I like the idea Will work on a fix today and cherry-pick it as necessary

Discussion:
package csi import ( <ins> "context" </ins> "errors" "fmt" "os"

"strings" "time" <del> "context" </del> "k8s.io/klog" api "k8s.io/api/core/v1"

if utilfeature.DefaultFeatureGate.Enabled(features.CSINodeInfo) && utilfeature.DefaultFeatureGate.Enabled(features.CSIMigration) { <del> // This function prevents Kubelet from posting Ready status until CSINodeInfo </del> <ins> // This function prevents Kubelet from posting Ready status until CSINode </ins> // is both installed and initialized if err := initializeCSINode(host); err != nil { <del> return errors.New(log("failed to initialize CSINodeInfo: %v", err)) </del> <ins> return errors.New(log("failed to initialize CSINode: %v", err)) </ins> } }

func initializeCSINode(host volume.VolumeHost) error { kvh, ok := host.(volume.KubeletVolumeHost) if !ok { <del> klog.V(4).Info("Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINodeInfo initialization, not running on kubelet") </del> <ins> klog.V(4).Info("Cast from VolumeHost to KubeletVolumeHost failed. Skipping CSINode initialization, not running on kubelet") </ins> return nil } kubeClient := host.GetKubeClient() if kubeClient == nil { <del> // Kubelet running in standalone mode. Skip CSINodeInfo initialization klog.Warning("Skipping CSINodeInfo initialization, kubelet running in standalone mode") </del> <ins> // Kubelet running in standalone mode. Skip CSINode initialization klog.Warning("Skipping CSINode initialization, kubelet running in standalone mode") </ins> return nil } <del> kvh.SetKubeletError(errors.New("CSINodeInfo is not yet initialized")) </del> <ins> kvh.SetKubeletError(errors.New("CSINode is not yet initialized")) </ins> go func() { defer utilruntime.HandleCrash() <ins> // First wait indefinitely to talk to Kube APIServer nodeName := host.GetNodeName() err := waitForAPIServerForever(kubeClient, nodeName) if err != nil { klog.Fatalf("Failed to initialize CSINode while waiting for API server to report ok: %v", err) } </ins> // Backoff parameters tuned to retry over 140 seconds. Will fail and restart the Kubelet // after max retry steps. initBackoff := wait.Backoff{

Factor:   6.0, Jitter:   0.1, } <del> err := wait.ExponentialBackoff(initBackoff, func() (bool, error) { klog.V(4).Infof("Initializing migrated drivers on CSINodeInfo") </del> <ins> err = wait.ExponentialBackoff(initBackoff, func() (bool, error) { klog.V(4).Infof("Initializing migrated drivers on CSINode") </ins> err := nim.InitializeCSINodeWithAnnotation() if err != nil { <del> kvh.SetKubeletError(fmt.Errorf("Failed to initialize CSINodeInfo: %v", err)) klog.Errorf("Failed to initialize CSINodeInfo: %v", err) </del> <ins> kvh.SetKubeletError(fmt.Errorf("Failed to initialize CSINode: %v", err)) klog.Errorf("Failed to initialize CSINode: %v", err) </ins> return false, nil }

// using CSI for all Migrated volume plugins. Then all the CSINode initialization // code can be dropped from Kubelet. // Kill the Kubelet process and allow it to restart to retry initialization <del> klog.Fatalf("Failed to initialize CSINodeInfo after retrying") </del> <ins> klog.Fatalf("Failed to initialize CSINode after retrying: %v", err) </ins> } }() return nil

} return highestSupportedVersion, nil } <ins> // waitForAPIServerForever waits forever to get a CSINode instance as a proxy // for a healthy APIServer func waitForAPIServerForever(client clientset.Interface, nodeName types.NodeName) error { var lastErr error err := wait.PollImmediateInfinite(time.Second, func() (bool, error) { // Get a CSINode from API server to make sure 1) kubelet can reach API server // and 2) it has enough permissions. Kubelet may have restricted permissions // when it's bootstrapping TLS. // https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/ _, lastErr = client.StorageV1().CSINodes().Get(context.TODO(), string(nodeName), meta.GetOptions{}) if lastErr == nil || apierrors.IsNotFound(lastErr) { // API server contacted return true, nil } klog.V(2).Infof("Failed to contact API server when waiting for CSINode publishing: %s", lastErr) return false, nil }) if err != nil { // In theory this is unreachable, but just in case: return fmt.Errorf("%v: %v", err, lastErr) } return nil } </ins>

return goerrors.New("error getting CSI client") } <del> var updateErrs []error </del> <ins> var lastErr error </ins> err := wait.ExponentialBackoff(updateBackoff, func() (bool, error) { <del> if err := nim.tryInitializeCSINodeWithAnnotation(csiKubeClient); err != nil { updateErrs = append(updateErrs, err) </del> <ins> if lastErr = nim.tryInitializeCSINodeWithAnnotation(csiKubeClient); lastErr != nil { klog.V(2).Infof("Failed to publish CSINode: %v", lastErr) </ins> return false, nil } return true, nil }) if err != nil { <del> return fmt.Errorf("error updating CSINode annotation: %v; caused by: %v", err, utilerrors.NewAggregate(updateErrs)) </del> <ins> return fmt.Errorf("error updating CSINode annotation: %v; caused by: %v", err, lastErr) </ins> } return nil

} func (f *fakeVolumeHost) WaitForKubeletErrNil() error { <del> return wait.PollImmediate(100*time.Millisecond, 10*time.Second, func() (bool, error) { </del> <ins> return wait.PollImmediate(10*time.Millisecond, 10*time.Second, func() (bool, error) { </ins> f.mux.Lock() defer f.mux.Unlock() return f.kubeletErr == nil, nil