Change:
So today, cadvisor became wedged and just stopped paying attention to its socket. Not kubelet's fault. But when I would launch all I would get would be: It took me a bit of trolling through netstat and lsof to figure out that the problem was cadvisor being completely locked up. kill -9 cadvisor and everything started working well. I wonder if we should emit more messages during startup about what operation we are about to attempt, not just the results of operations, as I was blaming docker for a while....
kubelet shouldn't block on this. I'll take a look.
Any ideas what was locking cAdvisor? and was it just not responding to requests?
I don't have any idea why it was locked. If I find another one locked I'll hang onto it to debug. Journal had: obviously the 'kill' came from me. It would not respond to systemctl cadvisor stop. netstat looking at port 4194 showed me At which point I realized cadvisor wasn't handling its sockets right since so many long dead kubelet sockets hadn't been cleaned up, so I kill -9'd it.
I cut some interesting info from the cadvisor journal log (trying to cut hostnames) You can see the last message about failed to update stats for container ... happened Oct 08 01:06:01. It was then silent until I killed it Oct 08 19:20:13, even though I'm pretty sure dnf was still being launched every hour on the hour. So it was wedged for about 18 hours.
What kernel is this on? We've seen there is a kernel bug fixed in 3.12+ that sometimes hangs us forever on exit. We're not quite sure how it gets triggered...
16.3

Discussion:
glog.Fatal("Couldn't connect to docker.") } <del> cadvisorClient, err := cadvisor.NewClient("http://127.0.0.1:4194") if err != nil { glog.Errorf("Error on creating cadvisor client: %v", err) } </del> hostname := getHostname() if *rootDirectory == "" {

k := kubelet.NewMainKubelet( getHostname(), dockerClient, <del> cadvisorClient, </del> etcdClient, *rootDirectory, *networkContainerImage,

float32(*registryPullQPS), *registryBurst) <ins> go func() { defer util.HandleCrash() // TODO: Monitor this connection, reconnect if needed? glog.V(1).Infof("Trying to create cadvisor client.") cadvisorClient, err := cadvisor.NewClient("http://127.0.0.1:4194") if err != nil { glog.Errorf("Error on creating cadvisor client: %v", err) return } glog.V(1).Infof("Successfully created cadvisor client.") k.SetCadvisorClient(cadvisorClient) }() </ins> // TODO: These should probably become more plugin-ish: register a factory func // in each checker's init(), iterate those here. health.AddHealthChecker(health.NewExecHealthChecker(k))

func NewMainKubelet( hn string, dc dockertools.DockerInterface, <del> cc CadvisorInterface, </del> ec tools.EtcdClient, rd string, ni string,

return &Kubelet{ hostname:              hn, dockerClient:          dc, <del> cadvisorClient:        cc, </del> etcdClient:            ec, rootDirectory:         rd, resyncInterval:        ri,

// Optional, no events will be sent without it etcdClient tools.EtcdClient <del> // Optional, no statistics will be available if omitted cadvisorClient CadvisorInterface </del> // Optional, defaults to simple implementaiton healthChecker health.HealthChecker // Optional, defaults to simple Docker implementation

pullQPS float32 // Optional, maximum burst QPS from the docker registry, must be positive if QPS is > 0.0 pullBurst int <ins> // Optional, no statistics will be available if omitted cadvisorClient CadvisorInterface cadvisorLock   sync.RWMutex } // SetCadvisorClient sets the cadvisor client in a thread-safe way. func (kl *Kubelet) SetCadvisorClient(c CadvisorInterface) { kl.cadvisorLock.Lock() defer kl.cadvisorLock.Unlock() kl.cadvisorClient = c } // GetCadvisorClient gets the cadvisor client. func (kl *Kubelet) GetCadvisorClient() CadvisorInterface { kl.cadvisorLock.RLock() defer kl.cadvisorLock.RUnlock() return kl.cadvisorClient </ins> } // Run starts the kubelet reacting to config updates

// container.  The container's absolute path refers to its hierarchy in the // cgroup file system. e.g. The root container, which represents the whole // machine, has path "/"; all docker containers have path "/docker/<docker id>" <del> func (kl *Kubelet) statsFromContainerPath(containerPath string, req *info.ContainerInfoRequest) (*info.ContainerInfo, error) { cinfo, err := kl.cadvisorClient.ContainerInfo(containerPath, getCadvisorContainerInfoRequest(req)) </del> <ins> func (kl *Kubelet) statsFromContainerPath(cc CadvisorInterface, containerPath string, req *info.ContainerInfoRequest) (*info.ContainerInfo, error) { cinfo, err := cc.ContainerInfo(containerPath, getCadvisorContainerInfoRequest(req)) </ins> if err != nil { return nil, err }

// GetContainerInfo returns stats (from Cadvisor) for a container. func (kl *Kubelet) GetContainerInfo(podFullName, uuid, containerName string, req *info.ContainerInfoRequest) (*info.ContainerInfo, error) { <del> if kl.cadvisorClient == nil { </del> <ins> cc := kl.GetCadvisorClient() if cc == nil { </ins> return nil, nil } dockerContainers, err := dockertools.GetKubeletDockerContainers(kl.dockerClient, false)

if !found { return nil, errors.New("couldn't find container") } <del> return kl.statsFromContainerPath(fmt.Sprintf("/docker/%s", dockerContainer.ID), req) </del> <ins> return kl.statsFromContainerPath(cc, fmt.Sprintf("/docker/%s", dockerContainer.ID), req) </ins> } // GetRootInfo returns stats (from Cadvisor) of current machine (root container). func (kl *Kubelet) GetRootInfo(req *info.ContainerInfoRequest) (*info.ContainerInfo, error) { <del> return kl.statsFromContainerPath("/", req) </del> <ins> cc := kl.GetCadvisorClient() if cc == nil { return nil, fmt.Errorf("no cadvisor connection") } return kl.statsFromContainerPath(cc, "/", req) </ins> } func (kl *Kubelet) GetMachineInfo() (*info.MachineInfo, error) { <del> return kl.cadvisorClient.MachineInfo() </del> <ins> cc := kl.GetCadvisorClient() if cc == nil { return nil, fmt.Errorf("no cadvisor connection") } return cc.MachineInfo() </ins> } func (kl *Kubelet) healthy(podFullName string, currentState api.PodState, container api.Container, dockerContainer *docker.APIContainers) (health.Status, error) {