Change:
What happened: Windows kubelet panic occasionally on nodes which have high volume of pods run and delete. Full logs: Preliminary investigate result: How to reproduce it (as minimally and precisely as possible): Run high volume of pods and delete them when they completed in Windows nodes. Environment: Kubernetes version (use ): 1.17.7 Cloud provider or hardware configuration: Azure AKS OS (e.g: ): Windows Server 2019 Kernel (e.g. ): 10.0..1282
/assign I would like to give this a shot.
/remove-sig windows I think this is not a windows specific problem. /sig node /area kubelet
Hi! Thanks for your report, I have filed to address this issue.
This issue is a bit urgent for our situation, it occurs every few days in production clusters. Seems the PR is pending a few days, is it possible to increase the priority to merge it trunk and 1.17/1.18 branch? Thanks
I understand how urgent this issue is, but I am not privileged to do anything further :(
Do you know anyone who can help?
looks like a simple fix, add a lock for , I could help.
There is already a fix by , the problem is the PR is pending, not sure if you or someone can increase the priority to merge it trunk and 1.17/1.18 branch asap. Our AKS clusters are affecting by this frequently.
Thanks guys.

Discussion:
return &runtimeapi.ListContainersResponse{Containers: result}, nil } <ins> func (ds *dockerService) getContainerCleanupInfo(containerID string) (*containerCleanupInfo, bool) { ds.cleanupInfosLock.RLock() defer ds.cleanupInfosLock.RUnlock() info, ok := ds.containerCleanupInfos[containerID] return info, ok } func (ds *dockerService) setContainerCleanupInfo(containerID string, info *containerCleanupInfo) { ds.cleanupInfosLock.Lock() defer ds.cleanupInfosLock.Unlock() ds.containerCleanupInfos[containerID] = info } func (ds *dockerService) clearContainerCleanupInfo(containerID string) { ds.cleanupInfosLock.Lock() defer ds.cleanupInfosLock.Unlock() delete(ds.containerCleanupInfos, containerID) } </ins> // CreateContainer creates a new container in the given PodSandbox // Docker cannot store the log to an arbitrary location (yet), so we create an // symlink at LogPath, linking to the actual path of the log.

// we don't perform the clean up just yet at that could destroy information // needed for the container to start (e.g. Windows credentials stored in // registry keys); instead, we'll clean up when the container gets removed <del> ds.containerCleanupInfos[containerID] = cleanupInfo </del> <ins> ds.setContainerCleanupInfo(containerID, cleanupInfo) </ins> } return &runtimeapi.CreateContainerResponse{ContainerId: containerID}, nil }

} func (ds *dockerService) performPlatformSpecificContainerForContainer(containerID string) (errors []error) { <del> if cleanupInfo, present := ds.containerCleanupInfos[containerID]; present { </del> <ins> if cleanupInfo, present := ds.getContainerCleanupInfo(containerID); present { </ins> errors = ds.performPlatformSpecificContainerCleanupAndLogErrors(containerID, cleanupInfo) if len(errors) == 0 { <del> delete(ds.containerCleanupInfos, containerID) </del> <ins> ds.clearContainerCleanupInfo(containerID) </ins> } }

"fmt" "path/filepath" "strings" <ins> "sync" </ins> "testing" "time"

return context.Background() } <ins> // TestConcurrentlyCreateAndDeleteContainers is a regression test for #93771, which ensures // kubelet would not panic on concurrent writes to `dockerService.containerCleanupInfos`. func TestConcurrentlyCreateAndDeleteContainers(t *testing.T) { ds, _, _ := newTestDockerService() podName, namespace := "foo", "bar" containerName, image := "sidecar", "logger" const count = 20 configs := make([]*runtimeapi.ContainerConfig, 0, count) sConfigs := make([]*runtimeapi.PodSandboxConfig, 0, count) for i := 0; i < count; i++ { s := makeSandboxConfig(fmt.Sprintf("%s%d", podName, i), fmt.Sprintf("%s%d", namespace, i), fmt.Sprintf("%d", i), 0) labels := map[string]string{"concurrent-test": fmt.Sprintf("label%d", i)} c := makeContainerConfig(s, fmt.Sprintf("%s%d", containerName, i), fmt.Sprintf("%s:v%d", image, i), uint32(i), labels, nil) sConfigs = append(sConfigs, s) configs = append(configs, c) } containerIDs := make(chan string, len(configs)) // make channel non-blocking to simulate concurrent containers creation var ( creationWg sync.WaitGroup deletionWg sync.WaitGroup ) creationWg.Add(len(configs)) go func() { creationWg.Wait() close(containerIDs) }() for i := range configs { go func(i int) { defer creationWg.Done() // We don't care about the sandbox id; pass a bogus one. sandboxID := fmt.Sprintf("sandboxid%d", i) req := &runtimeapi.CreateContainerRequest{PodSandboxId: sandboxID, Config: configs[i], SandboxConfig: sConfigs[i]} createResp, err := ds.CreateContainer(getTestCTX(), req) if err != nil { t.Errorf("CreateContainer: %v", err) return } containerIDs <- createResp.ContainerId }(i) } for containerID := range containerIDs { deletionWg.Add(1) go func(id string) { defer deletionWg.Done() _, err := ds.RemoveContainer(getTestCTX(), &runtimeapi.RemoveContainerRequest{ContainerId: id}) if err != nil { t.Errorf("RemoveContainer: %v", err) } }(containerID) } deletionWg.Wait() } </ins> // TestListContainers creates several containers and then list them to check // whether the correct metadatas, states, and labels are returned. func TestListContainers(t *testing.T) {

dockertypes "github.com/docker/docker/api/types" "k8s.io/klog/v2" <del> "k8s.io/api/core/v1" </del> <ins> v1 "k8s.io/api/core/v1" </ins> runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager"

// (see `applyPlatformSpecificDockerConfig` and `performPlatformSpecificContainerCleanup` // methods for more info). containerCleanupInfos map[string]*containerCleanupInfo <ins> cleanupInfosLock      sync.RWMutex </ins> } // TODO: handle context.