Change:
assert.Nil(t, testMainKubelet.configMapManager, "configmap manager should be nil if kubelet is in standalone mode") assert.Nil(t, testMainKubelet.secretManager, "secret manager should be nil if kubelet is in standalone mode") } <ins> func TestSyncPodSpans(t *testing.T) { testKubelet := newTestKubelet(t, false) kubelet := testKubelet.kubelet recorder := record.NewFakeRecorder(20) nodeRef := &v1.ObjectReference{ Kind:      "Node", Name:      "testNode", UID:       types.UID("testNode"), Namespace: "", } kubelet.dnsConfigurer = dns.NewConfigurer(recorder, nodeRef, nil, nil, "TEST", "") kubeCfg := &kubeletconfiginternal.KubeletConfiguration{ SyncFrequency: metav1.Duration{Duration: time.Minute}, ConfigMapAndSecretChangeDetectionStrategy: kubeletconfiginternal.WatchChangeDetectionStrategy, ContainerLogMaxSize:                       "10Mi", ContainerLogMaxFiles:                      5, MemoryThrottlingFactor:                    utilpointer.Float64(0), } exp := tracetest.NewInMemoryExporter() tp := sdktrace.NewTracerProvider( sdktrace.WithSyncer(exp), ) kubelet.tracer = tp.Tracer(instrumentationScope) fakeRuntime, endpoint := createAndStartFakeRemoteRuntime(t) defer func() { fakeRuntime.Stop() }() runtimeSvc := createRemoteRuntimeService(endpoint, t, tp) kubelet.runtimeService = runtimeSvc fakeRuntime.ImageService.SetFakeImageSize(100) fakeRuntime.ImageService.SetFakeImages([]string{"test:latest"}) imageSvc, err := remote.NewRemoteImageService(endpoint, 15*time.Second, tp) assert.NoError(t, err) kubelet.containerRuntime, err = kuberuntime.NewKubeGenericRuntimeManager( kubelet.recorder, kubelet.livenessManager, kubelet.readinessManager, kubelet.startupManager, kubelet.rootDirectory, kubelet.machineInfo, kubelet.podWorkers, kubelet.os, kubelet, nil, kubelet.backOff, kubeCfg.SerializeImagePulls, kubeCfg.MaxParallelImagePulls, float32(kubeCfg.RegistryPullQPS), int(kubeCfg.RegistryBurst), "", "", kubeCfg.CPUCFSQuota, kubeCfg.CPUCFSQuotaPeriod, runtimeSvc, imageSvc, kubelet.containerManager, kubelet.containerLogManager, kubelet.runtimeClassManager, false, kubeCfg.MemorySwap.SwapBehavior, kubelet.containerManager.GetNodeAllocatableAbsolute, *kubeCfg.MemoryThrottlingFactor, kubeletutil.NewPodStartupLatencyTracker(), tp, ) assert.NoError(t, err) pod := podWithUIDNameNsSpec("12345678", "foo", "new", v1.PodSpec{ Containers: []v1.Container{ { Name:            "bar", Image:           "test:latest", ImagePullPolicy: v1.PullAlways, }, }, EnableServiceLinks: utilpointer.Bool(false), }) _, err = kubelet.SyncPod(context.Background(), kubetypes.SyncPodCreate, pod, nil, &kubecontainer.PodStatus{}) require.NoError(t, err) require.NoError(t, err) assert.NotEmpty(t, exp.GetSpans()) // find root span for syncPod var rootSpan *tracetest.SpanStub spans := exp.GetSpans() for i, span := range spans { if span.Name == "syncPod" { rootSpan = &spans[i] break } } assert.NotNil(t, rootSpan) imageServiceSpans := make([]tracetest.SpanStub, 0) runtimeServiceSpans := make([]tracetest.SpanStub, 0) for _, span := range exp.GetSpans() { if span.SpanContext.TraceID() == rootSpan.SpanContext.TraceID() { switch { case strings.HasPrefix(span.Name, "runtime.v1.ImageService"): imageServiceSpans = append(imageServiceSpans, span) case strings.HasPrefix(span.Name, "runtime.v1.RuntimeService"): runtimeServiceSpans = append(runtimeServiceSpans, span) } } } assert.NotEmpty(t, imageServiceSpans, "syncPod trace should have image service spans") assert.NotEmpty(t, runtimeServiceSpans, "syncPod trace should have runtime service spans") for _, span := range imageServiceSpans { assert.Equal(t, span.Parent.SpanID(), rootSpan.SpanContext.SpanID(), fmt.Sprintf("image service span %s %s should be child of root span", span.Name, span.Parent.SpanID())) } for _, span := range runtimeServiceSpans { assert.Equal(t, span.Parent.SpanID(), rootSpan.SpanContext.SpanID(), fmt.Sprintf("runtime service span %s %s should be child of root span", span.Name, span.Parent.SpanID())) } } </ins>

Discussion:
spans are not being parented to other spans - propagation of context seems broken. More context: spans should be parented to other spans enable kubelet tracing at v1.28 and hook up a tracing backend. No response <detailsv1.28 at least </details<detailsNot sure this matters. </details<details</details<details</details<details</details<details</details
/sig instrumentation /assign
/triage accepted
!
repro steps:
/assign Thanks for helping to look into this! Feel free to post the findings here you shared with me on slack
GitHub didn't allow me to assign the following users: fatsheep9146. Note that only with read permissions, repo collaborators and people who have commented on this issue/PR can be assigned. Additionally, issues/PRs can only have 10 assignees at the same time. For more information please see <detailsIn response to : /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Thanks! I will post this ASAP
The broken issue is due to the pr , which try to propagate context brought by pod worker to syncPod function in order to let the syncPod can be interrupted in time by other asynchronous routines, for example eviction. But in the line we did not pass the context which contain the parent span and trace into function, so the trace is broken. So I want to fix this by passing the into , which is also desired in comment But passing the ctx into syncPod may have something other to do as described in comments, so I tried to fix as comments advised.
cc