Change:
spans are not being parented to other spans - propagation of context seems broken. More context: spans should be parented to other spans enable kubelet tracing at v1.28 and hook up a tracing backend. No response <detailsv1.28 at least </details<detailsNot sure this matters. </details<details</details<details</details<details</details<details</details
/sig instrumentation /assign
/triage accepted
!
repro steps:
/assign Thanks for helping to look into this! Feel free to post the findings here you shared with me on slack
GitHub didn't allow me to assign the following users: fatsheep9146. Note that only with read permissions, repo collaborators and people who have commented on this issue/PR can be assigned. Additionally, issues/PRs can only have 10 assignees at the same time. For more information please see <detailsIn response to : /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Thanks! I will post this ASAP
The broken issue is due to the pr , which try to propagate context brought by pod worker to syncPod function in order to let the syncPod can be interrupted in time by other asynchronous routines, for example eviction. But in the line we did not pass the context which contain the parent span and trace into function, so the trace is broken. So I want to fix this by passing the into , which is also desired in comment But passing the ctx into syncPod may have something other to do as described in comments, so I tried to fix as comments advised.
cc

Discussion:
} } <del> // TODO(#113606): connect this with the incoming context parameter, which comes from the pod worker. // Currently, using that context causes test failures. To remove this todoCtx, any wait.Interrupted // errors need to be filtered from result and bypass the reasonCache - cancelling the context for // SyncPod is a known and deliberate error, not a generic error. todoCtx := context.TODO() </del> <ins> // TODO(#113606): use cancellation from the incoming context parameter, which comes from the pod worker. // Currently, using cancellation from that context causes test failures. To remove this WithoutCancel, // any wait.Interrupted errors need to be filtered from result and bypass the reasonCache - cancelling // the context for SyncPod is a known and deliberate error, not a generic error. // Use WithoutCancel instead of a new context.TODO() to propagate trace context </ins> // Call the container runtime's SyncPod callback <del> result := kl.containerRuntime.SyncPod(todoCtx, pod, podStatus, pullSecrets, kl.backOff) </del> <ins> sctx := context.WithoutCancel(ctx) result := kl.containerRuntime.SyncPod(sctx, pod, podStatus, pullSecrets, kl.backOff) </ins> kl.reasonCache.Update(pod.UID, result) if err := result.Error(); err != nil { // Do not return error if the only failures were pods in backoff

"testing" "time" <ins> sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/sdk/trace/tracetest" </ins> oteltrace "go.opentelemetry.io/otel/trace" "github.com/golang/mock/gomock"

fakeremote "k8s.io/kubernetes/pkg/kubelet/cri/remote/fake" "k8s.io/kubernetes/pkg/kubelet/eviction" "k8s.io/kubernetes/pkg/kubelet/images" <ins> "k8s.io/kubernetes/pkg/kubelet/kuberuntime" </ins> "k8s.io/kubernetes/pkg/kubelet/lifecycle" "k8s.io/kubernetes/pkg/kubelet/logs" "k8s.io/kubernetes/pkg/kubelet/network/dns"

return fakeRuntime, endpoint } <del> func createRemoteRuntimeService(endpoint string, t *testing.T) internalapi.RuntimeService { runtimeService, err := remote.NewRemoteRuntimeService(endpoint, 15*time.Second, oteltrace.NewNoopTracerProvider()) </del> <ins> func createRemoteRuntimeService(endpoint string, t *testing.T, tp oteltrace.TracerProvider) internalapi.RuntimeService { runtimeService, err := remote.NewRemoteRuntimeService(endpoint, 15*time.Second, tp) </ins> require.NoError(t, err) return runtimeService }

fakeRuntime.Stop() }() fakeRecorder := &record.FakeRecorder{} <del> rtSvc := createRemoteRuntimeService(endpoint, t) </del> <ins> rtSvc := createRemoteRuntimeService(endpoint, t, oteltrace.NewNoopTracerProvider()) </ins> kubeDep := &Dependencies{ Auth:                 nil, CAdvisorInterface:    cadvisor,

assert.Nil(t, testMainKubelet.configMapManager, "configmap manager should be nil if kubelet is in standalone mode") assert.Nil(t, testMainKubelet.secretManager, "secret manager should be nil if kubelet is in standalone mode") } <ins> func TestSyncPodSpans(t *testing.T) { testKubelet := newTestKubelet(t, false) kubelet := testKubelet.kubelet recorder := record.NewFakeRecorder(20) nodeRef := &v1.ObjectReference{ Kind:      "Node", Name:      "testNode", UID:       types.UID("testNode"), Namespace: "", } kubelet.dnsConfigurer = dns.NewConfigurer(recorder, nodeRef, nil, nil, "TEST", "") kubeCfg := &kubeletconfiginternal.KubeletConfiguration{ SyncFrequency: metav1.Duration{Duration: time.Minute}, ConfigMapAndSecretChangeDetectionStrategy: kubeletconfiginternal.WatchChangeDetectionStrategy, ContainerLogMaxSize:                       "10Mi", ContainerLogMaxFiles:                      5, MemoryThrottlingFactor:                    utilpointer.Float64(0), } exp := tracetest.NewInMemoryExporter() tp := sdktrace.NewTracerProvider( sdktrace.WithSyncer(exp), ) kubelet.tracer = tp.Tracer(instrumentationScope) fakeRuntime, endpoint := createAndStartFakeRemoteRuntime(t) defer func() { fakeRuntime.Stop() }() runtimeSvc := createRemoteRuntimeService(endpoint, t, tp) kubelet.runtimeService = runtimeSvc fakeRuntime.ImageService.SetFakeImageSize(100) fakeRuntime.ImageService.SetFakeImages([]string{"test:latest"}) imageSvc, err := remote.NewRemoteImageService(endpoint, 15*time.Second, tp) assert.NoError(t, err) kubelet.containerRuntime, err = kuberuntime.NewKubeGenericRuntimeManager( kubelet.recorder, kubelet.livenessManager, kubelet.readinessManager, kubelet.startupManager, kubelet.rootDirectory, kubelet.machineInfo, kubelet.podWorkers, kubelet.os, kubelet, nil, kubelet.backOff, kubeCfg.SerializeImagePulls, kubeCfg.MaxParallelImagePulls, float32(kubeCfg.RegistryPullQPS), int(kubeCfg.RegistryBurst), "", "", kubeCfg.CPUCFSQuota, kubeCfg.CPUCFSQuotaPeriod, runtimeSvc, imageSvc, kubelet.containerManager, kubelet.containerLogManager, kubelet.runtimeClassManager, false, kubeCfg.MemorySwap.SwapBehavior, kubelet.containerManager.GetNodeAllocatableAbsolute, *kubeCfg.MemoryThrottlingFactor, kubeletutil.NewPodStartupLatencyTracker(), tp, ) assert.NoError(t, err) pod := podWithUIDNameNsSpec("12345678", "foo", "new", v1.PodSpec{ Containers: []v1.Container{ { Name:            "bar", Image:           "test:latest", ImagePullPolicy: v1.PullAlways, }, }, EnableServiceLinks: utilpointer.Bool(false), }) _, err = kubelet.SyncPod(context.Background(), kubetypes.SyncPodCreate, pod, nil, &kubecontainer.PodStatus{}) require.NoError(t, err) require.NoError(t, err) assert.NotEmpty(t, exp.GetSpans()) // find root span for syncPod var rootSpan *tracetest.SpanStub spans := exp.GetSpans() for i, span := range spans { if span.Name == "syncPod" { rootSpan = &spans[i] break } } assert.NotNil(t, rootSpan) imageServiceSpans := make([]tracetest.SpanStub, 0) runtimeServiceSpans := make([]tracetest.SpanStub, 0) for _, span := range exp.GetSpans() { if span.SpanContext.TraceID() == rootSpan.SpanContext.TraceID() { switch { case strings.HasPrefix(span.Name, "runtime.v1.ImageService"): imageServiceSpans = append(imageServiceSpans, span) case strings.HasPrefix(span.Name, "runtime.v1.RuntimeService"): runtimeServiceSpans = append(runtimeServiceSpans, span) } } } assert.NotEmpty(t, imageServiceSpans, "syncPod trace should have image service spans") assert.NotEmpty(t, runtimeServiceSpans, "syncPod trace should have runtime service spans") for _, span := range imageServiceSpans { assert.Equal(t, span.Parent.SpanID(), rootSpan.SpanContext.SpanID(), fmt.Sprintf("image service span %s %s should be child of root span", span.Name, span.Parent.SpanID())) } for _, span := range runtimeServiceSpans { assert.Equal(t, span.Parent.SpanID(), rootSpan.SpanContext.SpanID(), fmt.Sprintf("runtime service span %s %s should be child of root span", span.Name, span.Parent.SpanID())) } } </ins>