Change:
While trying to enable API Server Tracing while using k3s, the API Server component crashes. The only way to avoid this crash is to disable egress selectors in k3s entirely. I originally opened an issue in the k3s repo (see ), and we determined that the issue seems to be an assumption that a dialer will always be returned if there's no error, but in this case there's no grpc egress configured so is returning and causing a panic at API Server Tracing to function properly. Please see the linked k3s issue for a repro case; However, I believe the minimal setup for this would be to configure cluster egress without defining a grpc egress selector and then attempting to enable API server tracing. panic stack trace -- contents of egress selector config file in direct mode- contents of egress selector config in 'agent' mode - contents of api server tracing config file -- <details25.2+k3s1 </details<detailsDigital Ocean Droplet </details<details</details<details</details<details</details<details</details
/sig instrumentation
/assign /triage accepted Thanks

Discussion:
if err != nil { return err } <del> otelDialer := func(ctx context.Context, addr string) (net.Conn, error) { return egressDialer(ctx, "tcp", addr) </del> <ins> if egressDialer != nil { otelDialer := func(ctx context.Context, addr string) (net.Conn, error) { return egressDialer(ctx, "tcp", addr) } opts = append(opts, otlptracegrpc.WithDialOption(grpc.WithContextDialer(otelDialer))) </ins> } <del> opts = append(opts, otlptracegrpc.WithDialOption(grpc.WithContextDialer(otelDialer))) </del> } resourceOpts := []resource.Option{

) func TestAPIServerTracing(t *testing.T) { <del> defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.APIServerTracing, true)() </del> // Listen for traces from the API Server before starting it, so the // API Server will successfully connect right away during the test. listener, err := net.Listen("tcp", "localhost:") if err != nil { t.Fatal(err) } <ins> // Write the configuration for tracing to a file tracingConfigFile, err := os.CreateTemp("", "tracing-config.yaml") if err != nil { t.Fatal(err) } defer os.Remove(tracingConfigFile.Name()) </ins> <del> traceFound := make(chan struct{}) defer close(traceFound) srv := grpc.NewServer() traceservice.RegisterTraceServiceServer(srv, &traceServer{ traceFound: traceFound, filterFunc: containsNodeListSpan}) </del> <ins> if err := os.WriteFile(tracingConfigFile.Name(), []byte(fmt.Sprintf(` apiVersion: apiserver.config.k8s.io/v1alpha1 kind: TracingConfiguration endpoint: %s`, listener.Addr().String())), os.FileMode(0755)); err != nil { t.Fatal(err) } testAPIServerTracing(t, listener, []string{"--tracing-config-file=" + tracingConfigFile.Name()}, ) } </ins> <del> go srv.Serve(listener) defer srv.Stop() </del> <ins> func TestAPIServerTracingWithEgressSelector(t *testing.T) { // Listen for traces from the API Server before starting it, so the // API Server will successfully connect right away during the test. listener, err := net.Listen("tcp", "localhost:") if err != nil { t.Fatal(err) } // Use an egress selector which doesn't have a controlplane config to ensure // tracing works in that context. Write the egress selector configuration to a file. egressSelectorConfigFile, err := os.CreateTemp("", "egress_selector_configuration.yaml") if err != nil { t.Fatal(err) } defer os.Remove(egressSelectorConfigFile.Name()) if err := os.WriteFile(egressSelectorConfigFile.Name(), []byte(` apiVersion: apiserver.config.k8s.io/v1beta1 kind: EgressSelectorConfiguration egressSelections: - name: cluster connection: proxyProtocol: Direct transport:`), os.FileMode(0755)); err != nil { t.Fatal(err) } </ins> // Write the configuration for tracing to a file tracingConfigFile, err := os.CreateTemp("", "tracing-config.yaml")

endpoint: %s`, listener.Addr().String())), os.FileMode(0755)); err != nil { t.Fatal(err) } <ins> testAPIServerTracing(t, listener, []string{ "--tracing-config-file=" + tracingConfigFile.Name(), "--egress-selector-config-file=" + egressSelectorConfigFile.Name(), }, ) } func testAPIServerTracing(t *testing.T, listener net.Listener, apiserverArgs []string) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, genericfeatures.APIServerTracing, true)() traceFound := make(chan struct{}) defer close(traceFound) srv := grpc.NewServer() traceservice.RegisterTraceServiceServer(srv, &traceServer{ traceFound: traceFound, filterFunc: containsNodeListSpan}) go srv.Serve(listener) defer srv.Stop() </ins> // Start the API Server with our tracing configuration testServer := kubeapiservertesting.StartTestServerOrDie(t, kubeapiservertesting.NewDefaultTestServerOptions(), <del> []string{"--tracing-config-file=" + tracingConfigFile.Name()}, </del> <ins> apiserverArgs, </ins> framework.SharedEtcd(), ) defer testServer.TearDownFn()